ScoreStringEvalChain Class — langchain Architecture

Source Code

libs/langchain/langchain_classic/evaluation/scoring/eval_chain.py lines 150–411

class ScoreStringEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """A chain for scoring on a scale of 1-10 the output of a model.

    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.

    Example:
        >>> from langchain_openai import ChatOpenAI
        >>> from langchain_classic.evaluation.scoring import ScoreStringEvalChain
        >>> model = ChatOpenAI(temperature=0, model_name="gpt-4")
        >>> chain = ScoreStringEvalChain.from_llm(llm=model)
        >>> result = chain.evaluate_strings(
        ...     input="What is the chemical formula for water?",
        ...     prediction="H2O",
        ...     reference="The chemical formula for water is H2O.",
        ... )
        >>> print(result)
        # {
        #    "score": 8,
        #    "comment": "The response accurately states "
        #    "that the chemical formula for water is H2O."
        #    "However, it does not provide an explanation of what the formula means."
        # }

    """

    output_key: str = "results"
    output_parser: BaseOutputParser = Field(
        default_factory=ScoreStringResultOutputParser,
    )
    normalize_by: float | None = None
    """The value to normalize the score by, if specified."""
    criterion_name: str
    """The name of the criterion being evaluated."""

    model_config = ConfigDict(
        extra="ignore",
    )

    @classmethod
    @override
    def is_lc_serializable(cls) -> bool:
        return False

    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.

        Returns:
            `True` if the chain requires a reference, `False` otherwise.

        """
        return False

    @property
    def requires_input(self) -> bool:
        """Return whether the chain requires an input.

        Returns:
            `True` if the chain requires an input, `False` otherwise.

        """
        return True

    @property
    def evaluation_name(self) -> str:
        """Get the name of the evaluation.

        Returns:
        -------
        str
            The name of the evaluation.
        """
        return f"score_string:{self.criterion_name}"

    @property
    def _skip_reference_warning(self) -> str:
        """Return the warning to show when reference is ignored.

        Returns:
            The warning to show when reference is ignored.