PairwiseStringEvalChain Class — langchain Architecture

Source Code

libs/langchain/langchain_classic/evaluation/comparison/eval_chain.py lines 156–408

class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
    r"""Pairwise String Evaluation Chain.

    A chain for comparing two outputs, such as the outputs
     of two models, prompts, or outputs of a single model on similar inputs.

    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.

    Example:
        >>> from langchain_openai import ChatOpenAI
        >>> from langchain_classic.evaluation.comparison import PairwiseStringEvalChain
        >>> model = ChatOpenAI(
        ...     temperature=0, model_name="gpt-4", model_kwargs={"random_seed": 42}
        ... )
        >>> chain = PairwiseStringEvalChain.from_llm(llm=model)
        >>> result = chain.evaluate_string_pairs(
        ...     input = "What is the chemical formula for water?",
        ...     prediction = "H2O",
        ...     prediction_b = (
        ...        "The chemical formula for water is H2O, which means"
        ...        " there are two hydrogen atoms and one oxygen atom."
        ...     reference = "The chemical formula for water is H2O.",
        ... )
        >>> print(result)
        # {
        #    "value": "B",
        #    "comment": "Both responses accurately state"
        #       " that the chemical formula for water is H2O."
        #       " However, Response B provides additional information"
        # .     " by explaining what the formula means.\n[[B]]"
        # }

    """

    output_key: str = "results"
    output_parser: BaseOutputParser = Field(
        default_factory=PairwiseStringResultOutputParser,
    )

    @classmethod
    @override
    def is_lc_serializable(cls) -> bool:
        return False

    model_config = ConfigDict(
        extra="ignore",
    )

    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.

        Returns:
            `True` if the chain requires a reference, `False` otherwise.

        """
        return False

    @property
    def requires_input(self) -> bool:
        """Return whether the chain requires an input.

        Returns:
            `True` if the chain requires an input, `False` otherwise.

        """
        return True

    @property
    def _skip_reference_warning(self) -> str:
        """Return the warning to show when reference is ignored.

        Returns:
            The warning to show when reference is ignored.

        """
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use a reference, use the LabeledPairwiseStringEvalChain"
            " (EvaluatorType.LABELED_PAIRWISE_STRING) instead."