LabeledCriteriaEvalChain Class — langchain Architecture

Architecture documentation for the LabeledCriteriaEvalChain class in eval_chain.py from the langchain codebase.

Class python

Entity Profile

Dependency Diagram

graph TD
  530b536b_29e3_b2c0_9528_f38501e896c1["LabeledCriteriaEvalChain"]
  e476a8f9_5ced_15c1_3631_8a65948b94ed["CriteriaEvalChain"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|extends| e476a8f9_5ced_15c1_3631_8a65948b94ed
  8e14b60d_5880_d030_e540_4a171da7ff70["eval_chain.py"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|defined in| 8e14b60d_5880_d030_e540_4a171da7ff70
  c4e24174_d1b0_f8d5_887a_3f4c0659c7b5["is_lc_serializable()"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|method| c4e24174_d1b0_f8d5_887a_3f4c0659c7b5
  3720c93d_ebc1_1c13_4fc5_ff1d2e45f8b4["requires_reference()"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|method| 3720c93d_ebc1_1c13_4fc5_ff1d2e45f8b4
  d7b47090_5602_93cf_8322_76a617b649a4["_resolve_prompt()"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|method| d7b47090_5602_93cf_8322_76a617b649a4
  52373e3f_ed16_e930_78b1_4cfd309eb9b6["from_llm()"]
  530b536b_29e3_b2c0_9528_f38501e896c1 -->|method| 52373e3f_ed16_e930_78b1_4cfd309eb9b6

Relationship Graph

Source Code

libs/langchain/langchain_classic/evaluation/criteria/eval_chain.py lines 508–593

class LabeledCriteriaEvalChain(CriteriaEvalChain):
    """Criteria evaluation chain that requires references."""

    @classmethod
    @override
    def is_lc_serializable(cls) -> bool:
        return False

    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return True

    @classmethod
    def _resolve_prompt(
        cls,
        prompt: BasePromptTemplate | None = None,
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria", "reference"}
        prompt_ = prompt or PROMPT_WITH_REFERENCES
        if expected_input_vars != set(prompt_.input_variables):
            msg = (
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
            raise ValueError(msg)
        return prompt_

    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: CRITERIA_TYPE | None = None,
        *,
        prompt: BasePromptTemplate | None = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `LabeledCriteriaEvalChain` instance from an llm and criteria.

        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.

        Returns:
        -------
        LabeledCriteriaEvalChain
            An instance of the `LabeledCriteriaEvalChain` class.

        Examples:
        --------
        >>> from langchain_openai import OpenAI
        >>> from langchain_classic.evaluation.criteria import LabeledCriteriaEvalChain
        >>> model = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=model,
                criteria=criteria,
            )
        """
        prompt = cls._resolve_prompt(prompt)
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt.partial(criteria=criteria_str)
        return cls(