from_run_and_data_type() — langchain Function Reference

Source Code

libs/langchain/langchain_classic/smith/evaluation/string_run_evaluator.py lines 406–477

    def from_run_and_data_type(
        cls,
        evaluator: StringEvaluator,
        run_type: str,
        data_type: DataType,
        input_key: str | None = None,
        prediction_key: str | None = None,
        reference_key: str | None = None,
        tags: list[str] | None = None,
    ) -> StringRunEvaluatorChain:
        """Create a StringRunEvaluatorChain.

        Create a StringRunEvaluatorChain from an evaluator and the run and dataset
        types.

        This method provides an easy way to instantiate a StringRunEvaluatorChain, by
        taking an evaluator and information about the type of run and the data.
        The method supports LLM and chain runs.

        Args:
            evaluator: The string evaluator to use.
            run_type: The type of run being evaluated.
                Supported types are LLM and Chain.
            data_type: The type of dataset used in the run.
            input_key: The key used to map the input from the run.
            prediction_key: The key used to map the prediction from the run.
            reference_key: The key used to map the reference from the dataset.
            tags: List of tags to attach to the evaluation chain.

        Returns:
            The instantiated evaluation chain.

        Raises:
            ValueError: If the run type is not supported, or if the evaluator requires a
                reference from the dataset but the reference key is not provided.

        """
        # Configure how run inputs/predictions are passed to the evaluator
        if run_type == "llm":
            run_mapper: StringRunMapper = LLMStringRunMapper()
        elif run_type == "chain":
            run_mapper = ChainStringRunMapper(
                input_key=input_key,
                prediction_key=prediction_key,
            )
        else:
            msg = f"Unsupported run type {run_type}. Expected one of 'llm' or 'chain'."
            raise ValueError(msg)

        # Configure how example rows are fed as a reference string to the evaluator
        if (
            reference_key is not None
            or data_type in (DataType.llm, DataType.chat)
            or evaluator.requires_reference
        ):
            example_mapper = StringExampleMapper(reference_key=reference_key)
        elif evaluator.requires_reference:
            msg = (  # type: ignore[unreachable]
                f"Evaluator {evaluator.evaluation_name} requires a reference"
                " example from the dataset. Please specify the reference key from"
                " amongst the dataset outputs keys."
            )
            raise ValueError(msg)
        else:
            example_mapper = None
        return cls(
            name=evaluator.evaluation_name,
            run_mapper=run_mapper,
            example_mapper=example_mapper,
            string_evaluator=evaluator,
            tags=tags,
        )