StringRunEvaluatorChain Class — langchain Architecture

Architecture documentation for the StringRunEvaluatorChain class in string_run_evaluator.py from the langchain codebase.

Class python

Entity Profile

Dependency Diagram

graph TD
  9be76341_f857_9929_c92b_0369bee6b002["StringRunEvaluatorChain"]
  097a4781_5519_0b5d_6244_98c64eadc0d6["Chain"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|extends| 097a4781_5519_0b5d_6244_98c64eadc0d6
  d4197ac3_b22a_ddb0_5b6e_314ddf206db2["string_run_evaluator.py"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|defined in| d4197ac3_b22a_ddb0_5b6e_314ddf206db2
  77097aae_573b_f8fc_2e52_1ee7fd635d01["input_keys()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 77097aae_573b_f8fc_2e52_1ee7fd635d01
  ef073e4c_3ca5_87b5_df9c_b86cadd75d9f["output_keys()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| ef073e4c_3ca5_87b5_df9c_b86cadd75d9f
  1e0c79ae_0af5_7d8e_e4a0_0394c74a2152["_prepare_input()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 1e0c79ae_0af5_7d8e_e4a0_0394c74a2152
  0faefd96_9864_af80_9290_22a7c49d8ac4["_prepare_output()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 0faefd96_9864_af80_9290_22a7c49d8ac4
  78d3b2b8_d186_f335_82f5_ff948ca8fe6a["_call()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 78d3b2b8_d186_f335_82f5_ff948ca8fe6a
  26cead46_d9e1_d011_de0a_4db5afa3f394["_acall()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 26cead46_d9e1_d011_de0a_4db5afa3f394
  028617a7_19c9_7ddb_8865_e082e52a7f32["_prepare_evaluator_output()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 028617a7_19c9_7ddb_8865_e082e52a7f32
  a5c90749_382d_3ec0_c965_098c740e580e["evaluate_run()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| a5c90749_382d_3ec0_c965_098c740e580e
  2b1cb497_2da6_2965_c0d8_1c7779cd0593["aevaluate_run()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| 2b1cb497_2da6_2965_c0d8_1c7779cd0593
  d4e3b139_7e98_4e28_b736_ff037e74259e["from_run_and_data_type()"]
  9be76341_f857_9929_c92b_0369bee6b002 -->|method| d4e3b139_7e98_4e28_b736_ff037e74259e

Relationship Graph

Source Code

libs/langchain/langchain_classic/smith/evaluation/string_run_evaluator.py lines 275–477

class StringRunEvaluatorChain(Chain, RunEvaluator):
    """Evaluate Run and optional examples."""

    run_mapper: StringRunMapper
    """Maps the Run to a dictionary with 'input' and 'prediction' strings."""
    example_mapper: StringExampleMapper | None = None
    """Maps the Example (dataset row) to a dictionary
    with a 'reference' string."""
    name: str
    """The name of the evaluation metric."""
    string_evaluator: StringEvaluator
    """The evaluation chain."""

    @property
    @override
    def input_keys(self) -> list[str]:
        return ["run", "example"]

    @property
    @override
    def output_keys(self) -> list[str]:
        return ["feedback"]

    def _prepare_input(self, inputs: dict[str, Any]) -> dict[str, str]:
        run: Run = inputs["run"]
        example: Example | None = inputs.get("example")
        evaluate_strings_inputs = self.run_mapper(run)
        if not self.string_evaluator.requires_input:
            # Hide warning about unused input
            evaluate_strings_inputs.pop("input", None)
        if example and self.example_mapper and self.string_evaluator.requires_reference:
            evaluate_strings_inputs.update(self.example_mapper(example))
        elif self.string_evaluator.requires_reference:
            msg = (
                f"Evaluator {self.name} requires an reference"
                " example from the dataset,"
                f" but none was provided for run {run.id}."
            )
            raise ValueError(msg)
        return evaluate_strings_inputs

    def _prepare_output(self, output: dict[str, Any]) -> dict[str, Any]:
        evaluation_result = EvaluationResult(
            key=self.name,
            comment=output.get("reasoning"),
            **output,
        )
        if RUN_KEY in output:
            # TODO: Not currently surfaced. Update
            evaluation_result.evaluator_info[RUN_KEY] = output[RUN_KEY]
        return {"feedback": evaluation_result}

    def _call(
        self,
        inputs: dict[str, str],
        run_manager: CallbackManagerForChainRun | None = None,
    ) -> dict[str, Any]:
        """Call the evaluation chain."""
        evaluate_strings_inputs = self._prepare_input(inputs)
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = self.string_evaluator.evaluate_strings(
            **evaluate_strings_inputs,
            callbacks=callbacks,
            include_run_info=True,
        )
        return self._prepare_output(chain_output)

    async def _acall(
        self,
        inputs: dict[str, str],
        run_manager: AsyncCallbackManagerForChainRun | None = None,
    ) -> dict[str, Any]:
        """Call the evaluation chain."""
        evaluate_strings_inputs = self._prepare_input(inputs)
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = await self.string_evaluator.aevaluate_strings(
            **evaluate_strings_inputs,
            callbacks=callbacks,
            include_run_info=True,