StringExampleMapper Class — langchain Architecture

Architecture documentation for the StringExampleMapper class in string_run_evaluator.py from the langchain codebase.

Class python

Entity Profile

Dependency Diagram

graph TD
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe["StringExampleMapper"]
  e3623fbf_23b3_a8fc_0b7c_43b53f33b606["Serializable"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|extends| e3623fbf_23b3_a8fc_0b7c_43b53f33b606
  2f0b23f2_7760_d68c_7feb_721c5231c4ec["string_run_evaluator.py"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|defined in| 2f0b23f2_7760_d68c_7feb_721c5231c4ec
  049c89ef_0ad2_3788_aeba_88ffacedb13a["output_keys()"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|method| 049c89ef_0ad2_3788_aeba_88ffacedb13a
  da073bde_7e9c_b610_6d77_18f3b0290349["serialize_chat_messages()"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|method| da073bde_7e9c_b610_6d77_18f3b0290349
  b15ccccb_1344_8d38_767a_922bd0d352ca["map()"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|method| b15ccccb_1344_8d38_767a_922bd0d352ca
  f56f231b_b6eb_b4a1_5171_9aa21c13ee17["__call__()"]
  e9537ec5_dd39_fde8_64f7_e16f805b8dbe -->|method| f56f231b_b6eb_b4a1_5171_9aa21c13ee17

Relationship Graph

Source Code

libs/langchain/langchain_classic/smith/evaluation/string_run_evaluator.py lines 225–272

class StringExampleMapper(Serializable):
    """Map an example, or row in the dataset, to the inputs of an evaluation."""

    reference_key: str | None = None

    @property
    def output_keys(self) -> list[str]:
        """The keys to extract from the run."""
        return ["reference"]

    def serialize_chat_messages(self, messages: list[dict]) -> str:
        """Extract the input messages from the run."""
        chat_messages = _get_messages_from_run_dict(messages)
        return get_buffer_string(chat_messages)

    def map(self, example: Example) -> dict[str, str]:
        """Maps the Example, or dataset row to a dictionary."""
        if not example.outputs:
            msg = f"Example {example.id} has no outputs to use as a reference."
            raise ValueError(msg)
        if self.reference_key is None:
            if len(example.outputs) > 1:
                msg = (
                    f"Example {example.id} has multiple outputs, so you must"
                    " specify a reference_key."
                )
                raise ValueError(msg)
            output = next(iter(example.outputs.values()))
        elif self.reference_key not in example.outputs:
            msg = (
                f"Example {example.id} does not have reference key"
                f" {self.reference_key}."
            )
            raise ValueError(msg)
        else:
            output = example.outputs[self.reference_key]
        return {
            "reference": self.serialize_chat_messages([output])
            if isinstance(output, dict) and output.get("type") and output.get("data")
            else output,
        }

    def __call__(self, example: Example) -> dict[str, str]:
        """Maps the Run and Example to a dictionary."""
        if not example.outputs:
            msg = f"Example {example.id} has no outputs to use as areference label."
            raise ValueError(msg)
        return self.map(example)