TestResult Class — langchain Architecture

Source Code

libs/langchain/langchain_classic/smith/evaluation/runner_utils.py lines 80–149

class TestResult(dict):
    """A dictionary of the results of a single test run."""

    def get_aggregate_feedback(
        self,
    ) -> pd.DataFrame:
        """Return quantiles for the feedback scores.

        This method calculates and prints the quantiles for the feedback scores
        across all feedback keys.

        Returns:
            A DataFrame containing the quantiles for each feedback key.
        """
        df = self.to_dataframe()
        # Drop all things starting with inputs., outputs., and reference
        to_drop = [
            col
            for col in df.columns
            if col.startswith(("inputs.", "outputs.", "reference"))
            or col in {"input", "output"}
        ]
        return df.describe(include="all").drop(to_drop, axis=1)

    def to_dataframe(self) -> pd.DataFrame:
        """Convert the results to a dataframe."""
        try:
            import pandas as pd
        except ImportError as e:
            msg = (
                "Pandas is required to convert the results to a dataframe."
                " to install pandas, run `pip install pandas`."
            )
            raise ImportError(msg) from e

        indices = []
        records = []
        for example_id, result in self["results"].items():
            feedback = result["feedback"]
            output_ = result.get("output")
            if isinstance(output_, dict):
                output = {f"outputs.{k}": v for k, v in output_.items()}
            elif output_ is None:
                output = {}
            else:
                output = {"output": output_}

            r = {
                **{f"inputs.{k}": v for k, v in result["input"].items()},
                **output,
            }
            if "reference" in result:
                if isinstance(result["reference"], dict):
                    r.update(
                        {f"reference.{k}": v for k, v in result["reference"].items()},
                    )
                else:
                    r["reference"] = result["reference"]
            r.update(
                {
                    **{f"feedback.{f.key}": f.score for f in feedback},
                    "error": result.get("Error"),
                    "execution_time": result["execution_time"],
                    "run_id": result.get("run_id"),
                },
            )
            records.append(r)
            indices.append(example_id)

        return pd.DataFrame(records, index=indices)