test_pairwise_string_comparison_chain() — langchain Function Reference

Source Code

libs/langchain/tests/unit_tests/evaluation/scoring/test_eval_chain.py lines 41–63

def test_pairwise_string_comparison_chain() -> None:
    llm = FakeLLM(
        queries={
            "a": "This is a rather good answer. Rating: [[9]]",
            "b": "This is a rather bad answer. Rating: [[1]]",
        },
        sequential_responses=True,
    )
    chain = ScoreStringEvalChain.from_llm(llm=llm)
    res = chain.evaluate_strings(
        prediction="I like pie.",
        input="What is your favorite food?",
    )
    assert res["score"] == 9
    assert res["reasoning"] == "This is a rather good answer. Rating: [[9]]"
    with pytest.warns(UserWarning, match=re.escape(chain._skip_reference_warning)):
        res = chain.evaluate_strings(
            prediction="I like pie.",
            input="What is your favorite food?",
            reference="I enjoy pie.",
        )
    assert res["score"] == 1
    assert res["reasoning"] == "This is a rather bad answer. Rating: [[1]]"