base.py — langchain Source File

Source Code

"""Evaluators for parsing strings."""

import json
import logging
from collections.abc import Callable
from operator import eq
from typing import Any, cast

from langchain_core.utils.json import parse_json_markdown
from typing_extensions import override

from langchain_classic.evaluation.schema import StringEvaluator

_logger = logging.getLogger(__name__)


class JsonValidityEvaluator(StringEvaluator):
    """Evaluate whether the prediction is valid JSON.

    This evaluator checks if the prediction is a valid JSON string. It does not
        require any input or reference.

    Attributes:
        requires_input: Whether this evaluator requires an input
            string. Always False.
        requires_reference: Whether this evaluator requires a
            reference string. Always False.
        evaluation_name: The name of the evaluation metric.
            Always "json".

    Examples:
        >>> evaluator = JsonValidityEvaluator()
        >>> prediction = '{"name": "John", "age": 30, "city": "New York"}'
        >>> evaluator.evaluate(prediction)
        {'score': 1}

        >>> prediction = '{"name": "John", "age": 30, "city": "New York",}'
        >>> evaluator.evaluate(prediction)
        {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
    """

    def __init__(self, **_: Any) -> None:
        """Initialize the JsonValidityEvaluator."""
        super().__init__()

    @property
    @override
    def requires_input(self) -> bool:
        return False

    @property
    @override
    def requires_reference(self) -> bool:
        return False

    @property
    @override
    def evaluation_name(self) -> str:
        return "json_validity"

// ... (122 more lines)