reduce.py — langchain Source File

Source Code

"""Combine many documents together by recursively reducing them."""

from __future__ import annotations

from collections.abc import Callable
from typing import Any, Protocol

from langchain_core._api import deprecated
from langchain_core.callbacks import Callbacks
from langchain_core.documents import Document
from pydantic import ConfigDict

from langchain_classic.chains.combine_documents.base import BaseCombineDocumentsChain


class CombineDocsProtocol(Protocol):
    """Interface for the combine_docs method."""

    def __call__(self, docs: list[Document], **kwargs: Any) -> str:
        """Interface for the combine_docs method."""


class AsyncCombineDocsProtocol(Protocol):
    """Interface for the combine_docs method."""

    async def __call__(self, docs: list[Document], **kwargs: Any) -> str:
        """Async interface for the combine_docs method."""


def split_list_of_docs(
    docs: list[Document],
    length_func: Callable,
    token_max: int,
    **kwargs: Any,
) -> list[list[Document]]:
    """Split `Document` objects to subsets that each meet a cumulative len. constraint.

    Args:
        docs: The full list of `Document` objects.
        length_func: Function for computing the cumulative length of a set of `Document`
            objects.
        token_max: The maximum cumulative length of any subset of `Document` objects.
        **kwargs: Arbitrary additional keyword params to pass to each call of the
            `length_func`.

    Returns:
        A `list[list[Document]]`.
    """
    new_result_doc_list = []
    _sub_result_docs = []
    for doc in docs:
        _sub_result_docs.append(doc)
        _num_tokens = length_func(_sub_result_docs, **kwargs)
        if _num_tokens > token_max:
            if len(_sub_result_docs) == 1:
                msg = (
                    "A single document was longer than the context length,"
                    " we cannot handle this."
                )
                raise ValueError(msg)
// ... (330 more lines)