base.py — langchain Source File

Source Code

"""Abstract interface for document loader implementations."""

from __future__ import annotations

from abc import ABC, abstractmethod
from typing import TYPE_CHECKING

from langchain_core.runnables import run_in_executor

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Iterator

    from langchain_text_splitters import TextSplitter

    from langchain_core.documents import Document
    from langchain_core.documents.base import Blob

try:
    from langchain_text_splitters import RecursiveCharacterTextSplitter

    _HAS_TEXT_SPLITTERS = True
except ImportError:
    _HAS_TEXT_SPLITTERS = False


class BaseLoader(ABC):  # noqa: B024
    """Interface for document loader.

    Implementations should implement the lazy-loading method using generators to avoid
    loading all documents into memory at once.

    `load` is provided just for user convenience and should not be overridden.
    """

    # Sub-classes should not implement this method directly. Instead, they
    # should implement the lazy load method.
    def load(self) -> list[Document]:
        """Load data into `Document` objects.

        Returns:
            The documents.
        """
        return list(self.lazy_load())

    async def aload(self) -> list[Document]:
        """Load data into `Document` objects.

        Returns:
            The documents.
        """
        return [document async for document in self.alazy_load()]

    def load_and_split(
        self, text_splitter: TextSplitter | None = None
    ) -> list[Document]:
        """Load `Document` and split into chunks. Chunks are returned as `Document`.

        !!! danger

            Do not override this method. It should be considered to be deprecated!
// ... (96 more lines)