parent_document_retriever.py — langchain Source File

Source Code

import uuid
from collections.abc import Sequence
from typing import Any

from langchain_core.documents import Document
from langchain_text_splitters import TextSplitter

from langchain_classic.retrievers import MultiVectorRetriever


class ParentDocumentRetriever(MultiVectorRetriever):
    """Retrieve small chunks then retrieve their parent documents.

    When splitting documents for retrieval, there are often conflicting desires:

    1. You may want to have small documents, so that their embeddings can most
        accurately reflect their meaning. If too long, then the embeddings can
        lose meaning.
    2. You want to have long enough documents that the context of each chunk is
        retained.

    The ParentDocumentRetriever strikes that balance by splitting and storing
    small chunks of data. During retrieval, it first fetches the small chunks
    but then looks up the parent IDs for those chunks and returns those larger
    documents.

    Note that "parent document" refers to the document that a small chunk
    originated from. This can either be the whole raw document OR a larger
    chunk.

    Examples:
        ```python
        from langchain_chroma import Chroma
        from langchain_community.embeddings import OpenAIEmbeddings
        from langchain_text_splitters import RecursiveCharacterTextSplitter
        from langchain_classic.storage import InMemoryStore

        # This text splitter is used to create the parent documents
        parent_splitter = RecursiveCharacterTextSplitter(
            chunk_size=2000, add_start_index=True
        )
        # This text splitter is used to create the child documents
        # It should create documents smaller than the parent
        child_splitter = RecursiveCharacterTextSplitter(
            chunk_size=400, add_start_index=True
        )
        # The VectorStore to use to index the child chunks
        vectorstore = Chroma(embedding_function=OpenAIEmbeddings())
        # The storage layer for the parent documents
        store = InMemoryStore()

        # Initialize the retriever
        retriever = ParentDocumentRetriever(
            vectorstore=vectorstore,
            docstore=store,
            child_splitter=child_splitter,
            parent_splitter=parent_splitter,
        )
        ```
    """
// ... (117 more lines)