test_incremental_cleanup_with_different_batchsize() — langchain Function Reference

Source Code

libs/core/tests/unit_tests/indexing/test_indexing.py lines 2217–2264

def test_incremental_cleanup_with_different_batchsize(
    record_manager: InMemoryRecordManager, vector_store: VectorStore
) -> None:
    """Check that we can clean up with different batch size."""
    docs = [
        Document(
            page_content="This is a test document.",
            metadata={"source": str(d)},
        )
        for d in range(1000)
    ]

    assert index(
        docs,
        record_manager,
        vector_store,
        source_id_key="source",
        cleanup="incremental",
        key_encoder="sha256",
    ) == {
        "num_added": 1000,
        "num_deleted": 0,
        "num_skipped": 0,
        "num_updated": 0,
    }

    docs = [
        Document(
            page_content="Different doc",
            metadata={"source": str(d)},
        )
        for d in range(1001)
    ]

    assert index(
        docs,
        record_manager,
        vector_store,
        source_id_key="source",
        cleanup="incremental",
        cleanup_batch_size=17,
        key_encoder="sha256",
    ) == {
        "num_added": 1001,
        "num_deleted": 1000,
        "num_skipped": 0,
        "num_updated": 0,
    }