test_nlp_text_splitters.py — langchain Source File

Source Code

"""Test text splitting functionality using NLTK and Spacy based sentence splitters."""

import re

import nltk
import pytest
from langchain_core.documents import Document

from langchain_text_splitters.nltk import NLTKTextSplitter
from langchain_text_splitters.spacy import SpacyTextSplitter


def setup_module() -> None:
    nltk.download("punkt_tab")


@pytest.fixture
def spacy() -> None:
    spacy = pytest.importorskip("spacy")

    # Check if en_core_web_sm model is available
    try:
        spacy.load("en_core_web_sm")
    except OSError:
        pytest.skip(
            "en_core_web_sm model not installed. Install with: "
            "uv add --group test_integration "
            "https://github.com/explosion/spacy-models/releases/download/"
            "en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"
        )


def test_nltk_text_splitting_args() -> None:
    """Test invalid arguments."""
    with pytest.raises(
        ValueError,
        match=re.escape(
            "Got a larger chunk overlap (4) than chunk size (2), should be smaller."
        ),
    ):
        NLTKTextSplitter(chunk_size=2, chunk_overlap=4)


@pytest.mark.usefixtures("spacy")
def test_spacy_text_splitting_args() -> None:
    """Test invalid arguments."""
    with pytest.raises(
        ValueError,
        match=re.escape(
            "Got a larger chunk overlap (4) than chunk size (2), should be smaller."
        ),
    ):
        SpacyTextSplitter(chunk_size=2, chunk_overlap=4)


def test_nltk_text_splitter() -> None:
    """Test splitting by sentence using NLTK."""
    text = "This is sentence one. And this is sentence two."
    separator = "|||"
    splitter = NLTKTextSplitter(separator=separator)
// ... (64 more lines)