test_character_text_splitter_keep_separator_regex() — langchain Function Reference

Source Code

libs/text-splitters/tests/unit_tests/test_text_splitters.py lines 149–167

def test_character_text_splitter_keep_separator_regex(
    *, separator: str, is_separator_regex: bool
) -> None:
    """Test CharacterTextSplitter keep separator regex.

    Test splitting by characters while keeping the separator
    that is a regex special character.
    """
    text = "foo.bar.baz.123"
    splitter = CharacterTextSplitter(
        separator=separator,
        chunk_size=1,
        chunk_overlap=0,
        keep_separator=True,
        is_separator_regex=is_separator_regex,
    )
    output = splitter.split_text(text)
    expected_output = ["foo", ".bar", ".baz", ".123"]
    assert output == expected_output