test_character_text_splitter_discard_separator_regex() — langchain Function Reference

Source Code

libs/text-splitters/tests/unit_tests/test_text_splitters.py lines 221–239

def test_character_text_splitter_discard_separator_regex(
    *, separator: str, is_separator_regex: bool
) -> None:
    """Test CharacterTextSplitter discard separator regex.

    Test splitting by characters discarding the separator
    that is a regex special character.
    """
    text = "foo.bar.baz.123"
    splitter = CharacterTextSplitter(
        separator=separator,
        chunk_size=1,
        chunk_overlap=0,
        keep_separator=False,
        is_separator_regex=is_separator_regex,
    )
    output = splitter.split_text(text)
    expected_output = ["foo", "bar", "baz", "123"]
    assert output == expected_output