test_character_text_splitter_keep_separator_regex_end() — langchain Function Reference

Source Code

libs/text-splitters/tests/unit_tests/test_text_splitters.py lines 197–215

def test_character_text_splitter_keep_separator_regex_end(
    *, separator: str, is_separator_regex: bool
) -> None:
    """Test CharacterTextSplitter keep separator regex and put at end.

    Test splitting by characters while keeping the separator
    that is a regex special character and placing it at the end of each chunk.
    """
    text = "foo.bar.baz.123"
    splitter = CharacterTextSplitter(
        separator=separator,
        chunk_size=1,
        chunk_overlap=0,
        keep_separator="end",
        is_separator_regex=is_separator_regex,
    )
    output = splitter.split_text(text)
    expected_output = ["foo.", "bar.", "baz.", "123"]
    assert output == expected_output