test_md_header_text_splitter_with_custom_headers() — langchain Function Reference

Source Code

libs/text-splitters/tests/unit_tests/test_text_splitters.py lines 1556–1609

def test_md_header_text_splitter_with_custom_headers() -> None:
    """Test markdown splitter with custom header patterns like **Header**."""
    markdown_document = """**Chapter 1**

This is the content for chapter 1.

***Section 1.1***

This is the content for section 1.1.

**Chapter 2**

This is the content for chapter 2.

***Section 2.1***

This is the content for section 2.1.
"""

    headers_to_split_on = [
        ("**", "Bold Header"),
        ("***", "Bold Italic Header"),
    ]

    custom_header_patterns = {
        "**": 1,  # Level 1 headers
        "***": 2,  # Level 2 headers
    }
    markdown_splitter = MarkdownHeaderTextSplitter(
        headers_to_split_on=headers_to_split_on,
        custom_header_patterns=custom_header_patterns,
    )
    output = markdown_splitter.split_text(markdown_document)

    expected_output = [
        Document(
            page_content="This is the content for chapter 1.",
            metadata={"Bold Header": "Chapter 1"},
        ),
        Document(
            page_content="This is the content for section 1.1.",
            metadata={"Bold Header": "Chapter 1", "Bold Italic Header": "Section 1.1"},
        ),
        Document(
            page_content="This is the content for chapter 2.",
            metadata={"Bold Header": "Chapter 2"},
        ),
        Document(
            page_content="This is the content for section 2.1.",
            metadata={"Bold Header": "Chapter 2", "Bold Italic Header": "Section 2.1"},
        ),
    ]

    assert output == expected_output