HTMLSemanticPreservingSplitter Class — langchain Architecture

Source Code

libs/text-splitters/langchain_text_splitters/html.py lines 557–1060

class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
    """Split HTML content preserving semantic structure.

    Splits HTML content by headers into generalized chunks, preserving semantic
    structure. If chunks exceed the maximum chunk size, it uses
    `RecursiveCharacterTextSplitter` for further splitting.

    The splitter preserves full HTML elements and converts links to Markdown-like links.
    It can also preserve images, videos, and audio elements by converting them into
    Markdown format. Note that some chunks may exceed the maximum size to maintain
    semantic integrity.

    !!! version-added "Added in `langchain-text-splitters` 0.3.5"

    Example:
        ```python
        from langchain_text_splitters.html import HTMLSemanticPreservingSplitter

        def custom_iframe_extractor(iframe_tag):
            ```
            Custom handler function to extract the 'src' attribute from an <iframe> tag.
            Converts the iframe to a Markdown-like link: [iframe:<src>](src).

            Args:
                iframe_tag (bs4.element.Tag): The <iframe> tag to be processed.

            Returns:
                str: A formatted string representing the iframe in Markdown-like format.
            ```
            iframe_src = iframe_tag.get('src', '')
            return f"[iframe:{iframe_src}]({iframe_src})"

        text_splitter = HTMLSemanticPreservingSplitter(
            headers_to_split_on=[("h1", "Header 1"), ("h2", "Header 2")],
            max_chunk_size=500,
            preserve_links=True,
            preserve_images=True,
            custom_handlers={"iframe": custom_iframe_extractor}
        )
        ```
    """  # noqa: D214

    def __init__(
        self,
        headers_to_split_on: list[tuple[str, str]],
        *,
        max_chunk_size: int = 1000,
        chunk_overlap: int = 0,
        separators: list[str] | None = None,
        elements_to_preserve: list[str] | None = None,
        preserve_links: bool = False,
        preserve_images: bool = False,
        preserve_videos: bool = False,
        preserve_audio: bool = False,
        custom_handlers: dict[str, Callable[[Tag], str]] | None = None,
        stopword_removal: bool = False,
        stopword_lang: str = "english",
        normalize_text: bool = False,
        external_metadata: dict[str, str] | None = None,
        allowlist_tags: list[str] | None = None,
        denylist_tags: list[str] | None = None,
        preserve_parent_metadata: bool = False,
        keep_separator: bool | Literal["start", "end"] = True,
    ) -> None:
        """Initialize splitter.

        Args:
            headers_to_split_on: HTML headers (e.g., `h1`, `h2`) that define content
                sections.
            max_chunk_size: Maximum size for each chunk, with allowance for exceeding
                this limit to preserve semantics.
            chunk_overlap: Number of characters to overlap between chunks to ensure
                contextual continuity.
            separators: Delimiters used by `RecursiveCharacterTextSplitter` for
                further splitting.
            elements_to_preserve: HTML tags (e.g., `table`, `ul`) to remain
                intact during splitting.
            preserve_links: Converts `a` tags to Markdown links (`[text](url)`).
            preserve_images: Converts `img` tags to Markdown images (`![alt](src)`).
            preserve_videos: Converts `video` tags to Markdown video links
                (`![video](src)`).