crawler.py — langchain Source File

Source Code

import logging
import time
from sys import platform
from typing import (
    TYPE_CHECKING,
    Any,
    TypedDict,
)

if TYPE_CHECKING:
    from playwright.sync_api import Browser, CDPSession, Page

logger = logging.getLogger(__name__)

black_listed_elements: set[str] = {
    "html",
    "head",
    "title",
    "meta",
    "iframe",
    "body",
    "script",
    "style",
    "path",
    "svg",
    "br",
    "::marker",
}


class ElementInViewPort(TypedDict):
    """A typed dictionary containing information about elements in the viewport."""

    node_index: str
    backend_node_id: int
    node_name: str | None
    node_value: str | None
    node_meta: list[str]
    is_clickable: bool
    origin_x: int
    origin_y: int
    center_x: int
    center_y: int


class Crawler:
    """A crawler for web pages.

    **Security Note**: This is an implementation of a crawler that uses a browser via
        Playwright.

        This crawler can be used to load arbitrary webpages INCLUDING content
        from the local file system.

        Control access to who can submit crawling requests and what network access
        the crawler has.

        Make sure to scope permissions to the minimal permissions necessary for
        the application.

// ... (420 more lines)