Crawler Class — langchain Architecture

Source Code

libs/langchain/langchain_classic/chains/natbot/crawler.py lines 46–479

class Crawler:
    """A crawler for web pages.

    **Security Note**: This is an implementation of a crawler that uses a browser via
        Playwright.

        This crawler can be used to load arbitrary webpages INCLUDING content
        from the local file system.

        Control access to who can submit crawling requests and what network access
        the crawler has.

        Make sure to scope permissions to the minimal permissions necessary for
        the application.

        See https://docs.langchain.com/oss/python/security-policy for more information.
    """

    def __init__(self) -> None:
        """Initialize the crawler."""
        try:
            from playwright.sync_api import sync_playwright
        except ImportError as e:
            msg = (
                "Could not import playwright python package. "
                "Please install it with `pip install playwright`."
            )
            raise ImportError(msg) from e
        self.browser: Browser = (
            sync_playwright().start().chromium.launch(headless=False)
        )
        self.page: Page = self.browser.new_page()
        self.page.set_viewport_size({"width": 1280, "height": 1080})
        self.page_element_buffer: dict[int, ElementInViewPort]
        self.client: CDPSession

    def go_to_page(self, url: str) -> None:
        """Navigate to the given URL.

        Args:
            url: The URL to navigate to. If it does not contain a scheme, it will be
                prefixed with "http://".
        """
        self.page.goto(url=url if "://" in url else "http://" + url)
        self.client = self.page.context.new_cdp_session(self.page)
        self.page_element_buffer = {}

    def scroll(self, direction: str) -> None:
        """Scroll the page in the given direction.

        Args:
            direction: The direction to scroll in, either "up" or "down".
        """
        if direction == "up":
            self.page.evaluate(
                "(document.scrollingElement || document.body).scrollTop = "
                "(document.scrollingElement || document.body).scrollTop - "
                "window.innerHeight;"
            )
        elif direction == "down":
            self.page.evaluate(
                "(document.scrollingElement || document.body).scrollTop = "
                "(document.scrollingElement || document.body).scrollTop + "
                "window.innerHeight;"
            )

    def click(self, id_: str | int) -> None:
        """Click on an element with the given id.

        Args:
            id_: The id of the element to click on.
        """
        # Inject javascript into the page which removes the target= attribute from links
        js = """
		links = document.getElementsByTagName("a");
		for (var i = 0; i < links.length; i++) {
			links[i].removeAttribute("target");
		}
		"""
        self.page.evaluate(js)