HuggingFaceEndpoint Class — langchain Architecture

Source Code

libs/partners/huggingface/langchain_huggingface/llms/huggingface_endpoint.py lines 29–455

class HuggingFaceEndpoint(LLM):
    """Hugging Face Endpoint. This works with any model that supports text generation (i.e. text completion) task.

    To use this class, you should have installed the `huggingface_hub` package, and
    the environment variable `HUGGINGFACEHUB_API_TOKEN` set with your API token,
    or given as a named parameter to the constructor.

    Example:
        ```python
        # Basic Example (no streaming)
        model = HuggingFaceEndpoint(
            endpoint_url="http://localhost:8010/",
            max_new_tokens=512,
            top_k=10,
            top_p=0.95,
            typical_p=0.95,
            temperature=0.01,
            repetition_penalty=1.03,
            huggingfacehub_api_token="my-api-key",
        )
        print(model.invoke("What is Deep Learning?"))

        # Streaming response example
        from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler

        callbacks = [StreamingStdOutCallbackHandler()]
        model = HuggingFaceEndpoint(
            endpoint_url="http://localhost:8010/",
            max_new_tokens=512,
            top_k=10,
            top_p=0.95,
            typical_p=0.95,
            temperature=0.01,
            repetition_penalty=1.03,
            callbacks=callbacks,
            streaming=True,
            huggingfacehub_api_token="my-api-key",
        )
        print(model.invoke("What is Deep Learning?"))

        # Basic Example (no streaming) with Mistral-Nemo-Base-2407 model using a third-party provider (Novita).
        model = HuggingFaceEndpoint(
            repo_id="mistralai/Mistral-Nemo-Base-2407",
            provider="novita",
            max_new_tokens=100,
            do_sample=False,
            huggingfacehub_api_token="my-api-key",
        )
        print(model.invoke("What is Deep Learning?"))
        ```
    """  # noqa: E501

    endpoint_url: str | None = None
    """Endpoint URL to use. If repo_id is not specified then this needs to given or
    should be pass as env variable in `HF_INFERENCE_ENDPOINT`"""

    repo_id: str | None = None
    """Repo to use. If endpoint_url is not specified then this needs to given"""

    provider: str | None = None
    """Name of the provider to use for inference with the model specified in `repo_id`.
        e.g. "cerebras". if not specified, Defaults to "auto" i.e. the first of the
        providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers.
        available providers can be found in the [huggingface_hub documentation](https://huggingface.co/docs/huggingface_hub/guides/inference#supported-providers-and-tasks)."""

    huggingfacehub_api_token: str | None = Field(
        default_factory=from_env("HUGGINGFACEHUB_API_TOKEN", default=None)
    )

    max_new_tokens: int = 512
    """Maximum number of generated tokens"""

    top_k: int | None = None
    """The number of highest probability vocabulary tokens to keep for
    top-k-filtering."""

    top_p: float | None = 0.95
    """If set to < 1, only the smallest set of most probable tokens with probabilities
    that add up to `top_p` or higher are kept for generation."""

    typical_p: float | None = 0.95