test_count_tokens_approximately_with_unknown_block_type() — langchain Function Reference

Source Code

libs/core/tests/unit_tests/messages/test_utils.py lines 2848–2863

def test_count_tokens_approximately_with_unknown_block_type() -> None:
    """Test that unknown multimodal block types still contribute to token count."""
    text_only = count_tokens_approximately([HumanMessage(content="hello")])

    message_with_unknown_block = HumanMessage(
        content=[
            {"type": "text", "text": "hello"},
            {"type": "foo", "bar": "baz"},  # unknown type, falls back to repr(block)
        ]
    )

    mixed = count_tokens_approximately([message_with_unknown_block])

    # The message with an extra unknown block should be counted as more expensive
    # than the text-only version.
    assert mixed > text_only