getEncryptedPacketLength() — netty Function Reference

Source Code

handler/src/main/java/io/netty/handler/ssl/SslUtils.java lines 353–419

    static int getEncryptedPacketLength(ByteBuf buffer, int offset, boolean probeSSLv2) {
        assert offset >= buffer.readerIndex();
        int remaining = buffer.writerIndex() - offset;
        if (remaining < SSL_RECORD_HEADER_LENGTH) {
            return NOT_ENOUGH_DATA;
        }
        int packetLength = 0;
        // SSLv3 or TLS - Check ContentType
        boolean tls;
        switch (buffer.getUnsignedByte(offset)) {
            case SSL_CONTENT_TYPE_CHANGE_CIPHER_SPEC:
            case SSL_CONTENT_TYPE_ALERT:
            case SSL_CONTENT_TYPE_HANDSHAKE:
            case SSL_CONTENT_TYPE_APPLICATION_DATA:
            case SSL_CONTENT_TYPE_EXTENSION_HEARTBEAT:
                tls = true;
                break;
            default:
                // SSLv2 or bad data
                if (!probeSSLv2) {
                    return NOT_ENCRYPTED;
                }
                tls = false;
        }

        if (tls) {
            // SSLv3 or TLS or GMSSLv1.0 or GMSSLv1.1 - Check ProtocolVersion
            int majorVersion = buffer.getUnsignedByte(offset + 1);
            int version = buffer.getShort(offset + 1);
            if (majorVersion == 3 || version == GMSSL_PROTOCOL_VERSION) {
                // SSLv3 or TLS or GMSSLv1.0 or GMSSLv1.1
                packetLength = unsignedShortBE(buffer, offset + 3) + SSL_RECORD_HEADER_LENGTH;
                if (packetLength <= SSL_RECORD_HEADER_LENGTH) {
                    // Neither SSLv3 or TLSv1 (i.e. SSLv2 or bad data)
                    tls = false;
                }
            } else if (version == DTLS_1_0 || version == DTLS_1_2 || version == DTLS_1_3) {
                if (remaining < DTLS_RECORD_HEADER_LENGTH) {
                    return NOT_ENOUGH_DATA;
                }
                // length is the last 2 bytes in the 13 byte header.
                packetLength = unsignedShortBE(buffer, offset + DTLS_RECORD_HEADER_LENGTH - 2) +
                        DTLS_RECORD_HEADER_LENGTH;
            } else {
                // Neither SSLv3 or TLSv1 (i.e. SSLv2 or bad data)
                tls = false;
            }
        }

        if (!tls) {
            // SSLv2 or bad data - Check the version
            int headerLength = (buffer.getUnsignedByte(offset) & 0x80) != 0 ? 2 : 3;
            int majorVersion = buffer.getUnsignedByte(offset + headerLength + 1);
            if (majorVersion == 2 || majorVersion == 3) {
                // SSLv2
                packetLength = headerLength == 2 ?
                        (shortBE(buffer, offset) & 0x7FFF) + 2 : (shortBE(buffer, offset) & 0x3FFF) + 3;
                if (packetLength <= headerLength) {
                    // If there's no data then consider this package as not encrypted.
                    return NOT_ENCRYPTED;
                }
            } else {
                return NOT_ENCRYPTED;
            }
        }
        return packetLength;
    }