unwrap() — netty Function Reference

Source Code

handler/src/main/java/io/netty/handler/ssl/SslHandler.java lines 1492–1618

    private int unwrap(ChannelHandlerContext ctx, ByteBuf packet, int length) throws SSLException {
        final int originalLength = length;
        boolean wrapLater = false;
        boolean notifyClosure = false;
        boolean executedRead = false;
        ByteBuf decodeOut = allocate(ctx, length);
        try {
            // Only continue to loop if the handler was not removed in the meantime.
            // See https://github.com/netty/netty/issues/5860
            do {
                final SSLEngineResult result = engineType.unwrap(this, packet, length, decodeOut);
                final Status status = result.getStatus();
                final HandshakeStatus handshakeStatus = result.getHandshakeStatus();
                final int produced = result.bytesProduced();
                final int consumed = result.bytesConsumed();

                // Skip bytes now in case unwrap is called in a re-entry scenario. For example LocalChannel.read()
                // may entry this method in a re-entry fashion and if the peer is writing into a shared buffer we may
                // unwrap the same data multiple times.
                packet.skipBytes(consumed);
                length -= consumed;

                // The expected sequence of events is:
                // 1. Notify of handshake success
                // 2. fireChannelRead for unwrapped data
                if (handshakeStatus == HandshakeStatus.FINISHED || handshakeStatus == HandshakeStatus.NOT_HANDSHAKING) {
                    wrapLater |= (decodeOut.isReadable() ?
                            setHandshakeSuccessUnwrapMarkReentry() : setHandshakeSuccess()) ||
                            handshakeStatus == HandshakeStatus.FINISHED ||
                            // We need to check if pendingUnecryptedWrites is null as the SslHandler
                            // might have been removed in the meantime.
                            (pendingUnencryptedWrites != null  && !pendingUnencryptedWrites.isEmpty());
                }

                // Dispatch decoded data after we have notified of handshake success. If this method has been invoked
                // in a re-entry fashion we execute a task on the executor queue to process after the stack unwinds
                // to preserve order of events.
                if (decodeOut.isReadable()) {
                    setState(STATE_FIRE_CHANNEL_READ);
                    if (isStateSet(STATE_UNWRAP_REENTRY)) {
                        executedRead = true;
                        executeChannelRead(ctx, decodeOut);
                    } else {
                        ctx.fireChannelRead(decodeOut);
                    }
                    decodeOut = null;
                }

                if (status == Status.CLOSED) {
                    notifyClosure = true; // notify about the CLOSED state of the SSLEngine. See #137
                } else if (status == Status.BUFFER_OVERFLOW) {
                    if (decodeOut != null) {
                        decodeOut.release();
                    }
                    final int applicationBufferSize = engine.getSession().getApplicationBufferSize();
                    // Allocate a new buffer which can hold all the rest data and loop again.
                    // It may happen that applicationBufferSize < produced while there is still more to unwrap, in this
                    // case we will just allocate a new buffer with the capacity of applicationBufferSize and call
                    // unwrap again.
                    decodeOut = allocate(ctx, engineType.calculatePendingData(this, applicationBufferSize < produced ?
                            applicationBufferSize : applicationBufferSize - produced));
                    continue;
                }

                if (handshakeStatus == HandshakeStatus.NEED_TASK) {
                    boolean pending = runDelegatedTasks(true);
                    if (!pending) {
                        // We scheduled a task on the delegatingTaskExecutor, so stop processing as we will
                        // resume once the task completes.
                        //
                        // We break out of the loop only and do NOT return here as we still may need to notify
                        // about the closure of the SSLEngine.
                        wrapLater = false;
                        break;
                    }
                } else if (handshakeStatus == HandshakeStatus.NEED_WRAP) {
                    // If the wrap operation transitions the status to NOT_HANDSHAKING and there is no more data to
                    // unwrap then the next call to unwrap will not produce any data. We can avoid the potentially
                    // costly unwrap operation and break out of the loop.
                    if (wrapNonAppData(ctx, true) && length == 0) {
                        break;