handshake() — netty Function Reference

Source Code

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java lines 1951–2021

    private SSLEngineResult.HandshakeStatus handshake() throws SSLException {
        if (needTask) {
            return NEED_TASK;
        }
        if (handshakeState == HandshakeState.FINISHED) {
            return FINISHED;
        }

        checkEngineClosed();

        if (pendingException != null) {
            // Let's call SSL.doHandshake(...) again in case there is some async operation pending that would fill the
            // outbound buffer.
            if (SSL.doHandshake(ssl) <= 0) {
                // Clear any error that was put on the stack by the handshake
                SSL.clearError();
            }
            return handshakeException();
        }

        // Adding the OpenSslEngine to the OpenSslEngineMap so it can be used in the AbstractCertificateVerifier.
        engines.put(sslPointer(), this);

        if (!sessionSet) {
            if (!parentContext.sessionContext().setSessionFromCache(ssl, session, getPeerHost(), getPeerPort())) {
                // The session was not reused via the cache. Call prepareHandshake() to ensure we remove all previous
                // stored key-value pairs.
                session.prepareHandshake();
            }
            sessionSet = true;
        }

        int code = SSL.doHandshake(ssl);
        if (code <= 0) {
            int sslError = SSL.getError(ssl, code);
            if (sslError == SSL.SSL_ERROR_WANT_READ || sslError == SSL.SSL_ERROR_WANT_WRITE) {
                return pendingStatus(SSL.bioLengthNonApplication(networkBIO));
            }

            if (sslError == SSL.SSL_ERROR_WANT_X509_LOOKUP ||
                    sslError == SSL.SSL_ERROR_WANT_CERTIFICATE_VERIFY ||
                    sslError == SSL.SSL_ERROR_WANT_PRIVATE_KEY_OPERATION) {
                return NEED_TASK;
            }

            int errorNumber = SSL.getLastErrorNumber();
            if (needWrapAgain(errorNumber)) {
                // There is something that needs to be send to the remote peer before we can teardown.
                // This is most likely some alert.
                return NEED_WRAP;
            }
            // Check if we have a pending exception that was created during the handshake and if so throw it after
            // shutdown the connection.
            if (pendingException != null) {
                return handshakeException();
            }

            // Everything else is considered as error
            throw shutdownWithError("SSL_do_handshake", sslError, errorNumber);
        }
        // We have produced more data as part of the handshake if this is the case the user should call wrap(...)
        if (SSL.bioLengthNonApplication(networkBIO) > 0) {
            return NEED_WRAP;
        }
        // if SSL_do_handshake returns > 0 or sslError == SSL.SSL_ERROR_NAME it means the handshake was finished.
        session.handshakeFinished(SSL.getSessionId(ssl), SSL.getCipherForSSL(ssl), SSL.getVersion(ssl),
                SSL.getPeerCertificate(ssl), SSL.getPeerCertChain(ssl),
                SSL.getTime(ssl) * 1000L, parentContext.sessionTimeout() * 1000L);
        selectApplicationProtocol();
        return FINISHED;
    }