From 62d1687aa439dc4fdc2712917210705339357e3a Mon Sep 17 00:00:00 2001
From: Thomas Bechtold <thomasbechtold@jpberlin.de>
Date: Mon, 2 Nov 2026 14:35:19 +0100
Subject: [PATCH] [fix] send the alert after a failed client handshake (#399)

When the client rejects the server certificate, BCJSSE buffers the
fatal alert and stores the failure: the next wrap() throws it, and
only the wrap() after that emits the alert. doShutdown gave up after
the first wrap() threw, so the alert was never sent. A server whose
client kept the TCP socket open then blocked in accept indefinitely.

Before 0.19.2 the server did not hang, but only because shutdown
re-sent stale output (the ClientHello and zero bytes), which the
server rejected.

doShutdown now wraps once more after an SSLException while outbound
is not done, which flushes the buffered alert; the server fails with
certificate_unknown.
---
 .../java/org/jruby/ext/openssl/SSLSocket.java | 10 ++++++-
 test/ssl/test_ssl.rb                          | 30 +++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/jruby/ext/openssl/SSLSocket.java b/src/main/java/org/jruby/ext/openssl/SSLSocket.java
index 6b4e0821..b1ea06c7 100644
--- a/src/main/java/org/jruby/ext/openssl/SSLSocket.java
+++ b/src/main/java/org/jruby/ext/openssl/SSLSocket.java
@@ -975,7 +975,15 @@ private void doShutdown() throws IOException {
             try {
                 SSLEngineResult result;
                 do {
-                    result = engine.wrap(EMPTY_DATA.duplicate(), netWriteData); // send close (after engine.closeOutbound)
+                    try {
+                        result = engine.wrap(EMPTY_DATA.duplicate(), netWriteData); // send close (after engine.closeOutbound)
+                    }
+                    catch (SSLException e) {
+                        // BCJSSE throws a deferred handshake failure first, the next wrap flushes its buffered alert
+                        if (engine.isOutboundDone()) throw e;
+                        LOG.debug(getRuntime(), "doShutdown", e);
+                        result = engine.wrap(EMPTY_DATA.duplicate(), netWriteData);
+                    }
                     if (result.getStatus() == SSLEngineResult.Status.BUFFER_OVERFLOW) {
                         netWriteData = ensureCapacity(netWriteData, engine.getSession().getPacketBufferSize());
                         continue;
diff --git a/test/ssl/test_ssl.rb b/test/ssl/test_ssl.rb
index aa7317a7..4a23fb74 100644
--- a/test/ssl/test_ssl.rb
+++ b/test/ssl/test_ssl.rb
@@ -465,6 +465,36 @@ def test_server_verify_peer_allows_missing_client_cert
     end
   end
 
+  # a client rejecting the server certificate must send its alert, even with the socket still open
+  def test_client_verify_failure_sends_alert_to_server
+    [:TLSv1_2, :TLSv1_3].each do |version|
+      begin
+        server_ctx = OpenSSL::SSL::SSLContext.new
+        server_ctx.cert = @svr_cert
+        server_ctx.key = @svr_key
+        server_ctx.ssl_version = version if jruby?
+        tcp_server = TCPServer.new("127.0.0.1", 0)
+        ssl_server = OpenSSL::SSL::SSLServer.new(tcp_server, server_ctx)
+        server_thread = Thread.new { ssl_server.accept rescue $! }
+
+        client_ctx = OpenSSL::SSL::SSLContext.new
+        client_ctx.ssl_version = version if jruby?
+        client_ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        client_ctx.cert_store = OpenSSL::X509::Store.new # trusts nothing
+        sock = TCPSocket.new("127.0.0.1", tcp_server.addr[1])
+        assert_raise(OpenSSL::SSL::SSLError) { OpenSSL::SSL::SSLSocket.new(sock, client_ctx).connect }
+
+        assert server_thread.join(5), "#{version}: server accept blocked, no alert from client"
+        assert_kind_of OpenSSL::SSL::SSLError, server_thread.value, version.to_s
+      ensure
+        sock&.close rescue nil # unblocks a still waiting server
+        server_thread&.join rescue nil
+        ssl_server&.close rescue nil
+        tcp_server&.close rescue nil
+      end
+    end
+  end
+
   def test_unexpected_eof_is_not_accepted_by_default
     server_ctx = OpenSSL::SSL::SSLContext.new
     server_ctx.cert = @svr_cert