Fix: Akka.Remote should not shutdown on invalid TLS traffic (#7939)

When TLS is enabled, invalid traffic (like HTTP requests) hitting the
Akka.Remote port would cause the entire ActorSystem to shut down with
exit code 79. This was due to overly aggressive TLS handshake failure
handling introduced in #7839.

Changes:
- Modified TcpTransport to only trigger CoordinatedShutdown for client-side
  TLS handshake failures (outbound connections we initiate)
- Server-side TLS failures (incoming invalid connections) now just log a
  warning and reject the connection without shutting down
- Added test to verify servers remain running when invalid traffic hits
  the TLS port

This makes Akka.Remote resilient to port scanners, misconfigured clients,
or malicious traffic while maintaining strict security for legitimate
connections.

Fixes #7938

Author: Aaronontheweb

src/core/Akka.Remote.Tests/Transport/DotNettyTlsHandshakeFailureSpec.cs

@@ -7,7 +7,9 @@
 
 using System;
 using System.IO;
+using System.Net.Sockets;
 using System.Security.Cryptography.X509Certificates;
+using System.Text;
 using System.Threading.Tasks;
 using Akka.Actor;
 using Akka.Configuration;
@@ -28,13 +30,13 @@ public DotNettyTlsHandshakeFailureSpec(ITestOutputHelper output) : base(Configur
         {
         }
 
-        private static Config CreateConfig(bool enableSsl, string certPath, string certPassword, bool suppressValidation = true)
+        private static Config CreateConfig(bool enableSsl, string certPath, string certPassword, bool suppressValidation = true, int port = 0)
         {
             var baseConfig = ConfigurationFactory.ParseString(@"akka {
                 loglevel = DEBUG
                 actor.provider = ""Akka.Remote.RemoteActorRefProvider,Akka.Remote""
                 remote.dot-netty.tcp {
-                    port = 0
+                    port = " + port + @"
                     hostname = ""127.0.0.1""
                     enable-ssl = " + (enableSsl ? "on" : "off") + @"
                     log-transport = off
@@ -147,7 +149,66 @@ await AwaitAssertAsync(async () =>
             }
         }
 
+        [Fact(DisplayName = "Server should NOT shutdown when invalid traffic (like HTTP) hits TLS port")]
+        public async Task Server_side_invalid_traffic_should_not_shutdown_server()
+        {
+            // This test addresses issue https://github.com/akkadotnet/akka.net/issues/7938
+            // When invalid traffic (like HTTP requests) hits a TLS-enabled port,
+            // the server should reject the connection but NOT shut down
+            ActorSystem server = null;
+
+            try
+            {
+                // Start server with TLS enabled on a specific port
+                var port = 15557; // Use a fixed port for this test
+                var serverConfig = CreateConfig(true, ValidCertPath, Password, suppressValidation: true, port: port);
+                server = ActorSystem.Create("ServerSystem", serverConfig);
+
+                var serverEcho = server.ActorOf(Props.Create(() => new EchoActor()), "echo");
+
+                // Ensure the server is ready by waiting for the remote transport to be bound
+                var serverAddress = RARP.For(server).Provider.DefaultAddress;
+                Assert.NotNull(serverAddress);
+                Assert.Equal(port, serverAddress.Port.Value);
+
+                // Send invalid HTTP traffic to the TLS port (simulating the issue)
+                try
+                {
+                    using var tcpClient = new TcpClient();
+                    await tcpClient.ConnectAsync("127.0.0.1", port);
+
+                    // Send an HTTP OPTIONS request (as described in the bug report)
+                    var httpRequest = Encoding.UTF8.GetBytes("OPTIONS / HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n");
+                    await tcpClient.GetStream().WriteAsync(httpRequest, 0, httpRequest.Length);
+                    await tcpClient.GetStream().FlushAsync();
 
+                    // Connection should be closed by server after rejecting invalid TLS
+                    tcpClient.Close();
+                }
+                catch
+                {
+                    // Connection might be closed by server, that's expected
+                }
+
+                // Verify the server hasn't initiated shutdown
+                // If it was going to shut down due to TLS failure, it would have done so immediately
+                await AwaitConditionAsync(() => !server.WhenTerminated.IsCompleted,
+                    TimeSpan.FromSeconds(3), TimeSpan.FromMilliseconds(100));
+
+                // CRITICAL ASSERTION: Server should NOT have shut down
+                Assert.False(server.WhenTerminated.IsCompleted,
+                    "Server should NOT shut down after receiving invalid HTTP traffic on TLS port");
+
+                // Also verify the system is still functional
+                var testActor = server.ActorOf(Props.Empty, "test-actor");
+                Assert.NotNull(testActor);
+            }
+            finally
+            {
+                if (server != null)
+                    Shutdown(server, TimeSpan.FromSeconds(10));
+            }
+        }
 
         private sealed class EchoActor : ReceiveActor
         {

src/core/Akka.Remote/Transport/DotNetty/TcpTransport.cs

@@ -92,9 +92,20 @@ public override void UserEventTriggered(IChannelHandlerContext context, object e
                     context.Channel.LocalAddress, context.Channel.RemoteAddress,
                     context.Channel.Id, detailedError);
 
-                // Shutdown the ActorSystem on TLS handshake failure
-                var cs = CoordinatedShutdown.Get(Transport.System);
-                cs.Run(new TlsHandshakeFailureReason($"TLS handshake failed on channel [{context.Channel.LocalAddress}->{context.Channel.RemoteAddress}](Id={context.Channel.Id})"));
+                // Only shutdown the ActorSystem if this is a client-side failure
+                // Server-side failures (incoming connections) should just reject the connection
+                if (isClient)
+                {
+                    // Client-side: We initiated the connection and TLS failed - this is critical
+                    var cs = CoordinatedShutdown.Get(Transport.System);
+                    cs.Run(new TlsHandshakeFailureReason($"TLS handshake failed on outbound connection to [{context.Channel.RemoteAddress}]"));
+                }
+                else
+                {
+                    // Server-side: Someone connected to us with invalid TLS - just reject them
+                    Log.Warning("Rejected incoming connection from [{0}] due to TLS handshake failure. This is likely invalid or malicious traffic.",
+                        context.Channel.RemoteAddress);
+                }
 
                 context.CloseAsync();
                 return; // don't pass to next handlers