// Go parity: golang/nats-server/server/norace_1_test.go // Covers: slow consumer detection, backpressure stats, rapid subscribe/unsubscribe // cycles, multi-client connection stress, large message delivery, and connection // lifecycle stability under load using real NatsServer instances. using System.Net; using System.Net.Sockets; using System.Text; using Microsoft.Extensions.Logging.Abstractions; using NATS.Server; using NATS.Server.TestUtilities; namespace NATS.Server.Core.Tests.Stress; /// /// Stress tests for slow consumer behaviour and connection lifecycle using real NatsServer /// instances wired with raw Socket connections following the same pattern as /// ClientSlowConsumerTests.cs and ServerTests.cs. /// /// Go ref: norace_1_test.go — slow consumer, connection churn, and load tests. /// public class SlowConsumerStressTests { // --------------------------------------------------------------- // Helpers // --------------------------------------------------------------- private static async Task ConnectRawAsync(int port) { var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); await sock.ConnectAsync(IPAddress.Loopback, port); // Drain the INFO line var buf = new byte[4096]; await sock.ReceiveAsync(buf, SocketFlags.None); return sock; } // --------------------------------------------------------------- // Go: TestNoRaceSlowConsumerStatIncrement norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Slow_consumer_stat_incremented_when_client_falls_behind() { // Go: TestNoClientLeakOnSlowConsumer — verify Stats.SlowConsumers increments. const long maxPending = 512; const int payloadSize = 256; const int floodCount = 30; var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port, MaxPending = maxPending }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var slowSub = await ConnectRawAsync(port); await slowSub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB sc.stat 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(slowSub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var payload = new string('Z', payloadSize); var sb = new StringBuilder(); for (var i = 0; i < floodCount; i++) sb.Append($"PUB sc.stat {payloadSize}\r\n{payload}\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); await Task.Delay(500); var stats = server.Stats; Interlocked.Read(ref stats.SlowConsumers).ShouldBeGreaterThan(0); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceSlowConsumerClientsTrackedIndependently norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Multiple_slow_consumers_tracked_independently_in_stats() { const long maxPending = 256; const int payloadSize = 128; const int floodCount = 20; var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port, MaxPending = maxPending }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { // Two independent slow subscribers using var slow1 = await ConnectRawAsync(port); using var slow2 = await ConnectRawAsync(port); await slow1.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB multi.slow 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(slow1, "PONG"); await slow2.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB multi.slow 2\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(slow2, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var payload = new string('A', payloadSize); var sb = new StringBuilder(); for (var i = 0; i < floodCount; i++) sb.Append($"PUB multi.slow {payloadSize}\r\n{payload}\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); await Task.Delay(600); var stats = server.Stats; Interlocked.Read(ref stats.SlowConsumers).ShouldBeGreaterThan(0); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRacePublisherBackpressure norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Fast_publisher_with_slow_reader_generates_backpressure_stats() { const long maxPending = 512; var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port, MaxPending = maxPending }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub = await ConnectRawAsync(port); await sub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB bp.test 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var payload = new string('P', 400); var sb = new StringBuilder(); for (var i = 0; i < 25; i++) sb.Append($"PUB bp.test 400\r\n{payload}\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); await Task.Delay(400); var stats = server.Stats; // At least the SlowConsumers counter or client count dropped (Interlocked.Read(ref stats.SlowConsumers) > 0 || server.ClientCount <= 2) .ShouldBeTrue(); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRace100RapidPublishes norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Subscriber_receives_messages_after_100_rapid_publishes() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub = await ConnectRawAsync(port); await sub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB rapid 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var sb = new StringBuilder(); for (var i = 0; i < 100; i++) sb.Append("PUB rapid 4\r\nping\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); var received = await SocketTestHelper.ReadUntilAsync(sub, "MSG rapid", 5000); received.ShouldContain("MSG rapid"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceConcurrentSubscribeStartup norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Concurrent_publish_and_subscribe_startup_does_not_crash_server() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { var tasks = Enumerable.Range(0, 10).Select(async i => { using var sock = await ConnectRawAsync(port); await sock.SendAsync( Encoding.ASCII.GetBytes($"CONNECT {{\"verbose\":false}}\r\nSUB conc.start.{i} {i + 1}\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sock, "PONG", 3000); }); await Task.WhenAll(tasks); server.ClientCount.ShouldBeGreaterThanOrEqualTo(0); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceLargeMessageMultipleSubscribers norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Large_message_published_and_received_by_multiple_subscribers() { // Use 8KB payload — large enough to span multiple TCP segments but small // enough to stay well within the default MaxPending limit in CI. var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); const int payloadSize = 8192; var payload = new string('L', payloadSize); try { using var sub1 = await ConnectRawAsync(port); using var sub2 = await ConnectRawAsync(port); await sub1.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB large.msg 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub1, "PONG"); await sub2.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB large.msg 2\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub2, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); await pub.SendAsync(Encoding.ASCII.GetBytes($"PUB large.msg {payloadSize}\r\n{payload}\r\nPING\r\n")); // Use a longer timeout for large message delivery await SocketTestHelper.ReadUntilAsync(pub, "PONG", 10000); var r1 = await SocketTestHelper.ReadUntilAsync(sub1, "MSG large.msg", 10000); var r2 = await SocketTestHelper.ReadUntilAsync(sub2, "MSG large.msg", 10000); r1.ShouldContain("MSG large.msg"); r2.ShouldContain("MSG large.msg"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceSubscribeUnsubscribeResubscribeCycle norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Subscribe_unsubscribe_resubscribe_cycle_100_times_without_error() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var client = await ConnectRawAsync(port); await client.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); for (var i = 1; i <= 100; i++) { await client.SendAsync( Encoding.ASCII.GetBytes($"SUB resub.cycle {i}\r\nUNSUB {i}\r\n")); } await client.SendAsync(Encoding.ASCII.GetBytes("PING\r\n")); var resp = await SocketTestHelper.ReadUntilAsync(client, "PONG", 5000); resp.ShouldContain("PONG"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceSubscriberReceivesAfterPause norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Subscriber_receives_messages_correctly_after_brief_pause() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub = await ConnectRawAsync(port); await sub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB pause.sub 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub, "PONG"); // Brief pause simulating a subscriber that drifts slightly await Task.Delay(100); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); await pub.SendAsync(Encoding.ASCII.GetBytes("PUB pause.sub 5\r\nhello\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); var received = await SocketTestHelper.ReadUntilAsync(sub, "hello", 5000); received.ShouldContain("hello"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceMultipleClientConnectDisconnect norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Multiple_client_connections_and_disconnections_leave_server_stable() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { // Connect and disconnect 20 clients sequentially to avoid hammering the port for (var i = 0; i < 20; i++) { using var sock = await ConnectRawAsync(port); await sock.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sock, "PONG", 3000); sock.Close(); } // Brief settle time await Task.Delay(200); // Server should still accept new connections using var final = await ConnectRawAsync(port); await final.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nPING\r\n")); var resp = await SocketTestHelper.ReadUntilAsync(final, "PONG", 3000); resp.ShouldContain("PONG"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceStatsCountersUnderLoad norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Stats_in_and_out_bytes_increment_correctly_under_load() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub = await ConnectRawAsync(port); await sub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB stats.load 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var sb = new StringBuilder(); for (var i = 0; i < 50; i++) sb.Append("PUB stats.load 10\r\n0123456789\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); await Task.Delay(200); var stats = server.Stats; Interlocked.Read(ref stats.InMsgs).ShouldBeGreaterThan(0); Interlocked.Read(ref stats.OutMsgs).ShouldBeGreaterThan(0); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceRapidConnectDisconnect norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Rapid_connect_disconnect_cycles_do_not_corrupt_server_state() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { // 30 rapid sequential connect + disconnect cycles for (var i = 0; i < 30; i++) { var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); await sock.ConnectAsync(IPAddress.Loopback, port); // Drain INFO var buf = new byte[512]; await sock.ReceiveAsync(buf, SocketFlags.None); // Immediately close — simulates a client that disconnects without CONNECT sock.Close(); sock.Dispose(); } await Task.Delay(300); // Server should still respond using var healthy = await ConnectRawAsync(port); await healthy.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nPING\r\n")); var resp = await SocketTestHelper.ReadUntilAsync(healthy, "PONG", 3000); resp.ShouldContain("PONG"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRacePublishWithCancellation norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Server_accepts_connection_after_cancelled_client_task() { var port = TestPortAllocator.GetFreePort(); using var serverCts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(serverCts.Token); await server.WaitForReadyAsync(); try { using var clientCts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); // Attempt a receive with a very short timeout — the token will cancel the read // but the server should not be destabilised by the abrupt disconnect. try { using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); await sock.ConnectAsync(IPAddress.Loopback, port); var buf = new byte[512]; await sock.ReceiveAsync(buf, SocketFlags.None, clientCts.Token); } catch (OperationCanceledException) { // Expected } await Task.Delay(200); // Server should still function using var good = await ConnectRawAsync(port); await good.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nPING\r\n")); var resp = await SocketTestHelper.ReadUntilAsync(good, "PONG", 3000); resp.ShouldContain("PONG"); } finally { await serverCts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceSlowConsumerClientCountDrops norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Slow_consumer_is_removed_from_client_count_after_detection() { const long maxPending = 512; const int payloadSize = 256; const int floodCount = 20; var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port, MaxPending = maxPending }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var slowSub = await ConnectRawAsync(port); await slowSub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB drop.test 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(slowSub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var payload = new string('D', payloadSize); var sb = new StringBuilder(); for (var i = 0; i < floodCount; i++) sb.Append($"PUB drop.test {payloadSize}\r\n{payload}\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); await Task.Delay(600); // Publisher is still alive; slow subscriber has been dropped server.ClientCount.ShouldBeLessThanOrEqualTo(2); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceSubjectMatchingUnderConcurrentConnections norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Server_delivers_to_correct_subscriber_when_multiple_subjects_active() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub1 = await ConnectRawAsync(port); using var sub2 = await ConnectRawAsync(port); await sub1.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB target.A 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub1, "PONG"); await sub2.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB target.B 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub2, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); await pub.SendAsync(Encoding.ASCII.GetBytes("PUB target.A 5\r\nhello\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 5000); var r1 = await SocketTestHelper.ReadUntilAsync(sub1, "hello", 3000); r1.ShouldContain("MSG target.A"); // sub2 should NOT have received the target.A message sub2.ReceiveTimeout = 200; var buf = new byte[512]; var n = 0; try { n = sub2.Receive(buf); } catch (SocketException) { } var s2Data = Encoding.ASCII.GetString(buf, 0, n); s2Data.ShouldNotContain("target.A"); } finally { await cts.CancelAsync(); server.Dispose(); } } // --------------------------------------------------------------- // Go: TestNoRaceServerRejectsPayloadOverLimit norace_1_test.go // --------------------------------------------------------------- [Fact] [Trait("Category", "Stress")] public async Task Server_remains_stable_after_processing_many_medium_sized_messages() { var port = TestPortAllocator.GetFreePort(); using var cts = new CancellationTokenSource(); var server = new NatsServer( new NatsOptions { Port = port }, NullLoggerFactory.Instance); _ = server.StartAsync(cts.Token); await server.WaitForReadyAsync(); try { using var sub = await ConnectRawAsync(port); await sub.SendAsync( Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\nSUB medium.msgs 1\r\nPING\r\n")); await SocketTestHelper.ReadUntilAsync(sub, "PONG"); using var pub = await ConnectRawAsync(port); await pub.SendAsync(Encoding.ASCII.GetBytes("CONNECT {\"verbose\":false}\r\n")); var payload = new string('M', 1024); // 1 KB each var sb = new StringBuilder(); for (var i = 0; i < 200; i++) sb.Append($"PUB medium.msgs 1024\r\n{payload}\r\n"); sb.Append("PING\r\n"); await pub.SendAsync(Encoding.ASCII.GetBytes(sb.ToString())); await SocketTestHelper.ReadUntilAsync(pub, "PONG", 10000); var stats = server.Stats; Interlocked.Read(ref stats.InMsgs).ShouldBeGreaterThanOrEqualTo(200); } finally { await cts.CancelAsync(); server.Dispose(); } } }