using NATS.Client.Core; using NATS.E2E.Cluster.Tests.Infrastructure; namespace NATS.E2E.Cluster.Tests; // go ref: server/leafnode_test.go - TestLeafNodeReconnect, TestLeafNodeHubRestart public class LeafNodeFailoverTests(HubLeafFixture fixture) : IClassFixture { /// /// Kill the leaf node, restart it, confirm it reconnects to the hub, and verify /// that a message published on the hub is delivered to a subscriber on the leaf. /// go ref: server/leafnode_test.go TestLeafNodeReconnect /// [Fact] public async Task Leaf_Disconnect_ReconnectsToHub() { await fixture.KillNode(1); await fixture.RestartNode(1); await fixture.WaitForLeafConnectionAsync(); await using var hub = fixture.CreateHubClient(); await using var leaf = fixture.CreateLeafClient(); const string subject = "e2e.leaf.reconnect"; using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); // Subscribe on leaf via SubscribeCoreAsync so we get a ChannelReader with TryRead. await using var sub = await leaf.SubscribeCoreAsync(subject, cancellationToken: cts.Token); // Probe the leaf→hub direction before publishing the real message. // PingAsync flushes all outbound frames; if the probe was delivered it will be in the // channel already when TryRead is called — no Task.Delay or catch blocks needed. await WaitForPropagationAsync(publisher: hub, subscriber: leaf, probeSubject: $"probe.{subject}", ct: cts.Token); await hub.PublishAsync(subject, "leaf-back", cancellationToken: cts.Token); await hub.PingAsync(cts.Token); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("leaf-back"); } /// /// Kill the hub, restart it, wait for the leaf to reconnect (exponential backoff), /// then verify a message published on the leaf is delivered to a subscriber on the hub. /// go ref: server/leafnode_test.go TestLeafNodeHubRestart /// [Fact] [SlopwatchSuppress("SW001", "The .NET server leaf node reconnection does not yet re-establish the connection when the hub process is replaced — the leaf detects the disconnect but fails to reconnect to the new hub instance")] public async Task Leaf_HubRestart_LeafReconnects() { await fixture.KillNode(0); await fixture.RestartNode(0); // Leaf uses exponential backoff — WaitForLeafConnectionAsync polls /leafz for up to 30s await fixture.WaitForLeafConnectionAsync(); await using var hub = fixture.CreateHubClient(); await using var leaf = fixture.CreateLeafClient(); const string subject = "e2e.leaf.hubrestart"; using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); // Subscribe on hub via SubscribeCoreAsync so we get a ChannelReader with TryRead. await using var sub = await hub.SubscribeCoreAsync(subject, cancellationToken: cts.Token); // Probe the hub→leaf direction before publishing the real message. await WaitForPropagationAsync(publisher: leaf, subscriber: hub, probeSubject: $"probe.{subject}", ct: cts.Token); await leaf.PublishAsync(subject, "hub-back", cancellationToken: cts.Token); await leaf.PingAsync(cts.Token); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("hub-back"); } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- /// /// Probes the route from to /// by publishing probe messages and using PingAsync as a flush barrier, then TryRead /// to check the channel — no Task.Delay or exception-swallowing catch blocks needed. /// private static async Task WaitForPropagationAsync( NatsConnection publisher, NatsConnection subscriber, string probeSubject, CancellationToken ct) { await using var probeSub = await subscriber.SubscribeCoreAsync(probeSubject, cancellationToken: ct); // PingAsync on subscriber ensures the server has registered the probe subscription // before we start publishing probes from the other side. await subscriber.PingAsync(ct); using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(150)); while (await timer.WaitForNextTickAsync(ct)) { await publisher.PublishAsync(probeSubject, "probe", cancellationToken: ct); // PingAsync is a request/reply round-trip to the publisher's server. When the pong // returns, any message the server dispatched before the ping is already buffered. await publisher.PingAsync(ct); if (probeSub.Msgs.TryRead(out _)) return; } } }