fix(communication): site-address refresh gets an overlap guard, per-load deadline and monotonic sequence (review 2026-10-21 03/M-1)

2026-10-22 06:16:15 -04:00
parent b0a97a4b67
commit 3ed71167b3
4 changed files with 244 additions and 27 deletions
@@ -0,0 +1,119 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Communication.Actors;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
+
+/// <summary>
+/// The 60 s site-address refresh (review 2026-10-21 03/M-1): at most one DB load in flight, a
+/// per-load deadline so a wedged read cannot pin the refresh forever, and a monotonic sequence so a
+/// late (older) result never overwrites a newer one.
+/// </summary>
+public class CentralCommunicationActorAddressRefreshTests : TestKit
+{
+    /// <summary>Initializes the test kit with a quieter Akka log level.</summary>
+    public CentralCommunicationActorAddressRefreshTests() : base(@"akka.loglevel = WARNING") { }
+
+    private static Site GrpcSite(string id) =>
+        new("Test " + id, id) { GrpcNodeAAddress = "http://a:8083" };
+
+    private (IActorRef Actor, ISiteCommandTransport Transport) CreateActor(
+        ISiteRepository repo, TimeSpan? loadTimeout = null)
+    {
+        var services = new ServiceCollection();
+        services.AddScoped(_ => repo);
+        if (loadTimeout is { } t)
+        {
+            services.AddSingleton(Options.Create(new CommunicationOptions { QueryTimeout = t }));
+        }
+        var sp = services.BuildServiceProvider();
+        var transport = Substitute.For<ISiteCommandTransport>();
+        var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
+        return (actor, transport);
+    }
+
+    /// <summary>Verifies that refresh ticks arriving while a load is in flight do not start more loads, and coalesce into one re-run.</summary>
+    [Fact]
+    public void RefreshWhileLoadInFlight_DoesNotStackLoads_AndCoalescesIntoOneRerun()
+    {
+        var gate = new TaskCompletionSource<IReadOnlyList<Site>>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var calls = 0;
+        var repo = Substitute.For<ISiteRepository>();
+        repo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(_ =>
+            Interlocked.Increment(ref calls) == 1
+                ? gate.Task
+                : Task.FromResult<IReadOnlyList<Site>>(new List<Site> { GrpcSite("site-b") }));
+
+        var (actor, transport) = CreateActor(repo);
+        AwaitCondition(() => Volatile.Read(ref calls) == 1, TimeSpan.FromSeconds(3));
+
+        for (var i = 0; i < 5; i++) actor.Tell(new RefreshSiteAddresses());
+        Thread.Sleep(300);
+        Assert.Equal(1, Volatile.Read(ref calls));
+
+        gate.SetResult(new List<Site> { GrpcSite("site-a") });
+
+        // The wedged load completes, then exactly ONE coalesced re-run picks up the newer state.
+        AwaitAssert(() => transport.Received().ReconcileSites(
+            Arg.Is<SiteAddressCacheLoaded>(c => c.GrpcContacts.ContainsKey("site-b"))),
+            TimeSpan.FromSeconds(3));
+        Thread.Sleep(300);
+        Assert.Equal(2, Volatile.Read(ref calls));
+    }
+
+    /// <summary>Verifies that a load which never completes is cancelled at its deadline so the next refresh can run.</summary>
+    [Fact]
+    public void HungLoad_IsCancelledAtItsDeadline_AndTheNextRefreshRuns()
+    {
+        var calls = 0;
+        var repo = Substitute.For<ISiteRepository>();
+        repo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(async ci =>
+        {
+            if (Interlocked.Increment(ref calls) == 1)
+            {
+                await Task.Delay(Timeout.Infinite, ci.Arg<CancellationToken>());
+            }
+            return (IReadOnlyList<Site>)new List<Site> { GrpcSite("site-a") };
+        });
+
+        var (actor, transport) = CreateActor(repo, loadTimeout: TimeSpan.FromMilliseconds(300));
+        AwaitCondition(() => Volatile.Read(ref calls) == 1, TimeSpan.FromSeconds(3));
+
+        // Let the deadline expire, then a fresh tick must be able to load.
+        Thread.Sleep(600);
+        actor.Tell(new RefreshSiteAddresses());
+
+        AwaitAssert(() => transport.Received().ReconcileSites(
+            Arg.Is<SiteAddressCacheLoaded>(c => c.GrpcContacts.ContainsKey("site-a"))),
+            TimeSpan.FromSeconds(3));
+    }
+
+    /// <summary>Verifies that a result older than the last applied one is dropped rather than reconciled.</summary>
+    [Fact]
+    public void OlderResult_ArrivingAfterANewerOne_IsDropped()
+    {
+        var repo = Substitute.For<ISiteRepository>();
+        repo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<Site>());
+        var (actor, transport) = CreateActor(repo);
+
+        static SiteAddressCacheLoaded Loaded(string site, long seq) => new(
+            new[] { site },
+            new Dictionary<string, SiteGrpcEndpoints> { [site] = new("http://" + site + ":8083", null) },
+            seq);
+
+        actor.Tell(Loaded("site-new", long.MaxValue - 1));
+        AwaitAssert(() => transport.Received().ReconcileSites(
+            Arg.Is<SiteAddressCacheLoaded>(c => c.GrpcContacts.ContainsKey("site-new"))),
+            TimeSpan.FromSeconds(3));
+
+        actor.Tell(Loaded("site-old", long.MaxValue - 2));
+        Thread.Sleep(300);
+        transport.DidNotReceive().ReconcileSites(
+            Arg.Is<SiteAddressCacheLoaded>(c => c.GrpcContacts.ContainsKey("site-old")));
+    }
+}
@@ -101,7 +101,7 @@ public class SitePairChannelProviderBadAddressTests : TestKit
            ["site-good"] = new("http://localhost:15222", null),
        };

-        transport.ReconcileSites(new SiteAddressCacheLoaded(new[] { " ", "site-good" }, contacts));
+        transport.ReconcileSites(new SiteAddressCacheLoaded(new[] { " ", "site-good" }, contacts, 1));

        _ = await CurrentChannel(provider, "site-good");
    }