perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

2026-08-14 21:10:19 -04:00
parent 1040dc0fcc
commit 2ce0ad7ed1
30 changed files with 1941 additions and 482 deletions
@@ -920,6 +920,149 @@ public class DebugStreamBridgeActorTests : TestKit
            DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
        }
    }
+
+    // ── WP2.3: bounded pre-snapshot buffer, hard snapshot deadline, timeout hygiene ──
+
+    [Fact]
+    public void PreSnapshotBuffer_IsCapped_DropsOldest_AndCountsTheDrops()
+    {
+        // Before the cap a snapshot that never arrived buffered every live event on the
+        // CENTRAL node without limit — one wedged session on a chatty instance was enough
+        // to grow unbounded. Now the oldest are evicted and counted.
+        var ctx = CreateBridgeActor();
+        ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe envelope; no snapshot is ever sent
+
+        const int cap = 20_000;
+        const int overflow = 250;
+        var before = Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped);
+
+        var t = DateTimeOffset.UtcNow;
+        for (var i = 0; i < cap + overflow; i++)
+        {
+            ctx.BridgeActor.Tell(new AttributeValueChanged(
+                InstanceName, "Modules.IO", $"Attr{i}", i, "Good", t.AddMilliseconds(i)));
+        }
+
+        // The overflow was evicted (drop-oldest) and counted.
+        AwaitCondition(
+            () => Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped) - before >= overflow,
+            TimeSpan.FromSeconds(10));
+
+        // The session is still healthy: the snapshot can still arrive and flush the
+        // (capped) buffer — the newest events, which the snapshot may predate, survived.
+        var snapshot = new DebugViewSnapshot(
+            InstanceName,
+            new List<AttributeValueChanged>(),
+            new List<AlarmStateChanged>(),
+            t.AddMilliseconds(-1));
+        ctx.BridgeActor.Tell(snapshot);
+
+        AwaitCondition(() =>
+        {
+            lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count >= cap; }
+        }, TimeSpan.FromSeconds(10));
+
+        lock (ctx.ReceivedEvents)
+        {
+            // Snapshot + exactly the retained (capped) events, and the newest survived.
+            Assert.Equal(cap + 1, ctx.ReceivedEvents.Count);
+            var lastAttr = ctx.ReceivedEvents.OfType<AttributeValueChanged>().Last();
+            Assert.Equal($"Attr{cap + overflow - 1}", lastAttr.AttributeName);
+        }
+    }
+
+    [Fact]
+    public void NoSnapshotWithinDeadline_FailsTheSession_InsteadOfBufferingForever()
+    {
+        // Nothing else ends a session wedged in the buffering phase: a lost site reply
+        // raises no gRPC error, and stream events no longer reset the orphan timeout.
+        DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
+        try
+        {
+            var ctx = CreateBridgeActor();
+            ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe request — never answered
+
+            Watch(ctx.BridgeActor);
+            ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
+
+            // The consumer is told, so the UI can surface the failure and reopen.
+            Assert.True(ctx.TerminatedFlag[0]);
+            // And the site-side relay was released rather than left as a zombie.
+            AwaitCondition(
+                () => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
+                TimeSpan.FromSeconds(3));
+        }
+        finally
+        {
+            DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
+        }
+    }
+
+    [Fact]
+    public void SnapshotArrival_StandsDownTheDeadline()
+    {
+        // The deadline must not fire after a healthy snapshot — a live session would
+        // otherwise be killed mid-stream.
+        DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
+        try
+        {
+            var ctx = CreateBridgeActor();
+            ctx.CommProbe.ExpectMsg<SiteEnvelope>();
+
+            ctx.BridgeActor.Tell(new DebugViewSnapshot(
+                InstanceName,
+                new List<AttributeValueChanged>(),
+                new List<AlarmStateChanged>(),
+                DateTimeOffset.UtcNow));
+
+            Thread.Sleep(700); // well past the deadline
+            Assert.False(ctx.TerminatedFlag[0]);
+
+            // Still serving: a post-snapshot event passes straight through.
+            ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AlarmStateChanged(
+                InstanceName, "PumpFault", Commons.Types.Enums.AlarmState.Active, 500,
+                DateTimeOffset.UtcNow));
+            AwaitCondition(() =>
+            {
+                lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AlarmStateChanged>().Any(); }
+            }, TimeSpan.FromSeconds(3));
+        }
+        finally
+        {
+            DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
+        }
+    }
+
+    [Fact]
+    public void StreamEvents_AreWrapped_SoTheyDoNotResetTheOrphanReceiveTimeout()
+    {
+        // Structural pin for the fix: the gRPC callback wraps every event in an envelope
+        // marked INotInfluenceReceiveTimeout, so a busy site can no longer keep an
+        // abandoned session alive indefinitely by feeding it events.
+        Assert.True(typeof(Akka.Actor.INotInfluenceReceiveTimeout)
+            .IsAssignableFrom(typeof(LiveDebugStreamEvent)));
+
+        var ctx = CreateBridgeActor();
+        ctx.CommProbe.ExpectMsg<SiteEnvelope>();
+        AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
+
+        ctx.BridgeActor.Tell(new DebugViewSnapshot(
+            InstanceName,
+            new List<AttributeValueChanged>(),
+            new List<AlarmStateChanged>(),
+            DateTimeOffset.UtcNow));
+
+        // An event delivered through the real gRPC callback path still reaches the
+        // consumer — the wrapper is transparent to delivery.
+        var evt = new AttributeValueChanged(
+            InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
+        ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
+
+        AwaitCondition(() =>
+        {
+            lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
+        }, TimeSpan.FromSeconds(3));
+    }
 }

 /// <summary>