perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

2026-08-14 21:10:19 -04:00
parent 1040dc0fcc
commit 2ce0ad7ed1
30 changed files with 1941 additions and 482 deletions
@@ -920,6 +920,149 @@ public class DebugStreamBridgeActorTests : TestKit
            DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
        }
    }
+
+    // ── WP2.3: bounded pre-snapshot buffer, hard snapshot deadline, timeout hygiene ──
+
+    [Fact]
+    public void PreSnapshotBuffer_IsCapped_DropsOldest_AndCountsTheDrops()
+    {
+        // Before the cap a snapshot that never arrived buffered every live event on the
+        // CENTRAL node without limit — one wedged session on a chatty instance was enough
+        // to grow unbounded. Now the oldest are evicted and counted.
+        var ctx = CreateBridgeActor();
+        ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe envelope; no snapshot is ever sent
+
+        const int cap = 20_000;
+        const int overflow = 250;
+        var before = Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped);
+
+        var t = DateTimeOffset.UtcNow;
+        for (var i = 0; i < cap + overflow; i++)
+        {
+            ctx.BridgeActor.Tell(new AttributeValueChanged(
+                InstanceName, "Modules.IO", $"Attr{i}", i, "Good", t.AddMilliseconds(i)));
+        }
+
+        // The overflow was evicted (drop-oldest) and counted.
+        AwaitCondition(
+            () => Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped) - before >= overflow,
+            TimeSpan.FromSeconds(10));
+
+        // The session is still healthy: the snapshot can still arrive and flush the
+        // (capped) buffer — the newest events, which the snapshot may predate, survived.
+        var snapshot = new DebugViewSnapshot(
+            InstanceName,
+            new List<AttributeValueChanged>(),
+            new List<AlarmStateChanged>(),
+            t.AddMilliseconds(-1));
+        ctx.BridgeActor.Tell(snapshot);
+
+        AwaitCondition(() =>
+        {
+            lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count >= cap; }
+        }, TimeSpan.FromSeconds(10));
+
+        lock (ctx.ReceivedEvents)
+        {
+            // Snapshot + exactly the retained (capped) events, and the newest survived.
+            Assert.Equal(cap + 1, ctx.ReceivedEvents.Count);
+            var lastAttr = ctx.ReceivedEvents.OfType<AttributeValueChanged>().Last();
+            Assert.Equal($"Attr{cap + overflow - 1}", lastAttr.AttributeName);
+        }
+    }
+
+    [Fact]
+    public void NoSnapshotWithinDeadline_FailsTheSession_InsteadOfBufferingForever()
+    {
+        // Nothing else ends a session wedged in the buffering phase: a lost site reply
+        // raises no gRPC error, and stream events no longer reset the orphan timeout.
+        DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
+        try
+        {
+            var ctx = CreateBridgeActor();
+            ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe request — never answered
+
+            Watch(ctx.BridgeActor);
+            ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
+
+            // The consumer is told, so the UI can surface the failure and reopen.
+            Assert.True(ctx.TerminatedFlag[0]);
+            // And the site-side relay was released rather than left as a zombie.
+            AwaitCondition(
+                () => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
+                TimeSpan.FromSeconds(3));
+        }
+        finally
+        {
+            DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
+        }
+    }
+
+    [Fact]
+    public void SnapshotArrival_StandsDownTheDeadline()
+    {
+        // The deadline must not fire after a healthy snapshot — a live session would
+        // otherwise be killed mid-stream.
+        DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
+        try
+        {
+            var ctx = CreateBridgeActor();
+            ctx.CommProbe.ExpectMsg<SiteEnvelope>();
+
+            ctx.BridgeActor.Tell(new DebugViewSnapshot(
+                InstanceName,
+                new List<AttributeValueChanged>(),
+                new List<AlarmStateChanged>(),
+                DateTimeOffset.UtcNow));
+
+            Thread.Sleep(700); // well past the deadline
+            Assert.False(ctx.TerminatedFlag[0]);
+
+            // Still serving: a post-snapshot event passes straight through.
+            ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AlarmStateChanged(
+                InstanceName, "PumpFault", Commons.Types.Enums.AlarmState.Active, 500,
+                DateTimeOffset.UtcNow));
+            AwaitCondition(() =>
+            {
+                lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AlarmStateChanged>().Any(); }
+            }, TimeSpan.FromSeconds(3));
+        }
+        finally
+        {
+            DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
+        }
+    }
+
+    [Fact]
+    public void StreamEvents_AreWrapped_SoTheyDoNotResetTheOrphanReceiveTimeout()
+    {
+        // Structural pin for the fix: the gRPC callback wraps every event in an envelope
+        // marked INotInfluenceReceiveTimeout, so a busy site can no longer keep an
+        // abandoned session alive indefinitely by feeding it events.
+        Assert.True(typeof(Akka.Actor.INotInfluenceReceiveTimeout)
+            .IsAssignableFrom(typeof(LiveDebugStreamEvent)));
+
+        var ctx = CreateBridgeActor();
+        ctx.CommProbe.ExpectMsg<SiteEnvelope>();
+        AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
+
+        ctx.BridgeActor.Tell(new DebugViewSnapshot(
+            InstanceName,
+            new List<AttributeValueChanged>(),
+            new List<AlarmStateChanged>(),
+            DateTimeOffset.UtcNow));
+
+        // An event delivered through the real gRPC callback path still reaches the
+        // consumer — the wrapper is transparent to delivery.
+        var evt = new AttributeValueChanged(
+            InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
+        ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
+
+        AwaitCondition(() =>
+        {
+            lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
+        }, TimeSpan.FromSeconds(3));
+    }
 }

 /// <summary>
@@ -79,11 +79,14 @@ public class SiteAlarmAggregatorActorTests : TestKit
        private readonly object _lock = new();
        public List<IReadOnlyList<AlarmStateChanged>> Snapshots { get; } = new();

-        public void Publish(IReadOnlyList<AlarmStateChanged> snapshot)
+        public void Publish(IReadOnlyList<AlarmStateChanged> snapshot, bool streamLive)
        {
-            lock (_lock) { Snapshots.Add(snapshot); }
+            lock (_lock) { Snapshots.Add(snapshot); StreamLive = streamLive; }
        }

+        /// <summary>Liveness reported with the most recent publish.</summary>
+        public bool StreamLive { get; private set; }
+
        public IReadOnlyList<AlarmStateChanged>? Latest
        {
            get { lock (_lock) { return Snapshots.Count == 0 ? null : Snapshots[^1]; } }
@@ -94,7 +97,7 @@ public class SiteAlarmAggregatorActorTests : TestKit

    private sealed record SiteSub(
        string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError,
-        Action OnCompleted, CancellationToken Ct);
+        Action OnCompleted, CancellationToken Ct, Action? OnConnected);

    private sealed class MockSiteAlarmStreamClient : SiteStreamGrpcClient
    {
@@ -107,11 +110,21 @@ public class SiteAlarmAggregatorActorTests : TestKit

        public MockSiteAlarmStreamClient() : base() { }

+        /// <summary>When false the stream never reports connected — the site accepted the TCP
+        /// call but never answered, so the aggregator's connect-driven re-seed must not run.</summary>
+        public bool AutoConnect { get; set; } = true;
+
        public override Task SubscribeSiteAsync(
            string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError,
-            Action onCompleted, CancellationToken ct)
+            Action onCompleted, CancellationToken ct, Action? onConnected = null)
        {
-            lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, onCompleted, ct)); }
+            lock (_lock)
+            {
+                _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, onCompleted, ct, onConnected));
+            }
+            // A healthy site accepts the subscription immediately (it flushes response headers
+            // as soon as its relay is attached), which is the aggregator's connect signal.
+            if (AutoConnect) onConnected?.Invoke();
            var tcs = new TaskCompletionSource();
            ct.Register(() => tcs.TrySetResult());
            return tcs.Task; // never completes until cancelled (simulates a live stream)
@@ -130,8 +143,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
        public MockSiteAlarmStreamClientFactory()
            : base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance) { }

+        /// <summary>Applied to every client this factory hands out (set before the actor starts).</summary>
+        public bool AutoConnect { get; set; } = true;
+
        public MockSiteAlarmStreamClient ClientFor(string endpoint) =>
-            _byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient());
+            _byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient { AutoConnect = AutoConnect });

        public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
            => ClientFor(grpcEndpoint);
@@ -141,18 +157,19 @@ public class SiteAlarmAggregatorActorTests : TestKit
    }

    private (IActorRef Actor, SeedStub Seed, PublishSink Sink, MockSiteAlarmStreamClientFactory Factory) CreateActor(
-        TimeSpan? reconcileInterval = null, TimeSpan? publishCoalesce = null)
+        TimeSpan? reconcileInterval = null, TimeSpan? publishCoalesce = null, bool autoConnect = true)
    {
        var seed = new SeedStub();
        var sink = new PublishSink();
-        var factory = new MockSiteAlarmStreamClientFactory();
+        var factory = new MockSiteAlarmStreamClientFactory { AutoConnect = autoConnect };

        var props = Props.Create(() => new SiteAlarmAggregatorActor(
            SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
            reconcileInterval ?? TimeSpan.FromMinutes(10),
            publishCoalesce ?? TimeSpan.Zero,
            TestReconnectDelay,
-            TestStabilityWindow));
+            TestStabilityWindow,
+            0.0)); // no reconcile jitter in tests — determinism

        var actor = Sys.ActorOf(props);
        return (actor, seed, sink, factory);
@@ -414,12 +431,16 @@ public class SiteAlarmAggregatorActorTests : TestKit
        AwaitAssert(() => Assert.Equal(1, sink.Count));
        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));

+        // The first tick after a fan-out is deliberately skipped (WP2.3: the initial seed
+        // already covered this window); the second one actually fans out.
+        actor.Tell(new RunReconcile());            // skipped — consumes the "already fanned out" flag
        actor.Tell(new RunReconcile());            // reconcile fan-out now in flight (CallCount 2)
        AwaitAssert(() => Assert.Equal(2, seed.CallCount));

-        // Stream error while the reconcile is in flight → the failover re-seed must not be
-        // silently skipped. Pre-fix: StartFanout no-ops and CallCount stays 2 until the next
-        // 60s reconcile tick.
+        // Stream error while the reconcile is in flight: the reconnect that follows
+        // reconnects (the mock accepts immediately) and its connect-driven re-seed must not
+        // be silently swallowed. Pre-fix: StartFanout no-ops and CallCount stays 2 until the
+        // next 60s reconcile tick.
        factory.ClientFor(GrpcNodeA).Subs.Last().OnError(new Exception("stream fault"));

        seed.CompleteNext();                       // finish the in-flight reconcile
@@ -577,4 +598,141 @@ public class SiteAlarmAggregatorActorTests : TestKit
        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Unsubscribed.Contains("corr-1"),
            TimeSpan.FromSeconds(3));
    }
+
+    // -- WP2.3: seed once per SUCCESSFUL (re)connect, and stream-truthful liveness --
+
+    [Fact]
+    public void ReconnectAttempts_DoNotEachReSeed_OnlyTheSuccessfulConnectDoes()
+    {
+        // Finding #10's per-attempt re-fan-out: every reconnect ATTEMPT used to kick a
+        // whole-site snapshot fan-out - against a site that is, by definition of the
+        // reconnect, unreachable. Now the re-seed is owed to the connect that succeeds.
+        var (_, seed, _, factory) = CreateActor(
+            reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3)); // initial seed
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+
+        // Three failed attempts, each flipping the node - none of them may re-seed.
+        factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("1"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
+        factory.ClientFor(GrpcNodeB).Subs[0].OnError(new Exception("2"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
+        factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
+
+        Thread.Sleep(300);
+        Assert.Equal(1, seed.CallCount); // pre-fix: 4
+
+        // The stream finally comes up - exactly ONE re-seed, on the connect.
+        factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!();
+        AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
+        seed.CompleteNext();
+        Thread.Sleep(300);
+        Assert.Equal(2, seed.CallCount); // and no second one
+    }
+
+    [Fact]
+    public void ReconcileTick_IsSkipped_WhenAFanoutAlreadyRanInTheWindow()
+    {
+        // The reconcile is a BACKSTOP, not an unconditional 60s whole-site snapshot: a
+        // window already covered by a seed costs nothing. Staleness stays bounded because
+        // the skip consumes the flag, so the next tick always fans out.
+        var (actor, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+        Thread.Sleep(200);
+
+        actor.Tell(new RunReconcile());   // skipped - the initial seed covered this window
+        Thread.Sleep(300);
+        Assert.Equal(1, seed.CallCount);
+
+        actor.Tell(new RunReconcile());   // and the next one runs
+        AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
+    }
+
+    [Fact]
+    public void UnchangedReconcileSnapshot_DoesNotRepublish()
+    {
+        // A reconcile that finds nothing changed used to wake every viewer's render path
+        // once a minute regardless. It now publishes as a diff.
+        var (actor, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+
+        var t = DateTimeOffset.UtcNow;
+        seed.CompleteNext(Alarm("PumpFault", "", 500, t));
+        AwaitAssert(() => Assert.Equal(1, sink.Count));
+
+        actor.Tell(new RunReconcile()); // skipped (seed covered the window)
+        actor.Tell(new RunReconcile()); // real fan-out
+        AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
+        seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot
+
+        Thread.Sleep(300);
+        Assert.Equal(1, sink.Count); // no second publish
+
+        // A genuinely changed snapshot still publishes.
+        actor.Tell(new RunReconcile());
+        actor.Tell(new RunReconcile());
+        AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(3));
+        seed.CompleteNext(Alarm("PumpFault", "", 900, t.AddSeconds(1)));
+        AwaitAssert(() => Assert.Equal(2, sink.Count));
+    }
+
+    [Fact]
+    public void StreamCompletion_DropsLiveness_AndTheReopenRestoresIt()
+    {
+        // WP2.3 carried residual: a completed (or given-up) stream kept reporting live
+        // until the next reconcile publish, so the page grafted a freezing snapshot over
+        // fresh poll data.
+        // Reconcile is held far away so the drop can be observed before the reopen
+        // restores liveness; the reopen is then driven explicitly.
+        var (actor, seed, sink, factory) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+        AwaitCondition(() => sink.Count >= 1 && sink.StreamLive, TimeSpan.FromSeconds(3));
+
+        // The site ends the stream cleanly (its 4h max lifetime).
+        factory.ClientFor(GrpcNodeA).Subs[0].OnCompleted();
+
+        // Liveness drops IMMEDIATELY - not at the next reconcile.
+        AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
+
+        // The reconcile tick reopens; the connect restores liveness.
+        actor.Tell(new RunReconcile());
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
+        AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(5));
+    }
+
+    [Fact]
+    public void StreamFault_DropsLiveness_Immediately()
+    {
+        var (_, seed, sink, factory) = CreateActor(
+            reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
+        seed.CompleteNext();
+        AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(3));
+
+        factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("site gone"));
+        AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
+    }
+
+    [Fact]
+    public void FailedSeed_IsRetried_WithBackoff_BeforeTheNextReconcile()
+    {
+        // The seed leg had no backoff: a failing fan-out simply waited a full reconcile
+        // interval. Now it retries on its own timer (reconnectDelay, doubling).
+        var (_, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+
+        seed.FaultNext();
+        // TestReconnectDelay is 50 ms, so the first retry lands far inside the 10-minute
+        // reconcile interval - pre-fix nothing would run until that interval elapsed.
+        AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
+        seed.FaultNext();
+        AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(5));
+    }
 }
@@ -530,4 +530,54 @@ public class SiteStreamGrpcServerTests : TestKit

        Assert.True(condition(), $"Condition not met within {timeoutMs}ms");
    }
+
+    // ── WP2.3: the site-wide alarm feed gets its OWN, larger send channel ──
+
+    [Fact]
+    public void SiteAlarmStream_HasItsOwnLargerChannel_ThanTheDebugView()
+    {
+        // Sharing the Debug View's 1000-slot DropOldest channel meant an alarm burst during
+        // a WAN stall silently evicted operator-visible transitions to make room for
+        // diagnostics traffic. The two feeds are now sized independently.
+        var options = Microsoft.Extensions.Options.Options.Create(new CommunicationOptions());
+        var server = new SiteStreamGrpcServer(_subscriber, _logger, options);
+
+        Assert.Equal(1000, server.InstanceChannelCapacity);
+        Assert.Equal(20_000, server.SiteAlarmChannelCapacity);
+        Assert.True(server.SiteAlarmChannelCapacity > server.InstanceChannelCapacity);
+    }
+
+    [Fact]
+    public void ChannelCapacities_AreBoundFromOptions_AndFloorAtOne()
+    {
+        var options = Microsoft.Extensions.Options.Options.Create(new CommunicationOptions
+        {
+            GrpcInstanceStreamChannelCapacity = 42,
+            GrpcSiteAlarmStreamChannelCapacity = 4242,
+        });
+        var server = new SiteStreamGrpcServer(_subscriber, _logger, options);
+
+        Assert.Equal(42, server.InstanceChannelCapacity);
+        Assert.Equal(4242, server.SiteAlarmChannelCapacity);
+
+        // A misconfigured zero/negative capacity must not throw at channel-construction
+        // time deep inside a live RPC — it floors at one instead.
+        var degenerate = new SiteStreamGrpcServer(_subscriber, _logger,
+            Microsoft.Extensions.Options.Options.Create(new CommunicationOptions
+            {
+                GrpcInstanceStreamChannelCapacity = 0,
+                GrpcSiteAlarmStreamChannelCapacity = -5,
+            }));
+        Assert.Equal(1, degenerate.InstanceChannelCapacity);
+        Assert.Equal(1, degenerate.SiteAlarmChannelCapacity);
+    }
+
+    [Fact]
+    public void DroppedStreamEventCount_StartsAtZero()
+    {
+        // The raw counter behind scadabridge.site.stream.events_dropped — a fresh node has
+        // evicted nothing.
+        var server = CreateServer();
+        Assert.Equal(0, server.DroppedStreamEventCount);
+    }
 }