perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/DebugStreamBridgeActor.cs

@@ -2,6 +2,7 @@ using Akka.Actor;
 using Akka.Event;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
 using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 
 namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
@@ -45,9 +46,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     private const int MaxRetries = 3;
     private const string ReconnectTimerKey = "grpc-reconnect";
     private const string StabilityTimerKey = "grpc-stability";
+    private const string SnapshotTimerKey = "debug-snapshot-deadline";
     /// <summary>Delay between gRPC reconnection attempts.</summary>
     internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
 
+    /// <summary>
+    /// Hard deadline on the initial <see cref="DebugViewSnapshot"/> (WP2.3). The site builds
+    /// it in milliseconds; if none arrives inside this window the site never answered (the
+    /// Ask was lost, the singleton moved mid-request, the instance actor is wedged) and the
+    /// session must FAIL rather than sit in the buffering phase accumulating live events
+    /// behind a snapshot that is never coming. Settable for tests.
+    /// </summary>
+    internal static TimeSpan SnapshotTimeout { get; set; } = TimeSpan.FromSeconds(60);
+
     /// <summary>
     /// How long a freshly-opened gRPC stream must stay up before its retry budget
     /// is considered "recovered" and <see cref="_retryCount"/> is reset to 0.
@@ -85,17 +96,35 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     /// Ordered buffer of live gRPC events (<see cref="AttributeValueChanged"/>/
     /// <see cref="AlarmStateChanged"/>) that arrived before the snapshot was delivered.
     /// Flushed (with per-entity dedup against the snapshot) when the snapshot arrives,
-    /// then never used again. Mutated only on the actor thread.
+    /// then never used again. Bounded by <see cref="MaxPreSnapshotBuffer"/> with drop-oldest
+    /// eviction (WP2.3): a snapshot that never arrives used to buffer without limit on the
+    /// central node. Mutated only on the actor thread.
     /// </summary>
-    private readonly List<object> _preSnapshotBuffer = new();
+    private readonly Queue<object> _preSnapshotBuffer = new();
 
     /// <summary>
-    /// Defensive log threshold: if the pre-snapshot buffer grows past this many events
-    /// during a slow snapshot we log once (events are NOT dropped — the window is short).
+    /// Defensive log threshold: the first warning fires when the pre-snapshot buffer grows
+    /// past this many events during a slow snapshot, before the hard cap starts evicting.
     /// </summary>
     private const int BufferWarnThreshold = 10_000;
     private bool _bufferWarned;
 
+    /// <summary>
+    /// Hard cap on the pre-snapshot buffer. Beyond it the OLDEST event is evicted — the
+    /// snapshot that ends the buffering phase is authoritative for anything that old, so
+    /// keeping the newest events is what preserves the post-snapshot delta chain.
+    /// </summary>
+    private const int MaxPreSnapshotBuffer = 20_000;
+
+    /// <summary>Events evicted from the pre-snapshot buffer in this session. Actor-thread only.</summary>
+    private long _preSnapshotDropped;
+
+    /// <summary>
+    /// Total pre-snapshot events dropped across all debug sessions on this node — the raw
+    /// counter behind <c>scadabridge.central.debug_view.presnapshot_dropped</c>.
+    /// </summary>
+    internal static long TotalPreSnapshotDropped;
+
     /// <summary>Timer scheduler for reconnect and stability window timers.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
 
@@ -176,6 +205,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
                 _instanceUniqueName, snapshot.AttributeValues.Count, snapshot.AlarmStates.Count,
                 _preSnapshotBuffer.Count);
 
+            // The snapshot arrived — stand the hard deadline down.
+            Timers.Cancel(SnapshotTimerKey);
+
             // Deliver the snapshot, then flush the gap-window buffer (deduped), then
             // switch to pass-through. Order matters: snapshot first, buffered events next.
             _onEvent(snapshot);
@@ -183,14 +215,40 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             _snapshotDelivered = true;
         });
 
-        // Domain events arriving via Self.Tell from gRPC callback.
-        // Receiving an event must NOT reset _retryCount — a
-        // flapping stream that delivers a single event between failures would
-        // otherwise never trip MaxRetries. The retry budget is recovered only by
-        // GrpcStreamStable (a stream that has stayed up for StabilityWindow).
-        // Before the snapshot has been delivered, BUFFER (in arrival order)
-        // rather than deliver — these may be gap-window events. After the snapshot has
-        // been flushed, pass through directly (same handler, phase-dependent behavior).
+        // Hard snapshot deadline (WP2.3). Nothing else ends a session stuck in the
+        // buffering phase: the site's reply was lost, so no gRPC error fires, the stream
+        // keeps delivering events, and (with the wrapper above) they no longer even reset
+        // the orphan timeout. Fail the session so the consumer is told and can reopen.
+        Receive<DebugSnapshotDeadline>(_ =>
+        {
+            if (_stopped || _snapshotDelivered) return;
+            _log.Error(
+                "No debug snapshot for {0} within {1}s ({2} event(s) buffered, {3} dropped); failing the session",
+                _instanceUniqueName, SnapshotTimeout.TotalSeconds,
+                _preSnapshotBuffer.Count, _preSnapshotDropped);
+            CleanupGrpc();
+            SendUnsubscribe();
+            _stopped = true;
+            _preSnapshotBuffer.Clear();
+            _onTerminated();
+            Context.Stop(Self);
+        });
+
+        // Domain events arriving via Self.Tell from the gRPC callback, wrapped so they do
+        // NOT influence the receive timeout (WP2.3): the orphan safety net exists to end a
+        // session whose CONSUMER is gone, and a busy site's event flood used to keep that
+        // net permanently reset — an abandoned session on a chatty instance never timed out.
+        // Receiving an event must not reset _retryCount either: a flapping stream that
+        // delivers a single event between failures would otherwise never trip MaxRetries.
+        // The retry budget is recovered only by GrpcStreamStable (a stream that has stayed
+        // up for StabilityWindow). Before the snapshot has been delivered, BUFFER (in arrival
+        // order) rather than deliver — these may be gap-window events; after the snapshot has
+        // been flushed, pass through directly (phase-dependent behavior).
+        Receive<LiveDebugStreamEvent>(wrapped => HandleStreamEvent(wrapped.Event));
+
+        // Unwrapped forms are still accepted (a direct Tell from a test or a future
+        // in-process producer); those DO influence the receive timeout, which is correct —
+        // they are not the high-volume stream path.
         Receive<AttributeValueChanged>(changed => HandleStreamEvent(changed));
         Receive<AlarmStateChanged>(changed => HandleStreamEvent(changed));
 
@@ -286,15 +344,30 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             return;
         }
 
-        _preSnapshotBuffer.Add(evt);
-        if (!_bufferWarned && _preSnapshotBuffer.Count > BufferWarnThreshold)
+        if (!_bufferWarned && _preSnapshotBuffer.Count + 1 > BufferWarnThreshold)
         {
             _bufferWarned = true;
             _log.Warning(
-                "Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot; " +
-                "events are still retained (not dropped).",
-                _instanceUniqueName, BufferWarnThreshold);
+                "Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot " +
+                "(hard cap {2}, drop-oldest beyond it).",
+                _instanceUniqueName, BufferWarnThreshold, MaxPreSnapshotBuffer);
         }
+
+        while (_preSnapshotBuffer.Count >= MaxPreSnapshotBuffer)
+        {
+            _preSnapshotBuffer.Dequeue();
+            _preSnapshotDropped++;
+            Interlocked.Increment(ref TotalPreSnapshotDropped);
+            ScadaBridgeTelemetry.RecordDebugPreSnapshotDrop();
+            if (_preSnapshotDropped == 1 || _preSnapshotDropped % 500 == 0)
+            {
+                _log.Warning(
+                    "Pre-snapshot debug-event buffer for {0} is at its {1}-event cap; {2} event(s) evicted so far",
+                    _instanceUniqueName, MaxPreSnapshotBuffer, _preSnapshotDropped);
+            }
+        }
+
+        _preSnapshotBuffer.Enqueue(evt);
     }
 
     /// <summary>
@@ -352,8 +425,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 
         if (dropped > 0 || flushed > 0)
         {
-            _log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot",
-                flushed, _instanceUniqueName, dropped);
+            _log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot" +
+                       " ({3} previously evicted at the buffer cap)",
+                flushed, _instanceUniqueName, dropped, _preSnapshotDropped);
         }
 
         _preSnapshotBuffer.Clear();
@@ -429,6 +503,10 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
         var request = new SubscribeDebugViewRequest(_instanceUniqueName, _correlationId);
         var envelope = new SiteEnvelope(_siteIdentifier, request);
         _centralCommunicationActor.Tell(envelope, Self);
+
+        // Arm the hard snapshot deadline alongside the request.
+        if (SnapshotTimeout > TimeSpan.Zero)
+            Timers.StartSingleTimer(SnapshotTimerKey, new DebugSnapshotDeadline(), SnapshotTimeout);
     }
 
     /// <inheritdoc />
@@ -468,7 +546,8 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             await client.SubscribeAsync(
                 _correlationId,
                 _instanceUniqueName,
-                evt => self.Tell(evt),
+                // Wrapped: stream traffic must not reset the orphan receive timeout.
+                evt => self.Tell(new LiveDebugStreamEvent(evt)),
                 ex => self.Tell(new GrpcStreamError(ex, generation)),
                 () => self.Tell(new GrpcStreamCompleted(generation)),
                 ct);
@@ -579,6 +658,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 /// </summary>
 public record StopDebugStream;
 
+/// <summary>
+/// Envelope for a live gRPC stream event (<c>AttributeValueChanged</c>/
+/// <c>AlarmStateChanged</c>). Implements <see cref="INotInfluenceReceiveTimeout"/> so a busy
+/// site's event flood cannot keep resetting the orphan-session receive timeout — the timeout
+/// measures consumer/session liveness, not site chatter (WP2.3).
+/// </summary>
+internal record LiveDebugStreamEvent(object Event) : INotInfluenceReceiveTimeout;
+
+/// <summary>
+/// Internal message: the hard deadline for the initial <c>DebugViewSnapshot</c> expired.
+/// </summary>
+internal record DebugSnapshotDeadline;
+
 /// <summary>
 /// Internal message indicating a gRPC stream error occurred, stamped with the stream
 /// generation it came from so a late error out of a cancelled stream can be ignored.