perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/DebugStreamBridgeActor.cs

@@ -2,6 +2,7 @@ using Akka.Actor;
 using Akka.Event;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
 using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 
 namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
@@ -45,9 +46,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     private const int MaxRetries = 3;
     private const string ReconnectTimerKey = "grpc-reconnect";
     private const string StabilityTimerKey = "grpc-stability";
+    private const string SnapshotTimerKey = "debug-snapshot-deadline";
     /// <summary>Delay between gRPC reconnection attempts.</summary>
     internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
 
+    /// <summary>
+    /// Hard deadline on the initial <see cref="DebugViewSnapshot"/> (WP2.3). The site builds
+    /// it in milliseconds; if none arrives inside this window the site never answered (the
+    /// Ask was lost, the singleton moved mid-request, the instance actor is wedged) and the
+    /// session must FAIL rather than sit in the buffering phase accumulating live events
+    /// behind a snapshot that is never coming. Settable for tests.
+    /// </summary>
+    internal static TimeSpan SnapshotTimeout { get; set; } = TimeSpan.FromSeconds(60);
+
     /// <summary>
     /// How long a freshly-opened gRPC stream must stay up before its retry budget
     /// is considered "recovered" and <see cref="_retryCount"/> is reset to 0.
@@ -85,17 +96,35 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     /// Ordered buffer of live gRPC events (<see cref="AttributeValueChanged"/>/
     /// <see cref="AlarmStateChanged"/>) that arrived before the snapshot was delivered.
     /// Flushed (with per-entity dedup against the snapshot) when the snapshot arrives,
-    /// then never used again. Mutated only on the actor thread.
+    /// then never used again. Bounded by <see cref="MaxPreSnapshotBuffer"/> with drop-oldest
+    /// eviction (WP2.3): a snapshot that never arrives used to buffer without limit on the
+    /// central node. Mutated only on the actor thread.
     /// </summary>
-    private readonly List<object> _preSnapshotBuffer = new();
+    private readonly Queue<object> _preSnapshotBuffer = new();
 
     /// <summary>
-    /// Defensive log threshold: if the pre-snapshot buffer grows past this many events
-    /// during a slow snapshot we log once (events are NOT dropped — the window is short).
+    /// Defensive log threshold: the first warning fires when the pre-snapshot buffer grows
+    /// past this many events during a slow snapshot, before the hard cap starts evicting.
     /// </summary>
     private const int BufferWarnThreshold = 10_000;
     private bool _bufferWarned;
 
+    /// <summary>
+    /// Hard cap on the pre-snapshot buffer. Beyond it the OLDEST event is evicted — the
+    /// snapshot that ends the buffering phase is authoritative for anything that old, so
+    /// keeping the newest events is what preserves the post-snapshot delta chain.
+    /// </summary>
+    private const int MaxPreSnapshotBuffer = 20_000;
+
+    /// <summary>Events evicted from the pre-snapshot buffer in this session. Actor-thread only.</summary>
+    private long _preSnapshotDropped;
+
+    /// <summary>
+    /// Total pre-snapshot events dropped across all debug sessions on this node — the raw
+    /// counter behind <c>scadabridge.central.debug_view.presnapshot_dropped</c>.
+    /// </summary>
+    internal static long TotalPreSnapshotDropped;
+
     /// <summary>Timer scheduler for reconnect and stability window timers.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
 
@@ -176,6 +205,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
                 _instanceUniqueName, snapshot.AttributeValues.Count, snapshot.AlarmStates.Count,
                 _preSnapshotBuffer.Count);
 
+            // The snapshot arrived — stand the hard deadline down.
+            Timers.Cancel(SnapshotTimerKey);
+
             // Deliver the snapshot, then flush the gap-window buffer (deduped), then
             // switch to pass-through. Order matters: snapshot first, buffered events next.
             _onEvent(snapshot);
@@ -183,14 +215,40 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             _snapshotDelivered = true;
         });
 
-        // Domain events arriving via Self.Tell from gRPC callback.
-        // Receiving an event must NOT reset _retryCount — a
-        // flapping stream that delivers a single event between failures would
-        // otherwise never trip MaxRetries. The retry budget is recovered only by
-        // GrpcStreamStable (a stream that has stayed up for StabilityWindow).
-        // Before the snapshot has been delivered, BUFFER (in arrival order)
-        // rather than deliver — these may be gap-window events. After the snapshot has
-        // been flushed, pass through directly (same handler, phase-dependent behavior).
+        // Hard snapshot deadline (WP2.3). Nothing else ends a session stuck in the
+        // buffering phase: the site's reply was lost, so no gRPC error fires, the stream
+        // keeps delivering events, and (with the wrapper above) they no longer even reset
+        // the orphan timeout. Fail the session so the consumer is told and can reopen.
+        Receive<DebugSnapshotDeadline>(_ =>
+        {
+            if (_stopped || _snapshotDelivered) return;
+            _log.Error(
+                "No debug snapshot for {0} within {1}s ({2} event(s) buffered, {3} dropped); failing the session",
+                _instanceUniqueName, SnapshotTimeout.TotalSeconds,
+                _preSnapshotBuffer.Count, _preSnapshotDropped);
+            CleanupGrpc();
+            SendUnsubscribe();
+            _stopped = true;
+            _preSnapshotBuffer.Clear();
+            _onTerminated();
+            Context.Stop(Self);
+        });
+
+        // Domain events arriving via Self.Tell from the gRPC callback, wrapped so they do
+        // NOT influence the receive timeout (WP2.3): the orphan safety net exists to end a
+        // session whose CONSUMER is gone, and a busy site's event flood used to keep that
+        // net permanently reset — an abandoned session on a chatty instance never timed out.
+        // Receiving an event must not reset _retryCount either: a flapping stream that
+        // delivers a single event between failures would otherwise never trip MaxRetries.
+        // The retry budget is recovered only by GrpcStreamStable (a stream that has stayed
+        // up for StabilityWindow). Before the snapshot has been delivered, BUFFER (in arrival
+        // order) rather than deliver — these may be gap-window events; after the snapshot has
+        // been flushed, pass through directly (phase-dependent behavior).
+        Receive<LiveDebugStreamEvent>(wrapped => HandleStreamEvent(wrapped.Event));
+
+        // Unwrapped forms are still accepted (a direct Tell from a test or a future
+        // in-process producer); those DO influence the receive timeout, which is correct —
+        // they are not the high-volume stream path.
         Receive<AttributeValueChanged>(changed => HandleStreamEvent(changed));
         Receive<AlarmStateChanged>(changed => HandleStreamEvent(changed));
 
@@ -286,15 +344,30 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             return;
         }
 
-        _preSnapshotBuffer.Add(evt);
-        if (!_bufferWarned && _preSnapshotBuffer.Count > BufferWarnThreshold)
+        if (!_bufferWarned && _preSnapshotBuffer.Count + 1 > BufferWarnThreshold)
         {
             _bufferWarned = true;
             _log.Warning(
-                "Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot; " +
-                "events are still retained (not dropped).",
-                _instanceUniqueName, BufferWarnThreshold);
+                "Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot " +
+                "(hard cap {2}, drop-oldest beyond it).",
+                _instanceUniqueName, BufferWarnThreshold, MaxPreSnapshotBuffer);
         }
+
+        while (_preSnapshotBuffer.Count >= MaxPreSnapshotBuffer)
+        {
+            _preSnapshotBuffer.Dequeue();
+            _preSnapshotDropped++;
+            Interlocked.Increment(ref TotalPreSnapshotDropped);
+            ScadaBridgeTelemetry.RecordDebugPreSnapshotDrop();
+            if (_preSnapshotDropped == 1 || _preSnapshotDropped % 500 == 0)
+            {
+                _log.Warning(
+                    "Pre-snapshot debug-event buffer for {0} is at its {1}-event cap; {2} event(s) evicted so far",
+                    _instanceUniqueName, MaxPreSnapshotBuffer, _preSnapshotDropped);
+            }
+        }
+
+        _preSnapshotBuffer.Enqueue(evt);
     }
 
     /// <summary>
@@ -352,8 +425,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 
         if (dropped > 0 || flushed > 0)
         {
-            _log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot",
-                flushed, _instanceUniqueName, dropped);
+            _log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot" +
+                       " ({3} previously evicted at the buffer cap)",
+                flushed, _instanceUniqueName, dropped, _preSnapshotDropped);
         }
 
         _preSnapshotBuffer.Clear();
@@ -429,6 +503,10 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
         var request = new SubscribeDebugViewRequest(_instanceUniqueName, _correlationId);
         var envelope = new SiteEnvelope(_siteIdentifier, request);
         _centralCommunicationActor.Tell(envelope, Self);
+
+        // Arm the hard snapshot deadline alongside the request.
+        if (SnapshotTimeout > TimeSpan.Zero)
+            Timers.StartSingleTimer(SnapshotTimerKey, new DebugSnapshotDeadline(), SnapshotTimeout);
     }
 
     /// <inheritdoc />
@@ -468,7 +546,8 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             await client.SubscribeAsync(
                 _correlationId,
                 _instanceUniqueName,
-                evt => self.Tell(evt),
+                // Wrapped: stream traffic must not reset the orphan receive timeout.
+                evt => self.Tell(new LiveDebugStreamEvent(evt)),
                 ex => self.Tell(new GrpcStreamError(ex, generation)),
                 () => self.Tell(new GrpcStreamCompleted(generation)),
                 ct);
@@ -579,6 +658,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 /// </summary>
 public record StopDebugStream;
 
+/// <summary>
+/// Envelope for a live gRPC stream event (<c>AttributeValueChanged</c>/
+/// <c>AlarmStateChanged</c>). Implements <see cref="INotInfluenceReceiveTimeout"/> so a busy
+/// site's event flood cannot keep resetting the orphan-session receive timeout — the timeout
+/// measures consumer/session liveness, not site chatter (WP2.3).
+/// </summary>
+internal record LiveDebugStreamEvent(object Event) : INotInfluenceReceiveTimeout;
+
+/// <summary>
+/// Internal message: the hard deadline for the initial <c>DebugViewSnapshot</c> expired.
+/// </summary>
+internal record DebugSnapshotDeadline;
+
 /// <summary>
 /// Internal message indicating a gRPC stream error occurred, stamped with the stream
 /// generation it came from so a late error out of a cancelled stream can be ignored.

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs

@@ -31,10 +31,14 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
 /// </para>
 /// <para>
 /// <b>Failover + drift:</b> a gRPC error flips NodeA↔NodeB with the same retry budget +
-/// stability window as <see cref="DebugStreamBridgeActor"/>, and each reconnect triggers
-/// a RE-SEED (never silently serve stale). A periodic reconcile snapshot
-/// (<see cref="_reconcileInterval"/>, default 60s) corrects instance-set drift and any
-/// missed delta.
+/// stability window as <see cref="DebugStreamBridgeActor"/>. A re-seed runs <b>once per
+/// successful (re)connect</b> — driven by the stream's connected callback, not by each
+/// reconnect ATTEMPT (WP2.3: a site outage used to fan a full snapshot out per retry, all
+/// of them against the very site that is unreachable). A periodic reconcile snapshot
+/// (<see cref="_reconcileInterval"/>, default 60s, jittered per site so N aggregators do
+/// not fan out in lockstep) remains the drift/backstop: it corrects instance-set drift and
+/// any missed delta, but it is <em>skipped</em> when a fan-out already completed inside the
+/// window, and it publishes to viewers only when the snapshot actually changed the cache.
 /// </para>
 /// All state is mutated only on the actor thread: gRPC callbacks and fan-out results are
 /// marshalled back via <c>Self.Tell</c>, so the cache needs no internal lock. The
@@ -47,18 +51,20 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     private readonly string _siteIdentifier;
     private readonly string _correlationId;
     private readonly Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> _seedFn;
-    private readonly Action<IReadOnlyList<AlarmStateChanged>> _publish;
+    private readonly Action<IReadOnlyList<AlarmStateChanged>, bool> _publish;
     private readonly SiteStreamGrpcClientFactory _grpcFactory;
     private readonly string _grpcNodeAAddress;
     private readonly string _grpcNodeBAddress;
     private readonly TimeSpan _reconcileInterval;
     private readonly TimeSpan _publishCoalesce;
+    private readonly double _reconcileJitterFraction;
 
     private const int MaxRetries = 3;
     private const string ReconnectTimerKey = "alarm-grpc-reconnect";
     private const string StabilityTimerKey = "alarm-grpc-stability";
     private const string ReconcileTimerKey = "alarm-reconcile";
     private const string PublishTimerKey = "alarm-publish-coalesce";
+    private const string SeedRetryTimerKey = "alarm-seed-retry";
 
     /// <summary>True while a coalesced publish is armed (dirty deltas awaiting one tick). Actor-thread only.</summary>
     private bool _publishPending;
@@ -83,9 +89,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     /// lifetime). Reconcile snapshots keep serving in the meantime; the next reconcile tick
     /// self-heals the stream by resetting the retry budget and reopening it, so neither a
     /// sustained site outage nor a routine 4h stream expiry permanently drops the live feed.
-    /// Actor-thread only.
+    /// Starts <c>true</c>: until the site accepts the first subscription there is no live
+    /// feed, and the owning cache must not advertise one. Actor-thread only.
     /// </summary>
-    private bool _streamDown;
+    private bool _streamDown = true;
 
     /// <summary>
     /// Why the stream is down: <c>true</c> = the retry budget was exhausted, so the
@@ -107,8 +114,57 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     /// <summary>True while a seed/reconcile snapshot fan-out is in flight (deltas buffer). Actor-thread only.</summary>
     private bool _fanoutInFlight;
 
-    /// <summary>Ordered buffer of live deltas that arrived while a fan-out was in flight. Actor-thread only.</summary>
-    private readonly List<AlarmStateChanged> _buffer = new();
+    /// <summary>
+    /// Ordered buffer of live deltas that arrived while a fan-out was in flight, capped at
+    /// <see cref="MaxBufferedDeltas"/> with drop-oldest eviction (WP2.3 — a stuck fan-out
+    /// used to buffer without bound). Dropping the OLDEST is the right eviction here: the
+    /// fan-out that follows rebuilds the cache authoritatively, so an evicted delta is
+    /// superseded rather than lost, while the newest transitions — the ones the snapshot may
+    /// predate — are the ones kept. Actor-thread only.
+    /// </summary>
+    private readonly Queue<AlarmStateChanged> _buffer = new();
+
+    /// <summary>Total deltas evicted from <see cref="_buffer"/> over this actor's life. Actor-thread only.</summary>
+    private long _bufferDropped;
+
+    /// <summary>
+    /// Hard cap on the pre-fan-out delta buffer. Beyond this the oldest entry is evicted and
+    /// counted; the counter is logged (first drop + every 500th) and exported as
+    /// <c>scadabridge.site.alarm_cache.buffer_dropped</c>.
+    /// </summary>
+    private const int MaxBufferedDeltas = 20_000;
+
+    /// <summary>
+    /// True when the next successful (re)connect must run a seed fan-out: set whenever the
+    /// stream is lost (fault or graceful completion) and cleared by the connect that consumes
+    /// it. This is what makes the re-seed happen once per successful reconnect rather than
+    /// once per reconnect ATTEMPT — the old code fanned a full snapshot out on every retry,
+    /// against a site that was by definition unreachable. Actor-thread only.
+    /// </summary>
+    private bool _seedOnConnect;
+
+    /// <summary>
+    /// Set whenever a fan-out finishes (success or failure); consumed by the next reconcile
+    /// tick, which skips its own fan-out when it finds the flag set. That makes the
+    /// connect-driven seed and the periodic backstop mutually exclusive — the pair used to
+    /// run BOTH, which is the "full unconditional snapshot every 60s" waste — while bounding
+    /// staleness at two intervals (a tick can be skipped at most once in a row).
+    /// Actor-thread only.
+    /// </summary>
+    private bool _fanoutSinceLastTick;
+
+    /// <summary>
+    /// True between issuing a stream open and its first outcome (connected / error /
+    /// completed), so a reconcile tick never stacks a second open on an in-flight one.
+    /// Actor-thread only.
+    /// </summary>
+    private bool _openInFlight;
+
+    /// <summary>Consecutive whole-fan-out failures, driving the seed-retry backoff. Actor-thread only.</summary>
+    private int _consecutiveSeedFailures;
+
+    /// <summary>Upper bound on the seed-retry backoff, as a multiple of the reconcile interval.</summary>
+    private const int MaxSeedRetryBackoffMultiplier = 8;
 
     /// <summary>
     /// A failover re-seed was requested while a fan-out was already in flight; it must run
@@ -144,6 +200,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     /// <param name="publish">
     /// Publishes a fresh immutable snapshot of the cache to the owning service, which
     /// stores it and raises viewer <c>onChanged</c> callbacks. Invoked on the actor thread.
+    /// The second argument is the live-stream liveness flag: <c>false</c> means the site-wide
+    /// gRPC stream is currently down (fault-exhausted or gracefully completed), so the cache
+    /// is only as fresh as the last reconcile and the page must fall back to polling
+    /// (WP2.3 carried residual — <c>IsLive</c> used to stay true across a dead stream).
     /// </param>
     /// <param name="grpcFactory">Factory caching one gRPC client per (site, endpoint).</param>
     /// <param name="grpcNodeAAddress">gRPC address of the site's node A.</param>
@@ -158,18 +218,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     /// <param name="stabilityWindow">
     /// How long a fresh gRPC stream must stay up before its retry budget recovers (production 60s).
     /// </param>
+    /// <param name="reconcileJitterFraction">
+    /// Fraction of <paramref name="reconcileInterval"/> added as a per-tick random offset so
+    /// N per-site aggregators on one central node do not fan out in lockstep (production 0.2 =
+    /// up to +20%). Zero disables jitter, which is what tests want for determinism.
+    /// </param>
     public SiteAlarmAggregatorActor(
         string siteIdentifier,
         string correlationId,
         Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> seedFn,
-        Action<IReadOnlyList<AlarmStateChanged>> publish,
+        Action<IReadOnlyList<AlarmStateChanged>, bool> publish,
         SiteStreamGrpcClientFactory grpcFactory,
         string grpcNodeAAddress,
         string grpcNodeBAddress,
         TimeSpan reconcileInterval,
         TimeSpan publishCoalesce,
         TimeSpan reconnectDelay,
-        TimeSpan stabilityWindow)
+        TimeSpan stabilityWindow,
+        double reconcileJitterFraction = 0.0)
     {
         _siteIdentifier = siteIdentifier;
         _correlationId = correlationId;
@@ -182,6 +248,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         _publishCoalesce = publishCoalesce;
         _reconnectDelay = reconnectDelay;
         _stabilityWindow = stabilityWindow;
+        _reconcileJitterFraction = Math.Clamp(reconcileJitterFraction, 0.0, 1.0);
 
         // Live delta from the site-wide alarm stream (marshalled in via Self.Tell).
         // A received delta must NOT reset the retry budget (a flapping stream that
@@ -198,6 +265,22 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Periodic reconcile tick (and the re-seed kicked after a reconnect).
         Receive<RunReconcile>(_ => OnReconcileTick());
 
+        // Backoff-scheduled retry of a fan-out that failed as a whole.
+        Receive<RetrySeed>(_ =>
+        {
+            if (_stopped) return;
+            StartFanout(isInitial: false);
+        });
+
+        // The site-wide stream is confirmed established (response headers received). This —
+        // not the reconnect attempt — is the "successful (re)connect" that earns one re-seed.
+        Receive<GrpcAlarmStreamConnected>(msg =>
+        {
+            if (_stopped) return;
+            if (msg.Generation != _streamGeneration) return;
+            OnStreamConnected();
+        });
+
         // Coalesced-publish tick: one publish for a batch of dirtying deltas (N6).
         Receive<PublishCoalesced>(_ =>
         {
@@ -270,11 +353,31 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // in the seed window are captured (buffered) rather than lost.
         OpenGrpcStream();
 
-        // Kick the initial seed fan-out.
+        // Kick the initial seed fan-out. The connect callback for this first stream must
+        // NOT seed again on top of it, so the flag starts cleared.
+        _seedOnConnect = false;
         StartFanout(isInitial: true);
 
-        // Periodic reconcile backstop.
-        Timers.StartPeriodicTimer(ReconcileTimerKey, new RunReconcile(), _reconcileInterval, _reconcileInterval);
+        // Periodic reconcile backstop. Single-shot and re-armed with fresh jitter each
+        // tick (a periodic timer would lock every site to the same phase forever).
+        ArmReconcileTimer();
+    }
+
+    /// <summary>
+    /// Arms the next reconcile tick at <see cref="_reconcileInterval"/> plus a random
+    /// offset of up to <see cref="_reconcileJitterFraction"/> of it, so N per-site
+    /// aggregators started together (a central failover restarts them all at once) spread
+    /// their fan-outs instead of stampeding the same 60s boundary.
+    /// </summary>
+    private void ArmReconcileTimer()
+    {
+        var delay = _reconcileInterval;
+        if (_reconcileJitterFraction > 0 && _reconcileInterval > TimeSpan.Zero)
+        {
+            var jitterTicks = (long)(_reconcileInterval.Ticks * _reconcileJitterFraction * Random.Shared.NextDouble());
+            delay += TimeSpan.FromTicks(jitterTicks);
+        }
+        Timers.StartSingleTimer(ReconcileTimerKey, new RunReconcile(), delay);
     }
 
     /// <inheritdoc />
@@ -294,16 +397,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     // ── Reconcile tick ──────────────────────────────────────────────────────────
 
     /// <summary>
-    /// Periodic reconcile: always re-run the snapshot fan-out (corrects drift + missed
-    /// deltas), and if the live stream was previously given up, self-heal it by resetting
-    /// the retry budget and reopening — so a sustained outage never permanently kills the
-    /// live feed.
+    /// Periodic reconcile backstop: re-run the snapshot fan-out (corrects instance-set drift
+    /// + any missed delta) UNLESS one already completed inside this window — a reconnect-driven
+    /// seed makes the tick redundant, and running both was the "full unconditional snapshot
+    /// every 60s" waste (WP2.3). If the live stream was previously given up, self-heal it by
+    /// resetting the retry budget and reopening, so a sustained outage never permanently kills
+    /// the live feed.
     /// </summary>
     private void OnReconcileTick()
     {
         if (_stopped) return;
-        StartFanout(isInitial: false);
-        if (_streamDown)
+
+        // Re-arm first: every exit path below must leave the backstop running.
+        ArmReconcileTimer();
+
+        // A down stream is reopened in preference to fanning out: the reopen's connect
+        // callback seeds by itself, so doing both would double the work.
+        var reopening = false;
+        if (_streamDown && !_openInFlight)
         {
             _log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
             if (_retryBudgetExhausted)
@@ -313,8 +424,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             }
             // Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
             ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
+            _seedOnConnect = true;
             OpenGrpcStream();
+            reopening = true;
         }
+
+        if (_fanoutSinceLastTick)
+        {
+            // A connect-driven seed (or the initial seed) already refreshed the cache inside
+            // this window. Clear the flag so the NEXT tick fans out regardless — staleness
+            // stays bounded at two intervals even if reconnects keep arriving.
+            _fanoutSinceLastTick = false;
+            _log.Debug("Site-alarm reconcile for {0} skipped; a fan-out already ran this window",
+                _siteIdentifier);
+            return;
+        }
+
+        if (!reopening)
+            StartFanout(isInitial: false);
     }
 
     // ── Seed / reconcile fan-out ────────────────────────────────────────────────
@@ -366,6 +493,12 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Rebuild the cache authoritatively from the fresh snapshot (this is what makes
         // reconcile able to DROP rows for instances/alarms that disappeared — a merge
         // could never remove a stale row since the live stream sends no "removed" event).
+        // The previous cache is kept alongside so the reconcile can publish as a DIFF:
+        // an unchanged snapshot must not wake every viewer's render path once a minute.
+        var previous = _cache.Count == 0
+            ? null
+            : new Dictionary<string, AlarmStateChanged>(_cache);
+
         _cache.Clear();
         foreach (var alarm in msg.Alarms)
         {
@@ -378,19 +511,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // buffered delta whose key is in the fresh snapshot with an equal-or-newer
         // timestamp is already reflected → drop; a strictly-newer (or new-key) delta is
         // applied. Inclusive-on-snapshot boundary matches DebugStreamBridgeActor.
-        FlushBuffer();
+        var flushChanged = FlushBuffer();
 
         _fanoutInFlight = false;
+        _fanoutSinceLastTick = true;
+        _consecutiveSeedFailures = 0;
+        Timers.Cancel(SeedRetryTimerKey);
+        var firstSeed = !_seeded;
         _seeded = true;
 
         _log.Debug("Site-alarm {0} {1} complete: {2} alarm row(s)",
             _siteIdentifier, msg.IsInitial ? "seed" : "reconcile", _cache.Count);
 
         // The fresh snapshot already carries the buffered deltas; drop any armed coalesce
-        // tick so we publish once, immediately.
+        // tick so we publish once, immediately — but only when something actually moved.
         Timers.Cancel(PublishTimerKey);
         _publishPending = false;
-        Publish();
+        if (firstSeed || flushChanged || DiffersFrom(previous))
+            Publish();
 
         // A failover re-seed requested while this fan-out was in flight runs now (N7.1).
         if (_reseedQueued)
@@ -400,22 +538,44 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    /// True when the just-installed snapshot differs from <paramref name="previous"/> in any
+    /// key or any row value — the reconcile's "is this publish worth a viewer fan-out?" test.
+    /// A null previous cache (nothing published yet) always counts as different.
+    /// </summary>
+    private bool DiffersFrom(Dictionary<string, AlarmStateChanged>? previous)
+    {
+        if (previous is null || previous.Count != _cache.Count) return true;
+
+        foreach (var (key, current) in _cache)
+        {
+            if (!previous.TryGetValue(key, out var old)) return true;
+            // AlarmStateChanged is a record: structural equality covers state, level,
+            // timestamp and every native-alarm enrichment field.
+            if (!Equals(old, current)) return true;
+        }
+        return false;
+    }
+
     private void OnSeedFailed(SeedFailed msg)
     {
         if (_stopped) return;
 
+        _consecutiveSeedFailures++;
+
         _log.Warning(msg.Exception,
-            "Site-alarm {0} {1} fan-out failed; keeping current cache and relying on the next reconcile",
-            _siteIdentifier, msg.IsInitial ? "seed" : "reconcile");
+            "Site-alarm {0} {1} fan-out failed ({2} consecutive); keeping current cache and retrying with backoff",
+            _siteIdentifier, msg.IsInitial ? "seed" : "reconcile", _consecutiveSeedFailures);
 
         // Don't lose deltas captured during the failed window — apply them pass-through
         // into the (possibly stale/empty) cache. The next reconcile re-seeds authoritatively.
         _fanoutInFlight = false;
-        FlushBuffer(dedupAgainstSeed: false);
+        _fanoutSinceLastTick = true;
+        var flushChanged = FlushBuffer(dedupAgainstSeed: false);
 
         // Only publish if we already had a seed (so IsLive doesn't flip true on a
         // failed initial seed — the page keeps its poll fallback until we truly seed).
-        if (_seeded)
+        if (_seeded && flushChanged)
         {
             Timers.Cancel(PublishTimerKey);
             _publishPending = false;
@@ -427,27 +587,54 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         {
             _reseedQueued = false;
             StartFanout(isInitial: false);
+            return;
         }
+
+        // Backoff retry (WP2.3): the seed leg used to have none — a site returning errors
+        // was re-fanned at full reconcile cadence forever. Retry at reconnectDelay doubling
+        // up to MaxSeedRetryBackoffMultiplier × the reconcile interval; the periodic
+        // reconcile remains the floor once the backoff saturates.
+        ScheduleSeedRetry();
+    }
+
+    /// <summary>
+    /// Arms the backoff retry for a failed fan-out: <c>reconnectDelay × 2^(failures-1)</c>,
+    /// capped at <see cref="MaxSeedRetryBackoffMultiplier"/> × the reconcile interval. Once
+    /// the cap is reached the periodic reconcile tick is doing the same work anyway, so no
+    /// extra timer is armed.
+    /// </summary>
+    private void ScheduleSeedRetry()
+    {
+        var cap = _reconcileInterval > TimeSpan.Zero
+            ? TimeSpan.FromTicks(_reconcileInterval.Ticks * MaxSeedRetryBackoffMultiplier)
+            : TimeSpan.FromMinutes(8);
+
+        var shift = Math.Min(_consecutiveSeedFailures - 1, 16);
+        var delayTicks = _reconnectDelay.Ticks * (1L << shift);
+        var delay = TimeSpan.FromTicks(Math.Min(delayTicks, cap.Ticks));
+        if (delay <= TimeSpan.Zero) return;
+
+        Timers.StartSingleTimer(SeedRetryTimerKey, new RetrySeed(), delay);
     }
 
     /// <summary>
     /// Flushes the pre-fan-out buffer in arrival order. When <paramref name="dedupAgainstSeed"/>
     /// is true (normal completion) a buffered delta already reflected in the just-installed
     /// snapshot (same key, timestamp &lt;= cache entry) is dropped; otherwise every buffered
-    /// delta is applied pass-through.
+    /// delta is applied pass-through. Returns <c>true</c> when at least one delta changed the
+    /// cache (the reconcile's publish-only-on-change test must count these too).
     /// </summary>
-    private void FlushBuffer(bool dedupAgainstSeed = true)
+    private bool FlushBuffer(bool dedupAgainstSeed = true)
     {
-        if (_buffer.Count == 0) return;
+        if (_buffer.Count == 0) return false;
 
+        var changed = false;
         foreach (var delta in _buffer)
         {
-            if (dedupAgainstSeed)
-                ApplyDelta(delta, requireStrictlyNewer: true);
-            else
-                ApplyDelta(delta, requireStrictlyNewer: false);
+            changed |= ApplyDelta(delta, requireStrictlyNewer: dedupAgainstSeed);
         }
         _buffer.Clear();
+        return changed;
     }
 
     // ── Live delta handling ─────────────────────────────────────────────────────
@@ -458,15 +645,32 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
 
         if (_fanoutInFlight)
         {
-            _buffer.Add(delta);
-            if (!_bufferWarned && _buffer.Count > BufferWarnThreshold)
+            if (!_bufferWarned && _buffer.Count + 1 > BufferWarnThreshold)
             {
                 _bufferWarned = true;
                 _log.Warning(
                     "Site-alarm pre-seed buffer for {0} exceeded {1} deltas while a fan-out was in flight " +
-                    "(deltas retained, not dropped).",
-                    _siteIdentifier, BufferWarnThreshold);
+                    "(hard cap {2}, drop-oldest beyond it).",
+                    _siteIdentifier, BufferWarnThreshold, MaxBufferedDeltas);
             }
+
+            // Hard cap with drop-oldest: an unbounded buffer behind a stuck fan-out was a
+            // straight memory leak. The evicted rows are superseded by the snapshot that
+            // ends the fan-out, so the drop costs freshness, never correctness.
+            while (_buffer.Count >= MaxBufferedDeltas)
+            {
+                _buffer.Dequeue();
+                _bufferDropped++;
+                ScadaBridgeTelemetry.RecordLiveAlarmBufferDrop();
+                if (_bufferDropped == 1 || _bufferDropped % 500 == 0)
+                {
+                    _log.Warning(
+                        "Site-alarm pre-seed buffer for {0} is at its {1}-delta cap; {2} delta(s) evicted so far",
+                        _siteIdentifier, MaxBufferedDeltas, _bufferDropped);
+                }
+            }
+
+            _buffer.Enqueue(delta);
             return;
         }
 
@@ -516,7 +720,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         var snapshot = _cache.Values.ToList();
         try
         {
-            _publish(snapshot);
+            // Liveness rides every publish so the owning service's IsLive tracks the STREAM,
+            // not merely "a snapshot was published once". A down stream means the cache is
+            // only as fresh as the last reconcile and the page must keep polling.
+            _publish(snapshot, !_streamDown);
         }
         catch (Exception ex)
         {
@@ -530,7 +737,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     {
         if (_stopped) return;
 
-        _streamDown = false;
+        // The stream is not live again until the server actually accepts it — liveness flips
+        // on GrpcAlarmStreamConnected, not on the attempt (WP2.3 carried residual).
         var endpoint = _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress;
         _log.Info("Opening site-alarm gRPC stream for {0} to {1}", _siteIdentifier, endpoint);
 
@@ -540,6 +748,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
 
         Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), _stabilityWindow);
 
+        _openInFlight = true;
         var generation = ++_streamGeneration;
         var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
         var self = Self;
@@ -556,7 +765,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
                 alarm => self.Tell(alarm),
                 ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
                 () => self.Tell(new GrpcAlarmStreamCompleted(generation)),
-                ct);
+                ct,
+                () => self.Tell(new GrpcAlarmStreamConnected(generation)));
         }, ct).ContinueWith(t =>
         {
             if (t.IsFaulted)
@@ -567,6 +777,32 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         }, TaskContinuationOptions.ExecuteSynchronously);
     }
 
+    /// <summary>
+    /// The site accepted the subscription — the only point at which the stream is genuinely
+    /// live. Consumes the pending re-seed (so exactly ONE fan-out runs per successful
+    /// (re)connect, instead of one per reconnect attempt against a site that is by definition
+    /// unreachable) and republishes so <c>IsLive</c> recovers.
+    /// </summary>
+    private void OnStreamConnected()
+    {
+        _openInFlight = false;
+
+        var wasDown = _streamDown;
+        _streamDown = false;
+
+        if (_seedOnConnect)
+        {
+            _seedOnConnect = false;
+            _log.Info("Site-alarm gRPC stream for {0} (re)connected; running one re-seed", _siteIdentifier);
+            StartFanout(isInitial: false);
+        }
+
+        // Liveness recovered — republish so viewers stop falling back to polling even if the
+        // seed that follows finds nothing changed.
+        if (wasDown && _seeded)
+            Publish();
+    }
+
     private void HandleGrpcError()
     {
         if (_stopped) return;
@@ -574,6 +810,19 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Stream failed before the stability window — retry budget NOT recovered.
         Timers.Cancel(StabilityTimerKey);
 
+        _openInFlight = false;
+        var wasLive = !_streamDown;
+        _streamDown = true;
+
+        // The next successful connect owes exactly one re-seed. Setting it here (rather than
+        // fanning out now) is the fix for the per-attempt re-fan-out: a site outage used to
+        // cost one whole-site snapshot per retry.
+        _seedOnConnect = true;
+
+        // Liveness lost — tell viewers immediately so a dead stream stops reading as live.
+        if (wasLive && _seeded)
+            Publish();
+
         _retryCount++;
 
         if (_retryCount > MaxRetries)
@@ -585,7 +834,6 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             _log.Error("Site-alarm gRPC stream for {0} exceeded max retries ({1}); leaving stream down, " +
                        "reconcile snapshots continue and the next reconcile tick will retry the stream",
                        _siteIdentifier, MaxRetries);
-            _streamDown = true;
             _retryBudgetExhausted = true;
             CleanupGrpc();
             return;
@@ -602,11 +850,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Telemetry: a NodeA↔NodeB failover flip is a reconnect + re-seed.
         ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
 
-        // A failover flip must RE-SEED (never silently serve stale) — kick a reconcile
-        // fan-out alongside the reconnect. Buffering during the fan-out keeps the new
-        // stream's deltas coherent with the fresh snapshot.
-        StartFanout(isInitial: false);
-
+        // The re-seed is owed to the CONNECT (see _seedOnConnect above), not to this
+        // attempt: a snapshot fan-out issued while the site is unreachable degrades to
+        // empty rows and would have to be redone on reconnect anyway.
         if (_retryCount == 1)
             Self.Tell(new ReconnectAlarmStream());
         else
@@ -629,8 +875,17 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         _log.Info("Site-alarm gRPC stream for {0} completed gracefully (server end of stream); " +
                   "reopening on the next reconcile tick", _siteIdentifier);
 
+        _openInFlight = false;
+        var wasLive = !_streamDown;
         _streamDown = true;
+        // The reopen owes one re-seed, run when it actually connects.
+        _seedOnConnect = true;
         CleanupGrpc();
+
+        // A completed stream is NOT live: without this the cache kept reporting live for up
+        // to a full reconcile interval after the site closed the feed (WP2.3 residual).
+        if (wasLive && _seeded)
+            Publish();
     }
 
     private void CleanupGrpc()
@@ -676,6 +931,14 @@ internal sealed record SeedFailed(Exception Exception, bool IsInitial);
 /// <summary>Internal: periodic reconcile tick (and the re-seed kicked after a reconnect).</summary>
 internal sealed record RunReconcile;
 
+/// <summary>Internal: backoff-scheduled retry of a fan-out that failed as a whole.</summary>
+internal sealed record RetrySeed;
+
+/// <summary>Internal: the site accepted the site-wide alarm subscription (response headers
+/// received), stamped with its stream generation so a late connect from a cancelled stream
+/// is ignored. This — not the reconnect attempt — is what earns a re-seed.</summary>
+internal sealed record GrpcAlarmStreamConnected(int Generation);
+
 /// <summary>Internal: coalesced-publish tick — flush the dirty cache to viewers once (N6).</summary>
 internal sealed record PublishCoalesced;