Merge branch 'worktree-agent-a465fb3cd6ec48cd3' into arch-review-remediation

src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SiteAuditBacklogReporter.cs

@@ -45,12 +45,27 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
     /// </summary>
     internal static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(30);
 
+    /// <summary>
+    /// Age at which the oldest still-Pending row is called out in the log. A Pending row is
+    /// one central has not acknowledged through EITHER path (telemetry push or reconciliation
+    /// pull), and <c>PurgeExpiredAsync</c> deliberately never purges Pending — so the site
+    /// store's floor depends on reconciliation actually running. Rows served by a pull but
+    /// never covered by a later cursor sit in exactly this state, which is why the age is
+    /// worth a signal rather than only a dashboard number. One day is comfortably longer than
+    /// any normal drain/reconcile outage and well inside the ~7-day site retention window.
+    /// </summary>
+    internal static readonly TimeSpan StalePendingThreshold = TimeSpan.FromHours(24);
+
+    /// <summary>How often the stale-pending warning may repeat (it is a standing condition).</summary>
+    internal static readonly TimeSpan StalePendingWarnInterval = TimeSpan.FromHours(1);
+
     private readonly ISiteAuditQueue _queue;
     private readonly ISiteHealthCollector _collector;
     private readonly ILogger<SiteAuditBacklogReporter> _logger;
     private readonly TimeSpan _refreshInterval;
     private CancellationTokenSource? _cts;
     private Task? _loop;
+    private DateTime _lastStalePendingWarnUtc = DateTime.MinValue;
 
     /// <summary>Initializes a new instance of <see cref="SiteAuditBacklogReporter"/>.</summary>
     /// <param name="queue">The site audit queue used to probe the backlog count.</param>
@@ -151,6 +166,7 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
         {
             var snapshot = await _queue.GetBacklogStatsAsync(ct).ConfigureAwait(false);
             _collector.UpdateSiteAuditBacklog(snapshot);
+            WarnIfPendingIsStale(snapshot.OldestPendingUtc, snapshot.PendingCount);
         }
         catch (OperationCanceledException)
         {
@@ -166,6 +182,34 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
         }
     }
 
+    /// <summary>
+    /// Logs a rate-limited warning when the oldest still-Pending audit row is older than
+    /// <see cref="StalePendingThreshold"/>. Pending rows are exempt from the retention purge
+    /// by design, so a standing Pending backlog is the one site-store condition that does not
+    /// self-heal on age — it clears only when central acknowledges the rows (telemetry ack or
+    /// a reconciliation cursor that covers them). Internal so tests can drive it directly.
+    /// </summary>
+    /// <param name="oldestPendingUtc">Oldest pending row's occurrence instant, or null when none.</param>
+    /// <param name="pendingCount">Number of rows currently pending.</param>
+    internal void WarnIfPendingIsStale(DateTime? oldestPendingUtc, int pendingCount)
+    {
+        if (oldestPendingUtc is null) return;
+
+        var age = DateTime.UtcNow - DateTime.SpecifyKind(oldestPendingUtc.Value, DateTimeKind.Utc);
+        if (age < StalePendingThreshold) return;
+
+        var now = DateTime.UtcNow;
+        if (now - _lastStalePendingWarnUtc < StalePendingWarnInterval) return;
+        _lastStalePendingWarnUtc = now;
+
+        _logger.LogWarning(
+            "Site audit backlog: oldest pending row is {AgeHours:F1}h old ({PendingCount} pending). " +
+            "Pending rows are never purged on age, so this backlog only clears when central " +
+            "acknowledges them — check the site→central audit telemetry drain and the central " +
+            "reconciliation pull.",
+            age.TotalHours, pendingCount);
+    }
+
     /// <summary>Signals the polling loop to stop and waits for it to complete.</summary>
     /// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
     /// <returns>A task that represents the asynchronous operation.</returns>

src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SqliteAuditWriter.cs

@@ -792,9 +792,14 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                 // ordering the query applies, so it is a strict "everything after this row".
                 : "(fs.OccurredAtUtc > $since OR (fs.OccurredAtUtc = $since AND ae.EventId > $afterId))";
 
+            // fs.rowid rides along as an 11th column purely to maintain
+            // _maxServedRowId — the insertion-order high-water mark that bounds
+            // MarkReconciledUpToAsync (see that method). It is never mapped onto
+            // the returned AuditEvent; rowid stays a storage-layer concept.
             cmd.CommandText = $"""
                 SELECT ae.EventId, ae.OccurredAtUtc, ae.Actor, ae.Action, ae.Outcome,
-                       ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson
+                       ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson,
+                       fs.rowid
                 FROM   audit_event ae
                 JOIN   audit_forward_state fs ON fs.EventId = ae.EventId
                 WHERE  fs.ForwardState IN ($pending, $forwarded)
@@ -816,10 +821,47 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
             }
             cmd.Parameters.AddWithValue("$limit", batchSize);
 
-            return Task.FromResult(ReadRows(cmd, batchSize));
+            return Task.FromResult(ReadServedRows(cmd, batchSize));
         }
     }
 
+    /// <summary>
+    /// Highest <c>audit_forward_state.rowid</c> this instance has ever SERVED from
+    /// <see cref="ReadPendingSinceAsync"/> — i.e. the insertion-order high-water mark of
+    /// rows central could possibly have received through the pull path. Bounds
+    /// <see cref="MarkReconciledUpToAsync"/> so a row inserted after that point can never be
+    /// retired by a cursor that happens to sit above its timestamp. Monotonic (a later, smaller
+    /// batch never lowers it) except for the purge clamp. Written under <c>_readLock</c> (the
+    /// read path) and read under <c>_writeLock</c> (the flip/purge paths) — two different
+    /// locks, so the accesses go through <see cref="Volatile"/> for visibility. A stale read
+    /// can only make the bound smaller, i.e. more conservative, never unsafe.
+    /// </summary>
+    private long _maxServedRowId;
+
+    /// <summary>
+    /// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
+    /// 11th column, materialising the canonical rows while advancing
+    /// <see cref="_maxServedRowId"/>. Serving a row is what makes it eligible for
+    /// cursor-proved retirement, so the two must happen in the same step.
+    /// </summary>
+    private IReadOnlyList<AuditEvent> ReadServedRows(SqliteCommand cmd, int capacityHint)
+    {
+        var rows = new List<AuditEvent>(Math.Min(capacityHint, 256));
+        var maxRowId = Volatile.Read(ref _maxServedRowId);
+        using (var reader = cmd.ExecuteReader())
+        {
+            while (reader.Read())
+            {
+                rows.Add(MapRow(reader));
+                var rowId = reader.GetInt64(10);
+                if (rowId > maxRowId) maxRowId = rowId;
+            }
+        }
+
+        Volatile.Write(ref _maxServedRowId, maxRowId);
+        return rows;
+    }
+
     /// <summary>
     /// Normalises a wire-supplied event id to the exact textual form stored in
     /// <c>audit_event.EventId</c> so the keyset comparison is apples-to-apples. An
@@ -845,16 +887,40 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                 ? "fs.OccurredAtUtc < $since"
                 : "(fs.OccurredAtUtc < $since OR (fs.OccurredAtUtc = $since AND fs.EventId <= $afterId))";
 
+            // INSERTION-ORDER BOUND — "only rows we actually served can retire".
+            //
+            // The timestamp cursor alone is not enough. OccurredAtUtc is stamped by the
+            // caller, so a row can be INSERTED after a batch was served yet carry a
+            // timestamp BELOW central's (by then advanced) cursor — a late-stamped insert.
+            // The blanket cursor UPDATE retired exactly those rows: never served, never
+            // servable again (ReadPendingSinceAsync's keyset has moved past them), and,
+            // being Reconciled, purged on age — silent audit loss, a failure mode the old
+            // explicit id-set flip could not produce.
+            //
+            // fs.rowid is insertion order, so bounding the flip at the highest rowid this
+            // instance has ever served restores the invariant exactly: rows inserted after
+            // that point are out of the flip's reach no matter where the cursor sits. A
+            // FORWARDED row is exempt from the bound — central ACKED it through the
+            // telemetry push path, which is receipt-proof independent of the pull.
+            //
+            // Bound state is per-process: after a restart it is 0 until this instance serves
+            // a batch, so the first pull retires only Forwarded rows and the pull after it
+            // resumes normal retirement. Conservative in the safe direction (delays
+            // retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
+            // interface doc + Component-AuditLog.md covers what a permanently halted
+            // reconciliation means for such rows.
             using var cmd = _connection.CreateCommand();
             cmd.CommandText = $"""
                 UPDATE audit_forward_state AS fs
                 SET    ForwardState = $reconciled
                 WHERE  fs.ForwardState IN ($pending, $forwarded)
+                  AND  (fs.ForwardState = $forwarded OR fs.rowid <= $maxServedRowId)
                   AND  {predicate};
                 """;
             cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
             cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
             cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
+            cmd.Parameters.AddWithValue("$maxServedRowId", Volatile.Read(ref _maxServedRowId));
             cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
                 "o", System.Globalization.CultureInfo.InvariantCulture));
             if (afterId is not null)
@@ -1063,6 +1129,22 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                         dropCmd.ExecuteNonQuery();
                     }
 
+                    // Clamp the served-rows high-water mark to what is still in the table.
+                    // SQLite hands a fresh insert MAX(rowid)+1, so a purge that empties the
+                    // sidecar (or removes its top rows) makes rowids REUSABLE — a stale-high
+                    // _maxServedRowId would then vouch for rows nobody has served. Cheap:
+                    // MAX(rowid) is an index-less O(1) lookup.
+                    if (purged > 0)
+                    {
+                        using var maxCmd = _connection.CreateCommand();
+                        maxCmd.Transaction = transaction;
+                        maxCmd.CommandText = "SELECT IFNULL(MAX(rowid), 0) FROM audit_forward_state;";
+                        var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
+                            System.Globalization.CultureInfo.InvariantCulture);
+                        if (liveMax < Volatile.Read(ref _maxServedRowId))
+                            Volatile.Write(ref _maxServedRowId, liveMax);
+                    }
+
                     transaction.Commit();
                 }
                 catch

src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Services/ISiteAuditQueue.cs

@@ -143,6 +143,29 @@ public interface ISiteAuditQueue
     /// <paramref name="sinceUtc"/> are proven received; rows at the boundary instant are left
     /// alone. Idempotent; already-Reconciled rows are untouched.
     /// </para>
+    /// <para>
+    /// <b>Only-served-rows-retire.</b> The cursor is a timestamp, and
+    /// <see cref="AuditEvent.OccurredAtUtc"/> is stamped by the caller, so a row can be
+    /// INSERTED after a batch was served yet carry a timestamp below the (by then advanced)
+    /// cursor. Implementations MUST NOT retire such a row: it was never served, the keyset
+    /// read has moved past it, and retiring it would make it purgeable — silent audit loss.
+    /// The SQLite implementation bounds the flip by insertion order (<c>rowid</c> high-water
+    /// mark of rows it has served), exempting rows already
+    /// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Forwarded"/>
+    /// because central ACKED those through the telemetry push path.
+    /// </para>
+    /// <para>
+    /// <b>Liveness dependency (documented, accepted).</b> A row that WAS served but is never
+    /// covered by a later cursor — central reconciliation stopped for good, or the site node
+    /// restarted and its bound reset — stays
+    /// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Pending"/>
+    /// indefinitely, and <see cref="PurgeExpiredAsync"/> never purges Pending. That is the
+    /// retention invariant working as designed (an unacknowledged row is not droppable), but
+    /// it means the site store's floor is bounded by reconciliation actually running. The
+    /// backlog is observable: <see cref="GetBacklogStatsAsync"/> reports the pending count and
+    /// oldest-pending instant on every site health report, and the site reporter logs a
+    /// warning once the oldest pending row exceeds its stale threshold.
+    /// </para>
     /// </remarks>
     /// <param name="sinceUtc">The cursor timestamp central has consumed up to (UTC).</param>
     /// <param name="afterId">The last consumed <see cref="AuditEvent.EventId"/> at that instant, or null.</param>

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/DebugStreamBridgeActor.cs

@@ -47,6 +47,7 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     private const string ReconnectTimerKey = "grpc-reconnect";
     private const string StabilityTimerKey = "grpc-stability";
     private const string SnapshotTimerKey = "debug-snapshot-deadline";
+    private const string ConsumerLivenessTimerKey = "debug-consumer-liveness";
     /// <summary>Delay between gRPC reconnection attempts.</summary>
     internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
 
@@ -69,6 +70,33 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     /// </summary>
     internal static TimeSpan StabilityWindow { get; set; } = TimeSpan.FromSeconds(60);
 
+    /// <summary>
+    /// Orphan window: how long the session may go without ANY sign of life from its CONSUMER
+    /// before it self-terminates. Renewed by <see cref="DebugStreamConsumerAlive"/>, which
+    /// <c>DebugStreamService</c> Tells on a timer for every session still attached to a
+    /// consumer (Blazor debug view or the SignalR hub) — so it measures the consumer, never
+    /// the stream. Settable for tests.
+    /// </summary>
+    internal static TimeSpan ConsumerIdleTimeout { get; set; } = TimeSpan.FromMinutes(5);
+
+    /// <summary>
+    /// How often the actor checks the consumer-last-seen stamp against
+    /// <see cref="ConsumerIdleTimeout"/>. A self-tick rather than <c>SetReceiveTimeout</c>:
+    /// the receive timeout measures the MAILBOX, which conflates site chatter with consumer
+    /// liveness — and once stream events were correctly excluded from it (via
+    /// <see cref="LiveDebugStreamEvent"/>) nothing recurring reset it at all, so every healthy
+    /// session self-terminated one window after its snapshot with a false "Site disconnected".
+    /// Settable for tests.
+    /// </summary>
+    internal static TimeSpan ConsumerLivenessCheckInterval { get; set; } = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// When the consumer was last known to be attached (UTC). Seeded in <see cref="PreStart"/>
+    /// so a session gets a full window to receive its first keepalive, then refreshed by every
+    /// <see cref="DebugStreamConsumerAlive"/>. Actor-thread only.
+    /// </summary>
+    private DateTime _consumerLastSeenUtc = DateTime.UtcNow;
+
     private int _retryCount;
     private bool _useNodeA = true;
     private bool _stopped;
@@ -188,7 +216,7 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
                 // non-deployed instance — cancel it (and any buffered gap events are
                 // discarded with the actor). No pass-through.
                 // _stopped is set AFTER CleanupGrpc() to match the ordering in the
-                // DebugStreamTerminated and ReceiveTimeout handlers (cosmetic consistency).
+                // DebugStreamTerminated and consumer-liveness handlers (cosmetic consistency).
                 CleanupGrpc();
                 _stopped = true;
                 _preSnapshotBuffer.Clear();
@@ -217,8 +245,8 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 
         // Hard snapshot deadline (WP2.3). Nothing else ends a session stuck in the
         // buffering phase: the site's reply was lost, so no gRPC error fires, the stream
-        // keeps delivering events, and (with the wrapper above) they no longer even reset
-        // the orphan timeout. Fail the session so the consumer is told and can reopen.
+        // keeps delivering events, and stream traffic does not renew the orphan net (which
+        // measures the consumer). Fail the session so the consumer is told and can reopen.
         Receive<DebugSnapshotDeadline>(_ =>
         {
             if (_stopped || _snapshotDelivered) return;
@@ -234,21 +262,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             Context.Stop(Self);
         });
 
-        // Domain events arriving via Self.Tell from the gRPC callback, wrapped so they do
-        // NOT influence the receive timeout (WP2.3): the orphan safety net exists to end a
-        // session whose CONSUMER is gone, and a busy site's event flood used to keep that
-        // net permanently reset — an abandoned session on a chatty instance never timed out.
-        // Receiving an event must not reset _retryCount either: a flapping stream that
-        // delivers a single event between failures would otherwise never trip MaxRetries.
-        // The retry budget is recovered only by GrpcStreamStable (a stream that has stayed
-        // up for StabilityWindow). Before the snapshot has been delivered, BUFFER (in arrival
-        // order) rather than deliver — these may be gap-window events; after the snapshot has
-        // been flushed, pass through directly (phase-dependent behavior).
+        // Domain events arriving via Self.Tell from the gRPC callback. Stream traffic never
+        // proves the CONSUMER is still there, so it deliberately does not touch the orphan
+        // net (which now measures the consumer keepalive, not the mailbox). Receiving an
+        // event must not reset _retryCount either: a flapping stream that delivers a single
+        // event between failures would otherwise never trip MaxRetries. The retry budget is
+        // recovered only by GrpcStreamStable (a stream that has stayed up for
+        // StabilityWindow). Before the snapshot has been delivered, BUFFER (in arrival order)
+        // rather than deliver — these may be gap-window events; after the snapshot has been
+        // flushed, pass through directly (phase-dependent behavior).
         Receive<LiveDebugStreamEvent>(wrapped => HandleStreamEvent(wrapped.Event));
 
         // Unwrapped forms are still accepted (a direct Tell from a test or a future
-        // in-process producer); those DO influence the receive timeout, which is correct —
-        // they are not the high-volume stream path.
+        // in-process producer) and take the identical path.
         Receive<AttributeValueChanged>(changed => HandleStreamEvent(changed));
         Receive<AlarmStateChanged>(changed => HandleStreamEvent(changed));
 
@@ -315,11 +341,32 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             Context.Stop(Self);
         });
 
-        // Orphan safety net — if nobody stops us within 5 minutes, self-terminate
-        Context.SetReceiveTimeout(TimeSpan.FromMinutes(5));
-        Receive<ReceiveTimeout>(_ =>
+        // Consumer keepalive: DebugStreamService Tells this on a timer for every session it
+        // still holds (i.e. still attached to a Blazor debug view / SignalR connection). It
+        // is the ONLY thing that renews the orphan window — deliberately, so neither a chatty
+        // site nor a silent one can influence it.
+        Receive<DebugStreamConsumerAlive>(_ =>
         {
-            _log.Warning("Debug stream for {0} timed out (orphaned session), stopping", _instanceUniqueName);
+            if (_stopped) return;
+            _consumerLastSeenUtc = DateTime.UtcNow;
+        });
+
+        // Orphan safety net, CONSUMER-measured (WP2.3 follow-up). A periodic self-tick
+        // compares the consumer-last-seen stamp against ConsumerIdleTimeout; the previous
+        // SetReceiveTimeout(5 min) measured the mailbox instead, and once stream events were
+        // (correctly) marked INotInfluenceReceiveTimeout nothing recurring reset it — the
+        // snapshot arrives once and GrpcStreamStable once, so EVERY healthy session died at
+        // ~6 minutes and the consumer was told "Site disconnected".
+        Receive<ConsumerLivenessTick>(_ =>
+        {
+            if (_stopped) return;
+            var idle = DateTime.UtcNow - _consumerLastSeenUtc;
+            if (idle < ConsumerIdleTimeout) return;
+
+            _log.Warning(
+                "Debug stream for {0} has had no consumer activity for {1:F0}s (orphaned session), stopping",
+                _instanceUniqueName, idle.TotalSeconds);
+            Timers.Cancel(ConsumerLivenessTimerKey);
             CleanupGrpc();
             SendUnsubscribe();
             _stopped = true;
@@ -507,6 +554,15 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
         // Arm the hard snapshot deadline alongside the request.
         if (SnapshotTimeout > TimeSpan.Zero)
             Timers.StartSingleTimer(SnapshotTimerKey, new DebugSnapshotDeadline(), SnapshotTimeout);
+
+        // Arm the consumer-liveness net. The stamp starts now, so the session always gets a
+        // full ConsumerIdleTimeout to see its first keepalive from DebugStreamService.
+        _consumerLastSeenUtc = DateTime.UtcNow;
+        if (ConsumerIdleTimeout > TimeSpan.Zero && ConsumerLivenessCheckInterval > TimeSpan.Zero)
+        {
+            Timers.StartPeriodicTimer(
+                ConsumerLivenessTimerKey, new ConsumerLivenessTick(), ConsumerLivenessCheckInterval);
+        }
     }
 
     /// <inheritdoc />
@@ -546,7 +602,7 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
             await client.SubscribeAsync(
                 _correlationId,
                 _instanceUniqueName,
-                // Wrapped: stream traffic must not reset the orphan receive timeout.
+                // Wrapped so the stream path is explicit (it never renews the orphan net).
                 evt => self.Tell(new LiveDebugStreamEvent(evt)),
                 ex => self.Tell(new GrpcStreamError(ex, generation)),
                 () => self.Tell(new GrpcStreamCompleted(generation)),
@@ -660,11 +716,26 @@ public record StopDebugStream;
 
 /// <summary>
 /// Envelope for a live gRPC stream event (<c>AttributeValueChanged</c>/
-/// <c>AlarmStateChanged</c>). Implements <see cref="INotInfluenceReceiveTimeout"/> so a busy
-/// site's event flood cannot keep resetting the orphan-session receive timeout — the timeout
-/// measures consumer/session liveness, not site chatter (WP2.3).
+/// <c>AlarmStateChanged</c>). Kept as a distinct envelope so the high-volume stream path is
+/// explicit at the call site; the orphan net no longer keys off the mailbox at all (it
+/// measures the consumer keepalive), so a busy site's event flood can neither hold a dead
+/// session open nor — as briefly happened — be the only thing keeping a healthy one alive.
 /// </summary>
-internal record LiveDebugStreamEvent(object Event) : INotInfluenceReceiveTimeout;
+internal record LiveDebugStreamEvent(object Event);
+
+/// <summary>
+/// Consumer keepalive: <c>DebugStreamService</c> Tells one of these to every bridge actor
+/// whose session is still attached to a consumer, on
+/// <see cref="DebugStreamBridgeActor.ConsumerLivenessCheckInterval"/>-scale cadence. Renewing
+/// the consumer-last-seen stamp is its ONLY effect.
+/// </summary>
+public record DebugStreamConsumerAlive;
+
+/// <summary>
+/// Internal self-tick that checks the consumer-last-seen stamp against
+/// <see cref="DebugStreamBridgeActor.ConsumerIdleTimeout"/>.
+/// </summary>
+internal record ConsumerLivenessTick;
 
 /// <summary>
 /// Internal message: the hard deadline for the initial <c>DebugViewSnapshot</c> expired.

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs

@@ -144,15 +144,38 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     private bool _seedOnConnect;
 
     /// <summary>
-    /// Set whenever a fan-out finishes (success or failure); consumed by the next reconcile
-    /// tick, which skips its own fan-out when it finds the flag set. That makes the
-    /// connect-driven seed and the periodic backstop mutually exclusive — the pair used to
+    /// Set when a CONNECT- OR FAILOVER-DRIVEN fan-out finishes (the initial seed, a re-seed
+    /// consumed by a successful (re)connect, or a re-seed queued behind one); consumed by the
+    /// next reconcile tick, which skips its own fan-out when it finds the flag set. That makes
+    /// the connect-driven seed and the periodic backstop mutually exclusive — the pair used to
     /// run BOTH, which is the "full unconditional snapshot every 60s" waste — while bounding
     /// staleness at two intervals (a tick can be skipped at most once in a row).
+    /// <para>
+    /// A TICK-driven fan-out deliberately does NOT set it. It used to: the tick's own fan-out
+    /// completion armed the skip, so steady state ran fan-out → skip → fan-out → skip, i.e.
+    /// one reconcile per TWO intervals — halving the not-reporting refresh rate and the alarm
+    /// reconcile backstop for no reason (nothing was duplicated to suppress).
+    /// </para>
     /// Actor-thread only.
     /// </summary>
     private bool _fanoutSinceLastTick;
 
+    /// <summary>
+    /// Whether the in-flight fan-out is connect/failover-driven and therefore makes the next
+    /// reconcile tick redundant (see <see cref="_fanoutSinceLastTick"/>). Stable for the
+    /// fan-out's lifetime: <see cref="StartFanout"/> never overwrites it while one is in
+    /// flight (a colliding request queues instead). Actor-thread only.
+    /// </summary>
+    private bool _fanoutSuppressesNextTick;
+
+    /// <summary>
+    /// Same flag for a fan-out that was queued behind an in-flight one
+    /// (<see cref="_reseedQueued"/>) — OR-ed across colliding requests so a queued
+    /// connect/failover re-seed keeps its suppression even if a tick collided too.
+    /// Actor-thread only.
+    /// </summary>
+    private bool _queuedFanoutSuppressesNextTick;
+
     /// <summary>
     /// True between issuing a stream open and its first outcome (connected / error /
     /// completed), so a reconcile tick never stacks a second open on an in-flight one.
@@ -269,7 +292,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         Receive<RetrySeed>(_ =>
         {
             if (_stopped) return;
-            StartFanout(isInitial: false);
+            // A backoff retry of a failed fan-out is not a connect/failover seed: it must
+            // not stand the next reconcile tick down.
+            StartFanout(isInitial: false, suppressesNextTick: false);
         });
 
         // The site-wide stream is confirmed established (response headers received). This —
@@ -356,7 +381,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Kick the initial seed fan-out. The connect callback for this first stream must
         // NOT seed again on top of it, so the flag starts cleared.
         _seedOnConnect = false;
-        StartFanout(isInitial: true);
+        StartFanout(isInitial: true, suppressesNextTick: true);
 
         // Periodic reconcile backstop. Single-shot and re-armed with fresh jitter each
         // tick (a periodic timer would lock every site to the same phase forever).
@@ -398,9 +423,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
 
     /// <summary>
     /// Periodic reconcile backstop: re-run the snapshot fan-out (corrects instance-set drift
-    /// + any missed delta) UNLESS one already completed inside this window — a reconnect-driven
-    /// seed makes the tick redundant, and running both was the "full unconditional snapshot
-    /// every 60s" waste (WP2.3). If the live stream was previously given up, self-heal it by
+    /// + any missed delta) UNLESS a CONNECT- OR FAILOVER-DRIVEN seed already completed inside
+    /// this window — that seed makes the tick redundant, and running both was the "full
+    /// unconditional snapshot every 60s" waste (WP2.3). A tick's OWN fan-out never arms that
+    /// skip: it used to, which made steady state alternate fan-out/skip and halved the
+    /// effective reconcile rate. If the live stream was previously given up, self-heal it by
     /// resetting the retry budget and reopening, so a sustained outage never permanently kills
     /// the live feed.
     /// </summary>
@@ -441,7 +468,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         }
 
         if (!reopening)
-            StartFanout(isInitial: false);
+        {
+            // Tick-driven: deliberately does NOT set the skip flag, so the NEXT tick fans
+            // out too. Setting it here was the "one reconcile per two intervals" bug.
+            StartFanout(isInitial: false, suppressesNextTick: false);
+        }
     }
 
     // ── Seed / reconcile fan-out ────────────────────────────────────────────────
@@ -451,7 +482,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     /// <c>Self.Tell</c>. While in flight, live deltas buffer. A reconcile that arrives
     /// while a fan-out is already running is skipped (no stacking).
     /// </summary>
-    private void StartFanout(bool isInitial)
+    private void StartFanout(bool isInitial, bool suppressesNextTick)
     {
         if (_stopped) return;
         if (_fanoutInFlight)
@@ -460,11 +491,16 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             // one — its snapshot read-time predates the stream death, so skipping it would
             // serve stale up to the next 60s reconcile (N7.1). An initial-seed collision
             // never queues (there is only ever one).
-            if (!isInitial) _reseedQueued = true;
+            if (!isInitial)
+            {
+                _reseedQueued = true;
+                _queuedFanoutSuppressesNextTick |= suppressesNextTick;
+            }
             return;
         }
 
         _fanoutInFlight = true;
+        _fanoutSuppressesNextTick = suppressesNextTick;
         var self = Self;
         var ct = _lifetimeCts?.Token ?? CancellationToken.None;
 
@@ -514,7 +550,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         var flushChanged = FlushBuffer();
 
         _fanoutInFlight = false;
-        _fanoutSinceLastTick = true;
+        // Only a connect/failover-driven seed makes the next tick redundant — a tick's own
+        // fan-out must never suppress the tick after it (that halved the reconcile rate).
+        if (_fanoutSuppressesNextTick) _fanoutSinceLastTick = true;
         _consecutiveSeedFailures = 0;
         Timers.Cancel(SeedRetryTimerKey);
         var firstSeed = !_seeded;
@@ -531,11 +569,21 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             Publish();
 
         // A failover re-seed requested while this fan-out was in flight runs now (N7.1).
-        if (_reseedQueued)
-        {
-            _reseedQueued = false;
-            StartFanout(isInitial: false);
-        }
+        RunQueuedFanoutIfAny();
+    }
+
+    /// <summary>
+    /// Runs a fan-out that collided with an in-flight one, carrying the suppression flag it
+    /// was queued with so a queued connect/failover re-seed still stands the next tick down
+    /// (and a queued TICK fan-out still does not).
+    /// </summary>
+    private void RunQueuedFanoutIfAny()
+    {
+        if (!_reseedQueued) return;
+        _reseedQueued = false;
+        var suppresses = _queuedFanoutSuppressesNextTick;
+        _queuedFanoutSuppressesNextTick = false;
+        StartFanout(isInitial: false, suppressesNextTick: suppresses);
     }
 
     /// <summary>
@@ -570,7 +618,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // Don't lose deltas captured during the failed window — apply them pass-through
         // into the (possibly stale/empty) cache. The next reconcile re-seeds authoritatively.
         _fanoutInFlight = false;
-        _fanoutSinceLastTick = true;
+        // Same rule as the success path: only a connect/failover-driven fan-out stands the
+        // next reconcile tick down.
+        if (_fanoutSuppressesNextTick) _fanoutSinceLastTick = true;
         var flushChanged = FlushBuffer(dedupAgainstSeed: false);
 
         // Only publish if we already had a seed (so IsLive doesn't flip true on a
@@ -585,8 +635,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         // A failover re-seed requested while this fan-out was in flight runs now (N7.1).
         if (_reseedQueued)
         {
-            _reseedQueued = false;
-            StartFanout(isInitial: false);
+            RunQueuedFanoutIfAny();
             return;
         }
 
@@ -794,7 +843,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         {
             _seedOnConnect = false;
             _log.Info("Site-alarm gRPC stream for {0} (re)connected; running one re-seed", _siteIdentifier);
-            StartFanout(isInitial: false);
+            StartFanout(isInitial: false, suppressesNextTick: true);
         }
 
         // Liveness recovered — republish so viewers stop falling back to polling even if the

src/ZB.MOM.WW.ScadaBridge.Communication/DebugStreamService.cs

@@ -13,14 +13,34 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
 /// Manages debug stream sessions by creating DebugStreamBridgeActors that persist
 /// as subscribers on the site side. Both the Blazor debug view and the SignalR hub
 /// use this service to start/stop streams.
+/// <para>
+/// <b>Consumer keepalive.</b> This service is the session registry, and holding a session
+/// here IS what "a consumer is attached" means: the Blazor debug view and the SignalR hub
+/// both release their session (<see cref="StopStream"/>) on dispose/disconnect. A single
+/// shared timer therefore Tells <see cref="DebugStreamConsumerAlive"/> to every registered
+/// bridge actor, which is the only thing that renews each actor's orphan window. A bridge
+/// actor that outlives its registration — the leak the orphan net exists for — stops
+/// receiving keepalives and self-terminates. The keepalive is independent of stream traffic
+/// by design: neither a chatty nor a silent site can influence the orphan decision.
+/// </para>
 /// </summary>
-public class DebugStreamService
+public class DebugStreamService : IDisposable
 {
+    /// <summary>
+    /// Cadence of the shared consumer keepalive. Comfortably shorter than
+    /// <see cref="DebugStreamBridgeActor.ConsumerIdleTimeout"/> so a few missed ticks (GC
+    /// pause, thread-pool starvation) can never orphan a live session. Settable for tests.
+    /// </summary>
+    internal static TimeSpan KeepaliveInterval { get; set; } = TimeSpan.FromSeconds(30);
+
     private readonly CommunicationService _communicationService;
     private readonly IServiceProvider _serviceProvider;
     private readonly SiteStreamGrpcClientFactory _grpcClientFactory;
     private readonly ILogger<DebugStreamService> _logger;
     private readonly ConcurrentDictionary<string, IActorRef> _sessions = new();
+    private readonly object _keepaliveLock = new();
+    private Timer? _keepaliveTimer;
+    private bool _disposed;
     private ActorSystem? _actorSystem;
 
     /// <summary>
@@ -135,6 +155,7 @@ public class DebugStreamService
         var bridgeActor = system.ActorOf(props, $"debug-stream-{sessionId}");
 
         _sessions[sessionId] = bridgeActor;
+        EnsureKeepaliveTimer();
 
         // Wait for the initial snapshot (with timeout)
         using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
@@ -184,6 +205,73 @@ public class DebugStreamService
             _logger.LogInformation("Debug stream {SessionId} stopped", sessionId);
         }
     }
+
+    /// <summary>
+    /// Sends one keepalive round to every attached session. Exposed (internal) so tests can
+    /// drive the keepalive deterministically instead of waiting on the timer.
+    /// </summary>
+    internal void SendConsumerKeepalives()
+    {
+        foreach (var session in _sessions)
+        {
+            session.Value.Tell(new DebugStreamConsumerAlive());
+        }
+    }
+
+    /// <summary>
+    /// Starts the shared keepalive timer on first use. One timer for the whole service (not
+    /// one per session): the payload is a Tell per attached session, so a single periodic
+    /// callback is the cheapest shape that covers every consumer surface.
+    /// </summary>
+    private void EnsureKeepaliveTimer()
+    {
+        if (_keepaliveTimer is not null) return;
+        lock (_keepaliveLock)
+        {
+            if (_keepaliveTimer is not null || _disposed) return;
+            _keepaliveTimer = new Timer(
+                _ =>
+                {
+                    try
+                    {
+                        SendConsumerKeepalives();
+                    }
+                    catch (Exception ex)
+                    {
+                        // A keepalive round must never take the timer down — a thrown Tell
+                        // would silently stop renewing EVERY session's orphan window.
+                        _logger.LogWarning(ex, "Debug stream consumer keepalive round failed");
+                    }
+                },
+                null,
+                KeepaliveInterval,
+                KeepaliveInterval);
+        }
+    }
+
+    /// <summary>Stops the keepalive timer.</summary>
+    public void Dispose()
+    {
+        Dispose(true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>Stops the keepalive timer.</summary>
+    /// <param name="disposing">True when called from <see cref="Dispose()"/>.</param>
+    protected virtual void Dispose(bool disposing)
+    {
+        if (_disposed) return;
+        lock (_keepaliveLock)
+        {
+            if (_disposed) return;
+            _disposed = true;
+            if (disposing)
+            {
+                _keepaliveTimer?.Dispose();
+                _keepaliveTimer = null;
+            }
+        }
+    }
 }
 
 public record DebugStreamSession(string SessionId, DebugViewSnapshot InitialSnapshot);

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/SiteStreamGrpcClient.cs

@@ -246,11 +246,12 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     /// </param>
     /// <param name="ct">Cancellation token to stop the subscription.</param>
     /// <param name="onConnected">
-    /// Optional callback invoked once when the site has ACCEPTED the subscription (response
-    /// headers received — the site writes them as soon as its relay actor is subscribed, so
-    /// no event can be missed after this point). The per-site aggregator uses it to run
-    /// exactly one re-seed per successful (re)connect instead of one per reconnect attempt.
-    /// Never invoked more than once per call, and never after <paramref name="onError"/>.
+    /// Optional callback invoked once when the site has ACCEPTED the subscription — response
+    /// headers received (the site writes them as soon as its relay actor is subscribed, so no
+    /// event can be missed after this point), or the first event received if the peer defers
+    /// its headers. A header timeout is NOT treated as connected. The per-site aggregator uses
+    /// this to run exactly one re-seed per successful (re)connect instead of one per reconnect
+    /// attempt. Never invoked more than once per call, and never after <paramref name="onError"/>.
     /// </param>
     /// <returns>A task that represents the asynchronous operation.</returns>
     public virtual async Task SubscribeSiteAsync(
@@ -311,11 +312,11 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     /// <param name="onError">Invoked once if the stream faulted.</param>
     /// <param name="onCompleted">Invoked once if the server ended the stream with OK.</param>
     /// <param name="onConnected">
-    /// Optional; invoked once when the server's response headers arrive — i.e. the site has
-    /// accepted the subscription and its relay actor is attached. Bounded by
-    /// <see cref="ConnectedHeaderTimeout"/> so a peer that defers headers (a pre-WP2.3 site,
-    /// which only flushes them with its first event) still reports connected instead of
-    /// leaving the caller waiting for a signal that may never come on a quiet site.
+    /// Optional; invoked AT MOST ONCE when the site has demonstrably accepted the
+    /// subscription — either the server's response headers arrived (bounded by
+    /// <see cref="ConnectedHeaderTimeout"/>) or, for a peer that defers headers until its
+    /// first message, the first event was received. A header timeout alone is never
+    /// reported as connected: that is also exactly what an unreachable site looks like.
     /// </param>
     /// <returns>A task that completes when the stream has ended and its outcome been reported.</returns>
     internal async Task ConsumeStreamAsync(
@@ -328,27 +329,49 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
         Action? onConnected = null)
     {
         var completedGracefully = false;
+        var connectedReported = false;
+
+        // Fires onConnected AT MOST ONCE, from whichever proof of a live peer arrives
+        // first: the response headers, or (for a peer that defers headers until its first
+        // message) the first event. A header TIMEOUT is deliberately NOT such a proof —
+        // see AwaitHeadersAsync.
+        void ReportConnected()
+        {
+            if (connectedReported || onConnected is null) return;
+            connectedReported = true;
+            onConnected();
+        }
+
         try
         {
             using (var call = openCall())
             {
-                if (onConnected is not null)
+                if (onConnected is not null && await AwaitHeadersAsync(call, cts.Token).ConfigureAwait(false))
                 {
-                    await AwaitHeadersAsync(call, cts.Token).ConfigureAwait(false);
-                    onConnected();
+                    ReportConnected();
                 }
 
                 await foreach (var evt in call.ResponseStream.ReadAllAsync(cts.Token))
                 {
+                    // Fallback connected signal: an event can only come from a peer that
+                    // accepted the subscription, so it proves what the headers would have.
+                    // Raised BEFORE the event is delivered so the consumer sees
+                    // connected-then-event ordering.
+                    ReportConnected();
                     onEvent(evt);
                 }
             }
 
             completedGracefully = true;
         }
-        catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
+        catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && cts.IsCancellationRequested)
         {
-            // Normal cancellation — not an error
+            // OUR OWN cancellation (Unsubscribe / reconnect / channel teardown we asked
+            // for) — not an error. The IsCancellationRequested guard matters: a Cancelled
+            // status can also originate at the PEER or from a channel disposed underneath
+            // us, and swallowing THAT fired none of onError/onCompleted/onConnected, so
+            // the consuming actor kept a dead stream marked live forever. Foreign
+            // Cancelled now falls through to the onError path below.
         }
         catch (OperationCanceledException) when (cts.IsCancellationRequested)
         {
@@ -372,26 +395,37 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
     }
 
     /// <summary>
-    /// How long to wait for response headers before treating the stream as connected anyway.
-    /// A peer that only flushes headers with its first message would otherwise hold the
-    /// connected signal — and with it the aggregator's re-seed — for as long as the site
-    /// happens to be quiet.
+    /// How long to wait for response headers before giving up on them as the connected
+    /// signal and falling back to the first received event. A peer that only flushes headers
+    /// with its first message would otherwise hold the connected signal — and with it the
+    /// aggregator's re-seed — for as long as the site happens to be quiet.
     /// </summary>
     internal static TimeSpan ConnectedHeaderTimeout { get; set; } = TimeSpan.FromSeconds(10);
 
     /// <summary>
     /// Awaits the call's response headers, bounded by <see cref="ConnectedHeaderTimeout"/>.
-    /// A fault propagates (the caller reports it through <c>onError</c> like any other stream
-    /// fault); a timeout returns normally. On timeout the abandoned headers task is observed
-    /// so a later fault on it can never surface as an unobserved task exception.
+    /// Returns <see langword="true"/> when the headers arrived (the site accepted the
+    /// subscription) and <see langword="false"/> on timeout. A fault propagates (the caller
+    /// reports it through <c>onError</c> like any other stream fault); on timeout the
+    /// abandoned headers task is observed so a later fault on it can never surface as an
+    /// unobserved task exception.
+    /// <para>
+    /// A timeout must NOT be reported as connected: an unreachable/wedged site produces
+    /// exactly that shape, and calling <c>onConnected</c> for it made the aggregator clear
+    /// <c>_streamDown</c>, consume its pending re-seed and fan a full snapshot out at a site
+    /// that never answered. The caller instead treats the FIRST RECEIVED EVENT as the
+    /// fallback connected signal — real proof of a live peer, and the quiet-site case the
+    /// timeout was added for is covered by the reconcile backstop.
+    /// </para>
     /// </summary>
-    private static async Task AwaitHeadersAsync(
+    private static async Task<bool> AwaitHeadersAsync(
         AsyncServerStreamingCall<SiteStreamEvent> call, CancellationToken ct)
     {
         var headers = call.ResponseHeadersAsync;
         try
         {
             await headers.WaitAsync(ConnectedHeaderTimeout, ct).ConfigureAwait(false);
+            return true;
         }
         catch (TimeoutException)
         {
@@ -400,6 +434,7 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
                 CancellationToken.None,
                 TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
                 TaskScheduler.Default);
+            return false;
         }
     }