fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs

@@ -78,12 +78,23 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
     private bool _stopped;
 
     /// <summary>
-    /// True once the live gRPC stream has been given up (retry budget exhausted). Reconcile
-    /// snapshots keep serving in the meantime; the next reconcile tick self-heals the stream
-    /// by resetting the retry budget and reopening it, so a sustained site outage does not
-    /// permanently drop the live feed. Actor-thread only.
+    /// True while there is no live gRPC stream — either it was given up (retry budget
+    /// exhausted) or it ended gracefully (server status OK at the site's max stream
+    /// lifetime). Reconcile snapshots keep serving in the meantime; the next reconcile tick
+    /// self-heals the stream by resetting the retry budget and reopening it, so neither a
+    /// sustained site outage nor a routine 4h stream expiry permanently drops the live feed.
+    /// Actor-thread only.
     /// </summary>
     private bool _streamDown;
+
+    /// <summary>
+    /// Why the stream is down: <c>true</c> = the retry budget was exhausted, so the
+    /// self-healing reopen must also reset it; <c>false</c> = it ended gracefully, and the
+    /// budget — which a completion neither spends nor refunds — is carried across the reopen
+    /// untouched (a stream flapping between faults and clean closes must still trip
+    /// MaxRetries). Actor-thread only.
+    /// </summary>
+    private bool _retryBudgetExhausted;
     private CancellationTokenSource? _grpcCts;
     private CancellationTokenSource? _lifetimeCts;
 
@@ -218,6 +229,22 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             HandleGrpcError();
         });
 
+        // gRPC stream ended GRACEFULLY (server status OK) — the site's 4h max stream
+        // lifetime elapsing or a graceful site shutdown. Not a fault: the stream is marked
+        // down so the reconcile tick reopens it (which also re-seeds), but the retry budget
+        // is untouched and the node is not flipped. Same generation fence as the error path.
+        Receive<GrpcAlarmStreamCompleted>(msg =>
+        {
+            if (_stopped) return;
+            if (msg.Generation != _streamGeneration)
+            {
+                _log.Debug("Ignoring stale gRPC completion from stream generation {0} (current {1})",
+                    msg.Generation, _streamGeneration);
+                return;
+            }
+            HandleGrpcCompleted();
+        });
+
         Receive<ReconnectAlarmStream>(_ => OpenGrpcStream());
 
         // Owning service asks us to stop (last viewer left + linger elapsed).
@@ -279,7 +306,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         if (_streamDown)
         {
             _log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
-            _retryCount = 0;
+            if (_retryBudgetExhausted)
+            {
+                _retryBudgetExhausted = false;
+                _retryCount = 0;
+            }
             // Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
             ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
             OpenGrpcStream();
@@ -514,14 +545,26 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
         var self = Self;
         var ct = _grpcCts.Token;
 
+        // The subscription task itself is observed (below): a fault escaping
+        // SubscribeSiteAsync — or a Task.Run that never started because ct was already
+        // cancelled — would otherwise leave the actor waiting on a stream that does not
+        // exist, with the exception silently unobserved.
         Task.Run(async () =>
         {
             await client.SubscribeSiteAsync(
                 _correlationId,
                 alarm => self.Tell(alarm),
                 ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
+                () => self.Tell(new GrpcAlarmStreamCompleted(generation)),
                 ct);
-        }, ct);
+        }, ct).ContinueWith(t =>
+        {
+            if (t.IsFaulted)
+                self.Tell(new GrpcAlarmStreamError(t.Exception!.GetBaseException(), generation));
+            else if (t.IsCanceled && !ct.IsCancellationRequested)
+                self.Tell(new GrpcAlarmStreamCompleted(generation));
+            // RanToCompletion: SubscribeSiteAsync already reported its own outcome.
+        }, TaskContinuationOptions.ExecuteSynchronously);
     }
 
     private void HandleGrpcError()
@@ -543,6 +586,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
                        "reconcile snapshots continue and the next reconcile tick will retry the stream",
                        _siteIdentifier, MaxRetries);
             _streamDown = true;
+            _retryBudgetExhausted = true;
             CleanupGrpc();
             return;
         }
@@ -569,6 +613,26 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
             Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectAlarmStream(), _reconnectDelay);
     }
 
+    /// <summary>
+    /// Handles a graceful end of stream. The stream is torn down and left down for the
+    /// reconcile tick to reopen — deliberately NOT reopened inline, so a site that keeps
+    /// completing streams immediately can never spin this actor into a hot reconnect loop.
+    /// The retry budget is neither spent nor reset here (completion is not a fault), and the
+    /// endpoint is not flipped: the node that just closed a stream cleanly is healthy.
+    /// </summary>
+    private void HandleGrpcCompleted()
+    {
+        // The stream is gone, so its armed stability timer must not later "recover" a
+        // budget that its successor has since spent.
+        Timers.Cancel(StabilityTimerKey);
+
+        _log.Info("Site-alarm gRPC stream for {0} completed gracefully (server end of stream); " +
+                  "reopening on the next reconcile tick", _siteIdentifier);
+
+        _streamDown = true;
+        CleanupGrpc();
+    }
+
     private void CleanupGrpc()
     {
         _grpcCts?.Cancel();
@@ -619,6 +683,11 @@ internal sealed record PublishCoalesced;
 /// generation it came from so a late error from a cancelled stream can be ignored (N7.2).</summary>
 internal sealed record GrpcAlarmStreamError(Exception Exception, int Generation);
 
+/// <summary>Internal: the site-alarm gRPC stream ended gracefully (server status OK — max
+/// stream lifetime or site shutdown), stamped with its stream generation so a late completion
+/// from a cancelled stream can be ignored.</summary>
+internal sealed record GrpcAlarmStreamCompleted(int Generation);
+
 /// <summary>Internal: reconnect the site-alarm gRPC stream (flip node).</summary>
 internal sealed record ReconnectAlarmStream;