fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

src/ZB.MOM.WW.ScadaBridge.Communication/Actors/DebugStreamBridgeActor.cs

@@ -63,6 +63,14 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
     private bool _stopped;
     private CancellationTokenSource? _grpcCts;
 
+    /// <summary>
+    /// Monotonic stream generation stamped on each opened gRPC stream and echoed back on its
+    /// error/completion callbacks: a late callback raced out of a previous (cancelled) stream
+    /// carries a stale generation and is ignored, so it can neither burn retry budget nor
+    /// open a duplicate stream. Mirrors <c>SiteAlarmAggregatorActor</c>. Actor-thread only.
+    /// </summary>
+    private int _streamGeneration;
+
     /// <summary>
     /// Phase flag. <see langword="false"/> until the initial
     /// <see cref="DebugViewSnapshot"/> has been delivered and the pre-snapshot buffer
@@ -198,10 +206,33 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
         // gRPC stream error — attempt reconnection
         Receive<GrpcStreamError>(msg =>
         {
+            // Ignore a late error raced out of a previous (cancelled) stream: it must not
+            // burn retry budget or flip the node a second time.
+            if (msg.Generation != _streamGeneration)
+            {
+                _log.Debug("Ignoring stale gRPC error from stream generation {0} (current {1})",
+                    msg.Generation, _streamGeneration);
+                return;
+            }
             _log.Warning("gRPC stream error for {0}: {1}", _instanceUniqueName, msg.Exception.Message);
             HandleGrpcError();
         });
 
+        // gRPC stream ended GRACEFULLY (server status OK) — the site's 4h max stream
+        // lifetime elapsing or a graceful site shutdown. Not a fault: reopen on the SAME
+        // node without spending retry budget. Without this the session went silently deaf.
+        Receive<GrpcStreamCompleted>(msg =>
+        {
+            if (_stopped) return;
+            if (msg.Generation != _streamGeneration)
+            {
+                _log.Debug("Ignoring stale gRPC completion from stream generation {0} (current {1})",
+                    msg.Generation, _streamGeneration);
+                return;
+            }
+            HandleGrpcCompleted();
+        });
+
         // Scheduled reconnection
         Receive<ReconnectGrpcStream>(_ => OpenGrpcStream());
 
@@ -424,20 +455,57 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
         // retry budget is recovered. Cancelled by HandleGrpcError.
         Timers.StartSingleTimer(StabilityTimerKey, new GrpcStreamStable(), StabilityWindow);
 
+        var generation = ++_streamGeneration;
         var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
         var self = Self;
         var ct = _grpcCts.Token;
 
-        // Launch as background task — onEvent and onError marshal back to actor via Tell
+        // Launch as background task — the callbacks marshal back to the actor via Tell.
+        // The task itself is observed below: a fault escaping SubscribeAsync would otherwise
+        // leave the session waiting on a stream that does not exist, exception unobserved.
         Task.Run(async () =>
         {
             await client.SubscribeAsync(
                 _correlationId,
                 _instanceUniqueName,
                 evt => self.Tell(evt),
-                ex => self.Tell(new GrpcStreamError(ex)),
+                ex => self.Tell(new GrpcStreamError(ex, generation)),
+                () => self.Tell(new GrpcStreamCompleted(generation)),
                 ct);
-        }, ct);
+        }, ct).ContinueWith(t =>
+        {
+            if (t.IsFaulted)
+                self.Tell(new GrpcStreamError(t.Exception!.GetBaseException(), generation));
+            else if (t.IsCanceled && !ct.IsCancellationRequested)
+                self.Tell(new GrpcStreamCompleted(generation));
+            // RanToCompletion: SubscribeAsync already reported its own outcome.
+        }, TaskContinuationOptions.ExecuteSynchronously);
+    }
+
+    /// <summary>
+    /// Handles a graceful end of stream (server status OK). The stream simply expired or the
+    /// site shut down cleanly, so the retry budget is left untouched and the endpoint is not
+    /// flipped; the reopen is scheduled through the existing reconnect timer, which also
+    /// rate-limits a pathological site that keeps closing streams immediately.
+    /// </summary>
+    private void HandleGrpcCompleted()
+    {
+        // The stream is gone, so its armed stability timer must not later "recover" a
+        // budget that its successor has since spent.
+        Timers.Cancel(StabilityTimerKey);
+
+        _log.Info("gRPC stream for {0} completed gracefully (server end of stream); reopening",
+            _instanceUniqueName);
+
+        // Release the site-side relay for the finished stream before reopening, so the site
+        // is not left with a zombie relay actor for this correlation id.
+        _grpcCts?.Cancel();
+        _grpcCts?.Dispose();
+        _grpcCts = null;
+        _grpcFactory.TryGet(_siteIdentifier, _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress)
+            ?.Unsubscribe(_correlationId);
+
+        Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectGrpcStream(), ReconnectDelay);
     }
 
     private void HandleGrpcError()
@@ -512,9 +580,17 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
 public record StopDebugStream;
 
 /// <summary>
-/// Internal message indicating a gRPC stream error occurred.
+/// Internal message indicating a gRPC stream error occurred, stamped with the stream
+/// generation it came from so a late error out of a cancelled stream can be ignored.
 /// </summary>
-internal record GrpcStreamError(Exception Exception);
+internal record GrpcStreamError(Exception Exception, int Generation);
+
+/// <summary>
+/// Internal message indicating the gRPC stream ended gracefully (server status OK — the
+/// site's max stream lifetime elapsed, or the site shut down cleanly), stamped with its
+/// stream generation.
+/// </summary>
+internal record GrpcStreamCompleted(int Generation);
 
 /// <summary>
 /// Internal message to trigger gRPC stream reconnection.