fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

2026-08-14 19:57:08 -04:00
parent ee193cd2bb
commit 34a3f4bb69
9 changed files with 507 additions and 45 deletions
@@ -167,13 +167,19 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
    /// <summary>
    /// Opens a server-streaming subscription for a specific instance.
    /// This is a long-running async method; the caller launches it as a background task.
-    /// The <paramref name="onEvent"/> callback delivers domain events, and
-    /// <paramref name="onError"/> lets the caller handle reconnection.
+    /// The <paramref name="onEvent"/> callback delivers domain events,
+    /// <paramref name="onError"/> lets the caller handle reconnection after a fault, and
+    /// <paramref name="onCompleted"/> reports a graceful end of stream.
    /// </summary>
    /// <param name="correlationId">Unique identifier for this subscription.</param>
    /// <param name="instanceUniqueName">Unique name of the instance to subscribe to.</param>
    /// <param name="onEvent">Callback invoked for each domain event received from the stream.</param>
    /// <param name="onError">Callback invoked when the subscription encounters an error.</param>
+    /// <param name="onCompleted">
+    /// Callback invoked when the server ended the RPC with OK — the site's max stream
+    /// lifetime elapsing or a graceful site shutdown. Mutually exclusive with
+    /// <paramref name="onError"/>; see <see cref="ConsumeStreamAsync"/>.
+    /// </param>
    /// <param name="ct">Cancellation token to stop the subscription.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public virtual async Task SubscribeAsync(
@@ -181,6 +187,7 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
        string instanceUniqueName,
        Action<object> onEvent,
        Action<Exception> onError,
+        Action onCompleted,
        CancellationToken ct)
    {
        if (_client is null)
@@ -195,30 +202,18 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
            InstanceUniqueName = instanceUniqueName
        };

-        try
-        {
-            using var call = _client.SubscribeInstance(request, cancellationToken: cts.Token);
-
-            await foreach (var evt in call.ResponseStream.ReadAllAsync(cts.Token))
+        await ConsumeStreamAsync(
+            correlationId,
+            cts,
+            () => _client.SubscribeInstance(request, cancellationToken: cts.Token),
+            evt =>
            {
                var domainEvent = ConvertToDomainEvent(evt);
                if (domainEvent != null)
                    onEvent(domainEvent);
-            }
-        }
-        catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
-        {
-            // Normal cancellation — not an error
-        }
-        catch (Exception ex)
-        {
-            onError(ex);
-        }
-        finally
-        {
-            // Remove only our own entry -- a racing reconnect may already own the slot.
-            RemoveSubscription(correlationId, cts);
-        }
+            },
+            onError,
+            onCompleted);
    }

    /// <summary>
@@ -244,12 +239,18 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
    /// <param name="correlationId">Unique identifier for this subscription.</param>
    /// <param name="onAlarmEvent">Callback invoked for each alarm delta received from the site-wide stream.</param>
    /// <param name="onError">Callback invoked when the subscription encounters an error.</param>
+    /// <param name="onCompleted">
+    /// Callback invoked when the server ended the RPC with OK — the site's max stream
+    /// lifetime elapsing or a graceful site shutdown. Mutually exclusive with
+    /// <paramref name="onError"/>; see <see cref="ConsumeStreamAsync"/>.
+    /// </param>
    /// <param name="ct">Cancellation token to stop the subscription.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public virtual async Task SubscribeSiteAsync(
        string correlationId,
        Action<AlarmStateChanged> onAlarmEvent,
        Action<Exception> onError,
+        Action onCompleted,
        CancellationToken ct)
    {
        if (_client is null)
@@ -263,21 +264,73 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
            CorrelationId = correlationId
        };

-        try
-        {
-            using var call = _client.SubscribeSite(request, cancellationToken: cts.Token);
-
-            await foreach (var evt in call.ResponseStream.ReadAllAsync(cts.Token))
+        await ConsumeStreamAsync(
+            correlationId,
+            cts,
+            () => _client.SubscribeSite(request, cancellationToken: cts.Token),
+            evt =>
            {
                // Site-wide stream is alarm-only by contract; defensively ignore anything else.
                if (ConvertToAlarmEvent(evt) is { } alarm)
                    onAlarmEvent(alarm);
+            },
+            onError,
+            onCompleted);
+    }
+
+    /// <summary>
+    /// Drains a server-streaming call to its end, classifying the outcome into exactly one
+    /// of three terminations: a fault (<paramref name="onError"/>), a graceful server-side
+    /// end of stream (<paramref name="onCompleted"/>), or our own cancellation (neither).
+    /// <para>
+    /// The graceful case is the one that matters operationally: the site server caps every
+    /// stream at <c>CommunicationOptions.GrpcMaxStreamLifetime</c> (4 h) and, when that
+    /// elapses, ends the RPC with status OK. The client-side <c>await foreach</c> then simply
+    /// finishes, so before <c>onCompleted</c> existed the consuming actor observed nothing at
+    /// all and the stream stayed silently dead until the process restarted.
+    /// </para>
+    /// Internal so the completion/fault classification can be unit-tested with a fake
+    /// <see cref="IAsyncStreamReader{T}"/> and no live channel.
+    /// </summary>
+    /// <param name="correlationId">Unique identifier for this subscription.</param>
+    /// <param name="cts">The subscription's linked token source (owned by the caller).</param>
+    /// <param name="openCall">
+    /// Opens the server-streaming call. Invoked inside the guarded region so a throw while
+    /// opening is reported through <paramref name="onError"/> like any other stream fault.
+    /// </param>
+    /// <param name="onEvent">Invoked per wire event.</param>
+    /// <param name="onError">Invoked once if the stream faulted.</param>
+    /// <param name="onCompleted">Invoked once if the server ended the stream with OK.</param>
+    /// <returns>A task that completes when the stream has ended and its outcome been reported.</returns>
+    internal async Task ConsumeStreamAsync(
+        string correlationId,
+        CancellationTokenSource cts,
+        Func<AsyncServerStreamingCall<SiteStreamEvent>> openCall,
+        Action<SiteStreamEvent> onEvent,
+        Action<Exception> onError,
+        Action onCompleted)
+    {
+        var completedGracefully = false;
+        try
+        {
+            using (var call = openCall())
+            {
+                await foreach (var evt in call.ResponseStream.ReadAllAsync(cts.Token))
+                {
+                    onEvent(evt);
+                }
            }
+
+            completedGracefully = true;
        }
        catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
        {
            // Normal cancellation — not an error
        }
+        catch (OperationCanceledException) when (cts.IsCancellationRequested)
+        {
+            // Our own Unsubscribe/reconnect cancelled the read — not an error either.
+        }
        catch (Exception ex)
        {
            onError(ex);
@@ -287,6 +340,12 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
            // Remove only our own entry -- a racing reconnect may already own the slot.
            RemoveSubscription(correlationId, cts);
        }
+
+        // Raised outside the try so a throwing callback is not misreported as a stream
+        // fault. Suppressed when our own token was cancelled as the stream ended: that is
+        // a teardown, and the caller has already moved on to a newer stream.
+        if (completedGracefully && !cts.IsCancellationRequested)
+            onCompleted();
    }

    /// <summary>