fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

2026-08-14 19:57:08 -04:00
parent ee193cd2bb
commit 34a3f4bb69
9 changed files with 507 additions and 45 deletions
@@ -1,4 +1,5 @@
 using Google.Protobuf.WellKnownTypes;
+using Grpc.Core;
 using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
@@ -277,7 +278,120 @@ public class SiteStreamGrpcClientTests
        var client = SiteStreamGrpcClient.CreateForTesting();

        await Assert.ThrowsAsync<InvalidOperationException>(() =>
-            client.SubscribeSiteAsync("corr", _ => { }, _ => { }, CancellationToken.None));
+            client.SubscribeSiteAsync("corr", _ => { }, _ => { }, () => { }, CancellationToken.None));
+    }
+
+    // --- WP1.1: graceful (status OK) stream completion is reported, not swallowed ---
+
+    [Fact]
+    public async Task ConsumeStream_ServerEndsStreamWithOk_InvokesOnCompleted_NotOnError()
+    {
+        // The site server caps every stream at GrpcMaxStreamLifetime (4h) and then ends the
+        // RPC with OK. That surfaces here as a read loop that simply runs out of events.
+        var client = SiteStreamGrpcClient.CreateForTesting();
+        var cts = new CancellationTokenSource();
+        var events = new List<SiteStreamEvent>();
+        Exception? error = null;
+        var completed = 0;
+
+        await client.ConsumeStreamAsync(
+            "corr-ok",
+            cts,
+            () => FakeCall(new StubStreamReader(new SiteStreamEvent { CorrelationId = "corr-ok" })),
+            events.Add,
+            ex => error = ex,
+            () => completed++);
+
+        Assert.Single(events);
+        Assert.Null(error);
+        Assert.Equal(1, completed);
+    }
+
+    [Fact]
+    public async Task ConsumeStream_StreamFaults_InvokesOnError_NotOnCompleted()
+    {
+        var client = SiteStreamGrpcClient.CreateForTesting();
+        var cts = new CancellationTokenSource();
+        Exception? error = null;
+        var completed = 0;
+
+        await client.ConsumeStreamAsync(
+            "corr-fault",
+            cts,
+            () => FakeCall(new StubStreamReader(
+                new RpcException(new Status(StatusCode.Unavailable, "site gone")))),
+            _ => { },
+            ex => error = ex,
+            () => completed++);
+
+        Assert.IsType<RpcException>(error);
+        Assert.Equal(0, completed);
+    }
+
+    [Fact]
+    public async Task ConsumeStream_OwnCancellation_InvokesNeitherCallback()
+    {
+        // Our own Unsubscribe/reconnect is a teardown, not a fault and not a graceful end:
+        // the caller has already moved on to a newer stream.
+        var client = SiteStreamGrpcClient.CreateForTesting();
+        var cts = new CancellationTokenSource();
+        await cts.CancelAsync();
+        Exception? error = null;
+        var completed = 0;
+
+        await client.ConsumeStreamAsync(
+            "corr-cancel",
+            cts,
+            () => FakeCall(new StubStreamReader()),
+            _ => { },
+            ex => error = ex,
+            () => completed++);
+
+        Assert.Null(error);
+        Assert.Equal(0, completed);
+    }
+
+    private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
+        new(reader,
+            Task.FromResult(new Metadata()),
+            () => Status.DefaultSuccess,
+            () => new Metadata(),
+            () => { });
+
+    /// <summary>
+    /// Server stream stand-in: yields the queued events, then either ends the stream (the
+    /// status-OK completion the site's max-lifetime cap produces) or throws.
+    /// </summary>
+    private sealed class StubStreamReader : IAsyncStreamReader<SiteStreamEvent>
+    {
+        private readonly Queue<SiteStreamEvent> _events;
+        private readonly Exception? _fault;
+
+        public StubStreamReader(params SiteStreamEvent[] events)
+        {
+            _events = new Queue<SiteStreamEvent>(events);
+        }
+
+        public StubStreamReader(Exception fault)
+            : this()
+        {
+            _fault = fault;
+        }
+
+        public SiteStreamEvent Current { get; private set; } = null!;
+
+        public Task<bool> MoveNext(CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            if (_events.Count > 0)
+            {
+                Current = _events.Dequeue();
+                return Task.FromResult(true);
+            }
+
+            return _fault is null ? Task.FromResult(false) : Task.FromException<bool>(_fault);
+        }
    }

    // --- Communication-003 regression tests ---