fix(galaxy): #865 review round 1 - stale-session pump, data-path stream wait

- EnsureEventPumpStarted replaces a pump created for an older registry session (the reopen -> replay window still held the dead session's pump, so an advise there lost its initial value and the replay skipped it); RestartEventPumpForReplay keeps a pump already on the current session. - SubscribeAsync, read-via-subscribe and ReplayAsync also await the bounded StreamRequested wait before advising, not just the probes. - TryEnsureEventPumpStarted tolerates a teardown flipping _runtimeStopping mid-check; the probe hook honours an already-cancelled seed token. - StreamRequested is documented as narrowing, not closing, the attach race. - Tests: reopen-window probe test, EventPump.StreamRequested tests, and the FixRound1 fresh-pump assertion now waits on a positive control. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 12:31:51 -05:00
parent e35f2a4e93
commit cc1a7ec687
5 changed files with 209 additions and 24 deletions
@@ -0,0 +1,62 @@
+using System.Runtime.CompilerServices;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+/// <summary>
+///     #865: <see cref="EventPump.StreamRequested"/> — what advise paths wait on so an initial value is
+///     never pushed before the stream is requested. It must complete once the RPC is started (before any
+///     event), and must never strand a waiter on a pump whose loop died before getting that far.
+/// </summary>
+public sealed class EventPumpStreamRequestedTests
+{
+    private sealed class ScriptedSubscriber(Func<CancellationToken, IAsyncEnumerable<MxEvent>> stream) : IGalaxySubscriber
+    {
+        public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
+            IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+            => Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
+
+        public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
+
+        public IAsyncEnumerable<MxEvent> StreamEventsAsync(CancellationToken cancellationToken) => stream(cancellationToken);
+    }
+
+    /// <summary>A stream that is requested and then never yields: StreamRequested completes; the RPC started first.</summary>
+    [Fact]
+    public async Task Completes_once_the_stream_is_requested_before_any_event()
+    {
+        var rpcStarted = 0;
+        async IAsyncEnumerable<MxEvent> Never([EnumeratorCancellation] CancellationToken ct)
+        {
+            Interlocked.Exchange(ref rpcStarted, 1); // the synchronous first step — where a real call is created
+            await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
+            yield break;
+        }
+
+        await using var pump = new EventPump(new ScriptedSubscriber(Never), new SubscriptionRegistry());
+        pump.StreamRequested.IsCompleted.ShouldBeFalse("not before Start");
+        pump.Start();
+
+        await TestWait.ForAsync(pump.StreamRequested, "StreamRequested must complete once the stream call is started");
+        Volatile.Read(ref rpcStarted).ShouldBe(1, "StreamRequested must not precede the stream call");
+    }
+
+    /// <summary>A subscriber that throws before reaching the RPC still completes StreamRequested (no stranded waiter).</summary>
+    [Fact]
+    public async Task Completes_when_the_stream_fails_before_the_rpc()
+    {
+        var faulted = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
+        await using var pump = new EventPump(
+            new ScriptedSubscriber(_ => throw new InvalidOperationException("not connected")),
+            new SubscriptionRegistry(),
+            onStreamFault: ex => faulted.TrySetResult(ex));
+        pump.Start();
+
+        await TestWait.ForAsync(pump.StreamRequested, "a dead pump must not strand a StreamRequested waiter");
+        (await TestWait.ForAsync(faulted.Task, "positive control: the failure is still reported as a stream fault"))
+            .ShouldBeOfType<InvalidOperationException>();
+    }
+}
@@ -333,7 +333,10 @@ public sealed class GalaxyDriverFixRound1Tests
        await driver.InitializeAsync("{}", CancellationToken.None);
        // #865: the new runtime's background probe seed starts a pump of its own (its advises need a live
        // stream) — but it must be a FRESH one, never the dead subscriber's.
-        driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump");
+        await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
+            "positive control: the new runtime's background seed must advise its platform");
+        driver.EventPumpInstanceForTests.ShouldNotBeNull("the seed's probe advise must have started a pump")
+            .ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump");
    }

    // ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
@@ -193,6 +193,51 @@ public sealed class ProbeInitialValueTests
        sub.DroppedInitialValues.ShouldBe(0);
    }

+    /// <summary>
+    ///     Review finding: a probe advised in the reopen → replay window (the session was swapped, the old
+    ///     pump still streams the dead session) must still get its initial value — the stale-session pump
+    ///     is replaced before the advise, and the replay keeps the replacement (it already streams the
+    ///     recreated session). Reusing the old pump bound the value to the old session, so it never routed,
+    ///     and the replay skipped the platform (its handle is current-session).
+    /// </summary>
+    [Fact]
+    public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value()
+    {
+        var sub = new GatewayLikeSubscriber();
+        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+            platformSource: new FixedPlatformSource(Platforms))
+        {
+            RecreateOverrideForTests = _ => Task.CompletedTask,
+        };
+        await driver.InitializeAsync("{}", CancellationToken.None);
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
+            "positive control: the initial seed reports Running",
+            () => Describe(driver, sub));
+        var oldPump = driver.EventPumpInstanceForTests;
+
+        await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet
+        sub.ScanStateRunning = false;
+        await driver.ProbeWatcherForTests.ShouldNotBeNull()
+            .SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None);
+
+        await TestWait.UntilAsync(
+            () => StateOf(driver, "LatePlatform") == HostState.Stopped,
+            "a probe advised in the reopen window must get its initial value",
+            () => Describe(driver, sub));
+        var windowPump = driver.EventPumpInstanceForTests;
+        windowPump.ShouldNotBeSameAs(oldPump, "the stale-session pump must have been replaced before the advise");
+
+        await driver.InvokeReplayForTestAsync(CancellationToken.None);
+        driver.EventPumpInstanceForTests.ShouldBeSameAs(windowPump, "the replay keeps a pump already on the current session");
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
+            "the replay re-advises the stale probes and their initial values arrive on the kept pump",
+            () => Describe(driver, sub));
+        StateOf(driver, "LatePlatform").ShouldBe(HostState.Stopped);
+        sub.DroppedInitialValues.ShouldBe(0);
+    }
+
    /// <summary>
    ///     A probe sync on a driver whose pump was never started (no platform source, e.g. the
    ///     DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just