fix(galaxy): probe advises start the event stream first so initial ScanState arrives (#865)

The gateway's StreamEvents with after_worker_sequence = 0 is a fresh stream and never replays, and the event pump started only on the first Read/Subscribe, so platform ScanState probes advised before then lost their initial value (and a driver with no authored tags never got any host status). PerPlatformProbeWatcher now awaits an ensure-event-stream hook before every probe SubscribeBulk (seed, discover sync, reconnect ResubscribeStaleAsync). The driver's hook starts the pump (refused while a teardown is in progress) and waits, bounded, on the new EventPump.StreamRequested signal - completed once the stream loop's first MoveNext has issued the StreamEvents RPC, or when the loop ends - so the advise never precedes the stream request. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 12:24:08 -05:00
parent c4bc1ef309
commit e35f2a4e93
5 changed files with 320 additions and 4 deletions
@@ -303,6 +303,7 @@ public sealed class GalaxyDriverFixRound1Tests

        await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
        driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
+        var oldPump = driver.EventPumpInstanceForTests;

        sub.ParkUnsubscribe = true;
        var shutdown = driver.ShutdownAsync(CancellationToken.None);
@@ -330,7 +331,9 @@ public sealed class GalaxyDriverFixRound1Tests
        driver.TrackedSubscriptionCountForTests.ShouldBe(0);

        await driver.InitializeAsync("{}", CancellationToken.None);
-        driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
+        // #865: the new runtime's background probe seed starts a pump of its own (its advises need a live
+        // stream) — but it must be a FRESH one, never the dead subscriber's.
+        driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump");
    }

    // ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
@@ -0,0 +1,218 @@
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Google.Protobuf.WellKnownTypes;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
+
+/// <summary>
+///     #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value
+///     once, as its first event, and a FRESH <c>StreamEvents</c> (<c>after_worker_sequence = 0</c>) never
+///     replays — so a probe advised before the driver's event stream is attached never gets a value, and
+///     ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe,
+///     so a driver with no authored tags (or any driver before its first read) had no host statuses.
+/// </summary>
+public sealed class ProbeInitialValueTests
+{
+    private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"];
+
+    private static GalaxyDriverOptions Opts() => new(
+        new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
+        new GalaxyMxAccessOptions("ProbeInitialValue"),
+        new GalaxyRepositoryOptions(WatchDeployEvents: false),
+        new GalaxyReconnectOptions());
+
+    private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource
+    {
+        public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
+            => Task.FromResult<IReadOnlyList<string>>(platforms);
+    }
+
+    /// <summary>
+    ///     Models the gateway's event semantics: every advise pushes its current value exactly once, and
+    ///     only to a stream that is attached at that moment (no replay for a fresh stream). Records the
+    ///     order in which the stream was requested and the advises were made.
+    /// </summary>
+    private sealed class GatewayLikeSubscriber : IGalaxySubscriber
+    {
+        private readonly Lock _gate = new();
+        private Channel<MxEvent>? _attached;
+        private int _nextHandle;
+
+        /// <summary>"stream" per StreamEvents call, "subscribe:&lt;refs&gt;" per SubscribeBulk, in call order.</summary>
+        public List<string> Calls { get; } = [];
+
+        /// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
+        public int DroppedInitialValues;
+
+        /// <summary>The ScanState value the next advise reports.</summary>
+        public volatile bool ScanStateRunning = true;
+
+        public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
+            IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
+        {
+            var results = new List<SubscribeResult>();
+            lock (_gate)
+            {
+                Calls.Add("subscribe:" + string.Join(",", fullReferences));
+                foreach (var r in fullReferences)
+                {
+                    var h = ++_nextHandle;
+                    results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
+                    var initial = new MxEvent
+                    {
+                        Family = MxEventFamily.OnDataChange,
+                        ItemHandle = h,
+                        Value = new MxValue { BoolValue = ScanStateRunning },
+                        Quality = 192,
+                        SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+                    };
+                    if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues);
+                }
+            }
+            return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
+        }
+
+        public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
+
+        public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            // Attached synchronously on the first MoveNext — the point a real gRPC call is issued.
+            var mine = Channel.CreateUnbounded<MxEvent>();
+            lock (_gate)
+            {
+                Calls.Add("stream");
+                _attached = mine;
+            }
+            try
+            {
+                await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    yield return ev;
+                }
+            }
+            finally
+            {
+                lock (_gate)
+                {
+                    if (ReferenceEquals(_attached, mine)) _attached = null;
+                }
+            }
+        }
+
+        public string[] CallsSnapshot()
+        {
+            lock (_gate) return [.. Calls];
+        }
+    }
+
+    private static HostState? StateOf(GalaxyDriver driver, string host) =>
+        driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
+
+    private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) =>
+        $"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " +
+        $"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}";
+
+    /// <summary>
+    ///     A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded
+    ///     platform with its initial ScanState. Before #865 the pump never started, every initial value
+    ///     was pushed to no stream, and only the transport entry ever appeared.
+    /// </summary>
+    [Fact]
+    public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState()
+    {
+        var sub = new GatewayLikeSubscriber();
+        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+            platformSource: new FixedPlatformSource(Platforms));
+
+        await driver.InitializeAsync("{}", CancellationToken.None);
+
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
+            "every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe",
+            () => Describe(driver, sub));
+        sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached");
+        driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made");
+    }
+
+    /// <summary>
+    ///     Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code
+    ///     the probe advise went out with no stream at all.
+    /// </summary>
+    [Fact]
+    public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested()
+    {
+        var sub = new GatewayLikeSubscriber();
+        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+            platformSource: new FixedPlatformSource(Platforms));
+
+        await driver.InitializeAsync("{}", CancellationToken.None);
+        await TestWait.UntilAsync(
+            () => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length,
+            "positive control: the background seed must advise every platform",
+            () => Describe(driver, sub));
+
+        var calls = sub.CallsSnapshot();
+        var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal));
+        var firstStream = Array.IndexOf(calls, "stream");
+        firstSubscribe.ShouldBeGreaterThanOrEqualTo(0);
+        firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]");
+        firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]");
+    }
+
+    /// <summary>
+    ///     A reconnect replay re-advises the probes on the recreated session; their new initial values must
+    ///     arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait
+    ///     can only pass on a post-replay event.
+    /// </summary>
+    [Fact]
+    public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values()
+    {
+        var sub = new GatewayLikeSubscriber();
+        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
+            platformSource: new FixedPlatformSource(Platforms));
+        await driver.InitializeAsync("{}", CancellationToken.None);
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
+            "positive control: the initial seed reports Running",
+            () => Describe(driver, sub));
+
+        sub.ScanStateRunning = false;
+        await driver.InvokeReplayForTestAsync(CancellationToken.None);
+
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
+            "the re-advised probes' initial values (Stopped) must reach the host statuses after a replay",
+            () => Describe(driver, sub));
+        sub.DroppedInitialValues.ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     A probe sync on a driver whose pump was never started (no platform source, e.g. the
+    ///     DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
+    ///     the init path.
+    /// </summary>
+    [Fact]
+    public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first()
+    {
+        var sub = new GatewayLikeSubscriber();
+        // No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync.
+        await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
+        await driver.InitializeAsync("{}", CancellationToken.None);
+        driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet");
+
+        await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None);
+
+        await TestWait.UntilAsync(
+            () => Platforms.All(p => StateOf(driver, p) == HostState.Running),
+            "the watcher must start the stream before advising, so the initial values arrive",
+            () => Describe(driver, sub));
+        sub.DroppedInitialValues.ShouldBe(0);
+    }
+}