using System.Runtime.CompilerServices; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using Shouldly; using Xunit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value /// once, as its first event, and a FRESH StreamEvents (after_worker_sequence = 0) never /// replays — so a probe advised before the driver's event stream is attached never gets a value, and /// ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe, /// so a driver with no authored tags (or any driver before its first read) had no host statuses. /// public sealed class ProbeInitialValueTests { private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"]; private static GalaxyDriverOptions Opts() => new( new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), new GalaxyMxAccessOptions("ProbeInitialValue"), new GalaxyRepositoryOptions(WatchDeployEvents: false), new GalaxyReconnectOptions()); private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource { public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) => Task.FromResult>(platforms); } /// /// Models the gateway's event semantics: every advise pushes its current value exactly once, and /// only to a stream that is attached at that moment (no replay for a fresh stream). Records the /// order in which the stream was requested and the advises were made. /// private sealed class GatewayLikeSubscriber : IGalaxySubscriber { private readonly Lock _gate = new(); private Channel? _attached; private int _nextHandle; /// "stream" per StreamEvents call, "subscribe:<refs>" per SubscribeBulk, in call order. public List Calls { get; } = []; /// Initial values pushed while no stream was attached — lost, as on the real gateway. public int DroppedInitialValues; private int _streams; /// Streams beyond this count end immediately after attaching (a gateway refusing/ending them). public int StreamsThatStayOpen = int.MaxValue; /// The ScanState value the next advise reports. public volatile bool ScanStateRunning = true; public Task> SubscribeBulkAsync( IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) { var results = new List(); lock (_gate) { Calls.Add("subscribe:" + string.Join(",", fullReferences)); foreach (var r in fullReferences) { var h = ++_nextHandle; results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); var initial = new MxEvent { Family = MxEventFamily.OnDataChange, ItemHandle = h, Value = new MxValue { BoolValue = ScanStateRunning }, Quality = 192, SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), }; if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues); } } return Task.FromResult>(results); } public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) { // Attached synchronously on the first MoveNext — the point a real gRPC call is issued. var mine = Channel.CreateUnbounded(); lock (_gate) { Calls.Add("stream"); _attached = mine; } if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen) { lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; } yield break; // the gateway completed this stream at once } try { await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } finally { lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; } } } /// Fault the currently attached stream (the gateway dropping it). public void FaultAttachedStream(Exception cause) { lock (_gate) _attached.ShouldNotBeNull("a stream must be attached").Writer.TryComplete(cause); } public string[] CallsSnapshot() { lock (_gate) return [.. Calls]; } } private static HostState? StateOf(GalaxyDriver driver, string host) => driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) => $"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " + $"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}"; /// /// A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded /// platform with its initial ScanState. Before #865 the pump never started, every initial value /// was pushed to no stream, and only the transport entry ever appeared. /// [Fact] public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: new FixedPlatformSource(Platforms)); await driver.InitializeAsync("{}", CancellationToken.None); await TestWait.UntilAsync( () => Platforms.All(p => StateOf(driver, p) == HostState.Running), "every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe", () => Describe(driver, sub)); sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached"); driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made"); } /// /// Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code /// the probe advise went out with no stream at all. /// [Fact] public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: new FixedPlatformSource(Platforms)); await driver.InitializeAsync("{}", CancellationToken.None); await TestWait.UntilAsync( () => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length, "positive control: the background seed must advise every platform", () => Describe(driver, sub)); var calls = sub.CallsSnapshot(); var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal)); var firstStream = Array.IndexOf(calls, "stream"); firstSubscribe.ShouldBeGreaterThanOrEqualTo(0); firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]"); firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]"); } /// /// A reconnect replay re-advises the probes on the recreated session; their new initial values must /// arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait /// can only pass on a post-replay event. /// [Fact] public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: new FixedPlatformSource(Platforms)); await driver.InitializeAsync("{}", CancellationToken.None); await TestWait.UntilAsync( () => Platforms.All(p => StateOf(driver, p) == HostState.Running), "positive control: the initial seed reports Running", () => Describe(driver, sub)); sub.ScanStateRunning = false; await driver.InvokeReplayForTestAsync(CancellationToken.None); await TestWait.UntilAsync( () => Platforms.All(p => StateOf(driver, p) == HostState.Stopped), "the re-advised probes' initial values (Stopped) must reach the host statuses after a replay", () => Describe(driver, sub)); sub.DroppedInitialValues.ShouldBe(0); } /// /// Review finding: a probe advised in the reopen → replay window (the session was swapped, the old /// pump still streams the dead session) must still get its initial value. The advise waits for the /// replay's restarted pump instead of advising onto the stale one (its value would route under the /// old session and be lost, and the replay skips a current-session handle) or starting a competing /// stream (the gateway's default single-subscriber mode refuses a second attach). /// [Fact] public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: new FixedPlatformSource(Platforms)) { RecreateOverrideForTests = _ => Task.CompletedTask, // Generous: the window advise must still be waiting when the replay below starts the pump, even // on a stalled agent; the wait ends the moment that pump exists. StreamRequestWait = TimeSpan.FromSeconds(20), }; await driver.InitializeAsync("{}", CancellationToken.None); await TestWait.UntilAsync( () => Platforms.All(p => StateOf(driver, p) == HostState.Running), "positive control: the initial seed reports Running", () => Describe(driver, sub)); var oldPump = driver.EventPumpInstanceForTests; await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet sub.ScanStateRunning = false; var callsBeforeWindow = sub.CallsSnapshot().Length; // The window advise must WAIT for the replay's current-session pump rather than advise onto the // stale one (or start a competing stream). var sync = driver.ProbeWatcherForTests.ShouldNotBeNull() .SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None); await driver.InvokeReplayForTestAsync(CancellationToken.None); await TestWait.ForAsync(sync, "the window advise completes once the replay restarted the pump"); await TestWait.UntilAsync( () => StateOf(driver, "LatePlatform") == HostState.Stopped && Platforms.All(p => StateOf(driver, p) == HostState.Stopped), "the window advise and the replay's re-advises all get their initial values", () => Describe(driver, sub)); driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the replay restarted the stale pump"); var windowCalls = sub.CallsSnapshot()[callsBeforeWindow..]; var lateAdvise = Array.FindIndex(windowCalls, c => c.Contains("LatePlatform", StringComparison.Ordinal)); var newStream = Array.IndexOf(windowCalls, "stream"); newStream.ShouldBeGreaterThanOrEqualTo(0, $"the replay must start a new stream; calls=[{string.Join(" | ", windowCalls)}]"); lateAdvise.ShouldBeGreaterThan(newStream, $"the window advise must follow the new stream; calls=[{string.Join(" | ", windowCalls)}]"); windowCalls.Count(c => c == "stream").ShouldBe(1, "no competing stream may be started for the window advise"); sub.DroppedInitialValues.ShouldBe(0); } /// /// The data paths too: a first SubscribeAsync and a first ReadAsync on a driver whose pump was never /// started request the stream BEFORE advising, so their initial values arrive (a read of a value that /// never changes again would otherwise time out). /// [Fact] public async Task First_subscribe_and_read_advise_only_after_the_stream_is_requested() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) { ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10), }; await driver.InitializeAsync("{}", CancellationToken.None); var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); driver.OnDataChange += (_, e) => delivered.TrySetResult(e); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); var calls = sub.CallsSnapshot(); Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the advise; calls=[{string.Join(" | ", calls)}]"); (await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub))) .FullReference.ShouldBe("Tank01.Level"); sub.DroppedInitialValues.ShouldBe(0); } /// A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise. [Fact] public async Task First_read_advises_only_after_the_stream_is_requested() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) { ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10), }; await driver.InitializeAsync("{}", CancellationToken.None); driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise"); var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None); var calls = sub.CallsSnapshot(); Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]"); read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)"); sub.DroppedInitialValues.ShouldBe(0); } /// /// The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted /// pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a /// post-replay event can satisfy the wait. /// [Fact] public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); var last = new System.Collections.Concurrent.ConcurrentDictionary(); driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value; await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true, "positive control: the subscribe's initial value arrives", () => Describe(driver, sub)); sub.ScanStateRunning = false; // the fake's next initial value await driver.InvokeReplayForTestAsync(CancellationToken.None); await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false, "the replay's re-advise must deliver its new initial value", () => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]"); sub.DroppedInitialValues.ShouldBe(0); } /// /// A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and /// stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next /// advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control: /// the first replay, whose stream stays open, succeeds. /// [Fact] public async Task Replay_fails_when_the_restarted_stream_ends_at_once() { var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1 await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds await Should.ThrowAsync( () => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once "a replay must not report success over a dead event stream"); } /// /// #865 review: a stream fault is escalated only from the LIVE pump. A fault from a pump superseded /// by a reopen (its session is no longer current) is ignored — it would fail a good replay — while /// the live pump's faults (before the reopen, and the replay's new pump after it) still escalate. /// [Fact] public async Task Only_the_live_pumps_stream_faults_are_escalated() { var sub = new GatewayLikeSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) { RecreateOverrideForTests = _ => Task.CompletedTask, }; await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // pump 1 // Positive control: the live pump's fault escalates. sub.FaultAttachedStream(new IOException("drop 1")); await TestWait.UntilAsync(() => driver.EscalatedStreamFaultsForTests == 1, "the live pump's fault must escalate"); // Recover onto pump 2 (the replay restarts the faulted pump), then reopen: pump 2 is now superseded. await driver.InvokeReplayForTestAsync(CancellationToken.None); var pump2 = driver.EventPumpInstanceForTests.ShouldNotBeNull(); await driver.InvokeReopenForTestAsync(CancellationToken.None); sub.FaultAttachedStream(new IOException("late fault from the superseded pump")); await TestWait.UntilAsync(() => ((EventPump)pump2).HasEnded, "the superseded pump's stream must have faulted"); // Positive control after the absence: the replay's new pump faults and that DOES escalate — so the // count can only be 2 if the superseded pump's fault was ignored. await driver.InvokeReplayForTestAsync(CancellationToken.None); var pump3 = (EventPump)driver.EventPumpInstanceForTests.ShouldNotBeNull(); pump3.ShouldNotBeSameAs(pump2); sub.FaultAttachedStream(new IOException("drop 3")); // HasEnded is set only after the fault callback ran, so once both pumps have ended every escalation // that will ever happen has happened. await TestWait.UntilAsync(() => pump3.HasEnded, "the new live pump's stream must have faulted"); driver.EscalatedStreamFaultsForTests.ShouldBe(2, "the new live pump's fault escalates; the superseded pump's did not"); } /// /// A probe sync on a driver whose pump was never started (no platform source, e.g. the /// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just /// the init path. /// [Fact] public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first() { var sub = new GatewayLikeSubscriber(); // No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync. await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet"); await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None); await TestWait.UntilAsync( () => Platforms.All(p => StateOf(driver, p) == HostState.Running), "the watcher must start the stream before advising, so the initial values arrive", () => Describe(driver, sub)); sub.DroppedInitialValues.ShouldBe(0); } }