using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// Lane F fix round 2 (re-review of #735): /// /// N1 — the init platform seed runs in the background: a slow or cancelled repository /// fetch can neither delay nor fail InitializeAsync (Galaxy's Tier-A Initialize budget is /// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown /// is discarded. /// N2 — a seed requested under a new runtime always gets a pass of its own, even when a /// worker from the previous runtime is still running. /// N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails /// and unsubscribes instead of registering into the next runtime's registry. /// The probe watcher's own subscribe re-checks its stop after the await. /// /// public sealed class GalaxyDriverFixRound2Tests { private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2); /// A platform source whose fetches park until released (optionally ignoring cancellation). private sealed class GatedPlatformSource : IGalaxyPlatformSource { private int _calls; private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); public int Calls => Volatile.Read(ref _calls); public int ParkCall { get; init; } = 1; public bool IgnoreCancellation { get; init; } public Func> Result { get; init; } = _ => ["Plat01"]; public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public void Release() => _release.TrySetResult(); public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) { var call = Interlocked.Increment(ref _calls); if (call == ParkCall) { ParkedToken.TrySetResult(cancellationToken); Entered.TrySetResult(); if (IgnoreCancellation) await _release.Task; else await _release.Task.WaitAsync(cancellationToken); } return Result(call); } } private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource { private int _calls; public int Calls => Volatile.Read(ref _calls); public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) { Interlocked.Increment(ref _calls); return Task.FromException>(ex); } } private static async Task Entered(TaskCompletionSource tcs, string because) => (await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because); // ───────────────────────────── N1: background init seed ───────────────────────────── /// /// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy /// inside its budget, and the teardown cancels the parked fetch. /// [Fact] public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes() { var sub = new ParkableSubscriber(); var platforms = new GatedPlatformSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); try { var sw = System.Diagnostics.Stopwatch.StartNew(); await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); sw.Stop(); driver.GetHealth().State.ShouldBe(DriverState.Healthy); await Entered(platforms.Entered, "positive control: the seed fetch was started"); sw.Elapsed.ShouldBeLessThan(InitBudget); await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); var token = await platforms.ParkedToken.Task; token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight"); } finally { platforms.Release(); } } /// A fetch that is cancelled (the gateway's own deadline) does not fail init. [Fact] public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync() { var sub = new ParkableSubscriber(); var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline")); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); driver.GetHealth().State.ShouldBe(DriverState.Healthy); await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran"); driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); // The driver keeps serving after the failed seed. await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.TrackedSubscriptionCountForTests.ShouldBe(1); } /// /// The production failure mode: the host's Tier-A Initialize timeout cancels the init token /// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and /// the seed — tied to the runtime, not to the init call — must survive that cancellation and /// still land. /// [Fact] public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed() { var sub = new ParkableSubscriber(); var platforms = new GatedPlatformSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); using var initCts = new CancellationTokenSource(); try { initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget); driver.GetHealth().State.ShouldBe(DriverState.Healthy); await Entered(platforms.Entered, "the seed fetch must be in flight"); await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked"); (await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse( "the seed belongs to the runtime; the init call's timeout must not cancel it"); } finally { platforms.Release(); } await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, "the seed must still land after the init token was cancelled"); } /// Positive control: a fetch that completes after init still populates host statuses. [Fact] public async Task A_fetch_completing_after_init_populates_host_statuses() { var sub = new ParkableSubscriber(); var platforms = new GatedPlatformSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); try { await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); await Entered(platforms.Entered, "the seed must be parked"); driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); } finally { platforms.Release(); } await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, "the late seed must reach the probe watcher"); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, "a ScanState event must surface the late-seeded platform"); } // ───────────────────────────── N1 generation + N2: straddling worker ───────────────────────────── /// /// A seed from the previous runtime that completes after a teardown + re-init is discarded, /// and the new runtime's own seed request still gets its own pass. /// [Fact] public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded() { var sub = new ParkableSubscriber(); var platforms = new GatedPlatformSource { IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it Result = call => call == 1 ? ["Stale01"] : ["Plat01"], }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); try { await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); await Entered(platforms.Entered, "runtime 1's seed must be parked"); await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); } finally { platforms.Release(); } await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass"); await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, "the new runtime must be seeded"); driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]); lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded"); } // ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ───────────────────────────── /// /// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry /// back into the registry, and must release the handles it just acquired. Positive control: /// the same sync without a stop registers. /// [Fact] public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry() { { var registry = new SubscriptionRegistry(); using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry); await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control"); } { var registry = new SubscriptionRegistry(); var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry); var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None); await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk"); await watcher.StopAsync(CancellationToken.None); sub.ReleaseSubscribe(); await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register"); watcher.WatchedPlatforms.ShouldBeEmpty(); lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"], "the handle acquired after the stop must be released"); } } // ───────────────────────────── N3: subscribe / read straddling a teardown ───────────────────────────── /// /// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing /// and unsubscribes its handles; a subscribe on the live runtime registers (positive control). /// [Fact] public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing() { var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers"); var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None); try { await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk"); await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); driver.TrackedSubscriptionCountForTests.ShouldBe(0); } finally { sub.ReleaseSubscribe(); } await Should.ThrowAsync(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry"); lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]); await driver.InitializeAsync("{}", CancellationToken.None); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry"); } /// The subscribe-once read path: same straddle, same outcome. [Fact] public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing() { var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None); try { await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk"); await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); } finally { sub.ReleaseSubscribe(); } await Should.ThrowAsync(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); driver.TrackedSubscriptionCountForTests.ShouldBe(0); lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]); } }