using System.Runtime.CompilerServices; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using Shouldly; using Xunit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// Lane F fix round 1 (review of #735): the probe watcher is seeded at InitializeAsync /// and re-seeded on rediscovery (the runtime never calls DiscoverAsync on a live driver); /// a SubscribeAsync landing mid-teardown cannot create a pump on the dying subscriber; a /// gateway close that never answers cannot park ShutdownAsync; a re-subscribe that /// straddles a teardown never rebinds into the next runtime's registry. /// public sealed class GalaxyDriverFixRound1Tests { internal const int WaitMs = 10_000; internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new( new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), new GalaxyMxAccessOptions("FixRound1"), new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), new GalaxyReconnectOptions()); // ───────────────────────────── fakes ───────────────────────────── private sealed class ScriptedPlatformSource : IGalaxyPlatformSource { private int _calls; public int Calls => Volatile.Read(ref _calls); public Func> Script { get; init; } = _ => ["Plat01"]; public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) => Task.FromResult(Script(Interlocked.Increment(ref _calls))); } internal sealed class ParkableSubscriber : IGalaxySubscriber { private readonly Channel _events = Channel.CreateUnbounded(); private int _nextHandle; private int _subscribeCalls; public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); public List Subscribed { get; } = []; public int SubscribeCalls => Volatile.Read(ref _subscribeCalls); /// When set, UnsubscribeBulk parks until . public bool ParkUnsubscribe { get; set; } public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult(); /// When > 0, that SubscribeBulk call (1-based) parks until . public int ParkSubscribeCall { get; set; } public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult(); public async Task> SubscribeBulkAsync( IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) { var call = Interlocked.Increment(ref _subscribeCalls); if (call == ParkSubscribeCall) { SubscribeEntered.TrySetResult(); await _releaseSubscribe.Task.WaitAsync(cancellationToken); } var results = new List(); lock (Subscribed) { foreach (var r in fullReferences) { var h = Interlocked.Increment(ref _nextHandle); Subscribed.Add(r); HandleByRef[r] = h; results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); } } return results; } /// Every handle passed to UnsubscribeBulk, in call order. public List Unsubscribed { get; } = []; public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) { lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); if (ParkUnsubscribe) { UnsubscribeEntered.TrySetResult(); await _releaseUnsubscribe.Task.WaitAsync(cancellationToken); } } public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent { Family = MxEventFamily.OnDataChange, ItemHandle = handle, Value = new MxValue { BoolValue = value }, Quality = 192, SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), }); } internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource { public Channel Events { get; } = Channel.CreateUnbounded(); public int Opened; public async IAsyncEnumerable WatchAsync( DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) { Interlocked.Increment(ref Opened); await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent { Sequence = sequence, TimeOfLastDeployPresent = true, TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), }); } internal static async Task WaitUntilAsync(Func condition, string because) { var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); while (!condition()) { if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); await Task.Delay(10); } } internal static HostState? StateOf(GalaxyDriver driver, string host) => driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; // ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ───────────────────────────── /// A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses. [Fact] public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync() { var sub = new ParkableSubscriber(); var platforms = new ScriptedPlatformSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); await driver.InitializeAsync("{}", CancellationToken.None); // The seed runs in the background (fix round 2, N1) — init does not wait on it. await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, "the probe watcher must be seeded after init"); platforms.Calls.ShouldBe(1); lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState"); driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, "a ScanState event must surface the seeded platform on GetHostStatuses"); } /// A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds. [Fact] public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds() { var sub = new ParkableSubscriber(); var deploys = new ChannelDeployWatchSource(); var platforms = new ScriptedPlatformSource { Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"], }; await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, subscriber: sub, platformSource: platforms, deployWatchSource: deploys); var raised = 0; driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); await driver.InitializeAsync("{}", CancellationToken.None); driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init"); await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run"); sub.Subscribed.ShouldBeEmpty(); await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed) await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery"); await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, "the rediscovery must re-seed the probe set"); platforms.Calls.ShouldBe(2); sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true); } private sealed class ParkingPlatformSource : IGalaxyPlatformSource { private int _calls; public int Calls => Volatile.Read(ref _calls); public int ParkCall { get; init; } public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); public void Release() => _release.TrySetResult(); public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) { var call = Interlocked.Increment(ref _calls); if (call == ParkCall) { Entered.TrySetResult(); await _release.Task.WaitAsync(cancellationToken); return ["Plat01"]; // the pre-deploy view this pass fetched } return call == 1 ? [] : ["Plat01", "Eng02"]; } } /// /// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts /// after it — otherwise the probe set would stay on the stale view until the next deploy. /// [Fact] public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass() { var sub = new ParkableSubscriber(); var deploys = new ChannelDeployWatchSource(); var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, subscriber: sub, platformSource: platforms, deployWatchSource: deploys); var raised = 0; driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); try { await driver.InitializeAsync("{}", CancellationToken.None); await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); (await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task, "the first rediscovery pass must be parked mid-fetch"); await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero)); await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery"); } finally { platforms.Release(); } await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed"); await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, "the final probe set must reflect the latest deploy, not the parked pass's stale view"); } /// The production path (owned runtime) queries the platform source at init, with no DiscoverAsync. [Fact] public async Task Production_path_queries_the_platform_source_at_init() { var platforms = new ScriptedPlatformSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms) { SessionOpenOverrideForTests = _ => Task.CompletedTask, }; await driver.InitializeAsync("{}", CancellationToken.None); await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init"); driver.GetHealth().State.ShouldBe(DriverState.Healthy); } // ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ───────────────────────────── /// /// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before /// the subscriber is released) is refused and leaves no pump; before the shutdown the same /// call works (positive control). /// [Fact] public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump() { var sub = new ParkableSubscriber(); var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); await driver.InitializeAsync("{}", CancellationToken.None); await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, "the (background) seed must give the probe watcher its handle first"); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime"); sub.ParkUnsubscribe = true; var shutdown = driver.ShutdownAsync(CancellationToken.None); var subscribeCallsBefore = sub.SubscribeCalls; try { (await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task, "the teardown must be parked inside the probe watcher's unsubscribe"); shutdown.IsCompleted.ShouldBeFalse(); // The pump is still live here (the teardown takes it after the probe watcher) — reusing it // is exactly the race: the call must be refused, not served on the dying subscriber. driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken"); await Should.ThrowAsync( () => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None)); sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway"); } finally { // Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs. sub.ReleaseUnsubscribe(); } await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown"); driver.TrackedSubscriptionCountForTests.ShouldBe(0); await driver.InitializeAsync("{}", CancellationToken.None); driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump"); } // ───────────────────────────── IMPORTANT 3: bounded gateway close ───────────────────────────── /// A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline. [Fact] public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers() { await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource()) { SessionOpenOverrideForTests = _ => Task.CompletedTask, SessionCloseDeadline = TimeSpan.FromMilliseconds(300), }; await driver.InitializeAsync("{}", CancellationToken.None); var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var closeToken = CancellationToken.None; driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct => { closeToken = ct; closeEntered.TrySetResult(); await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway }; var sw = System.Diagnostics.Stopwatch.StartNew(); await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); sw.Stop(); closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted"); closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline"); sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s"); driver.OwnedMxSessionForTests.ShouldBeNull(); driver.GetHealth().State.ShouldBe(DriverState.Unknown); } // ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ───────────────────────────── /// /// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared /// registry; the same replay without a teardown rebinds (positive control). /// [Fact] public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind() { // Positive control first. { var sub = new ParkableSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); await driver.InvokeReplayForTestAsync(CancellationToken.None); driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription"); sub.SubscribeCalls.ShouldBe(2); } { var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); var replay = driver.InvokeReplayForTestAsync(CancellationToken.None); (await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task, "the replay must be parked inside its SubscribeBulk"); await driver.ShutdownAsync(CancellationToken.None); driver.TrackedSubscriptionCountForTests.ShouldBe(0); sub.ReleaseSubscribe(); await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles"); } } }