using System.Runtime.CompilerServices; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using Shouldly; using Xunit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// used to be a no-op (review 2026-10-21, G-1/G-2/G-9): /// the gRPC runtime outlived the driver on every respawn, removal and /raw browse, and /// the host's comms-loss recovery (ShutdownAsync → InitializeAsync on the SAME /// object) neither tore down nor rebuilt anything. These tests pin the teardown at every /// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed, /// the owned channels + session — and the rebuild on the next InitializeAsync. /// public sealed class GalaxyDriverShutdownTests { private const int WaitMs = 10_000; private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new( new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), new GalaxyMxAccessOptions("ShutdownTest"), new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), new GalaxyReconnectOptions()); // ───────────────────────────── fakes ───────────────────────────── /// /// Subscriber whose event stream records the token it was opened with and completes a /// signal when that token fires — the observable for "the StreamEvents stream was cancelled". /// private sealed class StreamingSubscriber : IGalaxySubscriber { private readonly Channel _events = Channel.CreateUnbounded(); private int _nextHandle; public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); public List Unsubscribed { get; } = []; public int StreamsOpened; public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task> SubscribeBulkAsync( IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) { var results = new List(); foreach (var r in fullReferences) { var h = Interlocked.Increment(ref _nextHandle); HandleByRef[r] = h; results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); } return Task.FromResult>(results); } public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) { lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); return Task.CompletedTask; } public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) { Interlocked.Increment(ref StreamsOpened); using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult()); await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent { Family = MxEventFamily.OnDataChange, ItemHandle = handle, Value = new MxValue { DoubleValue = value }, Quality = 192, SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), }); } /// /// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when /// the enumerator is cancelled or completes) — the observable for "exactly one watcher". /// private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource { private int _open; private int _opened; public Channel Current { get; private set; } = Channel.CreateUnbounded(); public int Open => Volatile.Read(ref _open); public int Opened => Volatile.Read(ref _opened); public async IAsyncEnumerable WatchAsync( DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) { Interlocked.Increment(ref _open); Interlocked.Increment(ref _opened); var channel = Current = Channel.CreateUnbounded(); try { await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } finally { Interlocked.Decrement(ref _open); } } public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent { Sequence = sequence, TimeOfLastDeployPresent = true, TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), }); } /// Canned platform set so a production-path init never dials the (fake) gateway for one. private sealed class NoPlatforms : IGalaxyPlatformSource { public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) => Task.FromResult>([]); } private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed { public int Starts; public int Disposes; public event EventHandler? OnAlarmTransition; public void Start() => Starts++; public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition( "Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise, AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi")); public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; } } private static async Task WaitUntilAsync(Func condition, string because) { var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); while (!condition()) { if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); await Task.Delay(10); } } // ───────────────────────────── G-1: streams + pump ───────────────────────────── /// /// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the /// registry is empty. Positive control: an event delivered through that same stream /// reached OnDataChange before the shutdown. /// [Fact] public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump() { var sub = new StreamingSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); driver.OnDataChange += (_, e) => delivered.TrySetResult(e); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); driver.HasEventPumpForTests.ShouldBeTrue(); await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0); (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task, "positive control: the stream must deliver before the shutdown"); await driver.ShutdownAsync(CancellationToken.None); // Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the // enumerator's cancellation registration has run by the time it returns (observed flaky). (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task, "ShutdownAsync must cancel the StreamEvents stream"); driver.HasEventPumpForTests.ShouldBeFalse(); driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init"); driver.GetHealth().State.ShouldBe(DriverState.Unknown); driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown"); } /// /// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT /// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive /// control on both sides of the shutdown. /// [Fact] public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it() { var feed = new RecordingAlarmFeed(); var sub = new StreamingSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub, alarmFeed: feed); var received = 0; driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); feed.Emit(); received.ShouldBe(1, "positive control: the feed is wired before the shutdown"); await driver.ShutdownAsync(CancellationToken.None); feed.Emit(); received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent"); feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector"); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); feed.Emit(); received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed"); feed.Starts.ShouldBe(2); } // ───────────────────────────── G-2: one DeployWatcher ───────────────────────────── /// /// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch /// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly /// once — five stacked watchers would raise five times. /// [Fact] public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher() { var source = new CountingDeployWatchSource(); var sub = new StreamingSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, deployWatchSource: source); var raised = 0; driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); await driver.InitializeAsync("{}", CancellationToken.None); for (var cycle = 0; cycle < 5; cycle++) { // Let this cycle's watch stream actually open before the next shutdown, so every cycle // holds a live stream the teardown has to cancel (a loop cancelled before its first // open would make the count vacuous). await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open"); await driver.ShutdownAsync(CancellationToken.None); await driver.InitializeAsync("{}", CancellationToken.None); } await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1, $"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}"); driver.DeployWatcherForTests.ShouldNotBeNull(); // Bootstrap (suppressed) then a genuine change. await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery"); await Task.Delay(200); Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy"); await driver.ShutdownAsync(CancellationToken.None); await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream"); driver.DeployWatcherForTests.ShouldBeNull(); } /// A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one. [Fact] public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher() { var source = new CountingDeployWatchSource(); var sub = new StreamingSubscriber(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, deployWatchSource: source); await driver.InitializeAsync("{}", CancellationToken.None); var first = driver.DeployWatcherForTests; await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open"); await driver.InitializeAsync("{}", CancellationToken.None); await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1, $"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}"); driver.DeployWatcherForTests.ShouldNotBeSameAs(first); } // ───────────────────────────── G-1/G-9: owned runtime, channels + session ───────────────────────────── /// /// The production path (no injected seams): InitializeAsync builds the owned client, session /// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the /// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's /// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC /// channels are real but lazy (nothing dials until a call is made). /// [Fact] public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds() { var opens = 0; var source = new CountingDeployWatchSource(); await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source, platformSource: new NoPlatforms()) { SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; }, }; await driver.InitializeAsync("{}", CancellationToken.None); opens.ShouldBe(1); var client = driver.OwnedMxClientForTests.ShouldNotBeNull(); var session = driver.OwnedMxSessionForTests.ShouldNotBeNull(); session.IsConnected.ShouldBeTrue(); driver.SupervisorForTests.ShouldNotBeNull(); driver.ProbeWatcherForTests.ShouldNotBeNull(); driver.DeployWatcherForTests.ShouldNotBeNull(); driver.GetHealth().State.ShouldBe(DriverState.Healthy); var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); await driver.ShutdownAsync(CancellationToken.None); session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session"); await Should.ThrowAsync(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None), "the owned gateway channel must be disposed"); driver.OwnedMxClientForTests.ShouldBeNull(); driver.OwnedMxSessionForTests.ShouldBeNull(); driver.SupervisorForTests.ShouldBeNull(); driver.ProbeWatcherForTests.ShouldBeNull(); driver.DeployWatcherForTests.ShouldBeNull(); await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled"); driver.GetHealth().State.ShouldBe(DriverState.Unknown); driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp); // The watchdog's second half: a fresh runtime on the same object. await driver.InitializeAsync("{}", CancellationToken.None); opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one"); driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client); driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session); driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue(); driver.SupervisorForTests.ShouldNotBeNull(); driver.GetHealth().State.ShouldBe(DriverState.Healthy); await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher"); } /// /// A connect that fails mid-build releases the partial runtime (no orphaned client per /// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing. /// [Fact] public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds() { var attempts = 0; await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, platformSource: new NoPlatforms()) { SessionOpenOverrideForTests = _ => ++attempts == 1 ? throw new InvalidOperationException("gateway refused") : Task.CompletedTask, }; var ex = await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); ex.Message.ShouldBe("gateway refused"); driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client"); driver.OwnedMxSessionForTests.ShouldBeNull(); driver.GetHealth().State.ShouldBe(DriverState.Faulted); driver.GetHealth().LastError.ShouldBe("gateway refused"); await driver.InitializeAsync("{}", CancellationToken.None); attempts.ShouldBe(2); driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue(); driver.GetHealth().State.ShouldBe(DriverState.Healthy); } /// DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op. [Fact] public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless() { var sub = new StreamingSubscriber(); var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); await driver.InitializeAsync("{}", CancellationToken.None); await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); await driver.ShutdownAsync(CancellationToken.None); await Should.NotThrowAsync(async () => await driver.DisposeAsync()); await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None)); await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); } }