using System.Runtime.CompilerServices; using System.Threading.Channels; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse; /// /// Tests 's consumption of : /// bootstrap suppression, change detection, presence-flip handling, clean shutdown, /// and reconnect-on-error backoff. /// public sealed class DeployWatcherTests { /// /// Test helper exposing a as the event source plus an /// optional fault hook so reconnect / retry paths can be exercised deterministically. /// private sealed class FakeDeployWatchSource : IGalaxyDeployWatchSource { private readonly Func> _channelFactory; /// /// Gets the list of last-seen deployment times from each watch iteration. /// public List LastSeenTimes { get; } = []; /// /// Gets the number of times WatchAsync has been called. /// public int CallCount { get; private set; } /// /// Gets or sets a function that can throw an exception on specific iterations. /// public Func? ThrowOnIteration { get; init; } /// /// Initializes a new instance of the class with a single channel. /// /// The deploy event channel to use for all iterations. public FakeDeployWatchSource(Channel channel) { _channelFactory = _ => channel; } /// /// Initializes a new instance of the class with a channel factory. /// /// A factory function that creates channels for each iteration. public FakeDeployWatchSource(Func> channelFactory) { _channelFactory = channelFactory; } /// /// Watches for deploy events asynchronously. /// /// The last deployment time seen. /// The cancellation token. /// An async enumerable of deploy events. public async IAsyncEnumerable WatchAsync( DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) { int iteration = ++CallCount; LastSeenTimes.Add(lastSeenDeployTime); if (ThrowOnIteration?.Invoke(iteration) is { } ex) { throw ex; } var channel = _channelFactory(iteration); await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return ev; } } } private static DeployEvent Event(ulong sequence, DateTimeOffset? deployTime) { var ev = new DeployEvent { Sequence = sequence, ObservedAt = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), TimeOfLastDeployPresent = deployTime is not null, }; if (deployTime is { } t) { ev.TimeOfLastDeploy = Timestamp.FromDateTimeOffset(t); } return ev; } private static List CaptureRediscoverEvents(DeployWatcher watcher) { var captured = new List(); watcher.OnRediscoveryNeeded += (_, args) => { lock (captured) captured.Add(args); }; return captured; } // Presence wait with a generous ceiling and a self-describing failure (#858): under heavy parallel // load the watcher's Task.Run loop alone could take > 2 s to be scheduled. private static Task WaitUntilAsync( Func condition, string because, FakeDeployWatchSource source, List captured) => TestWait.UntilAsync(condition, because, () => Describe(source, captured)); private static string Describe(FakeDeployWatchSource source, List captured) { lock (captured) { return $"watchCalls={source.CallCount} raised=[{string.Join(", ", captured.Select(c => $"{c.Reason}@{c.ScopeHint}"))}]"; } } private static List Snapshot(List captured) { lock (captured) return [.. captured]; } /// Verifies that bootstrap deploy events are suppressed. [Fact] public async Task BootstrapEventIsSuppressed() { var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); using var watcher = new DeployWatcher(source); var captured = CaptureRediscoverEvents(watcher); var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z"); var t1 = DateTimeOffset.Parse("2026-01-03T00:00:00Z"); await watcher.StartAsync(CancellationToken.None); // The bootstrap event, then a positive control that MUST raise. Events are handled in order, // so once the control's raise is seen the bootstrap has been fully handled - no sleep needed // to prove the bootstrap did not raise (#858). await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); await WaitUntilAsync(() => Snapshot(captured).Count >= 1, "positive control: the post-bootstrap change must raise rediscovery", source, captured); var raised = Snapshot(captured).ShouldHaveSingleItem("the bootstrap event must not raise"); DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime()); await watcher.StopAsync(); } /// Verifies that a deployment time change fires a rediscovery event. [Fact] public async Task DeployTimeChangeFiresRediscover() { var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z"); var t1 = DateTimeOffset.Parse("2026-01-02T12:00:00Z"); var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); using var watcher = new DeployWatcher(source); var captured = CaptureRediscoverEvents(watcher); await watcher.StartAsync(CancellationToken.None); await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); // real change await WaitUntilAsync(() => Snapshot(captured).Count >= 1, "a deploy-time change must raise rediscovery", source, captured); captured.Count.ShouldBe(1); captured[0].Reason.ShouldBe("deploy-time-changed"); captured[0].ScopeHint.ShouldNotBeNull(); DateTimeOffset.Parse(captured[0].ScopeHint!).ToUniversalTime() .ShouldBe(t1.ToUniversalTime()); await watcher.StopAsync(); } /// Verifies that the same deployment time does not fire a rediscovery event. [Fact] public async Task SameDeployTimeDoesNotFire() { var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z"); var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); using var watcher = new DeployWatcher(source); var captured = CaptureRediscoverEvents(watcher); await watcher.StartAsync(CancellationToken.None); var t1 = DateTimeOffset.Parse("2026-01-05T00:00:00Z"); await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap await channel.Writer.WriteAsync(Event(2, t0), TestContext.Current.CancellationToken); // duplicate state — gateway re-sent // Positive control (#858): a genuine change after the duplicate MUST raise. Events are handled in // order, so once it is seen the duplicate has been handled - its absence is then a real result. await channel.Writer.WriteAsync(Event(3, t1), TestContext.Current.CancellationToken); await WaitUntilAsync(() => Snapshot(captured).Count >= 1, "positive control: the change after the duplicate must raise rediscovery", source, captured); var raised = Snapshot(captured).ShouldHaveSingleItem("the duplicate deploy time must not raise"); DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime()); await watcher.StopAsync(); } /// Verifies that a deployment time presence flip fires a rediscovery event. [Fact] public async Task TimeOfLastDeployPresentFlipFiresRediscover() { var t1 = DateTimeOffset.Parse("2026-03-01T08:00:00Z"); var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); using var watcher = new DeployWatcher(source); var captured = CaptureRediscoverEvents(watcher); await watcher.StartAsync(CancellationToken.None); // Bootstrap with absent deploy time (Galaxy never deployed). await channel.Writer.WriteAsync(Event(0, deployTime: null), TestContext.Current.CancellationToken); // Now a deploy lands and the present flag flips. await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); await WaitUntilAsync(() => Snapshot(captured).Count >= 1, "a presence flip must raise rediscovery", source, captured); captured.Count.ShouldBe(1); captured[0].Reason.ShouldBe("deploy-time-changed"); captured[0].ScopeHint.ShouldNotBeNull(); await watcher.StopAsync(); } /// Verifies that stop cancels the watcher loop cleanly. [Fact] public async Task StopCancelsLoopCleanly() { var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); using var watcher = new DeployWatcher(source); await watcher.StartAsync(CancellationToken.None); // Push bootstrap so the loop enters its enumeration body before stop. await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken); await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []); // StopAsync should complete without throwing (a hang fails at the presence ceiling). await TestWait.ForAsync(watcher.StopAsync(), "StopAsync must cancel the loop and return"); } /// Verifies that disposing stops a running watcher. [Fact] public async Task DisposeStopsRunningWatcher() { var channel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(channel); var watcher = new DeployWatcher(source); await watcher.StartAsync(CancellationToken.None); await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken); await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []); // Should not throw, should not hang. await TestWait.ForAsync(Task.Run(watcher.Dispose, TestContext.Current.CancellationToken), "Dispose must stop the loop and return"); } /// Verifies that a source exception triggers retry with backoff. [Fact] public async Task SourceExceptionTriggersRetryWithBackoff() { var t0 = DateTimeOffset.Parse("2026-04-01T00:00:00Z"); var t1 = DateTimeOffset.Parse("2026-04-02T00:00:00Z"); var firstChannel = Channel.CreateUnbounded(); var secondChannel = Channel.CreateUnbounded(); var source = new FakeDeployWatchSource(iteration => iteration switch { 1 => firstChannel, _ => secondChannel, }) { ThrowOnIteration = i => i == 1 ? new InvalidOperationException("transport drop") : null, }; // Tiny backoff so the test doesn't sit in Task.Delay. using var watcher = new DeployWatcher( source, logger: null, initialBackoff: TimeSpan.FromMilliseconds(10), maxBackoff: TimeSpan.FromMilliseconds(50), jitter: _ => TimeSpan.Zero); var captured = CaptureRediscoverEvents(watcher); await watcher.StartAsync(CancellationToken.None); // Wait for the second iteration (post-retry) to start. await WaitUntilAsync(() => source.CallCount >= 2, "the watcher must re-open the stream after the source threw", source, captured); // Now feed bootstrap + real event into the second channel. await secondChannel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); await secondChannel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); await WaitUntilAsync(() => Snapshot(captured).Count >= 1, "the re-opened stream's change must raise rediscovery", source, captured); captured.Count.ShouldBe(1); captured[0].Reason.ShouldBe("deploy-time-changed"); // The retry call passed null lastSeenDeployTime because no events were seen // before the throw — confirms baseline tracking is per-instance, not per-stream. source.LastSeenTimes.Count.ShouldBeGreaterThanOrEqualTo(2); source.LastSeenTimes[0].ShouldBeNull(); source.LastSeenTimes[1].ShouldBeNull(); await watcher.StopAsync(); } }