using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
///
/// Tests 's consumption of :
/// bootstrap suppression, change detection, presence-flip handling, clean shutdown,
/// and reconnect-on-error backoff.
///
public sealed class DeployWatcherTests
{
///
/// Test helper exposing a as the event source plus an
/// optional fault hook so reconnect / retry paths can be exercised deterministically.
///
private sealed class FakeDeployWatchSource : IGalaxyDeployWatchSource
{
private readonly Func> _channelFactory;
///
/// Gets the list of last-seen deployment times from each watch iteration.
///
public List LastSeenTimes { get; } = [];
///
/// Gets the number of times WatchAsync has been called.
///
public int CallCount { get; private set; }
///
/// Gets or sets a function that can throw an exception on specific iterations.
///
public Func? ThrowOnIteration { get; init; }
///
/// Initializes a new instance of the class with a single channel.
///
/// The deploy event channel to use for all iterations.
public FakeDeployWatchSource(Channel channel)
{
_channelFactory = _ => channel;
}
///
/// Initializes a new instance of the class with a channel factory.
///
/// A factory function that creates channels for each iteration.
public FakeDeployWatchSource(Func> channelFactory)
{
_channelFactory = channelFactory;
}
///
/// Watches for deploy events asynchronously.
///
/// The last deployment time seen.
/// The cancellation token.
/// An async enumerable of deploy events.
public async IAsyncEnumerable WatchAsync(
DateTimeOffset? lastSeenDeployTime,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
int iteration = ++CallCount;
LastSeenTimes.Add(lastSeenDeployTime);
if (ThrowOnIteration?.Invoke(iteration) is { } ex)
{
throw ex;
}
var channel = _channelFactory(iteration);
await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
yield return ev;
}
}
}
private static DeployEvent Event(ulong sequence, DateTimeOffset? deployTime)
{
var ev = new DeployEvent
{
Sequence = sequence,
ObservedAt = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
TimeOfLastDeployPresent = deployTime is not null,
};
if (deployTime is { } t)
{
ev.TimeOfLastDeploy = Timestamp.FromDateTimeOffset(t);
}
return ev;
}
private static List CaptureRediscoverEvents(DeployWatcher watcher)
{
var captured = new List();
watcher.OnRediscoveryNeeded += (_, args) =>
{
lock (captured) captured.Add(args);
};
return captured;
}
// Presence wait with a generous ceiling and a self-describing failure (#858): under heavy parallel
// load the watcher's Task.Run loop alone could take > 2 s to be scheduled.
private static Task WaitUntilAsync(
Func condition, string because, FakeDeployWatchSource source, List captured)
=> TestWait.UntilAsync(condition, because, () => Describe(source, captured));
private static string Describe(FakeDeployWatchSource source, List captured)
{
lock (captured)
{
return $"watchCalls={source.CallCount} raised=[{string.Join(", ", captured.Select(c => $"{c.Reason}@{c.ScopeHint}"))}]";
}
}
private static List Snapshot(List captured)
{
lock (captured) return [.. captured];
}
/// Verifies that bootstrap deploy events are suppressed.
[Fact]
public async Task BootstrapEventIsSuppressed()
{
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
using var watcher = new DeployWatcher(source);
var captured = CaptureRediscoverEvents(watcher);
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
var t1 = DateTimeOffset.Parse("2026-01-03T00:00:00Z");
await watcher.StartAsync(CancellationToken.None);
// The bootstrap event, then a positive control that MUST raise. Events are handled in order,
// so once the control's raise is seen the bootstrap has been fully handled - no sleep needed
// to prove the bootstrap did not raise (#858).
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken);
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
"positive control: the post-bootstrap change must raise rediscovery", source, captured);
var raised = Snapshot(captured).ShouldHaveSingleItem("the bootstrap event must not raise");
DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime());
await watcher.StopAsync();
}
/// Verifies that a deployment time change fires a rediscovery event.
[Fact]
public async Task DeployTimeChangeFiresRediscover()
{
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
var t1 = DateTimeOffset.Parse("2026-01-02T12:00:00Z");
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
using var watcher = new DeployWatcher(source);
var captured = CaptureRediscoverEvents(watcher);
await watcher.StartAsync(CancellationToken.None);
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); // real change
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
"a deploy-time change must raise rediscovery", source, captured);
captured.Count.ShouldBe(1);
captured[0].Reason.ShouldBe("deploy-time-changed");
captured[0].ScopeHint.ShouldNotBeNull();
DateTimeOffset.Parse(captured[0].ScopeHint!).ToUniversalTime()
.ShouldBe(t1.ToUniversalTime());
await watcher.StopAsync();
}
/// Verifies that the same deployment time does not fire a rediscovery event.
[Fact]
public async Task SameDeployTimeDoesNotFire()
{
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
using var watcher = new DeployWatcher(source);
var captured = CaptureRediscoverEvents(watcher);
await watcher.StartAsync(CancellationToken.None);
var t1 = DateTimeOffset.Parse("2026-01-05T00:00:00Z");
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap
await channel.Writer.WriteAsync(Event(2, t0), TestContext.Current.CancellationToken); // duplicate state — gateway re-sent
// Positive control (#858): a genuine change after the duplicate MUST raise. Events are handled in
// order, so once it is seen the duplicate has been handled - its absence is then a real result.
await channel.Writer.WriteAsync(Event(3, t1), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
"positive control: the change after the duplicate must raise rediscovery", source, captured);
var raised = Snapshot(captured).ShouldHaveSingleItem("the duplicate deploy time must not raise");
DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime());
await watcher.StopAsync();
}
/// Verifies that a deployment time presence flip fires a rediscovery event.
[Fact]
public async Task TimeOfLastDeployPresentFlipFiresRediscover()
{
var t1 = DateTimeOffset.Parse("2026-03-01T08:00:00Z");
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
using var watcher = new DeployWatcher(source);
var captured = CaptureRediscoverEvents(watcher);
await watcher.StartAsync(CancellationToken.None);
// Bootstrap with absent deploy time (Galaxy never deployed).
await channel.Writer.WriteAsync(Event(0, deployTime: null), TestContext.Current.CancellationToken);
// Now a deploy lands and the present flag flips.
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
"a presence flip must raise rediscovery", source, captured);
captured.Count.ShouldBe(1);
captured[0].Reason.ShouldBe("deploy-time-changed");
captured[0].ScopeHint.ShouldNotBeNull();
await watcher.StopAsync();
}
/// Verifies that stop cancels the watcher loop cleanly.
[Fact]
public async Task StopCancelsLoopCleanly()
{
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
using var watcher = new DeployWatcher(source);
await watcher.StartAsync(CancellationToken.None);
// Push bootstrap so the loop enters its enumeration body before stop.
await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []);
// StopAsync should complete without throwing (a hang fails at the presence ceiling).
await TestWait.ForAsync(watcher.StopAsync(), "StopAsync must cancel the loop and return");
}
/// Verifies that disposing stops a running watcher.
[Fact]
public async Task DisposeStopsRunningWatcher()
{
var channel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(channel);
var watcher = new DeployWatcher(source);
await watcher.StartAsync(CancellationToken.None);
await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []);
// Should not throw, should not hang.
await TestWait.ForAsync(Task.Run(watcher.Dispose, TestContext.Current.CancellationToken),
"Dispose must stop the loop and return");
}
/// Verifies that a source exception triggers retry with backoff.
[Fact]
public async Task SourceExceptionTriggersRetryWithBackoff()
{
var t0 = DateTimeOffset.Parse("2026-04-01T00:00:00Z");
var t1 = DateTimeOffset.Parse("2026-04-02T00:00:00Z");
var firstChannel = Channel.CreateUnbounded();
var secondChannel = Channel.CreateUnbounded();
var source = new FakeDeployWatchSource(iteration => iteration switch
{
1 => firstChannel,
_ => secondChannel,
})
{
ThrowOnIteration = i => i == 1 ? new InvalidOperationException("transport drop") : null,
};
// Tiny backoff so the test doesn't sit in Task.Delay.
using var watcher = new DeployWatcher(
source,
logger: null,
initialBackoff: TimeSpan.FromMilliseconds(10),
maxBackoff: TimeSpan.FromMilliseconds(50),
jitter: _ => TimeSpan.Zero);
var captured = CaptureRediscoverEvents(watcher);
await watcher.StartAsync(CancellationToken.None);
// Wait for the second iteration (post-retry) to start.
await WaitUntilAsync(() => source.CallCount >= 2,
"the watcher must re-open the stream after the source threw", source, captured);
// Now feed bootstrap + real event into the second channel.
await secondChannel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken);
await secondChannel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
"the re-opened stream's change must raise rediscovery", source, captured);
captured.Count.ShouldBe(1);
captured[0].Reason.ShouldBe("deploy-time-changed");
// The retry call passed null lastSeenDeployTime because no events were seen
// before the throw — confirms baseline tracking is per-instance, not per-stream.
source.LastSeenTimes.Count.ShouldBeGreaterThanOrEqualTo(2);
source.LastSeenTimes[0].ShouldBeNull();
source.LastSeenTimes[1].ShouldBeNull();
await watcher.StopAsync();
}
}