4fdc3708f4
Under heavy parallel load (14 concurrent test processes + 14 CPU burners, with DOTNET_ThreadPool_ForceMinWorkerThreads=1 as the reproducer) the Galaxy tests with 1-3 s fixed waits failed 16/70 runs. Every failure was a background loop started with Task.Run (alarm feed, DeployWatcher, EventPump, ReconnectSupervisor) not being SCHEDULED within the ceiling - e.g. DeployWatcher's stream had not even been opened after 2 s. No production race: the code under test was checked for each (alarm feed dispatch is synchronous in-order, DeployWatcher/ReconnectSupervisor state is lock- or loop-local, subscribe registers before returning). - New shared TestWait (UntilAsync / ForAsync / HealthyAsync): 30 s presence ceiling, failure reports elapsed ms + an observed-state snapshot (#856 shape; GalaxyDriverFixRound1Tests.WaitUntilAsync now delegates to it). - Converted the short waits in GatewayGalaxyAlarmFeedTests, DeployWatcherTests, GalaxyDriverSubscribeTests, GalaxyDriverRawPathKeyingTests (its WaitForAsync never threw on timeout), EventPumpStreamFaultTests, EventPumpReplayGapTests, ReconnectSupervisorTests; FixRound2 InitBudget 2 s -> 10 s (the parked fetch never completes, so any finite bound keeps the verdict). - Absence by short sleep replaced with positive controls: DeployWatcher bootstrap and same-deploy-time (follow with a real change, assert exactly that one raise - verified red when suppression is removed), subscribe failed-tag / unsubscribed handle (follow with a live handle's event), faulted pump (wait for its onStreamFault instead of 100 ms). - Fixed sleeps that raced scheduling: subscribe-once read tests wait for the read's registration (TrackedSubscriptionCountForTests) instead of 50 ms; supervisor Dispose test waits for reopen entry instead of 50 ms; the already-Healthy WaitForHealthy test asserts synchronous completion instead of a wall-clock bound. FakeSubscriber.Map is now a ConcurrentDictionary. Loaded run (14 parallel x 5, 14 burners, ForceMinWorkerThreads=1): before 16/70 failed runs; after 0/350 (70 + 210 + 70). Without the thread-pool override: 0/70 before and after on this host. Serial: 472 passed, 8 skipped. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
333 lines
14 KiB
C#
333 lines
14 KiB
C#
using System.Runtime.CompilerServices;
|
|
using System.Threading.Channels;
|
|
using Google.Protobuf.WellKnownTypes;
|
|
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse;
|
|
|
|
/// <summary>
|
|
/// Tests <see cref="DeployWatcher"/>'s consumption of <see cref="IGalaxyDeployWatchSource"/>:
|
|
/// bootstrap suppression, change detection, presence-flip handling, clean shutdown,
|
|
/// and reconnect-on-error backoff.
|
|
/// </summary>
|
|
public sealed class DeployWatcherTests
|
|
{
|
|
/// <summary>
|
|
/// Test helper exposing a <see cref="Channel{T}"/> as the event source plus an
|
|
/// optional fault hook so reconnect / retry paths can be exercised deterministically.
|
|
/// </summary>
|
|
private sealed class FakeDeployWatchSource : IGalaxyDeployWatchSource
|
|
{
|
|
private readonly Func<int, Channel<DeployEvent>> _channelFactory;
|
|
|
|
/// <summary>
|
|
/// Gets the list of last-seen deployment times from each watch iteration.
|
|
/// </summary>
|
|
public List<DateTimeOffset?> LastSeenTimes { get; } = [];
|
|
|
|
/// <summary>
|
|
/// Gets the number of times WatchAsync has been called.
|
|
/// </summary>
|
|
public int CallCount { get; private set; }
|
|
|
|
/// <summary>
|
|
/// Gets or sets a function that can throw an exception on specific iterations.
|
|
/// </summary>
|
|
public Func<int, Exception?>? ThrowOnIteration { get; init; }
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="FakeDeployWatchSource"/> class with a single channel.
|
|
/// </summary>
|
|
/// <param name="channel">The deploy event channel to use for all iterations.</param>
|
|
public FakeDeployWatchSource(Channel<DeployEvent> channel)
|
|
{
|
|
_channelFactory = _ => channel;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="FakeDeployWatchSource"/> class with a channel factory.
|
|
/// </summary>
|
|
/// <param name="channelFactory">A factory function that creates channels for each iteration.</param>
|
|
public FakeDeployWatchSource(Func<int, Channel<DeployEvent>> channelFactory)
|
|
{
|
|
_channelFactory = channelFactory;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Watches for deploy events asynchronously.
|
|
/// </summary>
|
|
/// <param name="lastSeenDeployTime">The last deployment time seen.</param>
|
|
/// <param name="cancellationToken">The cancellation token.</param>
|
|
/// <returns>An async enumerable of deploy events.</returns>
|
|
public async IAsyncEnumerable<DeployEvent> WatchAsync(
|
|
DateTimeOffset? lastSeenDeployTime,
|
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
|
{
|
|
int iteration = ++CallCount;
|
|
LastSeenTimes.Add(lastSeenDeployTime);
|
|
|
|
if (ThrowOnIteration?.Invoke(iteration) is { } ex)
|
|
{
|
|
throw ex;
|
|
}
|
|
|
|
var channel = _channelFactory(iteration);
|
|
await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
yield return ev;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static DeployEvent Event(ulong sequence, DateTimeOffset? deployTime)
|
|
{
|
|
var ev = new DeployEvent
|
|
{
|
|
Sequence = sequence,
|
|
ObservedAt = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
|
TimeOfLastDeployPresent = deployTime is not null,
|
|
};
|
|
if (deployTime is { } t)
|
|
{
|
|
ev.TimeOfLastDeploy = Timestamp.FromDateTimeOffset(t);
|
|
}
|
|
return ev;
|
|
}
|
|
|
|
private static List<RediscoveryEventArgs> CaptureRediscoverEvents(DeployWatcher watcher)
|
|
{
|
|
var captured = new List<RediscoveryEventArgs>();
|
|
watcher.OnRediscoveryNeeded += (_, args) =>
|
|
{
|
|
lock (captured) captured.Add(args);
|
|
};
|
|
return captured;
|
|
}
|
|
|
|
// Presence wait with a generous ceiling and a self-describing failure (#858): under heavy parallel
|
|
// load the watcher's Task.Run loop alone could take > 2 s to be scheduled.
|
|
private static Task WaitUntilAsync(
|
|
Func<bool> condition, string because, FakeDeployWatchSource source, List<RediscoveryEventArgs> captured)
|
|
=> TestWait.UntilAsync(condition, because, () => Describe(source, captured));
|
|
|
|
private static string Describe(FakeDeployWatchSource source, List<RediscoveryEventArgs> captured)
|
|
{
|
|
lock (captured)
|
|
{
|
|
return $"watchCalls={source.CallCount} raised=[{string.Join(", ", captured.Select(c => $"{c.Reason}@{c.ScopeHint}"))}]";
|
|
}
|
|
}
|
|
|
|
private static List<RediscoveryEventArgs> Snapshot(List<RediscoveryEventArgs> captured)
|
|
{
|
|
lock (captured) return [.. captured];
|
|
}
|
|
|
|
/// <summary>Verifies that bootstrap deploy events are suppressed.</summary>
|
|
[Fact]
|
|
public async Task BootstrapEventIsSuppressed()
|
|
{
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
using var watcher = new DeployWatcher(source);
|
|
var captured = CaptureRediscoverEvents(watcher);
|
|
|
|
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
|
|
var t1 = DateTimeOffset.Parse("2026-01-03T00:00:00Z");
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
// The bootstrap event, then a positive control that MUST raise. Events are handled in order,
|
|
// so once the control's raise is seen the bootstrap has been fully handled - no sleep needed
|
|
// to prove the bootstrap did not raise (#858).
|
|
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken);
|
|
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
|
|
|
|
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
|
|
"positive control: the post-bootstrap change must raise rediscovery", source, captured);
|
|
|
|
var raised = Snapshot(captured).ShouldHaveSingleItem("the bootstrap event must not raise");
|
|
DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime());
|
|
|
|
await watcher.StopAsync();
|
|
}
|
|
|
|
/// <summary>Verifies that a deployment time change fires a rediscovery event.</summary>
|
|
[Fact]
|
|
public async Task DeployTimeChangeFiresRediscover()
|
|
{
|
|
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
|
|
var t1 = DateTimeOffset.Parse("2026-01-02T12:00:00Z");
|
|
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
using var watcher = new DeployWatcher(source);
|
|
var captured = CaptureRediscoverEvents(watcher);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap
|
|
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken); // real change
|
|
|
|
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
|
|
"a deploy-time change must raise rediscovery", source, captured);
|
|
|
|
captured.Count.ShouldBe(1);
|
|
captured[0].Reason.ShouldBe("deploy-time-changed");
|
|
captured[0].ScopeHint.ShouldNotBeNull();
|
|
DateTimeOffset.Parse(captured[0].ScopeHint!).ToUniversalTime()
|
|
.ShouldBe(t1.ToUniversalTime());
|
|
|
|
await watcher.StopAsync();
|
|
}
|
|
|
|
/// <summary>Verifies that the same deployment time does not fire a rediscovery event.</summary>
|
|
[Fact]
|
|
public async Task SameDeployTimeDoesNotFire()
|
|
{
|
|
var t0 = DateTimeOffset.Parse("2026-01-01T00:00:00Z");
|
|
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
using var watcher = new DeployWatcher(source);
|
|
var captured = CaptureRediscoverEvents(watcher);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
var t1 = DateTimeOffset.Parse("2026-01-05T00:00:00Z");
|
|
await channel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken); // bootstrap
|
|
await channel.Writer.WriteAsync(Event(2, t0), TestContext.Current.CancellationToken); // duplicate state — gateway re-sent
|
|
// Positive control (#858): a genuine change after the duplicate MUST raise. Events are handled in
|
|
// order, so once it is seen the duplicate has been handled - its absence is then a real result.
|
|
await channel.Writer.WriteAsync(Event(3, t1), TestContext.Current.CancellationToken);
|
|
|
|
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
|
|
"positive control: the change after the duplicate must raise rediscovery", source, captured);
|
|
|
|
var raised = Snapshot(captured).ShouldHaveSingleItem("the duplicate deploy time must not raise");
|
|
DateTimeOffset.Parse(raised.ScopeHint!).ToUniversalTime().ShouldBe(t1.ToUniversalTime());
|
|
|
|
await watcher.StopAsync();
|
|
}
|
|
|
|
/// <summary>Verifies that a deployment time presence flip fires a rediscovery event.</summary>
|
|
[Fact]
|
|
public async Task TimeOfLastDeployPresentFlipFiresRediscover()
|
|
{
|
|
var t1 = DateTimeOffset.Parse("2026-03-01T08:00:00Z");
|
|
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
using var watcher = new DeployWatcher(source);
|
|
var captured = CaptureRediscoverEvents(watcher);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
// Bootstrap with absent deploy time (Galaxy never deployed).
|
|
await channel.Writer.WriteAsync(Event(0, deployTime: null), TestContext.Current.CancellationToken);
|
|
// Now a deploy lands and the present flag flips.
|
|
await channel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
|
|
|
|
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
|
|
"a presence flip must raise rediscovery", source, captured);
|
|
|
|
captured.Count.ShouldBe(1);
|
|
captured[0].Reason.ShouldBe("deploy-time-changed");
|
|
captured[0].ScopeHint.ShouldNotBeNull();
|
|
|
|
await watcher.StopAsync();
|
|
}
|
|
|
|
/// <summary>Verifies that stop cancels the watcher loop cleanly.</summary>
|
|
[Fact]
|
|
public async Task StopCancelsLoopCleanly()
|
|
{
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
using var watcher = new DeployWatcher(source);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
// Push bootstrap so the loop enters its enumeration body before stop.
|
|
await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken);
|
|
await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []);
|
|
|
|
// StopAsync should complete without throwing (a hang fails at the presence ceiling).
|
|
await TestWait.ForAsync(watcher.StopAsync(), "StopAsync must cancel the loop and return");
|
|
}
|
|
|
|
/// <summary>Verifies that disposing stops a running watcher.</summary>
|
|
[Fact]
|
|
public async Task DisposeStopsRunningWatcher()
|
|
{
|
|
var channel = Channel.CreateUnbounded<DeployEvent>();
|
|
var source = new FakeDeployWatchSource(channel);
|
|
var watcher = new DeployWatcher(source);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
await channel.Writer.WriteAsync(Event(0, DateTimeOffset.UtcNow), TestContext.Current.CancellationToken);
|
|
await WaitUntilAsync(() => source.CallCount > 0, "the watch stream must open", source, []);
|
|
|
|
// Should not throw, should not hang.
|
|
await TestWait.ForAsync(Task.Run(watcher.Dispose, TestContext.Current.CancellationToken),
|
|
"Dispose must stop the loop and return");
|
|
}
|
|
|
|
/// <summary>Verifies that a source exception triggers retry with backoff.</summary>
|
|
[Fact]
|
|
public async Task SourceExceptionTriggersRetryWithBackoff()
|
|
{
|
|
var t0 = DateTimeOffset.Parse("2026-04-01T00:00:00Z");
|
|
var t1 = DateTimeOffset.Parse("2026-04-02T00:00:00Z");
|
|
|
|
var firstChannel = Channel.CreateUnbounded<DeployEvent>();
|
|
var secondChannel = Channel.CreateUnbounded<DeployEvent>();
|
|
|
|
var source = new FakeDeployWatchSource(iteration => iteration switch
|
|
{
|
|
1 => firstChannel,
|
|
_ => secondChannel,
|
|
})
|
|
{
|
|
ThrowOnIteration = i => i == 1 ? new InvalidOperationException("transport drop") : null,
|
|
};
|
|
|
|
// Tiny backoff so the test doesn't sit in Task.Delay.
|
|
using var watcher = new DeployWatcher(
|
|
source,
|
|
logger: null,
|
|
initialBackoff: TimeSpan.FromMilliseconds(10),
|
|
maxBackoff: TimeSpan.FromMilliseconds(50),
|
|
jitter: _ => TimeSpan.Zero);
|
|
var captured = CaptureRediscoverEvents(watcher);
|
|
|
|
await watcher.StartAsync(CancellationToken.None);
|
|
|
|
// Wait for the second iteration (post-retry) to start.
|
|
await WaitUntilAsync(() => source.CallCount >= 2,
|
|
"the watcher must re-open the stream after the source threw", source, captured);
|
|
|
|
// Now feed bootstrap + real event into the second channel.
|
|
await secondChannel.Writer.WriteAsync(Event(0, t0), TestContext.Current.CancellationToken);
|
|
await secondChannel.Writer.WriteAsync(Event(1, t1), TestContext.Current.CancellationToken);
|
|
|
|
await WaitUntilAsync(() => Snapshot(captured).Count >= 1,
|
|
"the re-opened stream's change must raise rediscovery", source, captured);
|
|
|
|
captured.Count.ShouldBe(1);
|
|
captured[0].Reason.ShouldBe("deploy-time-changed");
|
|
|
|
// The retry call passed null lastSeenDeployTime because no events were seen
|
|
// before the throw — confirms baseline tracking is per-instance, not per-stream.
|
|
source.LastSeenTimes.Count.ShouldBeGreaterThanOrEqualTo(2);
|
|
source.LastSeenTimes[0].ShouldBeNull();
|
|
source.LastSeenTimes[1].ShouldBeNull();
|
|
|
|
await watcher.StopAsync();
|
|
}
|
|
}
|