diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index 8d12dad7..c2aa6978 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient { while (!_stopCts.IsCancellationRequested) { - await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false); + await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false); if (State != WorkerClientState.Ready) { continue; diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 33a0ba9d..583f3165 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -967,20 +967,21 @@ public sealed class WorkerClientTests /// than , the /// heartbeat watchdog must NOT fault on heartbeat-expired alone — the /// gap is more likely caused by pipe-write contention than by a hung - /// worker. + /// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps. /// /// A task that represents the asynchronous operation. [Fact] public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat() { - ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); + TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) }; + clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); await using PipePair pipePair = await PipePair.CreateAsync(); await using WorkerClient client = CreateClient( pipePair, new WorkerClientOptions { - HeartbeatGrace = TimeSpan.FromMilliseconds(80), - HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20), + HeartbeatGrace = TimeSpan.FromSeconds(15), + HeartbeatCheckInterval = TimeSpan.FromSeconds(1), EventChannelCapacity = 8, HeartbeatStuckCeiling = TimeSpan.FromSeconds(30), }, @@ -996,16 +997,57 @@ public sealed class WorkerClientTests WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase); - // Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling. - clock.Advance(TimeSpan.FromSeconds(2)); - - // Give the heartbeat monitor a few real check-intervals to observe the gap. - await Task.Delay(TimeSpan.FromMilliseconds(150)); + // Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each + // step waits until the monitor has actually evaluated the advanced clock. + for (int i = 0; i < 20; i++) + { + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout); + } Assert.Equal(WorkerClientState.Ready, client.State); Assert.False(pendingInvoke.IsCompleted); } + /// + /// With nothing in flight the monitor must tolerate a silent worker up to + /// and fault as soon as the grace is exceeded. + /// The monitor's check timer runs on the injected + /// (mxaccessgw#217): with a real Task.Delay the 7-second check interval would not + /// fire from fake-time advances and this test would time out. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults() + { + TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) }; + clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient( + pipePair, + new WorkerClientOptions + { + HeartbeatGrace = TimeSpan.FromSeconds(21), + HeartbeatCheckInterval = TimeSpan.FromSeconds(7), + EventChannelCapacity = 8, + }, + timeProvider: clock); + await CompleteHandshakeAsync(client, pipePair); + + // Exactly at the grace boundary (21s, not greater) the client stays Ready. + for (int i = 0; i < 3; i++) + { + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout); + } + + Assert.Equal(WorkerClientState.Ready, client.State); + + // One more check interval pushes the gap past the grace. + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout); + await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout); + + Assert.Equal(WorkerClientState.Faulted, client.State); + } + /// /// Once the oldest pending command exceeds /// , the diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs new file mode 100644 index 00000000..ab1ad4ce --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs @@ -0,0 +1,78 @@ +using Microsoft.Extensions.Time.Testing; + +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// A that counts reads, so a test can wait +/// deterministically until a timer-driven loop has run one more iteration after an +/// instead of sleeping on the real clock and hoping. +/// +public sealed class TickObservingTimeProvider : FakeTimeProvider +{ + private long _utcNowReads; + private long _watchedTimersCreated; + private long _watchedTimersConsumed; + + /// Gets the number of times has been read. + public long UtcNowReads => Interlocked.Read(ref _utcNowReads); + + /// + public override DateTimeOffset GetUtcNow() + { + Interlocked.Increment(ref _utcNowReads); + return base.GetUtcNow(); + } + + /// + /// Gets or sets the due time of the timer the loop under test re-arms each iteration + /// (for Task.Delay(interval, timeProvider, ct) this is the interval). When set, + /// first waits until such a timer is armed, so an + /// advance can never be lost to a loop that has not reached its delay yet. + /// + public TimeSpan? WatchedDueTime { get; set; } + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ITimer timer = base.CreateTimer(callback, state, dueTime, period); + + // Count only after the base class has registered the timer, so a waiter that sees the + // count can advance the clock without racing the registration. + if (WatchedDueTime is { } watched && dueTime == watched) + { + Interlocked.Increment(ref _watchedTimersCreated); + } + + return timer; + } + + /// + /// Advances the clock and waits until a consumer has read again, + /// which proves the awakened loop evaluated its condition against the advanced time. + /// + /// Amount to advance. + /// Real-time bound on the wait. + /// A task that completes once a new read was observed. + public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout) + { + using CancellationTokenSource cts = new(timeout); + if (WatchedDueTime is not null) + { + while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed)) + { + await Task.Yield(); + cts.Token.ThrowIfCancellationRequested(); + } + + Interlocked.Increment(ref _watchedTimersConsumed); + } + + long before = UtcNowReads; + Advance(delta); + while (UtcNowReads == before) + { + await Task.Yield(); + cts.Token.ThrowIfCancellationRequested(); + } + } +}