From b5d11e5869171ba997df94e6dfe781ef251dc25a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:50:24 -0500 Subject: [PATCH] fix(worker): heartbeat check loop honours the injected TimeProvider (mxaccessgw#217) HeartbeatLoopAsync waited with a plain Task.Delay, so heartbeat grace/stuck detection could not be driven with a fake clock. It now uses Task.Delay(interval, _timeProvider, ct). Adds fake-time tests (grace boundary then fault; in-flight command within the stuck ceiling) built on a new TickObservingTimeProvider, and replaces the 150 ms real-clock sleep in the within-ceiling negative test (mxaccessgw#205). Both new tests fail without the fix. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .../Workers/WorkerClient.cs | 2 +- .../Gateway/Workers/WorkerClientTests.cs | 60 +++++++++++--- .../TestSupport/TickObservingTimeProvider.cs | 78 +++++++++++++++++++ 3 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index 8d12dad7..c2aa6978 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient { while (!_stopCts.IsCancellationRequested) { - await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false); + await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false); if (State != WorkerClientState.Ready) { continue; diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 33a0ba9d..583f3165 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -967,20 +967,21 @@ public sealed class WorkerClientTests /// than , the /// heartbeat watchdog must NOT fault on heartbeat-expired alone — the /// gap is more likely caused by pipe-write contention than by a hung - /// worker. + /// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps. /// /// A task that represents the asynchronous operation. [Fact] public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat() { - ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); + TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) }; + clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); await using PipePair pipePair = await PipePair.CreateAsync(); await using WorkerClient client = CreateClient( pipePair, new WorkerClientOptions { - HeartbeatGrace = TimeSpan.FromMilliseconds(80), - HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20), + HeartbeatGrace = TimeSpan.FromSeconds(15), + HeartbeatCheckInterval = TimeSpan.FromSeconds(1), EventChannelCapacity = 8, HeartbeatStuckCeiling = TimeSpan.FromSeconds(30), }, @@ -996,16 +997,57 @@ public sealed class WorkerClientTests WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase); - // Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling. - clock.Advance(TimeSpan.FromSeconds(2)); - - // Give the heartbeat monitor a few real check-intervals to observe the gap. - await Task.Delay(TimeSpan.FromMilliseconds(150)); + // Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each + // step waits until the monitor has actually evaluated the advanced clock. + for (int i = 0; i < 20; i++) + { + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout); + } Assert.Equal(WorkerClientState.Ready, client.State); Assert.False(pendingInvoke.IsCompleted); } + /// + /// With nothing in flight the monitor must tolerate a silent worker up to + /// and fault as soon as the grace is exceeded. + /// The monitor's check timer runs on the injected + /// (mxaccessgw#217): with a real Task.Delay the 7-second check interval would not + /// fire from fake-time advances and this test would time out. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults() + { + TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) }; + clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient( + pipePair, + new WorkerClientOptions + { + HeartbeatGrace = TimeSpan.FromSeconds(21), + HeartbeatCheckInterval = TimeSpan.FromSeconds(7), + EventChannelCapacity = 8, + }, + timeProvider: clock); + await CompleteHandshakeAsync(client, pipePair); + + // Exactly at the grace boundary (21s, not greater) the client stays Ready. + for (int i = 0; i < 3; i++) + { + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout); + } + + Assert.Equal(WorkerClientState.Ready, client.State); + + // One more check interval pushes the gap past the grace. + await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout); + await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout); + + Assert.Equal(WorkerClientState.Faulted, client.State); + } + /// /// Once the oldest pending command exceeds /// , the diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs new file mode 100644 index 00000000..ab1ad4ce --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs @@ -0,0 +1,78 @@ +using Microsoft.Extensions.Time.Testing; + +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// A that counts reads, so a test can wait +/// deterministically until a timer-driven loop has run one more iteration after an +/// instead of sleeping on the real clock and hoping. +/// +public sealed class TickObservingTimeProvider : FakeTimeProvider +{ + private long _utcNowReads; + private long _watchedTimersCreated; + private long _watchedTimersConsumed; + + /// Gets the number of times has been read. + public long UtcNowReads => Interlocked.Read(ref _utcNowReads); + + /// + public override DateTimeOffset GetUtcNow() + { + Interlocked.Increment(ref _utcNowReads); + return base.GetUtcNow(); + } + + /// + /// Gets or sets the due time of the timer the loop under test re-arms each iteration + /// (for Task.Delay(interval, timeProvider, ct) this is the interval). When set, + /// first waits until such a timer is armed, so an + /// advance can never be lost to a loop that has not reached its delay yet. + /// + public TimeSpan? WatchedDueTime { get; set; } + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ITimer timer = base.CreateTimer(callback, state, dueTime, period); + + // Count only after the base class has registered the timer, so a waiter that sees the + // count can advance the clock without racing the registration. + if (WatchedDueTime is { } watched && dueTime == watched) + { + Interlocked.Increment(ref _watchedTimersCreated); + } + + return timer; + } + + /// + /// Advances the clock and waits until a consumer has read again, + /// which proves the awakened loop evaluated its condition against the advanced time. + /// + /// Amount to advance. + /// Real-time bound on the wait. + /// A task that completes once a new read was observed. + public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout) + { + using CancellationTokenSource cts = new(timeout); + if (WatchedDueTime is not null) + { + while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed)) + { + await Task.Yield(); + cts.Token.ThrowIfCancellationRequested(); + } + + Interlocked.Increment(ref _watchedTimersConsumed); + } + + long before = UtcNowReads; + Advance(delta); + while (UtcNowReads == before) + { + await Task.Yield(); + cts.Token.ThrowIfCancellationRequested(); + } + } +}