fix(worker): heartbeat check loop honours the injected TimeProvider (mxaccessgw#217)

HeartbeatLoopAsync waited with a plain Task.Delay, so heartbeat grace/stuck
detection could not be driven with a fake clock. It now uses
Task.Delay(interval, _timeProvider, ct).

Adds fake-time tests (grace boundary then fault; in-flight command within the
stuck ceiling) built on a new TickObservingTimeProvider, and replaces the
150 ms real-clock sleep in the within-ceiling negative test (mxaccessgw#205).
Both new tests fail without the fix.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:50:24 -05:00
parent 21bab1a982
commit b5d11e5869
3 changed files with 130 additions and 10 deletions
@@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient
{
while (!_stopCts.IsCancellationRequested)
{
await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false);
await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false);
if (State != WorkerClientState.Ready)
{
continue;
@@ -967,20 +967,21 @@ public sealed class WorkerClientTests
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
/// gap is more likely caused by pipe-write contention than by a hung
/// worker.
/// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
{
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) };
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
HeartbeatGrace = TimeSpan.FromSeconds(15),
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
EventChannelCapacity = 8,
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
},
@@ -996,16 +997,57 @@ public sealed class WorkerClientTests
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
clock.Advance(TimeSpan.FromSeconds(2));
// Give the heartbeat monitor a few real check-intervals to observe the gap.
await Task.Delay(TimeSpan.FromMilliseconds(150));
// Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each
// step waits until the monitor has actually evaluated the advanced clock.
for (int i = 0; i < 20; i++)
{
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout);
}
Assert.Equal(WorkerClientState.Ready, client.State);
Assert.False(pendingInvoke.IsCompleted);
}
/// <summary>
/// With nothing in flight the monitor must tolerate a silent worker up to
/// <see cref="WorkerClientOptions.HeartbeatGrace"/> and fault as soon as the grace is exceeded.
/// The monitor's check timer runs on the injected <see cref="TimeProvider"/>
/// (mxaccessgw#217): with a real <c>Task.Delay</c> the 7-second check interval would not
/// fire from fake-time advances and this test would time out.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults()
{
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) };
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
HeartbeatGrace = TimeSpan.FromSeconds(21),
HeartbeatCheckInterval = TimeSpan.FromSeconds(7),
EventChannelCapacity = 8,
},
timeProvider: clock);
await CompleteHandshakeAsync(client, pipePair);
// Exactly at the grace boundary (21s, not greater) the client stays Ready.
for (int i = 0; i < 3; i++)
{
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
}
Assert.Equal(WorkerClientState.Ready, client.State);
// One more check interval pushes the gap past the grace.
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
Assert.Equal(WorkerClientState.Faulted, client.State);
}
/// <summary>
/// Once the oldest pending command exceeds
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
@@ -0,0 +1,78 @@
using Microsoft.Extensions.Time.Testing;
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// A <see cref="FakeTimeProvider"/> that counts <see cref="GetUtcNow"/> reads, so a test can wait
/// deterministically until a timer-driven loop has run one more iteration after an
/// <see cref="FakeTimeProvider.Advance"/> instead of sleeping on the real clock and hoping.
/// </summary>
public sealed class TickObservingTimeProvider : FakeTimeProvider
{
private long _utcNowReads;
private long _watchedTimersCreated;
private long _watchedTimersConsumed;
/// <summary>Gets the number of times <see cref="GetUtcNow"/> has been read.</summary>
public long UtcNowReads => Interlocked.Read(ref _utcNowReads);
/// <inheritdoc />
public override DateTimeOffset GetUtcNow()
{
Interlocked.Increment(ref _utcNowReads);
return base.GetUtcNow();
}
/// <summary>
/// Gets or sets the due time of the timer the loop under test re-arms each iteration
/// (for <c>Task.Delay(interval, timeProvider, ct)</c> this is the interval). When set,
/// <see cref="AdvanceAndAwaitTickAsync"/> first waits until such a timer is armed, so an
/// advance can never be lost to a loop that has not reached its delay yet.
/// </summary>
public TimeSpan? WatchedDueTime { get; set; }
/// <inheritdoc />
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
{
ITimer timer = base.CreateTimer(callback, state, dueTime, period);
// Count only after the base class has registered the timer, so a waiter that sees the
// count can advance the clock without racing the registration.
if (WatchedDueTime is { } watched && dueTime == watched)
{
Interlocked.Increment(ref _watchedTimersCreated);
}
return timer;
}
/// <summary>
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
/// which proves the awakened loop evaluated its condition against the advanced time.
/// </summary>
/// <param name="delta">Amount to advance.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>A task that completes once a new read was observed.</returns>
public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout)
{
using CancellationTokenSource cts = new(timeout);
if (WatchedDueTime is not null)
{
while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed))
{
await Task.Yield();
cts.Token.ThrowIfCancellationRequested();
}
Interlocked.Increment(ref _watchedTimersConsumed);
}
long before = UtcNowReads;
Advance(delta);
while (UtcNowReads == before)
{
await Task.Yield();
cts.Token.ThrowIfCancellationRequested();
}
}
}