fix(worker): heartbeat check loop honours the injected TimeProvider (mxaccessgw#217)

HeartbeatLoopAsync waited with a plain Task.Delay, so heartbeat grace/stuck
detection could not be driven with a fake clock. It now uses
Task.Delay(interval, _timeProvider, ct).

Adds fake-time tests (grace boundary then fault; in-flight command within the
stuck ceiling) built on a new TickObservingTimeProvider, and replaces the
150 ms real-clock sleep in the within-ceiling negative test (mxaccessgw#205).
Both new tests fail without the fix.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:50:24 -05:00
parent 21bab1a982
commit b5d11e5869
3 changed files with 130 additions and 10 deletions
@@ -967,20 +967,21 @@ public sealed class WorkerClientTests
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
/// gap is more likely caused by pipe-write contention than by a hung
/// worker.
/// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
{
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) };
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
HeartbeatGrace = TimeSpan.FromSeconds(15),
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
EventChannelCapacity = 8,
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
},
@@ -996,16 +997,57 @@ public sealed class WorkerClientTests
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
clock.Advance(TimeSpan.FromSeconds(2));
// Give the heartbeat monitor a few real check-intervals to observe the gap.
await Task.Delay(TimeSpan.FromMilliseconds(150));
// Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each
// step waits until the monitor has actually evaluated the advanced clock.
for (int i = 0; i < 20; i++)
{
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout);
}
Assert.Equal(WorkerClientState.Ready, client.State);
Assert.False(pendingInvoke.IsCompleted);
}
/// <summary>
/// With nothing in flight the monitor must tolerate a silent worker up to
/// <see cref="WorkerClientOptions.HeartbeatGrace"/> and fault as soon as the grace is exceeded.
/// The monitor's check timer runs on the injected <see cref="TimeProvider"/>
/// (mxaccessgw#217): with a real <c>Task.Delay</c> the 7-second check interval would not
/// fire from fake-time advances and this test would time out.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults()
{
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) };
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
HeartbeatGrace = TimeSpan.FromSeconds(21),
HeartbeatCheckInterval = TimeSpan.FromSeconds(7),
EventChannelCapacity = 8,
},
timeProvider: clock);
await CompleteHandshakeAsync(client, pipePair);
// Exactly at the grace boundary (21s, not greater) the client stays Ready.
for (int i = 0; i < 3; i++)
{
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
}
Assert.Equal(WorkerClientState.Ready, client.State);
// One more check interval pushes the gap past the grace.
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
Assert.Equal(WorkerClientState.Faulted, client.State);
}
/// <summary>
/// Once the oldest pending command exceeds
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the