fix(worker): heartbeat check loop honours the injected TimeProvider (mxaccessgw#217)
HeartbeatLoopAsync waited with a plain Task.Delay, so heartbeat grace/stuck detection could not be driven with a fake clock. It now uses Task.Delay(interval, _timeProvider, ct). Adds fake-time tests (grace boundary then fault; in-flight command within the stuck ceiling) built on a new TickObservingTimeProvider, and replaces the 150 ms real-clock sleep in the within-ceiling negative test (mxaccessgw#205). Both new tests fail without the fix. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
{
|
||||
while (!_stopCts.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false);
|
||||
await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false);
|
||||
if (State != WorkerClientState.Ready)
|
||||
{
|
||||
continue;
|
||||
|
||||
@@ -967,20 +967,21 @@ public sealed class WorkerClientTests
|
||||
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
|
||||
/// gap is more likely caused by pipe-write contention than by a hung
|
||||
/// worker.
|
||||
/// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) };
|
||||
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(15),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
|
||||
EventChannelCapacity = 8,
|
||||
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
|
||||
},
|
||||
@@ -996,16 +997,57 @@ public sealed class WorkerClientTests
|
||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
|
||||
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
|
||||
clock.Advance(TimeSpan.FromSeconds(2));
|
||||
|
||||
// Give the heartbeat monitor a few real check-intervals to observe the gap.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
||||
// Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each
|
||||
// step waits until the monitor has actually evaluated the advanced clock.
|
||||
for (int i = 0; i < 20; i++)
|
||||
{
|
||||
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout);
|
||||
}
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.False(pendingInvoke.IsCompleted);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// With nothing in flight the monitor must tolerate a silent worker up to
|
||||
/// <see cref="WorkerClientOptions.HeartbeatGrace"/> and fault as soon as the grace is exceeded.
|
||||
/// The monitor's check timer runs on the injected <see cref="TimeProvider"/>
|
||||
/// (mxaccessgw#217): with a real <c>Task.Delay</c> the 7-second check interval would not
|
||||
/// fire from fake-time advances and this test would time out.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults()
|
||||
{
|
||||
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) };
|
||||
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(21),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(7),
|
||||
EventChannelCapacity = 8,
|
||||
},
|
||||
timeProvider: clock);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
// Exactly at the grace boundary (21s, not greater) the client stays Ready.
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
|
||||
}
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
|
||||
// One more check interval pushes the gap past the grace.
|
||||
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
|
||||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Once the oldest pending command exceeds
|
||||
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
/// <summary>
|
||||
/// A <see cref="FakeTimeProvider"/> that counts <see cref="GetUtcNow"/> reads, so a test can wait
|
||||
/// deterministically until a timer-driven loop has run one more iteration after an
|
||||
/// <see cref="FakeTimeProvider.Advance"/> instead of sleeping on the real clock and hoping.
|
||||
/// </summary>
|
||||
public sealed class TickObservingTimeProvider : FakeTimeProvider
|
||||
{
|
||||
private long _utcNowReads;
|
||||
private long _watchedTimersCreated;
|
||||
private long _watchedTimersConsumed;
|
||||
|
||||
/// <summary>Gets the number of times <see cref="GetUtcNow"/> has been read.</summary>
|
||||
public long UtcNowReads => Interlocked.Read(ref _utcNowReads);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override DateTimeOffset GetUtcNow()
|
||||
{
|
||||
Interlocked.Increment(ref _utcNowReads);
|
||||
return base.GetUtcNow();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the due time of the timer the loop under test re-arms each iteration
|
||||
/// (for <c>Task.Delay(interval, timeProvider, ct)</c> this is the interval). When set,
|
||||
/// <see cref="AdvanceAndAwaitTickAsync"/> first waits until such a timer is armed, so an
|
||||
/// advance can never be lost to a loop that has not reached its delay yet.
|
||||
/// </summary>
|
||||
public TimeSpan? WatchedDueTime { get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
|
||||
{
|
||||
ITimer timer = base.CreateTimer(callback, state, dueTime, period);
|
||||
|
||||
// Count only after the base class has registered the timer, so a waiter that sees the
|
||||
// count can advance the clock without racing the registration.
|
||||
if (WatchedDueTime is { } watched && dueTime == watched)
|
||||
{
|
||||
Interlocked.Increment(ref _watchedTimersCreated);
|
||||
}
|
||||
|
||||
return timer;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
|
||||
/// which proves the awakened loop evaluated its condition against the advanced time.
|
||||
/// </summary>
|
||||
/// <param name="delta">Amount to advance.</param>
|
||||
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||
/// <returns>A task that completes once a new read was observed.</returns>
|
||||
public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout)
|
||||
{
|
||||
using CancellationTokenSource cts = new(timeout);
|
||||
if (WatchedDueTime is not null)
|
||||
{
|
||||
while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed))
|
||||
{
|
||||
await Task.Yield();
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _watchedTimersConsumed);
|
||||
}
|
||||
|
||||
long before = UtcNowReads;
|
||||
Advance(delta);
|
||||
while (UtcNowReads == before)
|
||||
{
|
||||
await Task.Yield();
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user