fix(worker): heartbeat check loop honours the injected TimeProvider (mxaccessgw#217)
HeartbeatLoopAsync waited with a plain Task.Delay, so heartbeat grace/stuck detection could not be driven with a fake clock. It now uses Task.Delay(interval, _timeProvider, ct). Adds fake-time tests (grace boundary then fault; in-flight command within the stuck ceiling) built on a new TickObservingTimeProvider, and replaces the 150 ms real-clock sleep in the within-ceiling negative test (mxaccessgw#205). Both new tests fail without the fix. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
{
|
{
|
||||||
while (!_stopCts.IsCancellationRequested)
|
while (!_stopCts.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false);
|
await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false);
|
||||||
if (State != WorkerClientState.Ready)
|
if (State != WorkerClientState.Ready)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
@@ -967,20 +967,21 @@ public sealed class WorkerClientTests
|
|||||||
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||||
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
|
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
|
||||||
/// gap is more likely caused by pipe-write contention than by a hung
|
/// gap is more likely caused by pipe-write contention than by a hung
|
||||||
/// worker.
|
/// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
|
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
|
||||||
{
|
{
|
||||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) };
|
||||||
|
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||||
await using WorkerClient client = CreateClient(
|
await using WorkerClient client = CreateClient(
|
||||||
pipePair,
|
pipePair,
|
||||||
new WorkerClientOptions
|
new WorkerClientOptions
|
||||||
{
|
{
|
||||||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
HeartbeatGrace = TimeSpan.FromSeconds(15),
|
||||||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
|
||||||
EventChannelCapacity = 8,
|
EventChannelCapacity = 8,
|
||||||
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
|
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
|
||||||
},
|
},
|
||||||
@@ -996,16 +997,57 @@ public sealed class WorkerClientTests
|
|||||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||||
|
|
||||||
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
|
// Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each
|
||||||
clock.Advance(TimeSpan.FromSeconds(2));
|
// step waits until the monitor has actually evaluated the advanced clock.
|
||||||
|
for (int i = 0; i < 20; i++)
|
||||||
// Give the heartbeat monitor a few real check-intervals to observe the gap.
|
{
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout);
|
||||||
|
}
|
||||||
|
|
||||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||||
Assert.False(pendingInvoke.IsCompleted);
|
Assert.False(pendingInvoke.IsCompleted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// With nothing in flight the monitor must tolerate a silent worker up to
|
||||||
|
/// <see cref="WorkerClientOptions.HeartbeatGrace"/> and fault as soon as the grace is exceeded.
|
||||||
|
/// The monitor's check timer runs on the injected <see cref="TimeProvider"/>
|
||||||
|
/// (mxaccessgw#217): with a real <c>Task.Delay</c> the 7-second check interval would not
|
||||||
|
/// fire from fake-time advances and this test would time out.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults()
|
||||||
|
{
|
||||||
|
TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) };
|
||||||
|
clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||||
|
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||||
|
await using WorkerClient client = CreateClient(
|
||||||
|
pipePair,
|
||||||
|
new WorkerClientOptions
|
||||||
|
{
|
||||||
|
HeartbeatGrace = TimeSpan.FromSeconds(21),
|
||||||
|
HeartbeatCheckInterval = TimeSpan.FromSeconds(7),
|
||||||
|
EventChannelCapacity = 8,
|
||||||
|
},
|
||||||
|
timeProvider: clock);
|
||||||
|
await CompleteHandshakeAsync(client, pipePair);
|
||||||
|
|
||||||
|
// Exactly at the grace boundary (21s, not greater) the client stays Ready.
|
||||||
|
for (int i = 0; i < 3; i++)
|
||||||
|
{
|
||||||
|
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||||
|
|
||||||
|
// One more check interval pushes the gap past the grace.
|
||||||
|
await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
|
||||||
|
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||||||
|
|
||||||
|
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Once the oldest pending command exceeds
|
/// Once the oldest pending command exceeds
|
||||||
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
using Microsoft.Extensions.Time.Testing;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A <see cref="FakeTimeProvider"/> that counts <see cref="GetUtcNow"/> reads, so a test can wait
|
||||||
|
/// deterministically until a timer-driven loop has run one more iteration after an
|
||||||
|
/// <see cref="FakeTimeProvider.Advance"/> instead of sleeping on the real clock and hoping.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class TickObservingTimeProvider : FakeTimeProvider
|
||||||
|
{
|
||||||
|
private long _utcNowReads;
|
||||||
|
private long _watchedTimersCreated;
|
||||||
|
private long _watchedTimersConsumed;
|
||||||
|
|
||||||
|
/// <summary>Gets the number of times <see cref="GetUtcNow"/> has been read.</summary>
|
||||||
|
public long UtcNowReads => Interlocked.Read(ref _utcNowReads);
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override DateTimeOffset GetUtcNow()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _utcNowReads);
|
||||||
|
return base.GetUtcNow();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the due time of the timer the loop under test re-arms each iteration
|
||||||
|
/// (for <c>Task.Delay(interval, timeProvider, ct)</c> this is the interval). When set,
|
||||||
|
/// <see cref="AdvanceAndAwaitTickAsync"/> first waits until such a timer is armed, so an
|
||||||
|
/// advance can never be lost to a loop that has not reached its delay yet.
|
||||||
|
/// </summary>
|
||||||
|
public TimeSpan? WatchedDueTime { get; set; }
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
|
||||||
|
{
|
||||||
|
ITimer timer = base.CreateTimer(callback, state, dueTime, period);
|
||||||
|
|
||||||
|
// Count only after the base class has registered the timer, so a waiter that sees the
|
||||||
|
// count can advance the clock without racing the registration.
|
||||||
|
if (WatchedDueTime is { } watched && dueTime == watched)
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _watchedTimersCreated);
|
||||||
|
}
|
||||||
|
|
||||||
|
return timer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
|
||||||
|
/// which proves the awakened loop evaluated its condition against the advanced time.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="delta">Amount to advance.</param>
|
||||||
|
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||||
|
/// <returns>A task that completes once a new read was observed.</returns>
|
||||||
|
public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout)
|
||||||
|
{
|
||||||
|
using CancellationTokenSource cts = new(timeout);
|
||||||
|
if (WatchedDueTime is not null)
|
||||||
|
{
|
||||||
|
while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed))
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
cts.Token.ThrowIfCancellationRequested();
|
||||||
|
}
|
||||||
|
|
||||||
|
Interlocked.Increment(ref _watchedTimersConsumed);
|
||||||
|
}
|
||||||
|
|
||||||
|
long before = UtcNowReads;
|
||||||
|
Advance(delta);
|
||||||
|
while (UtcNowReads == before)
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
cts.Token.ThrowIfCancellationRequested();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user