diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs
index 8d12dad7..c2aa6978 100644
--- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs
+++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs
@@ -599,7 +599,7 @@ public sealed class WorkerClient : IWorkerClient
{
while (!_stopCts.IsCancellationRequested)
{
- await Task.Delay(_options.HeartbeatCheckInterval, _stopCts.Token).ConfigureAwait(false);
+ await Task.Delay(_options.HeartbeatCheckInterval, _timeProvider, _stopCts.Token).ConfigureAwait(false);
if (State != WorkerClientState.Ready)
{
continue;
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
index 33a0ba9d..583f3165 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
@@ -967,20 +967,21 @@ public sealed class WorkerClientTests
/// than , the
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
/// gap is more likely caused by pipe-write contention than by a hung
- /// worker.
+ /// worker. Driven entirely by fake time (mxaccessgw#217/#205): no real sleeps.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
{
- ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
+ TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(1) };
+ clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(
pipePair,
new WorkerClientOptions
{
- HeartbeatGrace = TimeSpan.FromMilliseconds(80),
- HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
+ HeartbeatGrace = TimeSpan.FromSeconds(15),
+ HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
EventChannelCapacity = 8,
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
},
@@ -996,16 +997,57 @@ public sealed class WorkerClientTests
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
- // Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
- clock.Advance(TimeSpan.FromSeconds(2));
-
- // Give the heartbeat monitor a few real check-intervals to observe the gap.
- await Task.Delay(TimeSpan.FromMilliseconds(150));
+ // Step well past HeartbeatGrace (20s > 15s) but within HeartbeatStuckCeiling (30s). Each
+ // step waits until the monitor has actually evaluated the advanced clock.
+ for (int i = 0; i < 20; i++)
+ {
+ await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(1), TestTimeout);
+ }
Assert.Equal(WorkerClientState.Ready, client.State);
Assert.False(pendingInvoke.IsCompleted);
}
+ ///
+ /// With nothing in flight the monitor must tolerate a silent worker up to
+ /// and fault as soon as the grace is exceeded.
+ /// The monitor's check timer runs on the injected
+ /// (mxaccessgw#217): with a real Task.Delay the 7-second check interval would not
+ /// fire from fake-time advances and this test would time out.
+ ///
+ /// A task that represents the asynchronous operation.
+ [Fact]
+ public async Task HeartbeatMonitor_OnFakeClock_ToleratesGraceThenFaults()
+ {
+ TickObservingTimeProvider clock = new() { WatchedDueTime = TimeSpan.FromSeconds(7) };
+ clock.SetUtcNow(DateTimeOffset.Parse("2026-05-20T14:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
+ await using PipePair pipePair = await PipePair.CreateAsync();
+ await using WorkerClient client = CreateClient(
+ pipePair,
+ new WorkerClientOptions
+ {
+ HeartbeatGrace = TimeSpan.FromSeconds(21),
+ HeartbeatCheckInterval = TimeSpan.FromSeconds(7),
+ EventChannelCapacity = 8,
+ },
+ timeProvider: clock);
+ await CompleteHandshakeAsync(client, pipePair);
+
+ // Exactly at the grace boundary (21s, not greater) the client stays Ready.
+ for (int i = 0; i < 3; i++)
+ {
+ await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
+ }
+
+ Assert.Equal(WorkerClientState.Ready, client.State);
+
+ // One more check interval pushes the gap past the grace.
+ await clock.AdvanceAndAwaitTickAsync(TimeSpan.FromSeconds(7), TestTimeout);
+ await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
+
+ Assert.Equal(WorkerClientState.Faulted, client.State);
+ }
+
///
/// Once the oldest pending command exceeds
/// , the
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs
new file mode 100644
index 00000000..ab1ad4ce
--- /dev/null
+++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs
@@ -0,0 +1,78 @@
+using Microsoft.Extensions.Time.Testing;
+
+namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
+
+///
+/// A that counts reads, so a test can wait
+/// deterministically until a timer-driven loop has run one more iteration after an
+/// instead of sleeping on the real clock and hoping.
+///
+public sealed class TickObservingTimeProvider : FakeTimeProvider
+{
+ private long _utcNowReads;
+ private long _watchedTimersCreated;
+ private long _watchedTimersConsumed;
+
+ /// Gets the number of times has been read.
+ public long UtcNowReads => Interlocked.Read(ref _utcNowReads);
+
+ ///
+ public override DateTimeOffset GetUtcNow()
+ {
+ Interlocked.Increment(ref _utcNowReads);
+ return base.GetUtcNow();
+ }
+
+ ///
+ /// Gets or sets the due time of the timer the loop under test re-arms each iteration
+ /// (for Task.Delay(interval, timeProvider, ct) this is the interval). When set,
+ /// first waits until such a timer is armed, so an
+ /// advance can never be lost to a loop that has not reached its delay yet.
+ ///
+ public TimeSpan? WatchedDueTime { get; set; }
+
+ ///
+ public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
+ {
+ ITimer timer = base.CreateTimer(callback, state, dueTime, period);
+
+ // Count only after the base class has registered the timer, so a waiter that sees the
+ // count can advance the clock without racing the registration.
+ if (WatchedDueTime is { } watched && dueTime == watched)
+ {
+ Interlocked.Increment(ref _watchedTimersCreated);
+ }
+
+ return timer;
+ }
+
+ ///
+ /// Advances the clock and waits until a consumer has read again,
+ /// which proves the awakened loop evaluated its condition against the advanced time.
+ ///
+ /// Amount to advance.
+ /// Real-time bound on the wait.
+ /// A task that completes once a new read was observed.
+ public async Task AdvanceAndAwaitTickAsync(TimeSpan delta, TimeSpan timeout)
+ {
+ using CancellationTokenSource cts = new(timeout);
+ if (WatchedDueTime is not null)
+ {
+ while (Interlocked.Read(ref _watchedTimersCreated) <= Interlocked.Read(ref _watchedTimersConsumed))
+ {
+ await Task.Yield();
+ cts.Token.ThrowIfCancellationRequested();
+ }
+
+ Interlocked.Increment(ref _watchedTimersConsumed);
+ }
+
+ long before = UtcNowReads;
+ Advance(delta);
+ while (UtcNowReads == before)
+ {
+ await Task.Yield();
+ cts.Token.ThrowIfCancellationRequested();
+ }
+ }
+}