test: fake-time hosted-service wiring tests, drop a real-clock sleep (mxaccessgw#205)
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m19s
ci / java (push) Successful in 2m27s
ci / portable (push) Failing after 5m9s

R21-O32: SessionLeaseMonitorHostedService gets FakeTimeProvider tests (no sweep
before the interval, one sweep per interval with the clock's now, a failing
sweep does not stop the loop, interval floor, stop ends the loop), and
OrphanWorkerCleanupHostedService gets startup tests (sweeps once, failure
swallowed) plus a registration-order test (orphan cleanup before the session
hosted services). TickObservingTimeProvider gains WaitForWatchedTimersAsync so
Advance never races a service that has not armed its timer yet.

R21-O29: the 250 ms negative-assertion sleep in the stopping-WorkerClient test
is replaced by disposing the client and awaiting the pipe read-ahead to EOF.
Remaining real-clock sleeps (dashboard feed/publisher, alarm monitor tests)
need TimeProvider seams in production code and are left open.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:57:51 -05:00
parent 8c21678cb4
commit 0609e810f4
4 changed files with 278 additions and 8 deletions
@@ -0,0 +1,149 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
/// <summary>
/// Timer/startup wiring of <see cref="SessionLeaseMonitorHostedService"/> (mxaccessgw#205,
/// R21-O32), driven entirely by a <see cref="FakeTimeProvider"/>: no real-clock waits.
/// </summary>
public sealed class SessionLeaseMonitorHostedServiceTests
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
private static async Task<SessionLeaseMonitorHostedService> StartAsync(RecordingSessionManager manager, TickObservingTimeProvider clock, int sweepSeconds = 10)
{
// PeriodicTimer period == due time; watch it so Advance never races the service arming it.
clock.WatchedDueTime = TimeSpan.FromSeconds(Math.Max(1, sweepSeconds));
SessionLeaseMonitorHostedService service = new(
manager,
Options.Create(new GatewayOptions { Sessions = new SessionOptions { LeaseSweepIntervalSeconds = sweepSeconds } }),
NullLogger<SessionLeaseMonitorHostedService>.Instance,
clock);
await service.StartAsync(CancellationToken.None);
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
return service;
}
/// <summary>No sweep happens before the configured interval; one sweep fires per interval with the clock's now.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Sweeps_OncePerInterval_PassingTheCurrentTime()
{
TickObservingTimeProvider clock = new();
clock.SetUtcNow(DateTimeOffset.Parse("2026-11-09T00:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
RecordingSessionManager manager = new();
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
clock.Advance(TimeSpan.FromSeconds(10));
await manager.WaitForSweepsAsync(1);
clock.Advance(TimeSpan.FromSeconds(10));
await manager.WaitForSweepsAsync(2);
Assert.Equal(
[DateTimeOffset.Parse("2026-11-09T00:00:10Z", System.Globalization.CultureInfo.InvariantCulture),
DateTimeOffset.Parse("2026-11-09T00:00:20Z", System.Globalization.CultureInfo.InvariantCulture)],
manager.SweepTimes.ToArray());
await service.StopAsync(CancellationToken.None);
}
/// <summary>A sweep that throws is logged and does not stop later sweeps.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SweepFailure_DoesNotStopTheLoop()
{
TickObservingTimeProvider clock = new();
RecordingSessionManager manager = new() { ThrowOnFirstSweep = true };
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
clock.Advance(TimeSpan.FromSeconds(10));
await manager.WaitForSweepsAsync(1);
clock.Advance(TimeSpan.FromSeconds(10));
await manager.WaitForSweepsAsync(2);
await service.StopAsync(CancellationToken.None);
}
/// <summary>The configured interval is floored at one second.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task IntervalBelowOneSecond_IsFlooredToOneSecond()
{
TickObservingTimeProvider clock = new();
RecordingSessionManager manager = new();
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock, sweepSeconds: 0);
clock.Advance(TimeSpan.FromMilliseconds(999));
Assert.Empty(manager.SweepTimes);
clock.Advance(TimeSpan.FromMilliseconds(1));
await manager.WaitForSweepsAsync(1);
await service.StopAsync(CancellationToken.None);
}
/// <summary>Stopping cancels the loop: no further sweeps run and StopAsync completes.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StopAsync_EndsTheLoop_WithoutFurtherSweeps()
{
TickObservingTimeProvider clock = new();
RecordingSessionManager manager = new();
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
await service.StopAsync(CancellationToken.None).WaitAsync(TestTimeout);
clock.Advance(TimeSpan.FromMinutes(5));
Assert.Empty(manager.SweepTimes);
Assert.True(service.ExecuteTask is null or { IsCompleted: true });
}
private sealed class RecordingSessionManager : ISessionManager
{
private readonly SemaphoreSlim _swept = new(0);
private int _consumed;
public System.Collections.Concurrent.ConcurrentQueue<DateTimeOffset> SweepTimes { get; } = new();
public bool ThrowOnFirstSweep { get; init; }
public async Task WaitForSweepsAsync(int total)
{
while (_consumed < total)
{
Assert.True(await _swept.WaitAsync(TestTimeout), "Timed out waiting for a lease sweep.");
_consumed++;
}
}
public Task<int> CloseExpiredLeasesAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
bool first = SweepTimes.IsEmpty;
SweepTimes.Enqueue(now);
_swept.Release();
if (first && ThrowOnFirstSweep)
{
throw new InvalidOperationException("sweep failed");
}
return Task.FromResult(0);
}
public Task<GatewaySession> OpenSessionAsync(SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) => throw new NotSupportedException();
public bool TryGetSession(string sessionId, out GatewaySession session) => throw new NotSupportedException();
public Task<WorkerCommandReply> InvokeAsync(string sessionId, WorkerCommand command, CancellationToken cancellationToken) => throw new NotSupportedException();
public Task<SessionCloseResult> CloseSessionAsync(string sessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
public Task<SessionCloseResult> KillWorkerAsync(string sessionId, string reason, CancellationToken cancellationToken) => throw new NotSupportedException();
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
}
@@ -0,0 +1,102 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
/// <summary>
/// Startup wiring of <see cref="OrphanWorkerCleanupHostedService"/> and its registration order
/// relative to the session subsystem (mxaccessgw#205, R21-O32).
/// </summary>
public sealed class OrphanWorkerCleanupHostedServiceTests
{
private static readonly string WorkerPath = Path.GetFullPath(Path.Combine(
Path.GetTempPath(), "OrphanHostedTests", "ZB.MOM.WW.MxGateway.Worker.exe"));
private static OrphanWorkerCleanupHostedService Create(CountingInspector inspector) =>
new(
new OrphanWorkerTerminator(
Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = WorkerPath } }),
inspector,
new GatewayMetrics()),
NullLogger<OrphanWorkerCleanupHostedService>.Instance);
/// <summary>StartAsync sweeps for orphans exactly once and kills what the terminator finds.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StartAsync_TerminatesOrphansOnce()
{
CountingInspector inspector = new([new RunningProcessInfo(4242, WorkerPath)]);
OrphanWorkerCleanupHostedService service = Create(inspector);
await service.StartAsync(CancellationToken.None);
await service.StopAsync(CancellationToken.None);
Assert.Equal(1, inspector.EnumerateCalls);
Assert.Equal([4242], inspector.Killed);
}
/// <summary>A cleanup failure is swallowed so the gateway still starts.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StartAsync_WhenCleanupThrows_DoesNotFailStartup()
{
CountingInspector inspector = new([]) { ThrowOnEnumerate = true };
OrphanWorkerCleanupHostedService service = Create(inspector);
await service.StartAsync(CancellationToken.None);
Assert.Equal(1, inspector.EnumerateCalls);
}
/// <summary>
/// The worker launcher registration must put orphan cleanup ahead of the session
/// subsystem's hosted services, so leaked workers are gone before sessions are accepted.
/// </summary>
[Fact]
public void Registration_OrdersOrphanCleanupBeforeSessionHostedServices()
{
ServiceCollection services = new();
services.AddWorkerProcessLauncher();
services.AddGatewaySessions();
List<Type?> hosted = services
.Where(d => d.ServiceType == typeof(IHostedService))
.Select(d => d.ImplementationType)
.ToList();
int cleanup = hosted.IndexOf(typeof(OrphanWorkerCleanupHostedService));
int lease = hosted.IndexOf(typeof(SessionLeaseMonitorHostedService));
Assert.True(cleanup >= 0, "OrphanWorkerCleanupHostedService is not registered.");
Assert.True(lease >= 0, "SessionLeaseMonitorHostedService is not registered.");
Assert.True(cleanup < lease, "Orphan cleanup must start before the session lease monitor.");
Assert.Contains(typeof(SessionShutdownHostedService), hosted);
}
private sealed class CountingInspector(IReadOnlyList<RunningProcessInfo> processes) : IRunningProcessInspector
{
public int EnumerateCalls { get; private set; }
public bool ThrowOnEnumerate { get; init; }
public List<int> Killed { get; } = [];
public IReadOnlyList<RunningProcessInfo> GetProcessesByName(string processName)
{
EnumerateCalls++;
if (ThrowOnEnumerate)
{
throw new InvalidOperationException("enumeration failed");
}
return processes;
}
public void Kill(int processId) => Killed.Add(processId);
}
}
@@ -344,14 +344,14 @@ public sealed class WorkerClientTests
await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
await Task.Delay(TimeSpan.FromMilliseconds(250));
if (pendingRead.IsCompleted)
{
WorkerEnvelope? unexpected = await pendingRead;
Assert.True(
unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
"A stopping worker client must not put a WorkerCancel on the wire.");
}
// Disposing the client closes the gateway end of the pipe, so the read-ahead completes
// (EOF -> null) deterministically; whatever it saw before closing must not be a WorkerCancel. No
// real-clock grace period needed (mxaccessgw#205, R21-O29).
await client.DisposeAsync();
WorkerEnvelope? unexpected = await pendingRead.WaitAsync(TestTimeout);
Assert.True(
unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
"A stopping worker client must not put a WorkerCancel on the wire.");
}
/// <summary>
@@ -46,6 +46,25 @@ public sealed class TickObservingTimeProvider : FakeTimeProvider
return timer;
}
/// <summary>
/// Waits until at least <paramref name="count"/> timers with the <see cref="WatchedDueTime"/>
/// have been registered. For periodic consumers (<see cref="PeriodicTimer"/>) that arm a
/// single long-lived timer, so a later <c>Advance</c> cannot be lost to a service that has
/// not yet started waiting.
/// </summary>
/// <param name="count">Minimum number of watched timers.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>A task that completes once enough timers are armed.</returns>
public async Task WaitForWatchedTimersAsync(int count, TimeSpan timeout)
{
using CancellationTokenSource cts = new(timeout);
while (Interlocked.Read(ref _watchedTimersCreated) < count)
{
await Task.Yield();
cts.Token.ThrowIfCancellationRequested();
}
}
/// <summary>
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
/// which proves the awakened loop evaluated its condition against the advanced time.