From 0609e810f41cebb6601c2f6350b038120dc75a44 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:57:51 -0500 Subject: [PATCH] test: fake-time hosted-service wiring tests, drop a real-clock sleep (mxaccessgw#205) R21-O32: SessionLeaseMonitorHostedService gets FakeTimeProvider tests (no sweep before the interval, one sweep per interval with the clock's now, a failing sweep does not stop the loop, interval floor, stop ends the loop), and OrphanWorkerCleanupHostedService gets startup tests (sweeps once, failure swallowed) plus a registration-order test (orphan cleanup before the session hosted services). TickObservingTimeProvider gains WaitForWatchedTimersAsync so Advance never races a service that has not armed its timer yet. R21-O29: the 250 ms negative-assertion sleep in the stopping-WorkerClient test is replaced by disposing the client and awaiting the pipe read-ahead to EOF. Remaining real-clock sleeps (dashboard feed/publisher, alarm monitor tests) need TimeProvider seams in production code and are left open. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .../SessionLeaseMonitorHostedServiceTests.cs | 149 ++++++++++++++++++ .../OrphanWorkerCleanupHostedServiceTests.cs | 102 ++++++++++++ .../Gateway/Workers/WorkerClientTests.cs | 16 +- .../TestSupport/TickObservingTimeProvider.cs | 19 +++ 4 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionLeaseMonitorHostedServiceTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/OrphanWorkerCleanupHostedServiceTests.cs diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionLeaseMonitorHostedServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionLeaseMonitorHostedServiceTests.cs new file mode 100644 index 00000000..38a0733a --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionLeaseMonitorHostedServiceTests.cs @@ -0,0 +1,149 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Sessions; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions; + +/// +/// Timer/startup wiring of (mxaccessgw#205, +/// R21-O32), driven entirely by a : no real-clock waits. +/// +public sealed class SessionLeaseMonitorHostedServiceTests +{ + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); + + private static async Task StartAsync(RecordingSessionManager manager, TickObservingTimeProvider clock, int sweepSeconds = 10) + { + // PeriodicTimer period == due time; watch it so Advance never races the service arming it. + clock.WatchedDueTime = TimeSpan.FromSeconds(Math.Max(1, sweepSeconds)); + SessionLeaseMonitorHostedService service = new( + manager, + Options.Create(new GatewayOptions { Sessions = new SessionOptions { LeaseSweepIntervalSeconds = sweepSeconds } }), + NullLogger.Instance, + clock); + await service.StartAsync(CancellationToken.None); + await clock.WaitForWatchedTimersAsync(1, TestTimeout); + return service; + } + + /// No sweep happens before the configured interval; one sweep fires per interval with the clock's now. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Sweeps_OncePerInterval_PassingTheCurrentTime() + { + TickObservingTimeProvider clock = new(); + clock.SetUtcNow(DateTimeOffset.Parse("2026-11-09T00:00:00Z", System.Globalization.CultureInfo.InvariantCulture)); + RecordingSessionManager manager = new(); + SessionLeaseMonitorHostedService service = await StartAsync(manager, clock); + + clock.Advance(TimeSpan.FromSeconds(10)); + await manager.WaitForSweepsAsync(1); + clock.Advance(TimeSpan.FromSeconds(10)); + await manager.WaitForSweepsAsync(2); + + Assert.Equal( + [DateTimeOffset.Parse("2026-11-09T00:00:10Z", System.Globalization.CultureInfo.InvariantCulture), + DateTimeOffset.Parse("2026-11-09T00:00:20Z", System.Globalization.CultureInfo.InvariantCulture)], + manager.SweepTimes.ToArray()); + + await service.StopAsync(CancellationToken.None); + } + + /// A sweep that throws is logged and does not stop later sweeps. + /// A task that represents the asynchronous operation. + [Fact] + public async Task SweepFailure_DoesNotStopTheLoop() + { + TickObservingTimeProvider clock = new(); + RecordingSessionManager manager = new() { ThrowOnFirstSweep = true }; + SessionLeaseMonitorHostedService service = await StartAsync(manager, clock); + + clock.Advance(TimeSpan.FromSeconds(10)); + await manager.WaitForSweepsAsync(1); + clock.Advance(TimeSpan.FromSeconds(10)); + await manager.WaitForSweepsAsync(2); + + await service.StopAsync(CancellationToken.None); + } + + /// The configured interval is floored at one second. + /// A task that represents the asynchronous operation. + [Fact] + public async Task IntervalBelowOneSecond_IsFlooredToOneSecond() + { + TickObservingTimeProvider clock = new(); + RecordingSessionManager manager = new(); + SessionLeaseMonitorHostedService service = await StartAsync(manager, clock, sweepSeconds: 0); + + clock.Advance(TimeSpan.FromMilliseconds(999)); + Assert.Empty(manager.SweepTimes); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await manager.WaitForSweepsAsync(1); + + await service.StopAsync(CancellationToken.None); + } + + /// Stopping cancels the loop: no further sweeps run and StopAsync completes. + /// A task that represents the asynchronous operation. + [Fact] + public async Task StopAsync_EndsTheLoop_WithoutFurtherSweeps() + { + TickObservingTimeProvider clock = new(); + RecordingSessionManager manager = new(); + SessionLeaseMonitorHostedService service = await StartAsync(manager, clock); + + await service.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + clock.Advance(TimeSpan.FromMinutes(5)); + + Assert.Empty(manager.SweepTimes); + Assert.True(service.ExecuteTask is null or { IsCompleted: true }); + } + + private sealed class RecordingSessionManager : ISessionManager + { + private readonly SemaphoreSlim _swept = new(0); + private int _consumed; + + public System.Collections.Concurrent.ConcurrentQueue SweepTimes { get; } = new(); + + public bool ThrowOnFirstSweep { get; init; } + + public async Task WaitForSweepsAsync(int total) + { + while (_consumed < total) + { + Assert.True(await _swept.WaitAsync(TestTimeout), "Timed out waiting for a lease sweep."); + _consumed++; + } + } + + public Task CloseExpiredLeasesAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + bool first = SweepTimes.IsEmpty; + SweepTimes.Enqueue(now); + _swept.Release(); + if (first && ThrowOnFirstSweep) + { + throw new InvalidOperationException("sweep failed"); + } + + return Task.FromResult(0); + } + + public Task OpenSessionAsync(SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) => throw new NotSupportedException(); + + public bool TryGetSession(string sessionId, out GatewaySession session) => throw new NotSupportedException(); + + public Task InvokeAsync(string sessionId, WorkerCommand command, CancellationToken cancellationToken) => throw new NotSupportedException(); + + public Task CloseSessionAsync(string sessionId, CancellationToken cancellationToken) => throw new NotSupportedException(); + + public Task KillWorkerAsync(string sessionId, string reason, CancellationToken cancellationToken) => throw new NotSupportedException(); + + public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/OrphanWorkerCleanupHostedServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/OrphanWorkerCleanupHostedServiceTests.cs new file mode 100644 index 00000000..f41ba089 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/OrphanWorkerCleanupHostedServiceTests.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Metrics; +using ZB.MOM.WW.MxGateway.Server.Sessions; +using ZB.MOM.WW.MxGateway.Server.Workers; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers; + +/// +/// Startup wiring of and its registration order +/// relative to the session subsystem (mxaccessgw#205, R21-O32). +/// +public sealed class OrphanWorkerCleanupHostedServiceTests +{ + private static readonly string WorkerPath = Path.GetFullPath(Path.Combine( + Path.GetTempPath(), "OrphanHostedTests", "ZB.MOM.WW.MxGateway.Worker.exe")); + + private static OrphanWorkerCleanupHostedService Create(CountingInspector inspector) => + new( + new OrphanWorkerTerminator( + Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = WorkerPath } }), + inspector, + new GatewayMetrics()), + NullLogger.Instance); + + /// StartAsync sweeps for orphans exactly once and kills what the terminator finds. + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_TerminatesOrphansOnce() + { + CountingInspector inspector = new([new RunningProcessInfo(4242, WorkerPath)]); + OrphanWorkerCleanupHostedService service = Create(inspector); + + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, inspector.EnumerateCalls); + Assert.Equal([4242], inspector.Killed); + } + + /// A cleanup failure is swallowed so the gateway still starts. + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_WhenCleanupThrows_DoesNotFailStartup() + { + CountingInspector inspector = new([]) { ThrowOnEnumerate = true }; + OrphanWorkerCleanupHostedService service = Create(inspector); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, inspector.EnumerateCalls); + } + + /// + /// The worker launcher registration must put orphan cleanup ahead of the session + /// subsystem's hosted services, so leaked workers are gone before sessions are accepted. + /// + [Fact] + public void Registration_OrdersOrphanCleanupBeforeSessionHostedServices() + { + ServiceCollection services = new(); + services.AddWorkerProcessLauncher(); + services.AddGatewaySessions(); + + List hosted = services + .Where(d => d.ServiceType == typeof(IHostedService)) + .Select(d => d.ImplementationType) + .ToList(); + + int cleanup = hosted.IndexOf(typeof(OrphanWorkerCleanupHostedService)); + int lease = hosted.IndexOf(typeof(SessionLeaseMonitorHostedService)); + Assert.True(cleanup >= 0, "OrphanWorkerCleanupHostedService is not registered."); + Assert.True(lease >= 0, "SessionLeaseMonitorHostedService is not registered."); + Assert.True(cleanup < lease, "Orphan cleanup must start before the session lease monitor."); + Assert.Contains(typeof(SessionShutdownHostedService), hosted); + } + + private sealed class CountingInspector(IReadOnlyList processes) : IRunningProcessInspector + { + public int EnumerateCalls { get; private set; } + + public bool ThrowOnEnumerate { get; init; } + + public List Killed { get; } = []; + + public IReadOnlyList GetProcessesByName(string processName) + { + EnumerateCalls++; + if (ThrowOnEnumerate) + { + throw new InvalidOperationException("enumeration failed"); + } + + return processes; + } + + public void Kill(int processId) => Killed.Add(processId); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 583f3165..2262b00f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -344,14 +344,14 @@ public sealed class WorkerClientTests await Assert.ThrowsAnyAsync(async () => await invokeTask.WaitAsync(TestTimeout)); - await Task.Delay(TimeSpan.FromMilliseconds(250)); - if (pendingRead.IsCompleted) - { - WorkerEnvelope? unexpected = await pendingRead; - Assert.True( - unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel, - "A stopping worker client must not put a WorkerCancel on the wire."); - } + // Disposing the client closes the gateway end of the pipe, so the read-ahead completes + // (EOF -> null) deterministically; whatever it saw before closing must not be a WorkerCancel. No + // real-clock grace period needed (mxaccessgw#205, R21-O29). + await client.DisposeAsync(); + WorkerEnvelope? unexpected = await pendingRead.WaitAsync(TestTimeout); + Assert.True( + unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel, + "A stopping worker client must not put a WorkerCancel on the wire."); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs index ab1ad4ce..0ac09bd9 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TickObservingTimeProvider.cs @@ -46,6 +46,25 @@ public sealed class TickObservingTimeProvider : FakeTimeProvider return timer; } + /// + /// Waits until at least timers with the + /// have been registered. For periodic consumers () that arm a + /// single long-lived timer, so a later Advance cannot be lost to a service that has + /// not yet started waiting. + /// + /// Minimum number of watched timers. + /// Real-time bound on the wait. + /// A task that completes once enough timers are armed. + public async Task WaitForWatchedTimersAsync(int count, TimeSpan timeout) + { + using CancellationTokenSource cts = new(timeout); + while (Interlocked.Read(ref _watchedTimersCreated) < count) + { + await Task.Yield(); + cts.Token.ThrowIfCancellationRequested(); + } + } + /// /// Advances the clock and waits until a consumer has read again, /// which proves the awakened loop evaluated its condition against the advanced time.