test: fake-time hosted-service wiring tests, drop a real-clock sleep (mxaccessgw#205)
R21-O32: SessionLeaseMonitorHostedService gets FakeTimeProvider tests (no sweep before the interval, one sweep per interval with the clock's now, a failing sweep does not stop the loop, interval floor, stop ends the loop), and OrphanWorkerCleanupHostedService gets startup tests (sweeps once, failure swallowed) plus a registration-order test (orphan cleanup before the session hosted services). TickObservingTimeProvider gains WaitForWatchedTimersAsync so Advance never races a service that has not armed its timer yet. R21-O29: the 250 ms negative-assertion sleep in the stopping-WorkerClient test is replaced by disposing the client and awaiting the pipe read-ahead to EOF. Remaining real-clock sleeps (dashboard feed/publisher, alarm monitor tests) need TimeProvider seams in production code and are left open. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+149
@@ -0,0 +1,149 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Microsoft.Extensions.Time.Testing;
|
||||||
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||||
|
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Timer/startup wiring of <see cref="SessionLeaseMonitorHostedService"/> (mxaccessgw#205,
|
||||||
|
/// R21-O32), driven entirely by a <see cref="FakeTimeProvider"/>: no real-clock waits.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class SessionLeaseMonitorHostedServiceTests
|
||||||
|
{
|
||||||
|
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||||
|
|
||||||
|
private static async Task<SessionLeaseMonitorHostedService> StartAsync(RecordingSessionManager manager, TickObservingTimeProvider clock, int sweepSeconds = 10)
|
||||||
|
{
|
||||||
|
// PeriodicTimer period == due time; watch it so Advance never races the service arming it.
|
||||||
|
clock.WatchedDueTime = TimeSpan.FromSeconds(Math.Max(1, sweepSeconds));
|
||||||
|
SessionLeaseMonitorHostedService service = new(
|
||||||
|
manager,
|
||||||
|
Options.Create(new GatewayOptions { Sessions = new SessionOptions { LeaseSweepIntervalSeconds = sweepSeconds } }),
|
||||||
|
NullLogger<SessionLeaseMonitorHostedService>.Instance,
|
||||||
|
clock);
|
||||||
|
await service.StartAsync(CancellationToken.None);
|
||||||
|
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
|
||||||
|
return service;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>No sweep happens before the configured interval; one sweep fires per interval with the clock's now.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task Sweeps_OncePerInterval_PassingTheCurrentTime()
|
||||||
|
{
|
||||||
|
TickObservingTimeProvider clock = new();
|
||||||
|
clock.SetUtcNow(DateTimeOffset.Parse("2026-11-09T00:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||||
|
RecordingSessionManager manager = new();
|
||||||
|
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(10));
|
||||||
|
await manager.WaitForSweepsAsync(1);
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(10));
|
||||||
|
await manager.WaitForSweepsAsync(2);
|
||||||
|
|
||||||
|
Assert.Equal(
|
||||||
|
[DateTimeOffset.Parse("2026-11-09T00:00:10Z", System.Globalization.CultureInfo.InvariantCulture),
|
||||||
|
DateTimeOffset.Parse("2026-11-09T00:00:20Z", System.Globalization.CultureInfo.InvariantCulture)],
|
||||||
|
manager.SweepTimes.ToArray());
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A sweep that throws is logged and does not stop later sweeps.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task SweepFailure_DoesNotStopTheLoop()
|
||||||
|
{
|
||||||
|
TickObservingTimeProvider clock = new();
|
||||||
|
RecordingSessionManager manager = new() { ThrowOnFirstSweep = true };
|
||||||
|
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(10));
|
||||||
|
await manager.WaitForSweepsAsync(1);
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(10));
|
||||||
|
await manager.WaitForSweepsAsync(2);
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The configured interval is floored at one second.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task IntervalBelowOneSecond_IsFlooredToOneSecond()
|
||||||
|
{
|
||||||
|
TickObservingTimeProvider clock = new();
|
||||||
|
RecordingSessionManager manager = new();
|
||||||
|
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock, sweepSeconds: 0);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromMilliseconds(999));
|
||||||
|
Assert.Empty(manager.SweepTimes);
|
||||||
|
clock.Advance(TimeSpan.FromMilliseconds(1));
|
||||||
|
await manager.WaitForSweepsAsync(1);
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Stopping cancels the loop: no further sweeps run and StopAsync completes.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task StopAsync_EndsTheLoop_WithoutFurtherSweeps()
|
||||||
|
{
|
||||||
|
TickObservingTimeProvider clock = new();
|
||||||
|
RecordingSessionManager manager = new();
|
||||||
|
SessionLeaseMonitorHostedService service = await StartAsync(manager, clock);
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None).WaitAsync(TestTimeout);
|
||||||
|
clock.Advance(TimeSpan.FromMinutes(5));
|
||||||
|
|
||||||
|
Assert.Empty(manager.SweepTimes);
|
||||||
|
Assert.True(service.ExecuteTask is null or { IsCompleted: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class RecordingSessionManager : ISessionManager
|
||||||
|
{
|
||||||
|
private readonly SemaphoreSlim _swept = new(0);
|
||||||
|
private int _consumed;
|
||||||
|
|
||||||
|
public System.Collections.Concurrent.ConcurrentQueue<DateTimeOffset> SweepTimes { get; } = new();
|
||||||
|
|
||||||
|
public bool ThrowOnFirstSweep { get; init; }
|
||||||
|
|
||||||
|
public async Task WaitForSweepsAsync(int total)
|
||||||
|
{
|
||||||
|
while (_consumed < total)
|
||||||
|
{
|
||||||
|
Assert.True(await _swept.WaitAsync(TestTimeout), "Timed out waiting for a lease sweep.");
|
||||||
|
_consumed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<int> CloseExpiredLeasesAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
bool first = SweepTimes.IsEmpty;
|
||||||
|
SweepTimes.Enqueue(now);
|
||||||
|
_swept.Release();
|
||||||
|
if (first && ThrowOnFirstSweep)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("sweep failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
return Task.FromResult(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<GatewaySession> OpenSessionAsync(SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||||
|
|
||||||
|
public bool TryGetSession(string sessionId, out GatewaySession session) => throw new NotSupportedException();
|
||||||
|
|
||||||
|
public Task<WorkerCommandReply> InvokeAsync(string sessionId, WorkerCommand command, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||||
|
|
||||||
|
public Task<SessionCloseResult> CloseSessionAsync(string sessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||||
|
|
||||||
|
public Task<SessionCloseResult> KillWorkerAsync(string sessionId, string reason, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||||
|
|
||||||
|
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
+102
@@ -0,0 +1,102 @@
|
|||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||||
|
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Startup wiring of <see cref="OrphanWorkerCleanupHostedService"/> and its registration order
|
||||||
|
/// relative to the session subsystem (mxaccessgw#205, R21-O32).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class OrphanWorkerCleanupHostedServiceTests
|
||||||
|
{
|
||||||
|
private static readonly string WorkerPath = Path.GetFullPath(Path.Combine(
|
||||||
|
Path.GetTempPath(), "OrphanHostedTests", "ZB.MOM.WW.MxGateway.Worker.exe"));
|
||||||
|
|
||||||
|
private static OrphanWorkerCleanupHostedService Create(CountingInspector inspector) =>
|
||||||
|
new(
|
||||||
|
new OrphanWorkerTerminator(
|
||||||
|
Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = WorkerPath } }),
|
||||||
|
inspector,
|
||||||
|
new GatewayMetrics()),
|
||||||
|
NullLogger<OrphanWorkerCleanupHostedService>.Instance);
|
||||||
|
|
||||||
|
/// <summary>StartAsync sweeps for orphans exactly once and kills what the terminator finds.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task StartAsync_TerminatesOrphansOnce()
|
||||||
|
{
|
||||||
|
CountingInspector inspector = new([new RunningProcessInfo(4242, WorkerPath)]);
|
||||||
|
OrphanWorkerCleanupHostedService service = Create(inspector);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None);
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(1, inspector.EnumerateCalls);
|
||||||
|
Assert.Equal([4242], inspector.Killed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A cleanup failure is swallowed so the gateway still starts.</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task StartAsync_WhenCleanupThrows_DoesNotFailStartup()
|
||||||
|
{
|
||||||
|
CountingInspector inspector = new([]) { ThrowOnEnumerate = true };
|
||||||
|
OrphanWorkerCleanupHostedService service = Create(inspector);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(1, inspector.EnumerateCalls);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The worker launcher registration must put orphan cleanup ahead of the session
|
||||||
|
/// subsystem's hosted services, so leaked workers are gone before sessions are accepted.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Registration_OrdersOrphanCleanupBeforeSessionHostedServices()
|
||||||
|
{
|
||||||
|
ServiceCollection services = new();
|
||||||
|
services.AddWorkerProcessLauncher();
|
||||||
|
services.AddGatewaySessions();
|
||||||
|
|
||||||
|
List<Type?> hosted = services
|
||||||
|
.Where(d => d.ServiceType == typeof(IHostedService))
|
||||||
|
.Select(d => d.ImplementationType)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
int cleanup = hosted.IndexOf(typeof(OrphanWorkerCleanupHostedService));
|
||||||
|
int lease = hosted.IndexOf(typeof(SessionLeaseMonitorHostedService));
|
||||||
|
Assert.True(cleanup >= 0, "OrphanWorkerCleanupHostedService is not registered.");
|
||||||
|
Assert.True(lease >= 0, "SessionLeaseMonitorHostedService is not registered.");
|
||||||
|
Assert.True(cleanup < lease, "Orphan cleanup must start before the session lease monitor.");
|
||||||
|
Assert.Contains(typeof(SessionShutdownHostedService), hosted);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CountingInspector(IReadOnlyList<RunningProcessInfo> processes) : IRunningProcessInspector
|
||||||
|
{
|
||||||
|
public int EnumerateCalls { get; private set; }
|
||||||
|
|
||||||
|
public bool ThrowOnEnumerate { get; init; }
|
||||||
|
|
||||||
|
public List<int> Killed { get; } = [];
|
||||||
|
|
||||||
|
public IReadOnlyList<RunningProcessInfo> GetProcessesByName(string processName)
|
||||||
|
{
|
||||||
|
EnumerateCalls++;
|
||||||
|
if (ThrowOnEnumerate)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("enumeration failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
return processes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Kill(int processId) => Killed.Add(processId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -344,15 +344,15 @@ public sealed class WorkerClientTests
|
|||||||
|
|
||||||
await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
|
await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
|
||||||
|
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(250));
|
// Disposing the client closes the gateway end of the pipe, so the read-ahead completes
|
||||||
if (pendingRead.IsCompleted)
|
// (EOF -> null) deterministically; whatever it saw before closing must not be a WorkerCancel. No
|
||||||
{
|
// real-clock grace period needed (mxaccessgw#205, R21-O29).
|
||||||
WorkerEnvelope? unexpected = await pendingRead;
|
await client.DisposeAsync();
|
||||||
|
WorkerEnvelope? unexpected = await pendingRead.WaitAsync(TestTimeout);
|
||||||
Assert.True(
|
Assert.True(
|
||||||
unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
|
unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
|
||||||
"A stopping worker client must not put a WorkerCancel on the wire.");
|
"A stopping worker client must not put a WorkerCancel on the wire.");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A <c>WorkerFault</c> arriving in place of a handshake frame is the worker reporting a
|
/// A <c>WorkerFault</c> arriving in place of a handshake frame is the worker reporting a
|
||||||
|
|||||||
@@ -46,6 +46,25 @@ public sealed class TickObservingTimeProvider : FakeTimeProvider
|
|||||||
return timer;
|
return timer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Waits until at least <paramref name="count"/> timers with the <see cref="WatchedDueTime"/>
|
||||||
|
/// have been registered. For periodic consumers (<see cref="PeriodicTimer"/>) that arm a
|
||||||
|
/// single long-lived timer, so a later <c>Advance</c> cannot be lost to a service that has
|
||||||
|
/// not yet started waiting.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="count">Minimum number of watched timers.</param>
|
||||||
|
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||||
|
/// <returns>A task that completes once enough timers are armed.</returns>
|
||||||
|
public async Task WaitForWatchedTimersAsync(int count, TimeSpan timeout)
|
||||||
|
{
|
||||||
|
using CancellationTokenSource cts = new(timeout);
|
||||||
|
while (Interlocked.Read(ref _watchedTimersCreated) < count)
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
cts.Token.ThrowIfCancellationRequested();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
|
/// Advances the clock and waits until a consumer has read <see cref="GetUtcNow"/> again,
|
||||||
/// which proves the awakened loop evaluated its condition against the advanced time.
|
/// which proves the awakened loop evaluated its condition against the advanced time.
|
||||||
|
|||||||
Reference in New Issue
Block a user