fix(auditlog,health): harden hosted-service shutdown against disposed CTS
The host does not guarantee IHostedService.StopAsync is driven before the DI container is disposed — WebApplicationFactory's teardown reaches Dispose first — so cancelling the internal CTS from StopAsync threw ObjectDisposedException and aborted the host's whole shutdown sequence. Four services shared the same copy-pasted lifecycle and the same two races: StopAsync cancelling an already- disposed CTS, and StartAsync reading _cts.Token lazily inside the Task.Run lambda, which faults the loop task the host awaits when Dispose wins that race. Each service now captures the token on the caller's thread, tolerates a disposed CTS, and cancels-before-disposing so the loop is always signalled and its pending Task.Delay sees a cancelled token rather than a dead source. SiteAuditBacklogReporter also gains the outer OperationCanceledException guard its sibling SiteAuditRetentionService already carried (arch-review 04 R2, R7), without which a shutdown landing mid-probe threw TaskCanceledException out of Host.StopAsync. Surfaced while verifying the Gitea #15 test-harness fix: in Host.Tests the aborted teardown skipped the fixture's env-var restore, contaminating every later test in the run. Refs: Gitea #15
This commit is contained in:
+36
@@ -151,4 +151,40 @@ public class AuditLogPartitionMaintenanceServiceTests
|
||||
Assert.Equal(LogLevel.Error, errorEntry.Level);
|
||||
Assert.Equal(1, maintenance.EnsureCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopAsync_AfterDispose_DoesNotThrow()
|
||||
{
|
||||
// Regression (Gitea #15 follow-up): Dispose tears down the CTS that
|
||||
// StopAsync cancels, and the host does not guarantee that every
|
||||
// IHostedService.StopAsync is driven before the DI container is
|
||||
// disposed — WebApplicationFactory's teardown reaches Dispose first and
|
||||
// then calls StopAsync. Cancel() on a disposed CTS throws
|
||||
// ObjectDisposedException, and letting it escape aborts the host's whole
|
||||
// shutdown sequence (in Host.Tests it stranded the fixture's env-var
|
||||
// restore, contaminating every later test in the run). Stop-after-Dispose
|
||||
// must therefore be a no-op, not a throw.
|
||||
var opts = Options.Create(new AuditLogPartitionMaintenanceOptions
|
||||
{
|
||||
IntervalSeconds = 60,
|
||||
LookaheadMonths = 1,
|
||||
});
|
||||
var maintenance = new RecordingMaintenance();
|
||||
var sp = BuildProvider(maintenance);
|
||||
|
||||
var svc = new AuditLogPartitionMaintenanceService(
|
||||
sp.GetRequiredService<IServiceScopeFactory>(),
|
||||
opts,
|
||||
NullLogger<AuditLogPartitionMaintenanceService>.Instance);
|
||||
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
|
||||
svc.Dispose();
|
||||
|
||||
// The assertion is the absence of ObjectDisposedException.
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
|
||||
// Dispose is also idempotent — the host may reach it twice on the same path.
|
||||
svc.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
+55
@@ -1,8 +1,10 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
@@ -23,6 +25,59 @@ public class SiteAuditBacklogReporterCadenceTests
|
||||
explicitInterval,
|
||||
options);
|
||||
|
||||
[Fact]
|
||||
public async Task StopAsync_WhileProbeInFlight_LoopCompletesCleanly()
|
||||
{
|
||||
// SafeProbeAsync rethrows OperationCanceledException by design so a shutdown
|
||||
// aborts the probe promptly — but RunLoopAsync must absorb it, because StopAsync
|
||||
// hands _loop straight to the host and a canceled task there throws out of
|
||||
// Host.StopAsync. Mirrors the guard SiteAuditRetentionService already carries
|
||||
// (arch-review 04 round 2, R7).
|
||||
var probeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
queue.GetBacklogStatsAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(ci => BlockUntilCancelledAsync(probeStarted, ci.Arg<CancellationToken>()));
|
||||
|
||||
var reporter = new SiteAuditBacklogReporter(
|
||||
queue,
|
||||
Substitute.For<ISiteHealthCollector>(),
|
||||
NullLogger<SiteAuditBacklogReporter>.Instance,
|
||||
TimeSpan.FromHours(1),
|
||||
null);
|
||||
|
||||
await reporter.StartAsync(CancellationToken.None);
|
||||
await probeStarted.Task; // the immediate first probe is now in flight
|
||||
|
||||
// The assertion is that awaiting the loop task does not throw.
|
||||
await reporter.StopAsync(CancellationToken.None);
|
||||
reporter.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<SiteAuditBacklogSnapshot> BlockUntilCancelledAsync(
|
||||
TaskCompletionSource started, CancellationToken ct)
|
||||
{
|
||||
started.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
throw new UnreachableException("the delay above always throws on cancellation");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopAsync_AfterDispose_DoesNotThrow()
|
||||
{
|
||||
// Regression (Gitea #15 follow-up): Dispose tears down the CTS StopAsync
|
||||
// cancels, and the host does not guarantee StopAsync is driven before the DI
|
||||
// container is disposed. Cancel() on a disposed CTS throws, and letting that
|
||||
// escape an IHostedService aborts the host's whole shutdown sequence.
|
||||
var reporter = Create(Options.Create(new SqliteAuditWriterOptions()), TimeSpan.FromHours(1));
|
||||
|
||||
await reporter.StartAsync(CancellationToken.None);
|
||||
reporter.Dispose();
|
||||
|
||||
// The assertion is the absence of ObjectDisposedException.
|
||||
await reporter.StopAsync(CancellationToken.None);
|
||||
reporter.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cadence_ComesFromOptions_WhenConfigured()
|
||||
{
|
||||
|
||||
@@ -19,6 +19,30 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
/// </summary>
|
||||
public class SiteAuditRetentionServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task StopAsync_AfterDispose_DoesNotThrow()
|
||||
{
|
||||
// Regression (Gitea #15 follow-up): Dispose tears down the CTS StopAsync
|
||||
// cancels, and the host does not guarantee StopAsync is driven before the DI
|
||||
// container is disposed. Cancel() on a disposed CTS throws, and letting that
|
||||
// escape an IHostedService aborts the host's whole shutdown sequence.
|
||||
var queue = new RecordingSiteAuditQueue();
|
||||
var options = Options.Create(new SiteAuditRetentionOptions
|
||||
{
|
||||
RetentionDays = 7,
|
||||
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
||||
InitialDelay = TimeSpan.Zero,
|
||||
});
|
||||
var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
||||
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
svc.Dispose();
|
||||
|
||||
// The assertion is the absence of ObjectDisposedException.
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
svc.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tick_Purges_With_RetentionCutoff()
|
||||
{
|
||||
|
||||
+21
@@ -10,6 +10,27 @@ namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests;
|
||||
/// </summary>
|
||||
public class SiteEventLogFailureCountReporterTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task StopAsync_AfterDispose_DoesNotThrow()
|
||||
{
|
||||
// Regression (Gitea #15 follow-up): Dispose tears down the CTS StopAsync
|
||||
// cancels, and the host does not guarantee StopAsync is driven before the DI
|
||||
// container is disposed. Cancel() on a disposed CTS throws, and letting that
|
||||
// escape an IHostedService aborts the host's whole shutdown sequence.
|
||||
var reporter = new SiteEventLogFailureCountReporter(
|
||||
failedWriteCountProvider: () => 0L,
|
||||
collector: new SiteHealthCollector(),
|
||||
logger: NullLogger<SiteEventLogFailureCountReporter>.Instance,
|
||||
refreshInterval: TimeSpan.FromHours(1));
|
||||
|
||||
await reporter.StartAsync(CancellationToken.None);
|
||||
reporter.Dispose();
|
||||
|
||||
// The assertion is the absence of ObjectDisposedException.
|
||||
await reporter.StopAsync(CancellationToken.None);
|
||||
reporter.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StartAsync_ImmediatelyProbes_FailedWriteCount()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user