9110a4eb01
The host does not guarantee IHostedService.StopAsync is driven before the DI container is disposed — WebApplicationFactory's teardown reaches Dispose first — so cancelling the internal CTS from StopAsync threw ObjectDisposedException and aborted the host's whole shutdown sequence. Four services shared the same copy-pasted lifecycle and the same two races: StopAsync cancelling an already- disposed CTS, and StartAsync reading _cts.Token lazily inside the Task.Run lambda, which faults the loop task the host awaits when Dispose wins that race. Each service now captures the token on the caller's thread, tolerates a disposed CTS, and cancels-before-disposing so the loop is always signalled and its pending Task.Delay sees a cancelled token rather than a dead source. SiteAuditBacklogReporter also gains the outer OperationCanceledException guard its sibling SiteAuditRetentionService already carried (arch-review 04 R2, R7), without which a shutdown landing mid-probe threw TaskCanceledException out of Host.StopAsync. Surfaced while verifying the Gitea #15 test-harness fix: in Host.Tests the aborted teardown skipped the fixture's env-var restore, contaminating every later test in the run. Refs: Gitea #15
171 lines
7.4 KiB
C#
171 lines
7.4 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using AuditEvent = ZB.MOM.WW.Audit.AuditEvent;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
|
|
|
/// <summary>
|
|
/// Tests for <see cref="SiteAuditRetentionService"/> (PLAN-04 Task 3). The hosted
|
|
/// service runs the site SQLite retention purge on a timer: first tick after
|
|
/// <see cref="SiteAuditRetentionOptions.InitialDelay"/>, then every
|
|
/// <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>, passing a cutoff of
|
|
/// <c>UtcNow - ResolvedRetentionDays</c>. Per-tick failures are swallowed so a
|
|
/// transient SQLite error never tears the service down.
|
|
/// </summary>
|
|
public class SiteAuditRetentionServiceTests
|
|
{
|
|
[Fact]
|
|
public async Task StopAsync_AfterDispose_DoesNotThrow()
|
|
{
|
|
// Regression (Gitea #15 follow-up): Dispose tears down the CTS StopAsync
|
|
// cancels, and the host does not guarantee StopAsync is driven before the DI
|
|
// container is disposed. Cancel() on a disposed CTS throws, and letting that
|
|
// escape an IHostedService aborts the host's whole shutdown sequence.
|
|
var queue = new RecordingSiteAuditQueue();
|
|
var options = Options.Create(new SiteAuditRetentionOptions
|
|
{
|
|
RetentionDays = 7,
|
|
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
|
InitialDelay = TimeSpan.Zero,
|
|
});
|
|
var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
|
|
|
await svc.StartAsync(CancellationToken.None);
|
|
svc.Dispose();
|
|
|
|
// The assertion is the absence of ObjectDisposedException.
|
|
await svc.StopAsync(CancellationToken.None);
|
|
svc.Dispose();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_Purges_With_RetentionCutoff()
|
|
{
|
|
var queue = new RecordingSiteAuditQueue();
|
|
var options = Options.Create(new SiteAuditRetentionOptions
|
|
{
|
|
RetentionDays = 7,
|
|
// Override bypasses the 1-min production clamp so the loop ticks in ms.
|
|
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
|
InitialDelay = TimeSpan.Zero,
|
|
});
|
|
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
|
|
|
await svc.StartAsync(CancellationToken.None);
|
|
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
|
|
await svc.StopAsync(CancellationToken.None);
|
|
|
|
var cutoff = queue.PurgeCalls.First();
|
|
Assert.InRange(cutoff,
|
|
DateTime.UtcNow.AddDays(-7).AddMinutes(-1),
|
|
DateTime.UtcNow.AddDays(-7).AddMinutes(1));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_SwallowsExceptions_AndKeepsTicking()
|
|
{
|
|
var queue = new RecordingSiteAuditQueue { ThrowOnFirstCall = true };
|
|
var options = Options.Create(new SiteAuditRetentionOptions
|
|
{
|
|
RetentionDays = 7,
|
|
// Override bypasses the 1-min production clamp so the loop ticks in ms.
|
|
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
|
InitialDelay = TimeSpan.Zero,
|
|
});
|
|
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
|
|
|
await svc.StartAsync(CancellationToken.None);
|
|
// The first tick throws; a second tick must still arrive (service survives).
|
|
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 2, TimeSpan.FromSeconds(5));
|
|
await svc.StopAsync(CancellationToken.None);
|
|
|
|
Assert.True(queue.PurgeCalls.Count >= 2);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StopAsync_MidPurge_CompletesCleanly_WithoutSurfacingCancellation()
|
|
{
|
|
// PurgeExpiredAsync blocks until cancelled, so shutdown lands mid-purge: SafePurgeAsync
|
|
// rethrows the OCE by design (abort the purge promptly), but the loop task must still
|
|
// complete CLEANLY — StopAsync hands _loop straight to the host, and a canceled task
|
|
// there is shutdown-log noise (arch-review 04 round 2, R7).
|
|
var queue = new RecordingSiteAuditQueue { BlockUntilCancelled = true };
|
|
var options = Options.Create(new SiteAuditRetentionOptions
|
|
{
|
|
RetentionDays = 7,
|
|
PurgeInterval = TimeSpan.FromHours(24),
|
|
InitialDelay = TimeSpan.Zero,
|
|
});
|
|
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
|
|
|
await svc.StartAsync(CancellationToken.None);
|
|
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
|
|
|
|
// Must NOT throw TaskCanceledException back into the host's shutdown path.
|
|
await svc.StopAsync(CancellationToken.None);
|
|
}
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
|
{
|
|
var sw = Stopwatch.StartNew();
|
|
while (!condition())
|
|
{
|
|
if (sw.Elapsed > timeout)
|
|
{
|
|
throw new TimeoutException("Condition was not met within the timeout.");
|
|
}
|
|
|
|
await Task.Delay(10);
|
|
}
|
|
}
|
|
|
|
/// <summary>Stub queue that records purge cutoffs and can throw on the first tick.</summary>
|
|
private sealed class RecordingSiteAuditQueue : ISiteAuditQueue
|
|
{
|
|
private readonly ConcurrentQueue<DateTime> _purgeCalls = new();
|
|
private int _calls;
|
|
|
|
public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
|
|
public bool ThrowOnFirstCall { get; init; }
|
|
|
|
/// <summary>When set, PurgeExpiredAsync records the call then blocks until the token
|
|
/// cancels — simulating a shutdown that lands while a purge is in flight.</summary>
|
|
public bool BlockUntilCancelled { get; init; }
|
|
|
|
public async Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
|
{
|
|
var n = Interlocked.Increment(ref _calls);
|
|
_purgeCalls.Enqueue(olderThanUtc);
|
|
if (ThrowOnFirstCall && n == 1)
|
|
{
|
|
throw new InvalidOperationException("Simulated transient purge failure.");
|
|
}
|
|
|
|
if (BlockUntilCancelled)
|
|
{
|
|
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
public Task<IReadOnlyList<AuditEvent>> ReadPendingCachedTelemetryAsync(int limit, CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
public Task MarkForwardedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(DateTime sinceUtc, int batchSize, CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
public Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default)
|
|
=> throw new NotSupportedException();
|
|
}
|
|
}
|