9110a4eb01
The host does not guarantee IHostedService.StopAsync is driven before the DI container is disposed — WebApplicationFactory's teardown reaches Dispose first — so cancelling the internal CTS from StopAsync threw ObjectDisposedException and aborted the host's whole shutdown sequence. Four services shared the same copy-pasted lifecycle and the same two races: StopAsync cancelling an already- disposed CTS, and StartAsync reading _cts.Token lazily inside the Task.Run lambda, which faults the loop task the host awaits when Dispose wins that race. Each service now captures the token on the caller's thread, tolerates a disposed CTS, and cancels-before-disposing so the loop is always signalled and its pending Task.Delay sees a cancelled token rather than a dead source. SiteAuditBacklogReporter also gains the outer OperationCanceledException guard its sibling SiteAuditRetentionService already carried (arch-review 04 R2, R7), without which a shutdown landing mid-probe threw TaskCanceledException out of Host.StopAsync. Surfaced while verifying the Gitea #15 test-harness fix: in Host.Tests the aborted teardown skipped the fixture's env-var restore, contaminating every later test in the run. Refs: Gitea #15
175 lines
7.5 KiB
C#
175 lines
7.5 KiB
C#
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
|
|
/// <summary>
|
|
/// Site Event Logging — site-side hosted service that
|
|
/// periodically reads the cumulative event-log write-failure count and pushes
|
|
/// it into <see cref="ISiteHealthCollector"/> so the next
|
|
/// <see cref="ISiteHealthCollector.CollectReport"/> emits a fresh
|
|
/// <c>SiteEventLogWriteFailures</c> field on the site health report.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <b>Why a Func<long> and not ISiteEventLogger directly.</b>
|
|
/// A direct <c>HealthMonitoring → SiteEventLogging</c> reference is avoided
|
|
/// to prevent an undesirable low-level coupling: <c>SiteEventLogging</c> is a
|
|
/// leaf component that should not pull in higher-level infrastructure. Note that
|
|
/// <c>HealthMonitoring → StoreAndForward → SiteEventLogging</c> already
|
|
/// exists as a transitive path (confirmed: <c>StoreAndForward.csproj</c> references
|
|
/// <c>SiteEventLogging.csproj</c>), so a direct reference would NOT introduce a
|
|
/// cycle — the delegate is purely a coupling-avoidance measure. The
|
|
/// <see cref="Func{TResult}"/> seam lets the caller (Host site wiring) capture
|
|
/// <c>ISiteEventLogger.FailedWriteCount</c> as a lambda at registration time; this
|
|
/// service reads only the numeric result. The delegate approach is a standard
|
|
/// pattern for counter bridges and keeps the registration path self-documenting.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Cadence.</b> 30 s by default — the same cadence as
|
|
/// <c>SiteAuditBacklogReporter</c>, which is coarse enough to stay within
|
|
/// the health-report interval budget while keeping the central dashboard
|
|
/// current.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Failure containment.</b> Any unexpected exception during the probe is
|
|
/// caught and logged; the next tick retries. Mirrors
|
|
/// <c>SiteAuditBacklogReporter</c>'s "exception logged, not propagated"
|
|
/// contract.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class SiteEventLogFailureCountReporter : IHostedService, IDisposable
|
|
{
|
|
/// <summary>
|
|
/// Default poll cadence. Matches <c>SiteAuditBacklogReporter.DefaultRefreshInterval</c>
|
|
/// (30 s) — coarse enough to amortise the read across many reports, fine
|
|
/// enough that the central dashboard never lags by more than one
|
|
/// health-report interval.
|
|
/// </summary>
|
|
internal static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(30);
|
|
|
|
private readonly Func<long> _failedWriteCountProvider;
|
|
private readonly ISiteHealthCollector _collector;
|
|
private readonly ILogger<SiteEventLogFailureCountReporter> _logger;
|
|
private readonly TimeSpan _refreshInterval;
|
|
private CancellationTokenSource? _cts;
|
|
private Task? _loop;
|
|
|
|
/// <summary>Initializes a new instance of <see cref="SiteEventLogFailureCountReporter"/>.</summary>
|
|
/// <param name="failedWriteCountProvider">
|
|
/// A delegate that returns the current cumulative event-log write-failure count.
|
|
/// Typically wired as <c>() => sp.GetRequiredService<ISiteEventLogger>().FailedWriteCount</c>
|
|
/// in the Host site composition root.
|
|
/// </param>
|
|
/// <param name="collector">The site health collector that receives the failure-count snapshot.</param>
|
|
/// <param name="logger">Logger instance.</param>
|
|
/// <param name="refreshInterval">Poll interval override; defaults to <see cref="DefaultRefreshInterval"/> (30 s).</param>
|
|
public SiteEventLogFailureCountReporter(
|
|
Func<long> failedWriteCountProvider,
|
|
ISiteHealthCollector collector,
|
|
ILogger<SiteEventLogFailureCountReporter> logger,
|
|
TimeSpan? refreshInterval = null)
|
|
{
|
|
_failedWriteCountProvider = failedWriteCountProvider
|
|
?? throw new ArgumentNullException(nameof(failedWriteCountProvider));
|
|
_collector = collector ?? throw new ArgumentNullException(nameof(collector));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
_refreshInterval = refreshInterval ?? DefaultRefreshInterval;
|
|
}
|
|
|
|
/// <summary>Starts the background polling loop, running an immediate first probe before entering the timed cycle.</summary>
|
|
/// <param name="ct">Cancellation token signalling host shutdown.</param>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
public Task StartAsync(CancellationToken ct)
|
|
{
|
|
// Linked CTS lets StopAsync's cancellation AND the host's shutdown
|
|
// token both terminate the loop; either side firing aborts the
|
|
// pending Task.Delay.
|
|
var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
_cts = cts;
|
|
|
|
// Read Token on the caller's thread, not inside the lambda: the lambda runs
|
|
// whenever the thread pool gets to it, so a Dispose landing first would make
|
|
// the deferred _cts.Token read throw and fault the loop task the host awaits.
|
|
var token = cts.Token;
|
|
_loop = Task.Run(() => RunLoopAsync(token), CancellationToken.None);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private async Task RunLoopAsync(CancellationToken ct)
|
|
{
|
|
// First tick runs immediately so the very first health report after
|
|
// process start carries a real failure-count snapshot — without this
|
|
// the dashboard would show 0 for the first 30 s after a deploy even
|
|
// if failures had already accumulated.
|
|
SafeProbe();
|
|
|
|
while (!ct.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(_refreshInterval, ct).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
break;
|
|
}
|
|
|
|
SafeProbe();
|
|
}
|
|
}
|
|
|
|
private void SafeProbe()
|
|
{
|
|
try
|
|
{
|
|
var count = _failedWriteCountProvider();
|
|
_collector.SetSiteEventLogWriteFailures(count);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Catch-all is deliberate: the hosted service must survive every
|
|
// class of probe failure so the next tick gets a chance. Mirrors
|
|
// SiteAuditBacklogReporter's "exception logged, not propagated" contract.
|
|
_logger.LogWarning(ex, "SiteEventLogFailureCountReporter probe failed; next tick will retry.");
|
|
}
|
|
}
|
|
|
|
/// <summary>Signals the polling loop to stop and waits for it to complete.</summary>
|
|
/// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
public Task StopAsync(CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
_cts?.Cancel();
|
|
}
|
|
catch (ObjectDisposedException)
|
|
{
|
|
// Stop-after-Dispose is a legal ordering; Dispose already cancelled the
|
|
// loop. Letting this escape would abort the host's shutdown sequence.
|
|
}
|
|
|
|
return _loop ?? Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Releases the internal <see cref="CancellationTokenSource"/> used to stop the polling loop.</summary>
|
|
public void Dispose()
|
|
{
|
|
// Cancel before disposing so the loop is always signalled even when the host
|
|
// disposes the container without having driven StopAsync first, and so the
|
|
// loop's pending Task.Delay(interval, token) sees an already-cancelled token
|
|
// rather than registering against a dead source.
|
|
try
|
|
{
|
|
_cts?.Cancel();
|
|
}
|
|
catch (ObjectDisposedException)
|
|
{
|
|
// Already disposed — Dispose is idempotent.
|
|
}
|
|
|
|
_cts?.Dispose();
|
|
}
|
|
}
|