feat(audit-log): daily site SQLite retention purge job (closes unbounded site DB growth)

SiteAuditRetentionService (site-only IHostedService) ticks PurgeExpiredAsync on
InitialDelay(5m)/PurgeInterval(24h, clamped >=1m), cutoff = UtcNow - RetentionDays.
Registered site-only in AddAuditLogHealthMetricsBridge; options bound from
AuditLog:SiteRetention. Per-tick failures swallowed. Added a test-only
PurgeIntervalOverride (mirrors SiteCallAuditOptions) so the ms-cadence tick test
can bypass the 1-min production clamp. Doc updated. (PLAN-04 Task 3, S1/U2)

docs/requirements/Component-AuditLog.md

@@ -409,9 +409,19 @@ MS SQL for direct-write events). Unredacted secrets never persist.
   partition switch-out loop. Values are validated to be in
   `[30, RetentionDays]`; keys that are not a recognized `AuditChannel` enum name
   are rejected at startup.
-- **Sites:** daily site job; default 7-day retention (configurable, min 1,
+- **Sites:** `SiteAuditRetentionService` (site-only `IHostedService`) runs the
-  max 90). Respects the hard `ForwardState` invariant — `Pending` rows are
+  site SQLite retention purge on a timer — first tick after
-  never purged on age alone.
+  `AuditLog:SiteRetention:InitialDelay` (default 5 min, short so a daily-recycled
+  node still purges soon after start), then every
+  `AuditLog:SiteRetention:PurgeInterval` (default 24 h, clamped to ≥ 1 min).
+  Each tick calls `ISiteAuditQueue.PurgeExpiredAsync(UtcNow − RetentionDays)`,
+  which deletes eligible rows in one transaction (sidecar first, then the
+  canonical row) and then runs `PRAGMA incremental_vacuum` to return the freed
+  pages to the OS. `AuditLog:SiteRetention:RetentionDays` defaults to 7 and is
+  clamped to `[1, 90]`. Respects the hard `ForwardState` invariant — a `Pending`
+  row is never purged on age alone; only `Forwarded`/`Reconciled` rows older than
+  the cutoff are removed. Per-tick failures are swallowed and logged so a transient
+  SQLite fault never tears the service down.
 
 ## Security & Tamper-Evidence
 

src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs

@@ -101,6 +101,11 @@ public static class ServiceCollectionExtensions
             .Bind(config.GetSection(SiteWriterSectionName));
         services.AddOptions<SiteAuditTelemetryOptions>()
             .Bind(config.GetSection(SiteTelemetrySectionName));
+        // Retention purge options (the SiteAuditRetentionService hosted job is
+        // registered site-only in AddAuditLogHealthMetricsBridge). Binding here is
+        // inert on central — no hosted service consumes it there.
+        services.AddOptions<SiteAuditRetentionOptions>()
+            .Bind(config.GetSection(SiteAuditRetentionOptions.SectionName));
 
         // SqliteAuditWriter is a singleton with a single owned SqliteConnection
         // and a background writer Task; multiple instances would race on the
@@ -316,6 +321,12 @@ public static class ServiceCollectionExtensions
         // free of hosted-service registrations that would resolve a missing
         // ISiteHealthCollector on central.
         services.AddHostedService<SiteAuditBacklogReporter>();
+        // Site-only retention purge: runs ISiteAuditQueue.PurgeExpiredAsync on a
+        // daily timer so the ephemeral site SQLite store does not grow unbounded.
+        // Registered here (not in AddAuditLog, which also runs on central) so the
+        // hosted service only starts on site composition roots. Guarded by the same
+        // SiteAuditBacklogReporter sentinel above against double-registration.
+        services.AddHostedService<SiteAuditRetentionService>();
         return services;
     }
 

src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SiteAuditRetentionOptions.cs

@@ -10,8 +10,12 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
 /// </summary>
 public class SiteAuditRetentionOptions
 {
-    /// <summary>Configuration section path bound in the site host.</summary>
+    /// <summary>
-    public const string SectionName = "ScadaBridge:AuditLog:SiteRetention";
+    /// Configuration section path bound in the site host. Follows the component's
+    /// existing <c>AuditLog:*</c> convention (sibling of <c>AuditLog:SiteWriter</c>,
+    /// <c>AuditLog:Purge</c>, …), not an application-root-absolute path.
+    /// </summary>
+    public const string SectionName = "AuditLog:SiteRetention";
 
     /// <summary>
     /// Retention window in days. On each purge tick, Forwarded/Reconciled rows whose
@@ -42,14 +46,27 @@ public class SiteAuditRetentionOptions
     /// </summary>
     public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromHours(24);
 
+    /// <summary>
+    /// Test-only override for the purge cadence — bypasses the
+    /// <see cref="MinPurgeInterval"/> clamp so unit tests can drop the period to
+    /// milliseconds. Production config never sets this; leave null. Mirrors
+    /// <c>SiteCallAuditOptions.PurgeIntervalOverride</c>.
+    /// </summary>
+    public TimeSpan? PurgeIntervalOverride { get; set; }
+
     private static readonly TimeSpan MinPurgeInterval = TimeSpan.FromMinutes(1);
 
     /// <summary>
-    /// Effective purge period: <see cref="PurgeInterval"/> clamped to at least
+    /// Effective purge period: the test override when set (bypassing the clamp),
+    /// otherwise <see cref="PurgeInterval"/> clamped to at least
     /// <see cref="MinPurgeInterval"/> (the Timer/Akka zero-interval spin footgun).
     /// </summary>
     public TimeSpan ResolvedPurgeInterval =>
-        PurgeInterval < MinPurgeInterval ? MinPurgeInterval : PurgeInterval;
+        PurgeIntervalOverride is { } o
+            ? o
+            : PurgeInterval < MinPurgeInterval
+                ? MinPurgeInterval
+                : PurgeInterval;
 
     /// <summary>
     /// Delay before the first purge tick. Default 5 minutes — short so a node that

src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SiteAuditRetentionService.cs

@@ -0,0 +1,133 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+
+namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
+
+/// <summary>
+/// Site-side hosted service that runs the SQLite retention purge on a timer
+/// (PLAN-04 Task 3, S1/U2). Each tick calls
+/// <see cref="ISiteAuditQueue.PurgeExpiredAsync"/> with a cutoff of
+/// <c>UtcNow - <see cref="SiteAuditRetentionOptions.ResolvedRetentionDays"/></c>,
+/// deleting Forwarded/Reconciled rows past the retention window and reclaiming the
+/// freed pages. Without it the ephemeral site store grows unbounded until the next
+/// deployment recreates it.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>Cadence.</b> First tick after <see cref="SiteAuditRetentionOptions.InitialDelay"/>
+/// (default 5 min — short so a node that recycles daily still purges shortly after
+/// start), then every <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>
+/// (default 24 h). Mirrors the loop shape of <see cref="SiteAuditBacklogReporter"/>.
+/// </para>
+/// <para>
+/// <b>Failure containment.</b> Each purge is wrapped in a try/catch so a transient
+/// SQLite error never tears down the hosted service — the next tick retries.
+/// </para>
+/// </remarks>
+public sealed class SiteAuditRetentionService : IHostedService, IDisposable
+{
+    private readonly ISiteAuditQueue _queue;
+    private readonly SiteAuditRetentionOptions _options;
+    private readonly ILogger<SiteAuditRetentionService> _logger;
+    private CancellationTokenSource? _cts;
+    private Task? _loop;
+
+    /// <summary>Initializes a new instance of <see cref="SiteAuditRetentionService"/>.</summary>
+    /// <param name="queue">The site audit queue whose retention purge is driven.</param>
+    /// <param name="options">Retention cadence + window options.</param>
+    /// <param name="logger">Logger instance.</param>
+    public SiteAuditRetentionService(
+        ISiteAuditQueue queue,
+        IOptions<SiteAuditRetentionOptions> options,
+        ILogger<SiteAuditRetentionService> logger)
+    {
+        _queue = queue ?? throw new ArgumentNullException(nameof(queue));
+        _options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    /// <summary>Starts the background purge loop.</summary>
+    /// <param name="ct">Cancellation token signalling host shutdown.</param>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    public Task StartAsync(CancellationToken ct)
+    {
+        // Linked CTS so both StopAsync and the host shutdown token abort the loop.
+        _cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        _loop = Task.Run(() => RunLoopAsync(_cts.Token));
+        return Task.CompletedTask;
+    }
+
+    private async Task RunLoopAsync(CancellationToken ct)
+    {
+        // InitialDelay before the first purge — short so a daily-recycled node
+        // still purges soon after start rather than waiting a full interval it may
+        // never reach.
+        try
+        {
+            await Task.Delay(_options.InitialDelay, ct).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException)
+        {
+            return;
+        }
+
+        await SafePurgeAsync(ct).ConfigureAwait(false);
+
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                await Task.Delay(_options.ResolvedPurgeInterval, ct).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+
+            await SafePurgeAsync(ct).ConfigureAwait(false);
+        }
+    }
+
+    private async Task SafePurgeAsync(CancellationToken ct)
+    {
+        try
+        {
+            var cutoff = DateTime.UtcNow.AddDays(-_options.ResolvedRetentionDays);
+            var purged = await _queue.PurgeExpiredAsync(cutoff, ct).ConfigureAwait(false);
+            if (purged > 0)
+            {
+                _logger.LogInformation(
+                    "Site audit retention purge removed {Count} row(s) older than {Cutoff:o}.",
+                    purged, cutoff);
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            // Shutdown — let the outer loop exit cleanly.
+            throw;
+        }
+        catch (Exception ex)
+        {
+            // Catch-all is deliberate: a transient SQLite fault must not tear down
+            // the hosted service; the next tick retries.
+            _logger.LogWarning(ex, "SiteAuditRetentionService purge failed; next tick will retry.");
+        }
+    }
+
+    /// <summary>Signals the purge loop to stop and waits for it to complete.</summary>
+    /// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    public Task StopAsync(CancellationToken ct)
+    {
+        _cts?.Cancel();
+        return _loop ?? Task.CompletedTask;
+    }
+
+    /// <summary>Releases the internal <see cref="CancellationTokenSource"/>.</summary>
+    public void Dispose()
+    {
+        _cts?.Dispose();
+    }
+}

tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SiteAuditRetentionServiceTests.cs

@@ -0,0 +1,114 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Types;
+using AuditEvent = ZB.MOM.WW.Audit.AuditEvent;
+
+namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
+
+/// <summary>
+/// Tests for <see cref="SiteAuditRetentionService"/> (PLAN-04 Task 3). The hosted
+/// service runs the site SQLite retention purge on a timer: first tick after
+/// <see cref="SiteAuditRetentionOptions.InitialDelay"/>, then every
+/// <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>, passing a cutoff of
+/// <c>UtcNow - ResolvedRetentionDays</c>. Per-tick failures are swallowed so a
+/// transient SQLite error never tears the service down.
+/// </summary>
+public class SiteAuditRetentionServiceTests
+{
+    [Fact]
+    public async Task Tick_Purges_With_RetentionCutoff()
+    {
+        var queue = new RecordingSiteAuditQueue();
+        var options = Options.Create(new SiteAuditRetentionOptions
+        {
+            RetentionDays = 7,
+            // Override bypasses the 1-min production clamp so the loop ticks in ms.
+            PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
+            InitialDelay = TimeSpan.Zero,
+        });
+        using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
+
+        await svc.StartAsync(CancellationToken.None);
+        await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
+        await svc.StopAsync(CancellationToken.None);
+
+        var cutoff = queue.PurgeCalls.First();
+        Assert.InRange(cutoff,
+            DateTime.UtcNow.AddDays(-7).AddMinutes(-1),
+            DateTime.UtcNow.AddDays(-7).AddMinutes(1));
+    }
+
+    [Fact]
+    public async Task Tick_SwallowsExceptions_AndKeepsTicking()
+    {
+        var queue = new RecordingSiteAuditQueue { ThrowOnFirstCall = true };
+        var options = Options.Create(new SiteAuditRetentionOptions
+        {
+            RetentionDays = 7,
+            // Override bypasses the 1-min production clamp so the loop ticks in ms.
+            PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
+            InitialDelay = TimeSpan.Zero,
+        });
+        using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
+
+        await svc.StartAsync(CancellationToken.None);
+        // The first tick throws; a second tick must still arrive (service survives).
+        await WaitUntilAsync(() => queue.PurgeCalls.Count >= 2, TimeSpan.FromSeconds(5));
+        await svc.StopAsync(CancellationToken.None);
+
+        Assert.True(queue.PurgeCalls.Count >= 2);
+    }
+
+    private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
+    {
+        var sw = Stopwatch.StartNew();
+        while (!condition())
+        {
+            if (sw.Elapsed > timeout)
+            {
+                throw new TimeoutException("Condition was not met within the timeout.");
+            }
+
+            await Task.Delay(10);
+        }
+    }
+
+    /// <summary>Stub queue that records purge cutoffs and can throw on the first tick.</summary>
+    private sealed class RecordingSiteAuditQueue : ISiteAuditQueue
+    {
+        private readonly ConcurrentQueue<DateTime> _purgeCalls = new();
+        private int _calls;
+
+        public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
+        public bool ThrowOnFirstCall { get; init; }
+
+        public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
+        {
+            var n = Interlocked.Increment(ref _calls);
+            _purgeCalls.Enqueue(olderThanUtc);
+            if (ThrowOnFirstCall && n == 1)
+            {
+                throw new InvalidOperationException("Simulated transient purge failure.");
+            }
+
+            return Task.FromResult(0);
+        }
+
+        public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
+            => throw new NotSupportedException();
+        public Task<IReadOnlyList<AuditEvent>> ReadPendingCachedTelemetryAsync(int limit, CancellationToken ct = default)
+            => throw new NotSupportedException();
+        public Task MarkForwardedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
+            => throw new NotSupportedException();
+        public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(DateTime sinceUtc, int batchSize, CancellationToken ct = default)
+            => throw new NotSupportedException();
+        public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
+            => throw new NotSupportedException();
+        public Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default)
+            => throw new NotSupportedException();
+    }
+}