feat(kpihistory): one-shot rollup backfill of existing samples on recorder start (plan #22 T6)

Fold the whole raw-retention window of already-recorded KpiSample rows into
KpiRollupHourly once shortly after the recorder singleton starts, so long-range
(30 d/90 d) trend charts aren't blank until enough wall-clock passes after
deploy. Reuses Task 3's idempotent FoldHourlyRollupsAsync over
[TruncateToHour(now) - RetentionDays, TruncateToHour(now)); runs at most once
per actor lifetime (_backfillDone latch) and defers past / blocks the periodic
lookback fold via a dedicated _backfillInFlight guard so the two idempotent
upserts never race on overlapping rows. Best-effort: no exception escapes; logs
start, window, and elapsed on completion. No Host or options change.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs

@@ -28,7 +28,10 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
 /// <c>KpiRollupHourly</c> table via
 /// <see cref="IKpiHistoryRepository.FoldHourlyRollupsAsync"/> (idempotent upsert,
 /// in-progress hour excluded), so long-range trend charts read a pre-thinned
-/// series.
+/// series. A one-shot backfill timer additionally folds the whole raw-retention
+/// window of already-recorded samples once shortly after start, so 30 d/90 d
+/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
+/// same idempotent fold and runs at most once per actor lifetime.
 /// </summary>
 /// <remarks>
 /// <para>
@@ -66,6 +69,18 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     private const string SampleTimerKey = "kpi-sample";
     private const string PurgeTimerKey = "kpi-purge";
     private const string RollupTimerKey = "kpi-rollup";
+    private const string BackfillTimerKey = "kpi-rollup-backfill";
+
+    /// <summary>
+    /// Initial delay before the one-shot rollup backfill fires (see
+    /// <see cref="HandleBackfillTick"/>). Deliberately a touch longer than the periodic rollup
+    /// timer's own <c>10 s</c> initial delay so that, when both are armed at start, the periodic
+    /// tick claims the fold path first and the backfill simply defers (the two folds are
+    /// idempotent upserts, but they must not race on overlapping rows). Also comfortably beyond
+    /// the sub-second/second-scale <c>AwaitAssert</c> windows used by the unit tests, so the
+    /// one-shot never auto-fires mid-test — tests drive <see cref="BackfillTick"/> by hand.
+    /// </summary>
+    private static readonly TimeSpan BackfillInitialDelay = TimeSpan.FromSeconds(15);
 
     private readonly IServiceProvider _serviceProvider;
     private readonly KpiHistoryOptions _options;
@@ -102,6 +117,24 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     /// </summary>
     private bool _rollupInFlight;
 
+    /// <summary>
+    /// In-flight guard for the one-shot backfill fold. Set true when the backfill fold is
+    /// launched and cleared when its <see cref="BackfillComplete"/> arrives. While true, periodic
+    /// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so the (possibly
+    /// long) full-retention-window backfill never runs concurrently with a periodic lookback fold
+    /// on the same upsert rows.
+    /// </summary>
+    private bool _backfillInFlight;
+
+    /// <summary>
+    /// Once-only latch for the backfill. Set true when the backfill fold completes (success or
+    /// fault) so it runs at most once per actor lifetime — a fresh singleton on failover starts a
+    /// new instance and re-runs the (idempotent) backfill, which is the intended self-heal. Set on
+    /// completion rather than launch so that if the actor is torn down mid-backfill the next
+    /// lifetime still attempts it.
+    /// </summary>
+    private bool _backfillDone;
+
     /// <summary>Akka timer scheduler, assigned by the actor system via <see cref="IWithTimers"/>.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
 
@@ -126,6 +159,12 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         Receive<PurgeComplete>(_ => { }); // best-effort: no actor state to reset on completion
         Receive<RollupTick>(_ => HandleRollupTick());
         Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
+        Receive<BackfillTick>(_ => HandleBackfillTick());
+        Receive<BackfillComplete>(_ =>
+        {
+            _backfillInFlight = false; // lower the in-flight guard (success or fault)
+            _backfillDone = true;      // latch: at most once per actor lifetime
+        });
     }
 
     /// <inheritdoc />
@@ -161,6 +200,17 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
             RollupTick.Instance,
             initialDelay: TimeSpan.FromSeconds(10),
             interval: _options.RollupInterval);
+
+        // One-shot backfill: fold the whole raw-retention window of existing samples into the
+        // hourly rollup table once shortly after start, so long-range (30 d/90 d) trend charts
+        // aren't blank until enough wall-clock passes after deploy. The periodic tick only folds
+        // the trailing RollupLookbackHours, so without this the deep history stays unrolled. The
+        // fold is an idempotent upsert, so re-running on a failover (a fresh singleton re-arms
+        // this timer) is safe.
+        Timers.StartSingleTimer(
+            BackfillTimerKey,
+            BackfillTick.Instance,
+            timeout: BackfillInitialDelay);
     }
 
     /// <inheritdoc />
@@ -382,8 +432,11 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     /// </summary>
     private void HandleRollupTick()
     {
-        if (_rollupInFlight)
+        if (_rollupInFlight || _backfillInFlight)
         {
+            // Skip while either a prior periodic fold OR the one-shot full-window backfill is
+            // running, so two idempotent upserts never race on overlapping rows. A skipped
+            // periodic tick self-heals on the next fold (it re-processes the lookback window).
             _logger.LogDebug("KPI rollup tick skipped — a rollup fold is already in flight.");
             return;
         }
@@ -428,6 +481,82 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    /// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
+    /// has already completed (<see cref="_backfillDone"/>) or is in flight
+    /// (<see cref="_backfillInFlight"/>) the tick is ignored. If a periodic fold currently holds
+    /// the fold path (<see cref="_rollupInFlight"/>) the backfill defers by re-arming its single
+    /// timer, so the full-window backfill and the periodic lookback fold never race on overlapping
+    /// upsert rows. Otherwise it raises the backfill guard and folds the whole raw-retention
+    /// window <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> off the actor
+    /// thread, piping a <see cref="BackfillComplete"/> back to lower the guard and set the latch.
+    /// </summary>
+    private void HandleBackfillTick()
+    {
+        if (_backfillDone || _backfillInFlight)
+        {
+            return;
+        }
+
+        if (_rollupInFlight)
+        {
+            // A periodic fold is in flight; defer the one-shot backfill briefly so the two
+            // idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
+            Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
+            return;
+        }
+
+        _backfillInFlight = true;
+        var toHourUtc = TruncateToHour(DateTime.UtcNow);
+        var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
+        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
+
+        _logger.LogInformation(
+            "KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) ({RetentionDays}-day window).",
+            fromHourUtc, toHourUtc, _options.RetentionDays);
+
+        RunBackfillPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
+            Self,
+            success: () => BackfillComplete.Instance,
+            failure: ex =>
+            {
+                _logger.LogError(ex, "KPI rollup backfill faulted unexpectedly.");
+                return BackfillComplete.Instance;
+            });
+    }
+
+    /// <summary>
+    /// Runs the one-shot backfill fold: opens a DI scope, resolves the repository, and folds the
+    /// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
+    /// the hourly rollup table via the same idempotent upsert the periodic fold uses (Task 3). The
+    /// whole body is wrapped so the returned task never faults — best-effort observability must
+    /// never disrupt startup. The fold method returns no count, so completion is logged with the
+    /// window and elapsed wall time only.
+    /// </summary>
+    private async Task RunBackfillPass(
+        DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
+    {
+        var startedAt = DateTime.UtcNow;
+        try
+        {
+            await using var scope = _serviceProvider.CreateAsyncScope();
+            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
+            await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
+
+            _logger.LogInformation(
+                "KPI rollup backfill completed — folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
+                fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            // Shutdown interrupted the backfill; a failover re-runs it (idempotent). Not a failure.
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "KPI rollup backfill failed unexpectedly.");
+        }
+    }
+
     /// <summary>
     /// Truncates a UTC instant to the start of its hour (minutes/seconds/ticks dropped),
     /// preserving <see cref="DateTimeKind.Utc"/>. Used to derive the fold window's whole-hour
@@ -483,4 +612,26 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         public static readonly RollupComplete Instance = new();
         private RollupComplete() { }
     }
+
+    /// <summary>
+    /// Self-tick triggering the one-shot backfill fold over the full raw-retention window. Armed
+    /// once via a single timer in <see cref="PreStart"/> (and re-armed only to defer past an
+    /// in-flight periodic fold); runs at most once per actor lifetime.
+    /// </summary>
+    internal sealed class BackfillTick
+    {
+        public static readonly BackfillTick Instance = new();
+        private BackfillTick() { }
+    }
+
+    /// <summary>
+    /// Piped-back completion of the one-shot backfill fold; lets the fold run off the actor thread
+    /// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard and latches
+    /// <c>_backfillDone</c> (fires on success and fault).
+    /// </summary>
+    internal sealed class BackfillComplete
+    {
+        public static readonly BackfillComplete Instance = new();
+        private BackfillComplete() { }
+    }
 }