fix(kpi): backfill skips or shrinks to the rollup watermark on failover restarts (plan R2-04 T4)

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs

@@ -186,6 +186,7 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         Receive<RollupTick>(_ => HandleRollupTick());
         Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
         Receive<BackfillTick>(_ => HandleBackfillTick());
+        Receive<BackfillPlan>(HandleBackfillPlan);
         Receive<BackfillSliceComplete>(_ =>
         {
             _backfillInFlight = false; // release the fold path BETWEEN slices (R1)
@@ -546,26 +547,22 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
 
         if (_backfillSlices.Count == 0)
         {
-            // Build the slice plan once: the full raw-retention window in bounded 24 h folds,
-            // oldest-first. Never hand the repository fold the whole window at once (R1).
-            var toHourUtc = TruncateToHour(DateTime.UtcNow);
-            var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
-            EnqueueBackfillSlices(fromHourUtc, toHourUtc);
+            // No slice plan yet: resolve the rollup watermark off the actor thread, then plan on
+            // the actor thread (Receive<BackfillPlan>). Raise the guard so periodic ticks defer
+            // while planning is in flight; the plan handler lowers it. On a failover restart the
+            // watermark lets the plan SKIP or SHRINK to the un-rolled tail instead of re-folding
+            // the whole raw-retention window (arch-review 04 round 2, R1).
+            _backfillInFlight = true;
+            var planCancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
 
-            if (_backfillSlices.Count == 0)
-            {
-                // Degenerate window (no whole hours to fold) — nothing to backfill.
-                _backfillDone = true;
-                return;
-            }
-
-            _logger.LogInformation(
-                "KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) "
-                + "({RetentionDays}-day window) across {SliceCount} slice(s) of up to {SliceHours} h.",
-                fromHourUtc, toHourUtc, _options.RetentionDays, _backfillSlices.Count, BackfillSliceWidth.TotalHours);
-
-            // Start draining via the timer (uniform path with the between-slice re-arm).
-            Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
+            RunBackfillPlanPass(planCancellationToken).PipeTo(
+                Self,
+                success: watermark => new BackfillPlan(watermark),
+                failure: ex =>
+                {
+                    _logger.LogError(ex, "KPI rollup backfill plan pass faulted unexpectedly.");
+                    return new BackfillPlan(null); // null watermark ⇒ plan the full retention window
+                });
             return;
         }
 
@@ -605,6 +602,82 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    /// Plans the backfill from the resolved rollup watermark (actor thread). The window floor is
+    /// the newer of the watermark (re-folding the watermark hour itself is a cheap idempotent
+    /// safety margin) and the raw-retention floor; a null watermark (fresh install) keeps the full
+    /// retention window. If the floor is already within
+    /// <see cref="KpiHistoryOptions.RollupLookbackHours"/> of now the periodic fold's lookback
+    /// covers everything from there forward, so the whole historical pass is skipped (the failover
+    /// fast-path); otherwise the tail <c>[floor, TruncateToHour(now))</c> is enumerated into
+    /// bounded slices and draining is self-armed. Either way the plan-pass guard is lowered here
+    /// (arch-review 04 round 2, R1).
+    /// </summary>
+    private void HandleBackfillPlan(BackfillPlan plan)
+    {
+        var toHourUtc = TruncateToHour(DateTime.UtcNow);
+        var retentionFloor = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
+        // Shrink to the un-rolled tail; re-fold the watermark hour itself (idempotent) as a safety
+        // margin. A null watermark (fresh install) keeps the full retention window.
+        var floor = plan.Watermark is { } w && w > retentionFloor ? w : retentionFloor;
+
+        if (floor >= toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours))
+        {
+            // Failover fast-path: rollups are already current — the periodic fold's lookback
+            // window covers everything from the watermark forward, so the full historical pass is
+            // skipped entirely (arch-review 04 round 2, R1).
+            _backfillInFlight = false;
+            _backfillDone = true;
+            _logger.LogInformation(
+                "KPI rollup backfill skipped — rollups current through {Watermark:o}.", plan.Watermark);
+            return;
+        }
+
+        EnqueueBackfillSlices(floor, toHourUtc);
+        _backfillInFlight = false;
+
+        if (_backfillSlices.Count == 0)
+        {
+            // Degenerate window (no whole hours to fold) — nothing to backfill.
+            _backfillDone = true;
+            return;
+        }
+
+        _logger.LogInformation(
+            "KPI rollup backfill starting — folding [{FromHour:o}, {ToHour:o}) across {SliceCount} "
+            + "slice(s) of up to {SliceHours} h (watermark {Watermark:o}).",
+            floor, toHourUtc, _backfillSlices.Count, BackfillSliceWidth.TotalHours, plan.Watermark);
+
+        // Start draining via the timer (uniform path with the between-slice re-arm).
+        Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
+    }
+
+    /// <summary>
+    /// Resolves the newest folded rollup hour (the backfill watermark) off the actor thread in a
+    /// fresh DI scope. Never faults — on cancellation or error it returns <c>null</c>, which the
+    /// plan treats as "no rollups", keeping the full retention window (best-effort, mirrors the
+    /// other passes).
+    /// </summary>
+    private async Task<DateTime?> RunBackfillPlanPass(CancellationToken cancellationToken)
+    {
+        try
+        {
+            await using var scope = _serviceProvider.CreateAsyncScope();
+            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
+            return await repository.GetLatestRollupHourAsync(cancellationToken);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            // Shutdown interrupted the lookup; a fresh active node re-plans. Not a failure.
+            return null;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "KPI rollup backfill watermark lookup failed; planning the full retention window.");
+            return null;
+        }
+    }
+
     /// <summary>
     /// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
     /// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
@@ -706,6 +779,13 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         private BackfillTick() { }
     }
 
+    /// <summary>
+    /// Piped-back result of the backfill plan pass: the newest folded rollup hour (the watermark),
+    /// or <c>null</c> when no rollups exist. Consumed on the actor thread to skip or shrink the
+    /// backfill to the un-rolled tail (arch-review 04 round 2, R1).
+    /// </summary>
+    internal sealed record BackfillPlan(DateTime? Watermark);
+
     /// <summary>
     /// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
     /// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard BETWEEN slices — then