feat(kpihistory): hourly rollup recorder tick + rollup purge + options (plan #22 T4)

Add a third Akka periodic timer (kpi-rollup) to KpiHistoryRecorderActor that
folds the trailing RollupLookbackHours of raw KpiSample rows into the hourly
KpiRollupHourly table via the idempotent FoldHourlyRollupsAsync upsert, with the
in-progress hour excluded (toHourUtc = current hour start, exclusive). A
_rollupInFlight guard coalesces overlapping ticks (mirrors _sampleInFlight), the
fold is best-effort (no exception escapes a tick), and a missed/failover tick
self-heals via the lookback re-fold. The daily purge now runs BOTH the raw
PurgeOlderThanAsync (RetentionDays) and PurgeRollupsOlderThanAsync
(RollupRetentionDays), isolated so one failure never skips the other.

New KpiHistoryOptions: RollupInterval (1h), RollupLookbackHours (3),
RollupRetentionDays (365), RollupThresholdHours (168, consumed by Task 5).
Validator adds bounds incl. the coherence rule RollupRetentionDays >=
RetentionDays so long-range trends have no data hole.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryOptions.cs

@@ -35,4 +35,40 @@ public sealed class KpiHistoryOptions
     /// <c>[2, 5000]</c>). At least two points are required to draw a line.
     /// </summary>
     public int DefaultMaxSeriesPoints { get; set; } = 200;
+
+    /// <summary>
+    /// How often the recorder folds the trailing raw <c>KpiSample</c> rows into the
+    /// hourly <c>KpiRollupHourly</c> table (default 1 hour). Must be strictly
+    /// positive. Each tick re-folds the trailing
+    /// <see cref="RollupLookbackHours"/> hours via an idempotent upsert, so a tick
+    /// missed during a singleton-failover handover self-heals on the next fold.
+    /// </summary>
+    public TimeSpan RollupInterval { get; set; } = TimeSpan.FromHours(1);
+
+    /// <summary>
+    /// How many trailing whole hours each rollup fold re-processes (default 3,
+    /// range <c>[1, 168]</c>). Folding a lookback window rather than just the last
+    /// hour lets an idempotent re-fold recover the one or more complete hours a
+    /// missed/failover tick skipped. Never includes the in-progress hour (the fold's
+    /// upper bound is the current hour start, exclusive).
+    /// </summary>
+    public int RollupLookbackHours { get; set; } = 3;
+
+    /// <summary>
+    /// Central retention window in days for the hourly rollup rows (default 365,
+    /// range <c>[1, 3650]</c>). Rollups outlive raw samples so long-range trend
+    /// charts have data after raw <c>KpiSample</c> rows are purged; the validator
+    /// therefore requires <c>RollupRetentionDays &gt;= <see cref="RetentionDays"/></c>
+    /// so a window never falls into a hole where raw is purged but no rollup was
+    /// written. Dropped by the rollup purge sweep.
+    /// </summary>
+    public int RollupRetentionDays { get; set; } = 365;
+
+    /// <summary>
+    /// Query-routing boundary in hours (default 168 = 7 days, minimum 24). Trend
+    /// queries whose window is <c>&lt;=</c> this width read raw <c>KpiSample</c> rows
+    /// (preserving intra-minute detail); wider windows read the pre-aggregated
+    /// hourly rollups. Consumed by the Central UI query service's range routing.
+    /// </summary>
+    public int RollupThresholdHours { get; set; } = 168;
 }

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryOptionsValidator.cs

@@ -28,6 +28,21 @@ public sealed class KpiHistoryOptionsValidator : OptionsValidatorBase<KpiHistory
     /// <summary>Inclusive upper bound for <see cref="KpiHistoryOptions.DefaultMaxSeriesPoints"/>.</summary>
     public const int MaxDefaultMaxSeriesPoints = 5000;
 
+    /// <summary>Inclusive lower bound for <see cref="KpiHistoryOptions.RollupLookbackHours"/>.</summary>
+    public const int MinRollupLookbackHours = 1;
+
+    /// <summary>Inclusive upper bound for <see cref="KpiHistoryOptions.RollupLookbackHours"/> (a week).</summary>
+    public const int MaxRollupLookbackHours = 168;
+
+    /// <summary>Inclusive lower bound for <see cref="KpiHistoryOptions.RollupRetentionDays"/>.</summary>
+    public const int MinRollupRetentionDays = 1;
+
+    /// <summary>Inclusive upper bound for <see cref="KpiHistoryOptions.RollupRetentionDays"/>.</summary>
+    public const int MaxRollupRetentionDays = 3650;
+
+    /// <summary>Inclusive lower bound for <see cref="KpiHistoryOptions.RollupThresholdHours"/> (one day).</summary>
+    public const int MinRollupThresholdHours = 24;
+
     /// <inheritdoc />
     protected override void Validate(ValidationBuilder builder, KpiHistoryOptions options)
     {
@@ -54,5 +69,36 @@ public sealed class KpiHistoryOptionsValidator : OptionsValidatorBase<KpiHistory
               || options.DefaultMaxSeriesPoints > MaxDefaultMaxSeriesPoints),
             $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.DefaultMaxSeriesPoints)} ({options.DefaultMaxSeriesPoints}) " +
             $"must be in [{MinDefaultMaxSeriesPoints}, {MaxDefaultMaxSeriesPoints}].");
+
+        builder.RequireThat(options.RollupInterval > TimeSpan.Zero,
+            $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.RollupInterval)} ({options.RollupInterval}) " +
+            "must be > 0; it is the hourly rollup fold cadence.");
+
+        // Valid when RollupLookbackHours is within [Min, Max] inclusive.
+        builder.RequireThat(
+            !(options.RollupLookbackHours < MinRollupLookbackHours
+              || options.RollupLookbackHours > MaxRollupLookbackHours),
+            $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.RollupLookbackHours)} ({options.RollupLookbackHours}) " +
+            $"must be in [{MinRollupLookbackHours}, {MaxRollupLookbackHours}] hours.");
+
+        // Valid when RollupRetentionDays is within [Min, Max] inclusive.
+        builder.RequireThat(
+            !(options.RollupRetentionDays < MinRollupRetentionDays
+              || options.RollupRetentionDays > MaxRollupRetentionDays),
+            $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.RollupRetentionDays)} ({options.RollupRetentionDays}) " +
+            $"must be in [{MinRollupRetentionDays}, {MaxRollupRetentionDays}] days.");
+
+        // Coherence: rollups must be retained AT LEAST as long as raw samples, else a
+        // long-range chart hits a hole where raw is purged but the rollup was already
+        // dropped (or never written). Enforced independently of the range check above.
+        builder.RequireThat(
+            options.RollupRetentionDays >= options.RetentionDays,
+            $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.RollupRetentionDays)} ({options.RollupRetentionDays}) " +
+            $"must be >= {nameof(KpiHistoryOptions.RetentionDays)} ({options.RetentionDays}) so rollups outlive " +
+            "raw samples and long-range trends have no data hole.");
+
+        builder.RequireThat(options.RollupThresholdHours >= MinRollupThresholdHours,
+            $"ScadaBridge:KpiHistory:{nameof(KpiHistoryOptions.RollupThresholdHours)} ({options.RollupThresholdHours}) " +
+            $"must be >= {MinRollupThresholdHours} hours; it is the raw-vs-rollup query-routing boundary.");
     }
 }

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs

@@ -18,9 +18,17 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
 /// <item>Bulk-writes the combined batch through
 ///       <see cref="IKpiHistoryRepository.RecordSamplesAsync"/>.</item>
 /// </list>
-/// A separate daily timer (default 1 d) runs the retention purge, dropping rows
-/// older than <see cref="KpiHistoryOptions.RetentionDays"/> via
-/// <see cref="IKpiHistoryRepository.PurgeOlderThanAsync"/>.
+/// A separate daily timer (default 1 d) runs the retention purge, dropping raw
+/// sample rows older than <see cref="KpiHistoryOptions.RetentionDays"/> via
+/// <see cref="IKpiHistoryRepository.PurgeOlderThanAsync"/> and hourly rollup rows
+/// older than the longer <see cref="KpiHistoryOptions.RollupRetentionDays"/> via
+/// <see cref="IKpiHistoryRepository.PurgeRollupsOlderThanAsync"/>. A third hourly
+/// timer (default 1 h) folds the trailing
+/// <see cref="KpiHistoryOptions.RollupLookbackHours"/> hours of raw samples into the
+/// <c>KpiRollupHourly</c> table via
+/// <see cref="IKpiHistoryRepository.FoldHourlyRollupsAsync"/> (idempotent upsert,
+/// in-progress hour excluded), so long-range trend charts read a pre-thinned
+/// series.
 /// </summary>
 /// <remarks>
 /// <para>
@@ -57,6 +65,7 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
 {
     private const string SampleTimerKey = "kpi-sample";
     private const string PurgeTimerKey = "kpi-purge";
+    private const string RollupTimerKey = "kpi-rollup";
 
     private readonly IServiceProvider _serviceProvider;
     private readonly KpiHistoryOptions _options;
@@ -82,6 +91,17 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     /// </summary>
     private bool _sampleInFlight;
 
+    /// <summary>
+    /// In-flight guard for the hourly rollup fold. Set true at the start of a fold pass and
+    /// cleared when the pass's <see cref="RollupComplete"/> arrives. While true, further
+    /// <see cref="RollupTick"/>s are skipped so folds never overlap — the fold is a
+    /// group-by-and-upsert over a lookback window that can outlast the (default 1 h)
+    /// <see cref="KpiHistoryOptions.RollupInterval"/> on a slow/recovering store; without this
+    /// guard a subsequent tick would launch a second concurrent fold contending for the same
+    /// upsert key. Mirrors <see cref="_sampleInFlight"/>.
+    /// </summary>
+    private bool _rollupInFlight;
+
     /// <summary>Akka timer scheduler, assigned by the actor system via <see cref="IWithTimers"/>.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
 
@@ -104,6 +124,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         Receive<SampleComplete>(_ => _sampleInFlight = false); // lower the in-flight guard (success or fault)
         Receive<PurgeTick>(_ => HandlePurgeTick());
         Receive<PurgeComplete>(_ => { }); // best-effort: no actor state to reset on completion
+        Receive<RollupTick>(_ => HandleRollupTick());
+        Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
     }
 
     /// <inheritdoc />
@@ -129,6 +151,16 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
             PurgeTick.Instance,
             initialDelay: PurgeTimerSchedule.InitialDelay(_options.PurgeInterval),
             interval: _options.PurgeInterval);
+
+        // Fold the trailing hours into the hourly rollup table on a short initial delay
+        // (so a fresh active node catches up promptly after a failover), then settle into
+        // the periodic cadence. The fold re-processes a lookback window via an idempotent
+        // upsert, so a missed tick self-heals on the next fold.
+        Timers.StartPeriodicTimer(
+            RollupTimerKey,
+            RollupTick.Instance,
+            initialDelay: TimeSpan.FromSeconds(10),
+            interval: _options.RollupInterval);
     }
 
     /// <inheritdoc />
@@ -249,16 +281,20 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     }
 
     /// <summary>
-    /// Handles a purge tick: computes the retention cut-off on the actor thread, then runs
-    /// the bulk delete off-thread and pipes a completion back to <see cref="Self"/>. Purges
-    /// are daily and idempotent, so no in-flight guard is needed.
+    /// Handles a purge tick: computes both retention cut-offs on the actor thread (raw samples
+    /// at <see cref="KpiHistoryOptions.RetentionDays"/>, hourly rollups at the longer
+    /// <see cref="KpiHistoryOptions.RollupRetentionDays"/>), then runs both bulk deletes
+    /// off-thread and pipes a completion back to <see cref="Self"/>. Purges are daily and
+    /// idempotent, so no in-flight guard is needed.
     /// </summary>
     private void HandlePurgeTick()
     {
-        var before = DateTime.UtcNow - TimeSpan.FromDays(_options.RetentionDays);
+        var now = DateTime.UtcNow;
+        var sampleBefore = now - TimeSpan.FromDays(_options.RetentionDays);
+        var rollupBefore = now - TimeSpan.FromDays(_options.RollupRetentionDays);
         var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
 
-        RunPurgePass(before, cancellationToken).PipeTo(
+        RunPurgePass(sampleBefore, rollupBefore, cancellationToken).PipeTo(
             Self,
             success: deleted =>
             {
@@ -266,7 +302,7 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
                 {
                     _logger.LogInformation(
                         "KPI history purge removed {DeletedCount} sample(s) older than {Cutoff:o}.",
-                        deleted, before);
+                        deleted, sampleBefore);
                 }
 
                 return PurgeComplete.Instance;
@@ -280,22 +316,53 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
 
     /// <summary>
     /// Runs a single purge sweep: opens a DI scope, resolves the repository, and bulk-deletes
-    /// rows captured before <paramref name="before"/>, returning the deleted count. The whole
-    /// body is wrapped so the returned task never faults — on failure the exception is logged
-    /// and 0 is returned, mirroring <see cref="RunSamplePass"/>'s best-effort contract.
+    /// expired raw samples (before <paramref name="sampleBefore"/>) and expired hourly rollups
+    /// (before <paramref name="rollupBefore"/>), returning the deleted raw-sample count. The two
+    /// deletes are isolated so a failure of one never skips the other, and the whole body is
+    /// wrapped so the returned task never faults — on failure the exception is logged and 0 is
+    /// returned, mirroring <see cref="RunSamplePass"/>'s best-effort contract.
     /// </summary>
-    private async Task<int> RunPurgePass(DateTime before, CancellationToken cancellationToken)
+    private async Task<int> RunPurgePass(
+        DateTime sampleBefore, DateTime rollupBefore, CancellationToken cancellationToken)
     {
         try
         {
             await using var scope = _serviceProvider.CreateAsyncScope();
             var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
-            return await repository.PurgeOlderThanAsync(before, cancellationToken);
-        }
-        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-        {
-            // Shutdown interrupted the purge; the next active sweep retries. Not a failure.
-            return 0;
+
+            var deleted = 0;
+
+            // Raw-sample purge — isolated so a fault here does not skip the rollup purge below.
+            try
+            {
+                deleted = await repository.PurgeOlderThanAsync(sampleBefore, cancellationToken);
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                // Shutdown interrupted the purge; the next active sweep retries. Not a failure.
+                return deleted;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "KPI history raw-sample purge failed unexpectedly.");
+            }
+
+            // Rollup purge — retained longer than raw; isolated so its failure does not affect
+            // the raw-purge outcome already reported.
+            try
+            {
+                await repository.PurgeRollupsOlderThanAsync(rollupBefore, cancellationToken);
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                // Shutdown interrupted the rollup purge; the next active sweep retries. Not a failure.
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "KPI history rollup purge failed unexpectedly.");
+            }
+
+            return deleted;
         }
         catch (Exception ex)
         {
@@ -304,6 +371,71 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    /// Handles a rollup tick. If a fold is already in flight the tick is skipped (logged at
+    /// debug) so folds never overlap; otherwise the in-flight guard is raised and the fold is
+    /// launched off-thread with a <see cref="RollupComplete"/> piped back to <see cref="Self"/>
+    /// to lower the guard on the actor thread. The fold window's upper bound is the current
+    /// hour start (exclusive) so the in-progress hour is never folded, and its lower bound is
+    /// <see cref="KpiHistoryOptions.RollupLookbackHours"/> earlier so a missed/failover tick
+    /// self-heals via the idempotent upsert.
+    /// </summary>
+    private void HandleRollupTick()
+    {
+        if (_rollupInFlight)
+        {
+            _logger.LogDebug("KPI rollup tick skipped — a rollup fold is already in flight.");
+            return;
+        }
+
+        _rollupInFlight = true;
+        var toHourUtc = TruncateToHour(DateTime.UtcNow);
+        var fromHourUtc = toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours);
+        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
+
+        RunRollupPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
+            Self,
+            success: () => RollupComplete.Instance,
+            failure: ex =>
+            {
+                _logger.LogError(ex, "KPI rollup fold faulted unexpectedly.");
+                return RollupComplete.Instance;
+            });
+    }
+
+    /// <summary>
+    /// Runs a single rollup fold: opens a DI scope, resolves the repository, and folds the
+    /// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c>
+    /// into the hourly rollup table via the idempotent upsert. The whole body is wrapped so the
+    /// returned task never faults — best-effort observability must never disrupt anything.
+    /// </summary>
+    private async Task RunRollupPass(
+        DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
+    {
+        try
+        {
+            await using var scope = _serviceProvider.CreateAsyncScope();
+            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
+            await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            // Shutdown interrupted the fold; the next active node re-folds the lookback window. Not a failure.
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "KPI rollup fold failed unexpectedly.");
+        }
+    }
+
+    /// <summary>
+    /// Truncates a UTC instant to the start of its hour (minutes/seconds/ticks dropped),
+    /// preserving <see cref="DateTimeKind.Utc"/>. Used to derive the fold window's whole-hour
+    /// boundaries so the in-progress hour is excluded.
+    /// </summary>
+    private static DateTime TruncateToHour(DateTime utc) =>
+        new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
+
     /// <summary>Self-tick triggering a sampling pass across all registered sources.</summary>
     internal sealed class SampleTick
     {
@@ -334,4 +466,21 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         public static readonly PurgeComplete Instance = new();
         private PurgeComplete() { }
     }
+
+    /// <summary>Self-tick triggering an hourly rollup fold over the trailing lookback window.</summary>
+    internal sealed class RollupTick
+    {
+        public static readonly RollupTick Instance = new();
+        private RollupTick() { }
+    }
+
+    /// <summary>
+    /// Piped-back completion of a rollup fold; lets the fold run off the actor thread and
+    /// lowers the <c>_rollupInFlight</c> guard on the actor thread (fires on success and fault).
+    /// </summary>
+    internal sealed class RollupComplete
+    {
+        public static readonly RollupComplete Instance = new();
+        private RollupComplete() { }
+    }
 }