fix(kpi): backfill skips or shrinks to the rollup watermark on failover restarts (plan R2-04 T4)

2026-07-13 10:03:33 -04:00
parent 350d530135
commit 7b95ef3752
3 changed files with 204 additions and 22 deletions
@@ -112,6 +112,8 @@ public class KpiHistoryRecorderActorTests : TestKit
        private DateTime? _foldToHourUtc;
        private int _foldCount;
        private readonly List<(DateTime FromHourUtc, DateTime ToHourUtc)> _foldWindows = new();
+        private DateTime? _latestRollupHour;
+        private int _latestRollupHourQueryCount;

        public IReadOnlyList<KpiSample> Recorded
        {
@@ -200,8 +202,27 @@ public class KpiHistoryRecorderActorTests : TestKit
            return Task.CompletedTask;
        }

-        public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
-            Task.FromResult<DateTime?>(null);
+        /// <summary>Watermark the fake returns from <see cref="GetLatestRollupHourAsync"/> (settable).</summary>
+        public DateTime? LatestRollupHour
+        {
+            get { lock (_gate) { return _latestRollupHour; } }
+            set { lock (_gate) { _latestRollupHour = value; } }
+        }
+
+        /// <summary>Number of times the backfill plan pass queried the rollup watermark.</summary>
+        public int LatestRollupHourQueryCount
+        {
+            get { lock (_gate) { return _latestRollupHourQueryCount; } }
+        }
+
+        public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default)
+        {
+            lock (_gate)
+            {
+                _latestRollupHourQueryCount++;
+                return Task.FromResult(_latestRollupHour);
+            }
+        }
    }

    /// <summary>
@@ -985,4 +1006,85 @@ public class KpiHistoryRecorderActorTests : TestKit
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
    }
+
+    [Fact]
+    public void Backfill_Skips_WhenRollupsAlreadyCurrent()
+    {
+        // Watermark within RollupLookbackHours of now (the common failover case): the backfill
+        // must be skipped entirely — ZERO folds, the done-latch set — because the periodic fold's
+        // lookback window already covers everything from the watermark forward (arch-review 04
+        // round 2, R1).
+        var repository = new RecordingRepository();
+        var sp = BuildServiceProvider(repository, new HealthySource());
+        var actor = CreateActor(sp, new KpiHistoryOptions
+        {
+            SampleInterval = TimeSpan.FromHours(1),
+            PurgeInterval = TimeSpan.FromHours(1),
+            RollupInterval = TimeSpan.FromHours(1),
+            RetentionDays = 90,
+            RollupLookbackHours = 3,
+        });
+
+        // Rollups current through one hour ago — well inside the 3 h lookback tail.
+        var nowHour = new DateTime(
+            DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
+            DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
+        repository.LatestRollupHour = nowHour - TimeSpan.FromHours(1);
+
+        actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
+
+        // The plan pass must consult the watermark…
+        AwaitAssert(
+            () => Assert.True(repository.LatestRollupHourQueryCount >= 1),
+            duration: TimeSpan.FromSeconds(3),
+            interval: TimeSpan.FromMilliseconds(50));
+
+        // …and then skip: no slices are ever folded.
+        Thread.Sleep(300);
+        Assert.Equal(0, repository.FoldCount);
+
+        // A second BackfillTick is a no-op (the done-latch is set).
+        actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
+        Thread.Sleep(300);
+        Assert.Equal(0, repository.FoldCount);
+    }
+
+    [Fact]
+    public void Backfill_ShrinksToWatermark_InsteadOfRetentionFloor()
+    {
+        // RetentionDays = 90 but the watermark is only 2 days old: the first slice must start at
+        // the WATERMARK hour (inclusive — re-folding the newest already-folded hour is a cheap
+        // idempotent safety margin), NOT 90 days ago, and there must be <= 3 slices total
+        // (arch-review 04 round 2, R1).
+        var repository = new RecordingRepository();
+        var sp = BuildServiceProvider(repository, new HealthySource());
+        var actor = CreateActor(sp, new KpiHistoryOptions
+        {
+            SampleInterval = TimeSpan.FromHours(1),
+            PurgeInterval = TimeSpan.FromHours(1),
+            RollupInterval = TimeSpan.FromHours(1),
+            RetentionDays = 90,
+            RollupLookbackHours = 3,
+        });
+
+        var nowHour = new DateTime(
+            DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
+            DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
+        var watermark = nowHour - TimeSpan.FromDays(2);
+        repository.LatestRollupHour = watermark;
+
+        actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
+
+        AwaitAssert(
+            () =>
+            {
+                var windows = repository.FoldWindows;
+                Assert.NotEmpty(windows);
+                // Shrunk to the un-rolled tail: first slice starts AT the watermark, not 90 d ago.
+                Assert.Equal(watermark, windows[0].FromHourUtc);
+                Assert.True(windows.Count <= 3, $"expected <= 3 slices from the 2-day tail, got {windows.Count}");
+            },
+            duration: TimeSpan.FromSeconds(5),
+            interval: TimeSpan.FromMilliseconds(50));
+    }
 }