fix(kpi-history): fold raw samples one hour at a time; only a unique-key violation is the benign fold race (review 2026-10-21 06/PERF-6, 08/P4, 08/C3)

2026-10-22 09:26:17 -04:00
parent e7fb71c2de
commit e69be1654f
3 changed files with 187 additions and 40 deletions
@@ -107,27 +107,6 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
            return;
        }

-        // The caller passes a small trailing lookback (e.g. 3 h), so the window is bounded:
-        // fetch it and group IN MEMORY. This deliberately avoids provider-specific
-        // DATEADD/DATEPART hour-truncation translation (SQLite in tests, SQL Server in prod),
-        // keeping the hour-bucket arithmetic identical across providers and trivially correct.
-        // Projected, untracked fetch: the fold only reads these six fields and never
-        // mutates a KpiSample. A tracked ToListAsync registered ~5k-90k read-only
-        // entities in the change tracker per healthy 3h fold — all re-scanned by
-        // DetectChanges on the final SaveChanges (arch-review 04 round 2, R2). A
-        // projection is inherently untracked and materializes no entity at all.
-        var samples = await _context.KpiSamples
-            .Where(s => s.CapturedAtUtc >= from && s.CapturedAtUtc < to)
-            .Select(s => new FoldSample(s.Source, s.Metric, s.Scope, s.ScopeKey, s.CapturedAtUtc, s.Value))
-            .ToListAsync(cancellationToken);
-        if (samples.Count == 0)
-        {
-            return;
-        }
-
-        var groups = samples.GroupBy(s => new SeriesHourKey(
-            s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
-
        // Preload every rollup row already covering this window in ONE query and
        // index it by series+hour. The predecessor issued a
        // FirstOrDefaultAsync existence probe PER (series, hour) group — an N+1
@@ -159,6 +138,74 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
                DateTime.SpecifyKind(row.HourStartUtc, DateTimeKind.Utc))] = row;
        }

+        // Read and fold the raw samples ONE HOUR AT A TIME (review 2026-10-21 06/PERF-6,
+        // 08/P4). The periodic fold's 3 h lookback is small, but the startup backfill drains
+        // 24 h slices: materializing a whole slice at once held ~0.3-0.7 M FoldSample records
+        // resident on the active central node. Per-hour reads bound residency to one hour of
+        // samples for the price of one cheap range seek per hour. Grouping stays IN MEMORY
+        // within the hour: that avoids provider-specific DATEADD/DATEPART hour-truncation
+        // translation (SQLite in tests, SQL Server in prod) and keeps the gauge last-value
+        // pick trivially correct, and within one hour the series key alone is the group.
+        //
+        // Projected, untracked fetch: the fold only reads these six fields and never
+        // mutates a KpiSample. A tracked ToListAsync registered ~5k-90k read-only
+        // entities in the change tracker per healthy 3h fold — all re-scanned by
+        // DetectChanges on the final SaveChanges (arch-review 04 round 2, R2). A
+        // projection is inherently untracked and materializes no entity at all.
+        for (var hourStart = from; hourStart < to; hourStart = hourStart.AddHours(1))
+        {
+            var hourEnd = hourStart.AddHours(1);
+            var samples = await _context.KpiSamples
+                .Where(s => s.CapturedAtUtc >= hourStart && s.CapturedAtUtc < hourEnd)
+                .Select(s => new FoldSample(s.Source, s.Metric, s.Scope, s.ScopeKey, s.CapturedAtUtc, s.Value))
+                .ToListAsync(cancellationToken);
+
+            var groups = samples.GroupBy(s => new SeriesHourKey(
+                s.Source, s.Metric, s.Scope, s.ScopeKey, hourStart));
+            FoldHour(groups, existingByKey);
+        }
+
+        try
+        {
+            await _context.SaveChangesAsync(cancellationToken);
+        }
+        catch (DbUpdateException ex) when (IsUniqueKeyViolation(ex))
+        {
+            // Failover-overlap race: the old singleton incarnation's in-flight fold and the
+            // new node's first fold can both Add the same (series, hour) row; the unique
+            // IX_KpiRollupHourly_Series then faults the loser's ENTIRE SaveChanges — the
+            // failure grain is the PASS, not the row (arch-review 04 round 2, R5). A lost
+            // upsert race is repaired identically by the next idempotent re-fold, so it is
+            // classified at Information. ONLY a unique-key violation is: any other save
+            // fault (truncation, constraint, connectivity) propagates to the recorder, which
+            // logs it at Error — previously every DbUpdateException was swallowed here as a
+            // benign race, so a backfill slice faulting for a real reason stayed un-rolled and
+            // invisible until the next node start (review 2026-10-21 08/C3).
+            _logger.LogInformation(ex,
+                "KPI rollup fold lost a failover-overlap upsert race; pass discarded, next fold self-heals.");
+        }
+    }
+
+    /// <summary>
+    /// True when <paramref name="ex"/> is a unique-key violation: SQL Server 2601/2627 in
+    /// production, or the SQLite <c>UNIQUE constraint failed</c> fault on the test provider
+    /// (matched by type name, as <c>ScadaBridgeDbContext</c> does, so this project takes no
+    /// SQLite dependency).
+    /// </summary>
+    private static bool IsUniqueKeyViolation(DbUpdateException ex) =>
+        ex.InnerException is Microsoft.Data.SqlClient.SqlException sql
+            ? sql.Number is 2601 or 2627
+            : ex.InnerException?.GetType().Name == "SqliteException"
+                && ex.InnerException.Message.Contains("UNIQUE constraint failed", StringComparison.Ordinal);
+
+    /// <summary>
+    /// Applies one hour's grouped samples to the rollup table: adds a row per new series-hour
+    /// or overwrites the preloaded row in place.
+    /// </summary>
+    private void FoldHour(
+        IEnumerable<IGrouping<SeriesHourKey, FoldSample>> groups,
+        Dictionary<SeriesHourKey, KpiRollupHourly> existingByKey)
+    {
        foreach (var group in groups)
        {
            var key = group.Key;
@@ -216,23 +263,6 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
                existing.SampleCount = sampleCount;
            }
        }
-
-        try
-        {
-            await _context.SaveChangesAsync(cancellationToken);
-        }
-        catch (DbUpdateException ex)
-        {
-            // Failover-overlap race: the old singleton incarnation's in-flight fold and the
-            // new node's first fold can both Add the same (series, hour) row; the unique
-            // IX_KpiRollupHourly_Series then faults the loser's ENTIRE SaveChanges — the
-            // failure grain is the PASS, not the row (arch-review 04 round 2, R5). The fold
-            // only ever writes KpiRollupHourly rows, so any save fault here is a lost upsert
-            // race or a transient the next idempotent re-fold repairs identically; classify
-            // at Information instead of surfacing a scary error for a self-healing no-op.
-            _logger.LogInformation(ex,
-                "KPI rollup fold lost a failover-overlap upsert race; pass discarded, next fold self-heals.");
-        }
    }

    /// <inheritdoc />