dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kpi): slice the rollup backfill into bounded 24h folds that yield to periodic folds (plan R2-04 T3)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 09:57:42 -04:00
parent a1c7c4ef26
commit 350d530135
2 changed files with 367 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs
+123 -40
View File
@@ -31,7 +31,12 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
/// series. A one-shot backfill timer additionally folds the whole raw-retention
/// window of already-recorded samples once shortly after start, so 30 d/90 d
/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
/// same idempotent fold and runs at most once per actor lifetime.
/// same idempotent fold and runs at most once per actor lifetime. The backfill is
/// sliced into bounded 24 h fold windows (oldest-first) rather than one unbounded
/// pass, and lowers its in-flight guard between slices so periodic folds interleave
/// — periodic folds and backfill slices strictly alternate through the single
/// <c>_rollupInFlight</c>/<c>_backfillInFlight</c> gate pair, so the two idempotent
/// upserts never race on overlapping rows (arch-review 04 round 2, R1).
/// </summary>
/// <remarks>
/// <para>
@@ -118,11 +123,32 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
private bool _rollupInFlight;
/// <summary>
/// In-flight guard for the one-shot backfill fold. Set true when the backfill fold is
/// launched and cleared when its <see cref="BackfillComplete"/> arrives. While true, periodic
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so the (possibly
/// long) full-retention-window backfill never runs concurrently with a periodic lookback fold
/// on the same upsert rows.
/// Width of one backfill fold slice (24 h). The repository fold materializes its whole
/// window in memory, so the one-shot backfill must never hand it the full raw-retention
/// window (90 d ≈ tens of millions of rows at fleet volume — arch-review 04 round 2, R1).
/// One day per fold bounds each pass to roughly what a steady-state day writes, and the
/// guard is lowered between slices so periodic folds interleave instead of stalling
/// behind the historical pass.
/// </summary>
private static readonly TimeSpan BackfillSliceWidth = TimeSpan.FromHours(24);
/// <summary>
/// Queue of remaining backfill fold slices (each a bounded <see cref="BackfillSliceWidth"/>
/// window), oldest-first. Built once when the backfill plan is enumerated and drained one
/// slice per fold; a periodic <see cref="RollupTick"/> may claim the fold path between slices
/// (the two idempotent upserts strictly alternate through the
/// <see cref="_rollupInFlight"/>/<see cref="_backfillInFlight"/> gate pair, never racing on
/// overlapping rows — arch-review 04 round 2, R1).
/// </summary>
private readonly Queue<(DateTime FromHourUtc, DateTime ToHourUtc)> _backfillSlices = new();
/// <summary>
/// In-flight guard for a single backfill fold slice. Set true when a slice fold is launched
/// and cleared when its <see cref="BackfillSliceComplete"/> arrives. While true, periodic
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so a backfill
/// slice never runs concurrently with a periodic lookback fold on the same upsert rows; it is
/// lowered BETWEEN slices so periodic folds interleave instead of stalling behind the whole
/// historical pass.
/// </summary>
private bool _backfillInFlight;
@@ -160,10 +186,20 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
Receive<RollupTick>(_ => HandleRollupTick());
Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
Receive<BackfillTick>(_ => HandleBackfillTick());
Receive<BackfillComplete>(_ =>
Receive<BackfillSliceComplete>(_ =>
{
_backfillInFlight = false; // lower the in-flight guard (success or fault)
_backfillDone = true; // latch: at most once per actor lifetime
_backfillInFlight = false; // release the fold path BETWEEN slices (R1)
if (_backfillSlices.Count == 0)
{
_backfillDone = true; // latch: at most once per actor lifetime (unchanged semantics)
_logger.LogInformation("KPI rollup backfill completed — all slices folded.");
return;
}
// Next slice via the timer, not inline: a RollupTick already queued in the mailbox
// is processed first, so periodic folds interleave; the existing _rollupInFlight
// deferral branch in HandleBackfillTick then re-arms the backfill behind it.
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
});
}
@@ -483,13 +519,15 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
/// <summary>
/// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
/// has already completed (<see cref="_backfillDone"/>) or is in flight
/// has already completed (<see cref="_backfillDone"/>) or a slice is in flight
/// (<see cref="_backfillInFlight"/>) the tick is ignored. If a periodic fold currently holds
/// the fold path (<see cref="_rollupInFlight"/>) the backfill defers by re-arming its single
/// timer, so the full-window backfill and the periodic lookback fold never race on overlapping
/// upsert rows. Otherwise it raises the backfill guard and folds the whole raw-retention
/// window <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> off the actor
/// thread, piping a <see cref="BackfillComplete"/> back to lower the guard and set the latch.
/// timer, so a backfill slice and the periodic lookback fold never race on overlapping upsert
/// rows. When no slice plan exists yet it enumerates the raw-retention window
/// <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> into bounded
/// <see cref="BackfillSliceWidth"/> windows (oldest-first) and self-arms to start draining;
/// otherwise it dequeues ONE slice, raises the guard, and folds it off the actor thread,
/// piping a <see cref="BackfillSliceComplete"/> back to release the guard between slices.
/// </summary>
private void HandleBackfillTick()
{
@@ -500,38 +538,80 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
if (_rollupInFlight)
{
// A periodic fold is in flight; defer the one-shot backfill briefly so the two
// A periodic fold is in flight; defer the next backfill slice briefly so the two
// idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
return;
}
if (_backfillSlices.Count == 0)
{
// Build the slice plan once: the full raw-retention window in bounded 24 h folds,
// oldest-first. Never hand the repository fold the whole window at once (R1).
var toHourUtc = TruncateToHour(DateTime.UtcNow);
var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
EnqueueBackfillSlices(fromHourUtc, toHourUtc);
if (_backfillSlices.Count == 0)
{
// Degenerate window (no whole hours to fold) — nothing to backfill.
_backfillDone = true;
return;
}
_logger.LogInformation(
"KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) "
+ "({RetentionDays}-day window) across {SliceCount} slice(s) of up to {SliceHours} h.",
fromHourUtc, toHourUtc, _options.RetentionDays, _backfillSlices.Count, BackfillSliceWidth.TotalHours);
// Start draining via the timer (uniform path with the between-slice re-arm).
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
return;
}
// Dequeue and fold exactly ONE bounded slice.
var slice = _backfillSlices.Dequeue();
_backfillInFlight = true;
var toHourUtc = TruncateToHour(DateTime.UtcNow);
var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
_logger.LogInformation(
"KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) ({RetentionDays}-day window).",
fromHourUtc, toHourUtc, _options.RetentionDays);
RunBackfillPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
RunBackfillPass(slice.FromHourUtc, slice.ToHourUtc, cancellationToken).PipeTo(
Self,
success: () => BackfillComplete.Instance,
success: () => BackfillSliceComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "KPI rollup backfill faulted unexpectedly.");
return BackfillComplete.Instance;
_logger.LogError(ex, "KPI rollup backfill slice faulted unexpectedly.");
return BackfillSliceComplete.Instance;
});
}
/// <summary>
/// Runs the one-shot backfill fold: opens a DI scope, resolves the repository, and folds the
/// Enumerates <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
/// contiguous <see cref="BackfillSliceWidth"/> windows (the last possibly shorter),
/// oldest-first, and enqueues them onto <see cref="_backfillSlices"/>.
/// </summary>
private void EnqueueBackfillSlices(DateTime fromHourUtc, DateTime toHourUtc)
{
var sliceStart = fromHourUtc;
while (sliceStart < toHourUtc)
{
var sliceEnd = sliceStart + BackfillSliceWidth;
if (sliceEnd > toHourUtc)
{
sliceEnd = toHourUtc;
}
_backfillSlices.Enqueue((sliceStart, sliceEnd));
sliceStart = sliceEnd;
}
}
/// <summary>
/// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
/// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
/// the hourly rollup table via the same idempotent upsert the periodic fold uses (Task 3). The
/// whole body is wrapped so the returned task never faults — best-effort observability must
/// never disrupt startup. The fold method returns no count, so completion is logged with the
/// window and elapsed wall time only.
/// the hourly rollup table via the same idempotent upsert the periodic fold uses. The whole
/// body is wrapped so the returned task never faults — best-effort observability must never
/// disrupt startup. The fold method returns no count, so each slice is logged at Debug with its
/// window and elapsed wall time (the queue-drained completion is logged at Information).
/// </summary>
private async Task RunBackfillPass(
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
@@ -543,8 +623,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
_logger.LogInformation(
"KPI rollup backfill completed — folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
_logger.LogDebug(
"KPI rollup backfill slice folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -614,9 +694,11 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Self-tick triggering the one-shot backfill fold over the full raw-retention window. Armed
/// once via a single timer in <see cref="PreStart"/> (and re-armed only to defer past an
/// in-flight periodic fold); runs at most once per actor lifetime.
/// Self-tick triggering the one-shot backfill: on the first tick it enumerates the full
/// raw-retention window into bounded slices, and each subsequent tick folds one slice. Armed
/// once via a single timer in <see cref="PreStart"/> (and re-armed to drain the next slice or
/// to defer past an in-flight periodic fold); the whole pass runs at most once per actor
/// lifetime.
/// </summary>
internal sealed class BackfillTick
{
@@ -625,13 +707,14 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Piped-back completion of the one-shot backfill fold; lets the fold run off the actor thread
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard and latches
/// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard BETWEEN slices — then
/// either drains the next slice (self-armed timer) or, when the queue is empty, latches
/// <c>_backfillDone</c> (fires on success and fault).
/// </summary>
internal sealed class BackfillComplete
internal sealed class BackfillSliceComplete
{
public static readonly BackfillComplete Instance = new();
private BackfillComplete() { }
public static readonly BackfillSliceComplete Instance = new();
private BackfillSliceComplete() { }
}
}
tests/ZB.MOM.WW.ScadaBridge.KpiHistory.Tests/KpiHistoryRecorderActorTests.cs
+244 -35
View File
@@ -111,6 +111,7 @@ public class KpiHistoryRecorderActorTests : TestKit
private DateTime? _foldFromHourUtc;
private DateTime? _foldToHourUtc;
private int _foldCount;
private readonly List<(DateTime FromHourUtc, DateTime ToHourUtc)> _foldWindows = new();
public IReadOnlyList<KpiSample> Recorded
{
@@ -148,6 +149,12 @@ public class KpiHistoryRecorderActorTests : TestKit
get { lock (_gate) { return _foldCount; } }
}
/// <summary>Every fold window handed to <see cref="FoldHourlyRollupsAsync"/>, in call order.</summary>
public IReadOnlyList<(DateTime FromHourUtc, DateTime ToHourUtc)> FoldWindows
{
get { lock (_gate) { return _foldWindows.ToArray(); } }
}
public Task RecordSamplesAsync(
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default)
{
@@ -176,6 +183,7 @@ public class KpiHistoryRecorderActorTests : TestKit
{
_foldFromHourUtc = fromHourUtc;
_foldToHourUtc = toHourUtc;
_foldWindows.Add((fromHourUtc, toHourUtc));
_foldCount++;
}
return Task.CompletedTask;
@@ -354,6 +362,89 @@ public class KpiHistoryRecorderActorTests : TestKit
Task.FromResult<DateTime?>(null);
}
/// <summary>
/// Repository fake for the backfill-slice interleaving test. Classifies each fold as a backfill
/// slice (24 h window) or a periodic fold (the distinct <c>RollupLookbackHours</c>-wide window)
/// by window width. Backfill slices complete immediately (counted); the FIRST periodic fold
/// blocks in flight until released, so the test can observe that a periodic fold claimed the
/// fold path between backfill slices while backfill slices remained.
/// </summary>
private sealed class InterleaveFoldRepository : IKpiHistoryRepository
{
private readonly TimeSpan _periodicWindowWidth;
private readonly object _gate = new();
private readonly SemaphoreSlim _firstPeriodicRelease = new(initialCount: 0);
private int _backfillFoldCount;
private int _periodicFoldCount;
private int _backfillCountAtFirstPeriodic = -1;
public InterleaveFoldRepository(TimeSpan periodicWindowWidth) =>
_periodicWindowWidth = periodicWindowWidth;
/// <summary>Number of backfill-slice folds (24 h windows) entered so far.</summary>
public int BackfillFoldCount { get { lock (_gate) { return _backfillFoldCount; } } }
/// <summary>Number of periodic folds (the lookback-width window) entered so far.</summary>
public int PeriodicFoldCount { get { lock (_gate) { return _periodicFoldCount; } } }
/// <summary>Backfill-slice count observed at the instant the first periodic fold entered.</summary>
public int BackfillCountAtFirstPeriodic { get { lock (_gate) { return _backfillCountAtFirstPeriodic; } } }
/// <summary>Releases the first (held-in-flight) periodic fold so the deferred backfill resumes.</summary>
public void ReleaseFirstPeriodic() => _firstPeriodicRelease.Release();
public Task RecordSamplesAsync(
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<IReadOnlyList<KpiSeriesPoint>> GetRawSeriesAsync(
string source, string metric, string scope, string? scopeKey,
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
public Task<int> PurgeOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task FoldHourlyRollupsAsync(
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken = default)
{
var isPeriodic = (toHourUtc - fromHourUtc) == _periodicWindowWidth;
if (isPeriodic)
{
bool blockThis;
lock (_gate)
{
_periodicFoldCount++;
blockThis = _periodicFoldCount == 1;
if (blockThis)
{
_backfillCountAtFirstPeriodic = _backfillFoldCount;
}
}
// Hold the first periodic fold in flight (on a threadpool thread — PipeTo runs the
// pass off the actor thread) so the between-slice interleave state is observable.
return blockThis
? Task.Run(() => _firstPeriodicRelease.Wait(cancellationToken), cancellationToken)
: Task.CompletedTask;
}
lock (_gate) { _backfillFoldCount++; }
return Task.CompletedTask;
}
public Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
string source, string metric, string scope, string? scopeKey,
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<DateTime?>(null);
}
private IServiceProvider BuildServiceProvider(
IKpiHistoryRepository repository, params IKpiSampleSource[] sources)
{
@@ -683,13 +774,15 @@ public class KpiHistoryRecorderActorTests : TestKit
public void BackfillTick_FoldsFullRetentionWindow_Once()
{
// The one-shot backfill must fold the WHOLE raw-retention window once, so long-range
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy: the fold's
// upper bound is the current hour start (in-progress hour excluded) and its lower bound is
// RetentionDays earlier. A second BackfillTick to the same actor must be a no-op (the
// _backfillDone latch makes it run at most once per actor lifetime).
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy. It now
// does so in bounded <=24 h slices (R1) rather than a single unbounded fold: the union
// of the slice windows must cover exactly [truncate(now) − RetentionDays, truncate(now)),
// each slice must be <=24 h, and the slices must be contiguous and oldest-first. A second
// BackfillTick to the same actor must be a no-op (the _backfillDone latch makes the whole
// pass run at most once per actor lifetime).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
const int retentionDays = 90;
const int retentionDays = 5;
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
@@ -703,52 +796,168 @@ public class KpiHistoryRecorderActorTests : TestKit
AwaitAssert(
() =>
{
Assert.Equal(1, repository.FoldCount);
Assert.NotNull(repository.FoldToHourUtc);
Assert.NotNull(repository.FoldFromHourUtc);
var toHour = repository.FoldToHourUtc!.Value;
var fromHour = repository.FoldFromHourUtc!.Value;
// toHour is the truncated current hour (in-progress hour excluded).
Assert.Equal(DateTimeKind.Utc, toHour.Kind);
Assert.Equal(0, toHour.Minute);
Assert.Equal(0, toHour.Second);
Assert.Equal(0, toHour.Millisecond);
var windows = repository.FoldWindows;
// RetentionDays = 5 → exactly 5 contiguous 24 h slices.
Assert.Equal(retentionDays, windows.Count);
var expectedToHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
// Within one hour of "now truncated" (guards a rare hour-boundary crossing mid-test).
Assert.True(
Math.Abs((toHour - expectedToHour).TotalHours) <= 1.0,
$"backfill toHour {toHour:o} should be the current hour start {expectedToHour:o}");
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
// fromHour is exactly RetentionDays (the full raw-retention window) before toHour.
Assert.Equal(toHour - TimeSpan.FromDays(retentionDays), fromHour);
// Oldest-first, contiguous, each <=24 h, union == full retention window.
Assert.Equal(expectedFromHour, windows[0].FromHourUtc);
Assert.Equal(expectedToHour, windows[^1].ToHourUtc);
for (var i = 0; i < windows.Count; i++)
{
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
$"slice {i} width {windows[i].ToHourUtc - windows[i].FromHourUtc} exceeds 24 h");
if (i > 0)
{
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
}
}
},
duration: TimeSpan.FromSeconds(3),
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
// Second BackfillTick to the SAME actor: the once-only latch must suppress it, so the
// fold count stays at 1. (No periodic RollupTick is sent, and the periodic/backfill
// auto-timers don't fire within this sub-10 s window, so FoldCount is driven solely by
// the two backfill ticks under test.)
// Second BackfillTick to the SAME actor: the once-only latch must suppress the whole pass,
// so no further slices are folded. (No periodic RollupTick is sent, and the periodic/backfill
// auto-timers don't fire within this window, so the fold windows are driven solely by the
// backfill under test.)
var foldedCount = repository.FoldWindows.Count;
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
Thread.Sleep(300);
Assert.Equal(1, repository.FoldCount);
Assert.Equal(foldedCount, repository.FoldWindows.Count);
}
[Fact]
public void Backfill_SlicesRetentionWindow_IntoBoundedDayFolds_OldestFirst()
{
// RetentionDays = 3 → 3 fold calls, each window <=24 h, contiguous, oldest-first, whose
// union is exactly [TruncateToHour(now) − 3 d, TruncateToHour(now)). Pre-R1 a single fold
// call spanned the whole 3-day window (arch-review 04 round 2, R1).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
const int retentionDays = 3;
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = retentionDays,
});
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
AwaitAssert(
() =>
{
var windows = repository.FoldWindows;
Assert.Equal(retentionDays, windows.Count); // 3 × 24 h slices
var expectedToHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
Assert.Equal(expectedFromHour, windows[0].FromHourUtc); // oldest first
Assert.Equal(expectedToHour, windows[^1].ToHourUtc); // newest last
for (var i = 0; i < windows.Count; i++)
{
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
$"slice {i} exceeds 24 h");
if (i > 0)
{
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
}
}
},
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
}
[Fact]
public async Task PeriodicRollupTick_Interleaves_BetweenBackfillSlices()
{
// A periodic fold must be able to claim the fold path BETWEEN backfill slices: while the
// historical backfill runs slice-by-slice, a RollupTick that lands in a between-slice gap
// runs its (recent-window) fold and the remaining backfill slices defer behind it, then
// ALL backfill slices still complete afterwards. Pre-R1 _backfillInFlight blocked every
// periodic fold for the whole (single-fold) pass, so no interleaving was possible
// (arch-review 04 round 2, R1).
const int retentionDays = 8; // 8 backfill slices of 24 h
const int lookbackHours = 2; // periodic fold window width — distinct from the 24 h slices
var repository = new InterleaveFoldRepository(TimeSpan.FromHours(lookbackHours));
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(24), // periodic auto-timer won't fire during the test
RetentionDays = retentionDays,
RollupLookbackHours = lookbackHours,
});
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
// Continuously offer periodic ticks; one will win a between-slice gap and interleave.
using var stopFlood = new CancellationTokenSource();
var flood = Task.Run(() =>
{
while (!stopFlood.IsCancellationRequested)
{
actor.Tell(KpiHistoryRecorderActor.RollupTick.Instance);
Thread.Sleep(1);
}
});
try
{
// A periodic fold interleaved BEFORE the backfill finished all its slices.
AwaitAssert(
() =>
{
Assert.True(repository.PeriodicFoldCount >= 1, "no periodic fold interleaved");
Assert.True(
repository.BackfillCountAtFirstPeriodic < retentionDays,
"the periodic fold did not interleave before the backfill completed");
},
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(20));
}
finally
{
stopFlood.Cancel();
await flood;
}
// The first periodic fold is held in flight, deferring the remaining backfill slices;
// release it so the deferred backfill resumes and ALL slices still complete.
repository.ReleaseFirstPeriodic();
AwaitAssert(
() => Assert.Equal(retentionDays, repository.BackfillFoldCount),
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(50));
}
[Fact]
public void BackfillInFlight_SkipsPeriodicRollupTick()
{
// While the (possibly long) full-window backfill fold is in flight, a periodic RollupTick
// must be skipped so the two idempotent upserts never race on overlapping rows. With a
// gated fold holding the backfill open, a RollupTick must NOT enter a second fold — the
// fold count stays at 1 until the gate is released.
// While a backfill slice fold is in flight, a periodic RollupTick must be skipped so the
// two idempotent upserts never race on overlapping rows (unchanged semantics — a RollupTick
// arriving while a *slice* is in flight is still skipped). With a gated fold holding the
// single backfill slice open, a RollupTick must NOT enter a second fold — the fold count
// stays at 1 until the gate is released. RetentionDays = 1 → exactly one 24 h backfill slice.
var repository = new GatedFoldRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp);
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = 1,
});
// Backfill tick: raises the backfill guard, enters the (gated, blocking) fold — held in flight.
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
@@ -765,7 +974,7 @@ public class KpiHistoryRecorderActorTests : TestKit
// Release the gate so the backfill completes and its guard is lowered; a fresh periodic
// tick must now run a new fold (guards correctly reset, not stuck). Re-send on each poll so
// we don't race the asynchronous BackfillComplete that lowers the guard.
// we don't race the asynchronous BackfillSliceComplete that lowers the guard.
repository.Release();
AwaitAssert(
() =>