using Akka.Actor;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Kpi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
///
/// Central cluster-singleton ("KPI History & Trends") that drives the
/// KPI sampling backbone. On a periodic timer (default 60 s) it:
///
/// - Stamps a single capturedAtUtc for the pass.
/// - Fans out to every DI-registered ,
/// collecting each source's point-in-time samples.
/// - Bulk-writes the combined batch through
/// .
///
/// A separate daily timer (default 1 d) runs the retention purge, dropping raw
/// sample rows older than via
/// and hourly rollup rows
/// older than the longer via
/// . A third hourly
/// timer (default 1 h) folds the trailing
/// hours of raw samples into the
/// KpiRollupHourly table via
/// (idempotent upsert,
/// in-progress hour excluded), so long-range trend charts read a pre-thinned
/// series. A one-shot backfill timer additionally folds the whole raw-retention
/// window of already-recorded samples once shortly after start, so 30 d/90 d
/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
/// same idempotent fold and runs at most once per actor lifetime. The backfill is
/// sliced into bounded 24 h fold windows (oldest-first) rather than one unbounded
/// pass, and lowers its in-flight guard between slices so periodic folds interleave
/// — periodic folds and backfill slices strictly alternate through the single
/// _rollupInFlight/_backfillInFlight gate pair, so the two idempotent
/// upserts never race on overlapping rows (arch-review 04 round 2, R1).
///
///
///
/// Best-effort. KPI history is observability — it must NEVER disrupt the
/// system it observes. Every fault is contained: a throwing
/// is caught per-source so it neither aborts the
/// pass nor suppresses the other sources; a throwing repository write/purge is
/// caught and logged. No exception escapes either tick handler, so the singleton
/// stays alive across transient DB outages and misbehaving sources.
///
///
/// Off-thread work. The actual sampling and purge I/O runs off the actor
/// thread; the result is piped back to via
///
/// so the mailbox is never blocked while a DB round-trip is in flight. This
/// mirrors the
/// timer + scope-per-tick + PipeTo pattern.
///
///
/// DI scopes. s and
/// are scoped EF Core-backed services; the
/// recorder opens a fresh for the tick's repository
/// and a separate scope for each source collection. A timed-out collection owns
/// its scope until actual completion, while later sources use separate DbContexts.
/// Scopes are asynchronously disposed (CreateAsyncScope).
///
///
/// Singleton wiring. The Props is built in the Host on the
/// active central node with the constructor args resolved from DI — this actor
/// has no knowledge of Akka.Cluster.Tools, mirroring the Notification
/// Outbox singleton split.
///
///
public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
{
private const string SampleTimerKey = "kpi-sample";
private const string PurgeTimerKey = "kpi-purge";
private const string RollupTimerKey = "kpi-rollup";
private const string BackfillTimerKey = "kpi-rollup-backfill";
///
/// Initial delay before the one-shot rollup backfill fires (see
/// ). Deliberately a touch longer than the periodic rollup
/// timer's own 10 s initial delay so that, when both are armed at start, the periodic
/// tick claims the fold path first and the backfill simply defers (the two folds are
/// idempotent upserts, but they must not race on overlapping rows). Also comfortably beyond
/// the sub-second/second-scale AwaitAssert windows used by the unit tests, so the
/// one-shot never auto-fires mid-test — tests drive by hand.
///
private static readonly TimeSpan BackfillInitialDelay = TimeSpan.FromSeconds(15);
// Shared by every recorder incarnation on this root provider.
private readonly KpiSourceAdmission _sourceAdmission;
private readonly IServiceProvider _serviceProvider;
private readonly KpiHistoryOptions _options;
private readonly ILogger _logger;
///
/// Lifecycle-scoped cancellation source, cancelled in so any
/// in-flight sample/purge pass observes a coordinated shutdown / failover promptly
/// instead of blocking the singleton handover for a full DB round-trip timeout.
///
private CancellationTokenSource? _shutdownCts;
///
/// In-flight guard for the sample loop. Set true at the start of a sample pass and cleared
/// when the pass's arrives. While true, further
/// s are skipped so passes never overlap — Akka periodic timers
/// enqueue (not coalesce) missed ticks, so without this guard a pass running longer than
/// (slow/recovering DB) would let each
/// subsequent tick spawn another concurrent pass, amplifying load on the struggling store
/// and double-writing samples for overlapping windows. Mirrors the
/// dispatch
/// in-flight guard the timer pattern is modelled on.
///
private bool _sampleInFlight;
///
/// In-flight guard for the hourly rollup fold. Set true at the start of a fold pass and
/// cleared when the pass's arrives. While true, further
/// s are skipped so folds never overlap — the fold is a
/// group-by-and-upsert over a lookback window that can outlast the (default 1 h)
/// on a slow/recovering store; without this
/// guard a subsequent tick would launch a second concurrent fold contending for the same
/// upsert key. Mirrors .
///
private bool _rollupInFlight;
///
/// Width of one backfill fold slice (24 h). The repository fold materializes its whole
/// window in memory, so the one-shot backfill must never hand it the full raw-retention
/// window (90 d ≈ tens of millions of rows at fleet volume — arch-review 04 round 2, R1).
/// One day per fold bounds each pass to roughly what a steady-state day writes, and the
/// guard is lowered between slices so periodic folds interleave instead of stalling
/// behind the historical pass.
///
private static readonly TimeSpan BackfillSliceWidth = TimeSpan.FromHours(24);
///
/// Queue of remaining backfill fold slices (each a bounded
/// window), oldest-first. Built once when the backfill plan is enumerated and drained one
/// slice per fold; a periodic may claim the fold path between slices
/// (the two idempotent upserts strictly alternate through the
/// / gate pair, never racing on
/// overlapping rows — arch-review 04 round 2, R1).
///
private readonly Queue<(DateTime FromHourUtc, DateTime ToHourUtc)> _backfillSlices = new();
///
/// In-flight guard for a single backfill fold slice. Set true when a slice fold is launched
/// and cleared when its arrives. While true, periodic
/// s are skipped (like ) so a backfill
/// slice never runs concurrently with a periodic lookback fold on the same upsert rows; it is
/// lowered BETWEEN slices so periodic folds interleave instead of stalling behind the whole
/// historical pass.
///
private bool _backfillInFlight;
///
/// Once-only latch for the backfill. Set true when the backfill fold completes (success or
/// fault) so it runs at most once per actor lifetime — a fresh singleton on failover starts a
/// new instance and re-runs the (idempotent) backfill, which is the intended self-heal. Set on
/// completion rather than launch so that if the actor is torn down mid-backfill the next
/// lifetime still attempts it.
///
private bool _backfillDone;
/// Akka timer scheduler, assigned by the actor system via .
public ITimerScheduler Timers { get; set; } = null!;
///
/// Initializes the recorder with its dependencies and registers the tick handlers.
///
/// DI service provider used to open a scope per tick for the sample sources and repository.
/// KPI history configuration options.
/// Logger for this actor.
public KpiHistoryRecorderActor(
IServiceProvider serviceProvider,
KpiHistoryOptions options,
ILogger logger)
{
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_sourceAdmission = serviceProvider.GetRequiredService();
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
Receive(_ => HandleSampleTick());
Receive(_ => _sampleInFlight = false); // lower the in-flight guard (success or fault)
Receive(_ => HandlePurgeTick());
Receive(_ => { }); // best-effort: no actor state to reset on completion
Receive(_ => HandleRollupTick());
Receive(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
Receive(_ => HandleBackfillTick());
Receive(HandleBackfillPlan);
Receive(_ =>
{
_backfillInFlight = false; // release the fold path BETWEEN slices (R1)
if (_backfillSlices.Count == 0)
{
_backfillDone = true; // latch: at most once per actor lifetime (unchanged semantics)
_logger.LogInformation("KPI rollup backfill completed — all slices folded.");
return;
}
// Next slice via the timer, not inline: a RollupTick already queued in the mailbox
// is processed first, so periodic folds interleave; the existing _rollupInFlight
// deferral branch in HandleBackfillTick then re-arms the backfill behind it.
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
});
}
///
protected override void PreStart()
{
base.PreStart();
_shutdownCts = new CancellationTokenSource();
// Fire an initial sample shortly after start so the first trend point lands
// without waiting a full SampleInterval, then settle into the periodic cadence.
Timers.StartPeriodicTimer(
SampleTimerKey,
SampleTick.Instance,
initialDelay: TimeSpan.FromSeconds(5),
interval: _options.SampleInterval);
// The purge is idempotent, so it takes a short first tick (min(PurgeInterval, 5min)):
// scheduling the first sweep a full PurgeInterval out meant a node recycled more often
// than the interval (e.g. daily restart vs. daily cadence) would never purge
// (arch-review 04, S9). Fast test cadences (interval < 5min) are unchanged.
Timers.StartPeriodicTimer(
PurgeTimerKey,
PurgeTick.Instance,
initialDelay: PurgeTimerSchedule.InitialDelay(_options.PurgeInterval),
interval: _options.PurgeInterval);
// Fold the trailing hours into the hourly rollup table on a short initial delay
// (so a fresh active node catches up promptly after a failover), then settle into
// the periodic cadence. The fold re-processes a lookback window via an idempotent
// upsert, so a missed tick self-heals on the next fold.
Timers.StartPeriodicTimer(
RollupTimerKey,
RollupTick.Instance,
initialDelay: TimeSpan.FromSeconds(10),
interval: _options.RollupInterval);
// One-shot backfill: fold the whole raw-retention window of existing samples into the
// hourly rollup table once shortly after start, so long-range (30 d/90 d) trend charts
// aren't blank until enough wall-clock passes after deploy. The periodic tick only folds
// the trailing RollupLookbackHours, so without this the deep history stays unrolled. The
// fold is an idempotent upsert, so re-running on a failover (a fresh singleton re-arms
// this timer) is safe.
Timers.StartSingleTimer(
BackfillTimerKey,
BackfillTick.Instance,
timeout: BackfillInitialDelay);
}
///
protected override void PostStop()
{
// Cancel before disposing so an in-flight pass observes cancellation; disposing
// first would race with a pass registering against the token.
try
{
_shutdownCts?.Cancel();
}
catch (ObjectDisposedException)
{
// Already disposed under a restarted-actor race; nothing to do.
}
_shutdownCts?.Dispose();
_shutdownCts = null;
base.PostStop();
}
///
/// Handles a sample tick. If a sample pass is already in flight the tick is skipped
/// (logged at debug) so passes never overlap; otherwise the in-flight guard is raised,
/// the shared capturedAtUtc instant is captured on the actor thread, and the
/// asynchronous sampling pass is launched off-thread with a
/// piped back to to lower the guard on the actor thread — so the
/// mailbox is never blocked while sources are collected and the batch is written.
///
private void HandleSampleTick()
{
if (_sampleInFlight)
{
// A prior pass is still awaiting its DB round-trip; coalesce this tick rather than
// piling a second concurrent pass onto a slow/recovering store.
_logger.LogDebug("KPI sample tick skipped — a sample pass is already in flight.");
return;
}
_sampleInFlight = true;
var capturedAt = DateTime.UtcNow;
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
// RunSamplePass self-isolates its faults (it never throws), but the failure
// projection is kept as a belt-and-braces guard so even a faulted task still
// produces a SampleComplete that lowers the in-flight guard — otherwise the loop
// would wedge permanently.
RunSamplePass(capturedAt, cancellationToken).PipeTo(
Self,
success: () => SampleComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "KPI sample pass faulted unexpectedly.");
return SampleComplete.Instance;
});
}
///
/// Runs a single sampling pass: enumerates every registered ,
/// waits a bounded time for each separately scoped collection, and bulk-writes the combined batch. The whole body is wrapped so the
/// returned task never faults — best-effort observability must never disrupt anything.
///
private async Task RunSamplePass(DateTime capturedAt, CancellationToken cancellationToken)
{
try
{
await using var scope = _serviceProvider.CreateAsyncScope();
var sources = scope.ServiceProvider.GetServices().ToArray();
var repository = scope.ServiceProvider.GetRequiredService();
var samples = new List();
for (var sourceIndex = 0; sourceIndex < sources.Length; sourceIndex++)
{
var sourceName = sources[sourceIndex].Source;
// Each call owns a separate scope, so a cancellation-ignoring source
// neither shares a busy DbContext with healthy sources nor loses its scope.
var index = sourceIndex;
if (!_sourceAdmission.TryStart(index,
() => CollectSourceAsync(index, capturedAt, cancellationToken), out var collection))
continue;
_ = collection.ContinueWith(t =>
_logger.LogError(t.Exception, "KPI source {Source} collection failed.", sourceName),
CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
try
{
var collected = await collection.WaitAsync(_options.SourceTimeout, cancellationToken);
samples.AddRange(collected);
}
catch (TimeoutException)
{
_logger.LogWarning("KPI source {Source} exceeded its collection deadline; unfinished work remains owned and will not be restarted.", sourceName);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted collection; abandon the rest of the pass. The next
// active node samples on its own cadence. Not a failure.
return;
}
catch (Exception ex)
{
// A throwing source must NOT abort the pass or the other sources.
_logger.LogError(ex, "KPI source {Source} failed to collect samples.", sourceName);
}
}
if (samples.Count == 0)
{
return;
}
try
{
await repository.RecordSamplesAsync(samples, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the write; the batch is dropped (best-effort). Not a failure.
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI sample write failed for {SampleCount} sample(s).", samples.Count);
}
}
catch (Exception ex)
{
// Scope creation or service resolution faulted; swallow and log so the returned
// task completes normally and the singleton stays alive.
_logger.LogError(ex, "KPI sample pass failed unexpectedly.");
}
}
/// Owns a source's scope and cancellation source until its actual collection completes, even after the caller stops waiting.
private async Task> CollectSourceAsync(
int sourceIndex, DateTime capturedAt, CancellationToken cancellationToken)
{
await using var scope = _serviceProvider.CreateAsyncScope();
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
deadline.CancelAfter(_options.SourceTimeout);
var source = scope.ServiceProvider.GetServices().ElementAt(sourceIndex);
return await source.CollectAsync(capturedAt, deadline.Token).ConfigureAwait(false);
}
///
/// Handles a purge tick: computes both retention cut-offs on the actor thread (raw samples
/// at , hourly rollups at the longer
/// ), then runs both bulk deletes
/// off-thread and pipes a completion back to . Purges are daily and
/// idempotent, so no in-flight guard is needed.
///
private void HandlePurgeTick()
{
var now = DateTime.UtcNow;
var sampleBefore = now - TimeSpan.FromDays(_options.RetentionDays);
var rollupBefore = now - TimeSpan.FromDays(_options.RollupRetentionDays);
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
RunPurgePass(sampleBefore, rollupBefore, cancellationToken).PipeTo(
Self,
success: deleted =>
{
if (deleted > 0)
{
_logger.LogInformation(
"KPI history purge removed {DeletedCount} sample(s) older than {Cutoff:o}.",
deleted, sampleBefore);
}
return PurgeComplete.Instance;
},
failure: ex =>
{
_logger.LogError(ex, "KPI history purge faulted unexpectedly.");
return PurgeComplete.Instance;
});
}
///
/// Runs a single purge sweep: opens a DI scope, resolves the repository, and bulk-deletes
/// expired raw samples (before ) and expired hourly rollups
/// (before ), returning the deleted raw-sample count. The two
/// deletes are isolated so a failure of one never skips the other, and the whole body is
/// wrapped so the returned task never faults — on failure the exception is logged and 0 is
/// returned, mirroring 's best-effort contract.
///
private async Task RunPurgePass(
DateTime sampleBefore, DateTime rollupBefore, CancellationToken cancellationToken)
{
try
{
await using var scope = _serviceProvider.CreateAsyncScope();
var repository = scope.ServiceProvider.GetRequiredService();
var deleted = 0;
// Raw-sample purge — isolated so a fault here does not skip the rollup purge below.
try
{
deleted = await repository.PurgeOlderThanAsync(sampleBefore, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the purge; the next active sweep retries. Not a failure.
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI history raw-sample purge failed unexpectedly.");
}
// Rollup purge — retained longer than raw; isolated so its failure does not affect
// the raw-purge outcome already reported.
try
{
await repository.PurgeRollupsOlderThanAsync(rollupBefore, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the rollup purge; the next active sweep retries. Not a failure.
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI history rollup purge failed unexpectedly.");
}
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI history purge failed unexpectedly.");
return 0;
}
}
///
/// Handles a rollup tick. If a fold is already in flight the tick is skipped (logged at
/// debug) so folds never overlap; otherwise the in-flight guard is raised and the fold is
/// launched off-thread with a piped back to
/// to lower the guard on the actor thread. The fold window's upper bound is the current
/// hour start (exclusive) so the in-progress hour is never folded, and its lower bound is
/// earlier so a missed/failover tick
/// self-heals via the idempotent upsert.
///
private void HandleRollupTick()
{
if (_rollupInFlight || _backfillInFlight)
{
// Skip while either a prior periodic fold OR the one-shot full-window backfill is
// running, so two idempotent upserts never race on overlapping rows. A skipped
// periodic tick self-heals on the next fold (it re-processes the lookback window).
_logger.LogDebug("KPI rollup tick skipped — a rollup fold is already in flight.");
return;
}
_rollupInFlight = true;
var toHourUtc = TruncateToHour(DateTime.UtcNow);
var fromHourUtc = toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours);
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
RunRollupPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
Self,
success: () => RollupComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "KPI rollup fold faulted unexpectedly.");
return RollupComplete.Instance;
});
}
///
/// Runs a single rollup fold: opens a DI scope, resolves the repository, and folds the
/// complete hours in [, )
/// into the hourly rollup table via the idempotent upsert. The whole body is wrapped so the
/// returned task never faults — best-effort observability must never disrupt anything.
///
private async Task RunRollupPass(
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
{
try
{
await using var scope = _serviceProvider.CreateAsyncScope();
var repository = scope.ServiceProvider.GetRequiredService();
await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the fold; the next active node re-folds the lookback window. Not a failure.
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI rollup fold failed unexpectedly.");
}
}
///
/// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
/// has already completed () or a slice is in flight
/// () the tick is ignored. If a periodic fold currently holds
/// the fold path () the backfill defers by re-arming its single
/// timer, so a backfill slice and the periodic lookback fold never race on overlapping upsert
/// rows. When no slice plan exists yet it enumerates the raw-retention window
/// [TruncateToHour(now) − RetentionDays, TruncateToHour(now)) into bounded
/// windows (oldest-first) and self-arms to start draining;
/// otherwise it dequeues ONE slice, raises the guard, and folds it off the actor thread,
/// piping a back to release the guard between slices.
///
private void HandleBackfillTick()
{
if (_backfillDone || _backfillInFlight)
{
return;
}
if (_rollupInFlight)
{
// A periodic fold is in flight; defer the next backfill slice briefly so the two
// idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
return;
}
if (_backfillSlices.Count == 0)
{
// No slice plan yet: resolve the rollup watermark off the actor thread, then plan on
// the actor thread (Receive). Raise the guard so periodic ticks defer
// while planning is in flight; the plan handler lowers it. On a failover restart the
// watermark lets the plan SKIP or SHRINK to the un-rolled tail instead of re-folding
// the whole raw-retention window (arch-review 04 round 2, R1).
_backfillInFlight = true;
var planCancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
RunBackfillPlanPass(planCancellationToken).PipeTo(
Self,
success: watermark => new BackfillPlan(watermark),
failure: ex =>
{
_logger.LogError(ex, "KPI rollup backfill plan pass faulted unexpectedly.");
return new BackfillPlan(null); // null watermark ⇒ plan the full retention window
});
return;
}
// Dequeue and fold exactly ONE bounded slice.
var slice = _backfillSlices.Dequeue();
_backfillInFlight = true;
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
RunBackfillPass(slice.FromHourUtc, slice.ToHourUtc, cancellationToken).PipeTo(
Self,
success: () => BackfillSliceComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "KPI rollup backfill slice faulted unexpectedly.");
return BackfillSliceComplete.Instance;
});
}
///
/// Enumerates [, ) into
/// contiguous windows (the last possibly shorter),
/// oldest-first, and enqueues them onto .
///
private void EnqueueBackfillSlices(DateTime fromHourUtc, DateTime toHourUtc)
{
var sliceStart = fromHourUtc;
while (sliceStart < toHourUtc)
{
var sliceEnd = sliceStart + BackfillSliceWidth;
if (sliceEnd > toHourUtc)
{
sliceEnd = toHourUtc;
}
_backfillSlices.Enqueue((sliceStart, sliceEnd));
sliceStart = sliceEnd;
}
}
///
/// Plans the backfill from the resolved rollup watermark (actor thread). The window floor is
/// the newer of the watermark (re-folding the watermark hour itself is a cheap idempotent
/// safety margin) and the raw-retention floor; a null watermark (fresh install) keeps the full
/// retention window. If the floor is already within
/// of now the periodic fold's lookback
/// covers everything from there forward, so the whole historical pass is skipped (the failover
/// fast-path); otherwise the tail [floor, TruncateToHour(now)) is enumerated into
/// bounded slices and draining is self-armed. Either way the plan-pass guard is lowered here
/// (arch-review 04 round 2, R1).
///
private void HandleBackfillPlan(BackfillPlan plan)
{
var toHourUtc = TruncateToHour(DateTime.UtcNow);
var retentionFloor = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
// Shrink to the un-rolled tail; re-fold the watermark hour itself (idempotent) as a safety
// margin. A null watermark (fresh install) keeps the full retention window.
var floor = plan.Watermark is { } w && w > retentionFloor ? w : retentionFloor;
if (floor >= toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours))
{
// Failover fast-path: rollups are already current — the periodic fold's lookback
// window covers everything from the watermark forward, so the full historical pass is
// skipped entirely (arch-review 04 round 2, R1).
_backfillInFlight = false;
_backfillDone = true;
_logger.LogInformation(
"KPI rollup backfill skipped — rollups current through {Watermark:o}.", plan.Watermark);
return;
}
EnqueueBackfillSlices(floor, toHourUtc);
_backfillInFlight = false;
if (_backfillSlices.Count == 0)
{
// Degenerate window (no whole hours to fold) — nothing to backfill.
_backfillDone = true;
return;
}
_logger.LogInformation(
"KPI rollup backfill starting — folding [{FromHour:o}, {ToHour:o}) across {SliceCount} "
+ "slice(s) of up to {SliceHours} h (watermark {Watermark:o}).",
floor, toHourUtc, _backfillSlices.Count, BackfillSliceWidth.TotalHours, plan.Watermark);
// Start draining via the timer (uniform path with the between-slice re-arm).
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
}
///
/// Resolves the newest folded rollup hour (the backfill watermark) off the actor thread in a
/// fresh DI scope. Never faults — on cancellation or error it returns null, which the
/// plan treats as "no rollups", keeping the full retention window (best-effort, mirrors the
/// other passes).
///
private async Task RunBackfillPlanPass(CancellationToken cancellationToken)
{
try
{
await using var scope = _serviceProvider.CreateAsyncScope();
var repository = scope.ServiceProvider.GetRequiredService();
return await repository.GetLatestRollupHourAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the lookup; a fresh active node re-plans. Not a failure.
return null;
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI rollup backfill watermark lookup failed; planning the full retention window.");
return null;
}
}
///
/// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
/// complete hours in [, ) into
/// the hourly rollup table via the same idempotent upsert the periodic fold uses. The whole
/// body is wrapped so the returned task never faults — best-effort observability must never
/// disrupt startup. The fold method returns no count, so each slice is logged at Debug with its
/// window and elapsed wall time (the queue-drained completion is logged at Information).
///
private async Task RunBackfillPass(
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
{
var startedAt = DateTime.UtcNow;
try
{
await using var scope = _serviceProvider.CreateAsyncScope();
var repository = scope.ServiceProvider.GetRequiredService();
await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
_logger.LogDebug(
"KPI rollup backfill slice folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Shutdown interrupted the backfill; a failover re-runs it (idempotent). Not a failure.
}
catch (Exception ex)
{
_logger.LogError(ex, "KPI rollup backfill failed unexpectedly.");
}
}
///
/// Truncates a UTC instant to the start of its hour (minutes/seconds/ticks dropped),
/// preserving . Used to derive the fold window's whole-hour
/// boundaries so the in-progress hour is excluded.
///
private static DateTime TruncateToHour(DateTime utc) =>
new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
/// Self-tick triggering a sampling pass across all registered sources.
internal sealed class SampleTick
{
public static readonly SampleTick Instance = new();
private SampleTick() { }
}
///
/// Piped-back completion of a sampling pass; lets the pass run off the actor thread and
/// lowers the _sampleInFlight guard on the actor thread (fires on success and fault).
///
internal sealed class SampleComplete
{
public static readonly SampleComplete Instance = new();
private SampleComplete() { }
}
/// Self-tick triggering a retention purge sweep.
internal sealed class PurgeTick
{
public static readonly PurgeTick Instance = new();
private PurgeTick() { }
}
/// Piped-back completion of a purge sweep; lets the sweep run off the actor thread.
internal sealed class PurgeComplete
{
public static readonly PurgeComplete Instance = new();
private PurgeComplete() { }
}
/// Self-tick triggering an hourly rollup fold over the trailing lookback window.
internal sealed class RollupTick
{
public static readonly RollupTick Instance = new();
private RollupTick() { }
}
///
/// Piped-back completion of a rollup fold; lets the fold run off the actor thread and
/// lowers the _rollupInFlight guard on the actor thread (fires on success and fault).
///
internal sealed class RollupComplete
{
public static readonly RollupComplete Instance = new();
private RollupComplete() { }
}
///
/// Self-tick triggering the one-shot backfill: on the first tick it enumerates the full
/// raw-retention window into bounded slices, and each subsequent tick folds one slice. Armed
/// once via a single timer in (and re-armed to drain the next slice or
/// to defer past an in-flight periodic fold); the whole pass runs at most once per actor
/// lifetime.
///
internal sealed class BackfillTick
{
public static readonly BackfillTick Instance = new();
private BackfillTick() { }
}
///
/// Piped-back result of the backfill plan pass: the newest folded rollup hour (the watermark),
/// or null when no rollups exist. Consumed on the actor thread to skip or shrink the
/// backfill to the un-rolled tail (arch-review 04 round 2, R1).
///
internal sealed record BackfillPlan(DateTime? Watermark);
///
/// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
/// and, on the actor thread, lowers the _backfillInFlight guard BETWEEN slices — then
/// either drains the next slice (self-armed timer) or, when the queue is empty, latches
/// _backfillDone (fires on success and fault).
///
internal sealed class BackfillSliceComplete
{
public static readonly BackfillSliceComplete Instance = new();
private BackfillSliceComplete() { }
}
}