ScadaBridge/src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs

using Akka.Actor;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Kpi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types;

namespace ZB.MOM.WW.ScadaBridge.KpiHistory;

/// <summary>
/// Central cluster-singleton ("KPI History &amp; Trends") that drives the
/// KPI sampling backbone. On a periodic timer (default 60 s) it:
/// <list type="number">
/// <item>Stamps a single <c>capturedAtUtc</c> for the pass.</item>
/// <item>Fans out to every DI-registered <see cref="IKpiSampleSource"/>,
///       collecting each source's point-in-time samples.</item>
/// <item>Bulk-writes the combined batch through
///       <see cref="IKpiHistoryRepository.RecordSamplesAsync"/>.</item>
/// </list>
/// A separate daily timer (default 1 d) runs the retention purge, dropping raw
/// sample rows older than <see cref="KpiHistoryOptions.RetentionDays"/> via
/// <see cref="IKpiHistoryRepository.PurgeOlderThanAsync"/> and hourly rollup rows
/// older than the longer <see cref="KpiHistoryOptions.RollupRetentionDays"/> via
/// <see cref="IKpiHistoryRepository.PurgeRollupsOlderThanAsync"/>. A third hourly
/// timer (default 1 h) folds the trailing
/// <see cref="KpiHistoryOptions.RollupLookbackHours"/> hours of raw samples into the
/// <c>KpiRollupHourly</c> table via
/// <see cref="IKpiHistoryRepository.FoldHourlyRollupsAsync"/> (idempotent upsert,
/// in-progress hour excluded), so long-range trend charts read a pre-thinned
/// series. A one-shot backfill timer additionally folds the whole raw-retention
/// window of already-recorded samples once shortly after start, so 30 d/90 d
/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
/// same idempotent fold and runs at most once per actor lifetime. The backfill is
/// sliced into bounded 24 h fold windows (oldest-first) rather than one unbounded
/// pass, and lowers its in-flight guard between slices so periodic folds interleave
/// — periodic folds and backfill slices strictly alternate through the single
/// <c>_rollupInFlight</c>/<c>_backfillInFlight</c> gate pair, so the two idempotent
/// upserts never race on overlapping rows (arch-review 04 round 2, R1).
/// </summary>
/// <remarks>
/// <para>
/// <b>Best-effort.</b> KPI history is observability — it must NEVER disrupt the
/// system it observes. Every fault is contained: a throwing
/// <see cref="IKpiSampleSource"/> is caught per-source so it neither aborts the
/// pass nor suppresses the other sources; a throwing repository write/purge is
/// caught and logged. No exception escapes either tick handler, so the singleton
/// stays alive across transient DB outages and misbehaving sources.
/// </para>
/// <para>
/// <b>Off-thread work.</b> The actual sampling and purge I/O runs off the actor
/// thread; the result is piped back to <see cref="Self"/> via
/// <see cref="PipeToSupport.PipeTo{T}(Task{T}, ICanTell, IActorRef, System.Func{T, object}, System.Func{System.Exception, object})"/>
/// so the mailbox is never blocked while a DB round-trip is in flight. This
/// mirrors the <see cref="ZB.MOM.WW.ScadaBridge.NotificationOutbox.NotificationOutboxActor"/>
/// timer + scope-per-tick + PipeTo pattern.
/// </para>
/// <para>
/// <b>DI scopes.</b> <see cref="IKpiSampleSource"/>s and
/// <see cref="IKpiHistoryRepository"/> are scoped EF Core-backed services; the
/// recorder opens a fresh <see cref="IServiceScope"/> for the tick's repository
/// and a separate scope for each source collection. A timed-out collection owns
/// its scope until actual completion, while later sources use separate DbContexts.
/// Scopes are asynchronously disposed (<c>CreateAsyncScope</c>).
/// </para>
/// <para>
/// <b>Singleton wiring.</b> The <c>Props</c> is built in the Host on the
/// active central node with the constructor args resolved from DI — this actor
/// has no knowledge of <c>Akka.Cluster.Tools</c>, mirroring the Notification
/// Outbox singleton split.
/// </para>
/// </remarks>
public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
{
    private const string SampleTimerKey = "kpi-sample";
    private const string PurgeTimerKey = "kpi-purge";
    private const string RollupTimerKey = "kpi-rollup";
    private const string BackfillTimerKey = "kpi-rollup-backfill";

    /// <summary>
    /// Initial delay before the one-shot rollup backfill fires (see
    /// <see cref="HandleBackfillTick"/>). Deliberately a touch longer than the periodic rollup
    /// timer's own <c>10 s</c> initial delay so that, when both are armed at start, the periodic
    /// tick claims the fold path first and the backfill simply defers (the two folds are
    /// idempotent upserts, but they must not race on overlapping rows). Also comfortably beyond
    /// the sub-second/second-scale <c>AwaitAssert</c> windows used by the unit tests, so the
    /// one-shot never auto-fires mid-test — tests drive <see cref="BackfillTick"/> by hand.
    /// </summary>
    private static readonly TimeSpan BackfillInitialDelay = TimeSpan.FromSeconds(15);

    // Shared by every recorder incarnation on this root provider.
    private readonly KpiSourceAdmission _sourceAdmission;
    private readonly IServiceProvider _serviceProvider;
    private readonly KpiHistoryOptions _options;
    private readonly ILogger<KpiHistoryRecorderActor> _logger;

    /// <summary>
    /// Lifecycle-scoped cancellation source, cancelled in <see cref="PostStop"/> so any
    /// in-flight sample/purge pass observes a coordinated shutdown / failover promptly
    /// instead of blocking the singleton handover for a full DB round-trip timeout.
    /// </summary>
    private CancellationTokenSource? _shutdownCts;

    /// <summary>
    /// In-flight guard for the sample loop. Set true at the start of a sample pass and cleared
    /// when the pass's <see cref="SampleComplete"/> arrives. While true, further
    /// <see cref="SampleTick"/>s are skipped so passes never overlap — Akka periodic timers
    /// enqueue (not coalesce) missed ticks, so without this guard a pass running longer than
    /// <see cref="KpiHistoryOptions.SampleInterval"/> (slow/recovering DB) would let each
    /// subsequent tick spawn another concurrent pass, amplifying load on the struggling store
    /// and double-writing samples for overlapping windows. Mirrors the
    /// <see cref="ZB.MOM.WW.ScadaBridge.NotificationOutbox.NotificationOutboxActor"/> dispatch
    /// in-flight guard the timer pattern is modelled on.
    /// </summary>
    private bool _sampleInFlight;

    /// <summary>
    /// In-flight guard for the hourly rollup fold. Set true at the start of a fold pass and
    /// cleared when the pass's <see cref="RollupComplete"/> arrives. While true, further
    /// <see cref="RollupTick"/>s are skipped so folds never overlap — the fold is a
    /// group-by-and-upsert over a lookback window that can outlast the (default 1 h)
    /// <see cref="KpiHistoryOptions.RollupInterval"/> on a slow/recovering store; without this
    /// guard a subsequent tick would launch a second concurrent fold contending for the same
    /// upsert key. Mirrors <see cref="_sampleInFlight"/>.
    /// </summary>
    private bool _rollupInFlight;

    /// <summary>
    /// Width of one backfill fold slice (24 h). The repository fold materializes its whole
    /// window in memory, so the one-shot backfill must never hand it the full raw-retention
    /// window (90 d ≈ tens of millions of rows at fleet volume — arch-review 04 round 2, R1).
    /// One day per fold bounds each pass to roughly what a steady-state day writes, and the
    /// guard is lowered between slices so periodic folds interleave instead of stalling
    /// behind the historical pass.
    /// </summary>
    private static readonly TimeSpan BackfillSliceWidth = TimeSpan.FromHours(24);

    /// <summary>
    /// Queue of remaining backfill fold slices (each a bounded <see cref="BackfillSliceWidth"/>
    /// window), oldest-first. Built once when the backfill plan is enumerated and drained one
    /// slice per fold; a periodic <see cref="RollupTick"/> may claim the fold path between slices
    /// (the two idempotent upserts strictly alternate through the
    /// <see cref="_rollupInFlight"/>/<see cref="_backfillInFlight"/> gate pair, never racing on
    /// overlapping rows — arch-review 04 round 2, R1).
    /// </summary>
    private readonly Queue<(DateTime FromHourUtc, DateTime ToHourUtc)> _backfillSlices = new();

    /// <summary>
    /// In-flight guard for a single backfill fold slice. Set true when a slice fold is launched
    /// and cleared when its <see cref="BackfillSliceComplete"/> arrives. While true, periodic
    /// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so a backfill
    /// slice never runs concurrently with a periodic lookback fold on the same upsert rows; it is
    /// lowered BETWEEN slices so periodic folds interleave instead of stalling behind the whole
    /// historical pass.
    /// </summary>
    private bool _backfillInFlight;

    /// <summary>
    /// Once-only latch for the backfill. Set true when the backfill fold completes (success or
    /// fault) so it runs at most once per actor lifetime — a fresh singleton on failover starts a
    /// new instance and re-runs the (idempotent) backfill, which is the intended self-heal. Set on
    /// completion rather than launch so that if the actor is torn down mid-backfill the next
    /// lifetime still attempts it.
    /// </summary>
    private bool _backfillDone;

    /// <summary>Akka timer scheduler, assigned by the actor system via <see cref="IWithTimers"/>.</summary>
    public ITimerScheduler Timers { get; set; } = null!;

    /// <summary>
    /// Initializes the recorder with its dependencies and registers the tick handlers.
    /// </summary>
    /// <param name="serviceProvider">DI service provider used to open a scope per tick for the sample sources and repository.</param>
    /// <param name="options">KPI history configuration options.</param>
    /// <param name="logger">Logger for this actor.</param>
    public KpiHistoryRecorderActor(
        IServiceProvider serviceProvider,
        KpiHistoryOptions options,
        ILogger<KpiHistoryRecorderActor> logger)
    {
        _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
        _sourceAdmission = serviceProvider.GetRequiredService<KpiSourceAdmission>();
        _options = options ?? throw new ArgumentNullException(nameof(options));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));

        Receive<SampleTick>(_ => HandleSampleTick());
        Receive<SampleComplete>(_ => _sampleInFlight = false); // lower the in-flight guard (success or fault)
        Receive<PurgeTick>(_ => HandlePurgeTick());
        Receive<PurgeComplete>(_ => { }); // best-effort: no actor state to reset on completion
        Receive<RollupTick>(_ => HandleRollupTick());
        Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
        Receive<BackfillTick>(_ => HandleBackfillTick());
        Receive<BackfillPlan>(HandleBackfillPlan);
        Receive<BackfillSliceComplete>(_ =>
        {
            _backfillInFlight = false; // release the fold path BETWEEN slices (R1)
            if (_backfillSlices.Count == 0)
            {
                _backfillDone = true; // latch: at most once per actor lifetime (unchanged semantics)
                _logger.LogInformation("KPI rollup backfill completed — all slices folded.");
                return;
            }

            // Next slice via the timer, not inline: a RollupTick already queued in the mailbox
            // is processed first, so periodic folds interleave; the existing _rollupInFlight
            // deferral branch in HandleBackfillTick then re-arms the backfill behind it.
            Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
        });
    }

    /// <inheritdoc />
    protected override void PreStart()
    {
        base.PreStart();
        _shutdownCts = new CancellationTokenSource();

        // Fire an initial sample shortly after start so the first trend point lands
        // without waiting a full SampleInterval, then settle into the periodic cadence.
        Timers.StartPeriodicTimer(
            SampleTimerKey,
            SampleTick.Instance,
            initialDelay: TimeSpan.FromSeconds(5),
            interval: _options.SampleInterval);

        // The purge is idempotent, so it takes a short first tick (min(PurgeInterval, 5min)):
        // scheduling the first sweep a full PurgeInterval out meant a node recycled more often
        // than the interval (e.g. daily restart vs. daily cadence) would never purge
        // (arch-review 04, S9). Fast test cadences (interval < 5min) are unchanged.
        Timers.StartPeriodicTimer(
            PurgeTimerKey,
            PurgeTick.Instance,
            initialDelay: PurgeTimerSchedule.InitialDelay(_options.PurgeInterval),
            interval: _options.PurgeInterval);

        // Fold the trailing hours into the hourly rollup table on a short initial delay
        // (so a fresh active node catches up promptly after a failover), then settle into
        // the periodic cadence. The fold re-processes a lookback window via an idempotent
        // upsert, so a missed tick self-heals on the next fold.
        Timers.StartPeriodicTimer(
            RollupTimerKey,
            RollupTick.Instance,
            initialDelay: TimeSpan.FromSeconds(10),
            interval: _options.RollupInterval);

        // One-shot backfill: fold the whole raw-retention window of existing samples into the
        // hourly rollup table once shortly after start, so long-range (30 d/90 d) trend charts
        // aren't blank until enough wall-clock passes after deploy. The periodic tick only folds
        // the trailing RollupLookbackHours, so without this the deep history stays unrolled. The
        // fold is an idempotent upsert, so re-running on a failover (a fresh singleton re-arms
        // this timer) is safe.
        Timers.StartSingleTimer(
            BackfillTimerKey,
            BackfillTick.Instance,
            timeout: BackfillInitialDelay);
    }

    /// <inheritdoc />
    protected override void PostStop()
    {
        // Cancel before disposing so an in-flight pass observes cancellation; disposing
        // first would race with a pass registering against the token.
        try
        {
            _shutdownCts?.Cancel();
        }
        catch (ObjectDisposedException)
        {
            // Already disposed under a restarted-actor race; nothing to do.
        }

        _shutdownCts?.Dispose();
        _shutdownCts = null;

        base.PostStop();
    }

    /// <summary>
    /// Handles a sample tick. If a sample pass is already in flight the tick is skipped
    /// (logged at debug) so passes never overlap; otherwise the in-flight guard is raised,
    /// the shared <c>capturedAtUtc</c> instant is captured on the actor thread, and the
    /// asynchronous sampling pass is launched off-thread with a <see cref="SampleComplete"/>
    /// piped back to <see cref="Self"/> to lower the guard on the actor thread — so the
    /// mailbox is never blocked while sources are collected and the batch is written.
    /// </summary>
    private void HandleSampleTick()
    {
        if (_sampleInFlight)
        {
            // A prior pass is still awaiting its DB round-trip; coalesce this tick rather than
            // piling a second concurrent pass onto a slow/recovering store.
            _logger.LogDebug("KPI sample tick skipped — a sample pass is already in flight.");
            return;
        }

        _sampleInFlight = true;
        var capturedAt = DateTime.UtcNow;
        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;

        // RunSamplePass self-isolates its faults (it never throws), but the failure
        // projection is kept as a belt-and-braces guard so even a faulted task still
        // produces a SampleComplete that lowers the in-flight guard — otherwise the loop
        // would wedge permanently.
        RunSamplePass(capturedAt, cancellationToken).PipeTo(
            Self,
            success: () => SampleComplete.Instance,
            failure: ex =>
            {
                _logger.LogError(ex, "KPI sample pass faulted unexpectedly.");
                return SampleComplete.Instance;
            });
    }

    /// <summary>
    /// Runs a single sampling pass: enumerates every registered <see cref="IKpiSampleSource"/>,
    /// waits a bounded time for each separately scoped collection, and bulk-writes the combined batch. The whole body is wrapped so the
    /// returned task never faults — best-effort observability must never disrupt anything.
    /// </summary>
    private async Task RunSamplePass(DateTime capturedAt, CancellationToken cancellationToken)
    {
        try
        {
            await using var scope = _serviceProvider.CreateAsyncScope();
            var sources = scope.ServiceProvider.GetServices<IKpiSampleSource>().ToArray();
            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();

            var samples = new List<KpiSample>();
            for (var sourceIndex = 0; sourceIndex < sources.Length; sourceIndex++)
            {
                var sourceName = sources[sourceIndex].Source;
                // Each call owns a separate scope, so a cancellation-ignoring source
                // neither shares a busy DbContext with healthy sources nor loses its scope.
                var index = sourceIndex;
                if (!_sourceAdmission.TryStart(index,
                        () => CollectSourceAsync(index, capturedAt, cancellationToken), out var collection))
                    continue;
                _ = collection.ContinueWith(t =>
                    _logger.LogError(t.Exception, "KPI source {Source} collection failed.", sourceName),
                    CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
                    TaskScheduler.Default);
                try
                {
                    var collected = await collection.WaitAsync(_options.SourceTimeout, cancellationToken);
                    samples.AddRange(collected);
                }
                catch (TimeoutException)
                {
                    _logger.LogWarning("KPI source {Source} exceeded its collection deadline; unfinished work remains owned and will not be restarted.", sourceName);
                }
                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                {
                    // Shutdown interrupted collection; abandon the rest of the pass. The next
                    // active node samples on its own cadence. Not a failure.
                    return;
                }
                catch (Exception ex)
                {
                    // A throwing source must NOT abort the pass or the other sources.
                    _logger.LogError(ex, "KPI source {Source} failed to collect samples.", sourceName);
                }
            }

            if (samples.Count == 0)
            {
                return;
            }

            try
            {
                await repository.RecordSamplesAsync(samples, cancellationToken);
            }
            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
            {
                // Shutdown interrupted the write; the batch is dropped (best-effort). Not a failure.
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "KPI sample write failed for {SampleCount} sample(s).", samples.Count);
            }
        }
        catch (Exception ex)
        {
            // Scope creation or service resolution faulted; swallow and log so the returned
            // task completes normally and the singleton stays alive.
            _logger.LogError(ex, "KPI sample pass failed unexpectedly.");
        }
    }

    /// <summary>Owns a source's scope and cancellation source until its actual collection completes, even after the caller stops waiting.</summary>
    private async Task<IReadOnlyList<KpiSample>> CollectSourceAsync(
        int sourceIndex, DateTime capturedAt, CancellationToken cancellationToken)
    {
        await using var scope = _serviceProvider.CreateAsyncScope();
        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
        deadline.CancelAfter(_options.SourceTimeout);
        var source = scope.ServiceProvider.GetServices<IKpiSampleSource>().ElementAt(sourceIndex);
        return await source.CollectAsync(capturedAt, deadline.Token).ConfigureAwait(false);
    }

    /// <summary>
    /// Handles a purge tick: computes both retention cut-offs on the actor thread (raw samples
    /// at <see cref="KpiHistoryOptions.RetentionDays"/>, hourly rollups at the longer
    /// <see cref="KpiHistoryOptions.RollupRetentionDays"/>), then runs both bulk deletes
    /// off-thread and pipes a completion back to <see cref="Self"/>. Purges are daily and
    /// idempotent, so no in-flight guard is needed.
    /// </summary>
    private void HandlePurgeTick()
    {
        var now = DateTime.UtcNow;
        var sampleBefore = now - TimeSpan.FromDays(_options.RetentionDays);
        var rollupBefore = now - TimeSpan.FromDays(_options.RollupRetentionDays);
        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;

        RunPurgePass(sampleBefore, rollupBefore, cancellationToken).PipeTo(
            Self,
            success: deleted =>
            {
                if (deleted > 0)
                {
                    _logger.LogInformation(
                        "KPI history purge removed {DeletedCount} sample(s) older than {Cutoff:o}.",
                        deleted, sampleBefore);
                }

                return PurgeComplete.Instance;
            },
            failure: ex =>
            {
                _logger.LogError(ex, "KPI history purge faulted unexpectedly.");
                return PurgeComplete.Instance;
            });
    }

    /// <summary>
    /// Runs a single purge sweep: opens a DI scope, resolves the repository, and bulk-deletes
    /// expired raw samples (before <paramref name="sampleBefore"/>) and expired hourly rollups
    /// (before <paramref name="rollupBefore"/>), returning the deleted raw-sample count. The two
    /// deletes are isolated so a failure of one never skips the other, and the whole body is
    /// wrapped so the returned task never faults — on failure the exception is logged and 0 is
    /// returned, mirroring <see cref="RunSamplePass"/>'s best-effort contract.
    /// </summary>
    private async Task<int> RunPurgePass(
        DateTime sampleBefore, DateTime rollupBefore, CancellationToken cancellationToken)
    {
        try
        {
            await using var scope = _serviceProvider.CreateAsyncScope();
            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();

            var deleted = 0;

            // Raw-sample purge — isolated so a fault here does not skip the rollup purge below.
            try
            {
                deleted = await repository.PurgeOlderThanAsync(sampleBefore, cancellationToken);
            }
            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
            {
                // Shutdown interrupted the purge; the next active sweep retries. Not a failure.
                return deleted;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "KPI history raw-sample purge failed unexpectedly.");
            }

            // Rollup purge — retained longer than raw; isolated so its failure does not affect
            // the raw-purge outcome already reported.
            try
            {
                await repository.PurgeRollupsOlderThanAsync(rollupBefore, cancellationToken);
            }
            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
            {
                // Shutdown interrupted the rollup purge; the next active sweep retries. Not a failure.
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "KPI history rollup purge failed unexpectedly.");
            }

            return deleted;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "KPI history purge failed unexpectedly.");
            return 0;
        }
    }

    /// <summary>
    /// Handles a rollup tick. If a fold is already in flight the tick is skipped (logged at
    /// debug) so folds never overlap; otherwise the in-flight guard is raised and the fold is
    /// launched off-thread with a <see cref="RollupComplete"/> piped back to <see cref="Self"/>
    /// to lower the guard on the actor thread. The fold window's upper bound is the current
    /// hour start (exclusive) so the in-progress hour is never folded, and its lower bound is
    /// <see cref="KpiHistoryOptions.RollupLookbackHours"/> earlier so a missed/failover tick
    /// self-heals via the idempotent upsert.
    /// </summary>
    private void HandleRollupTick()
    {
        if (_rollupInFlight || _backfillInFlight)
        {
            // Skip while either a prior periodic fold OR the one-shot full-window backfill is
            // running, so two idempotent upserts never race on overlapping rows. A skipped
            // periodic tick self-heals on the next fold (it re-processes the lookback window).
            _logger.LogDebug("KPI rollup tick skipped — a rollup fold is already in flight.");
            return;
        }

        _rollupInFlight = true;
        var toHourUtc = TruncateToHour(DateTime.UtcNow);
        var fromHourUtc = toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours);
        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;

        RunRollupPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
            Self,
            success: () => RollupComplete.Instance,
            failure: ex =>
            {
                _logger.LogError(ex, "KPI rollup fold faulted unexpectedly.");
                return RollupComplete.Instance;
            });
    }

    /// <summary>
    /// Runs a single rollup fold: opens a DI scope, resolves the repository, and folds the
    /// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c>
    /// into the hourly rollup table via the idempotent upsert. The whole body is wrapped so the
    /// returned task never faults — best-effort observability must never disrupt anything.
    /// </summary>
    private async Task RunRollupPass(
        DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
    {
        try
        {
            await using var scope = _serviceProvider.CreateAsyncScope();
            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
            await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            // Shutdown interrupted the fold; the next active node re-folds the lookback window. Not a failure.
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "KPI rollup fold failed unexpectedly.");
        }
    }

    /// <summary>
    /// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
    /// has already completed (<see cref="_backfillDone"/>) or a slice is in flight
    /// (<see cref="_backfillInFlight"/>) the tick is ignored. If a periodic fold currently holds
    /// the fold path (<see cref="_rollupInFlight"/>) the backfill defers by re-arming its single
    /// timer, so a backfill slice and the periodic lookback fold never race on overlapping upsert
    /// rows. When no slice plan exists yet it enumerates the raw-retention window
    /// <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> into bounded
    /// <see cref="BackfillSliceWidth"/> windows (oldest-first) and self-arms to start draining;
    /// otherwise it dequeues ONE slice, raises the guard, and folds it off the actor thread,
    /// piping a <see cref="BackfillSliceComplete"/> back to release the guard between slices.
    /// </summary>
    private void HandleBackfillTick()
    {
        if (_backfillDone || _backfillInFlight)
        {
            return;
        }

        if (_rollupInFlight)
        {
            // A periodic fold is in flight; defer the next backfill slice briefly so the two
            // idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
            Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
            return;
        }

        if (_backfillSlices.Count == 0)
        {
            // No slice plan yet: resolve the rollup watermark off the actor thread, then plan on
            // the actor thread (Receive<BackfillPlan>). Raise the guard so periodic ticks defer
            // while planning is in flight; the plan handler lowers it. On a failover restart the
            // watermark lets the plan SKIP or SHRINK to the un-rolled tail instead of re-folding
            // the whole raw-retention window (arch-review 04 round 2, R1).
            _backfillInFlight = true;
            var planCancellationToken = _shutdownCts?.Token ?? CancellationToken.None;

            RunBackfillPlanPass(planCancellationToken).PipeTo(
                Self,
                success: watermark => new BackfillPlan(watermark),
                failure: ex =>
                {
                    _logger.LogError(ex, "KPI rollup backfill plan pass faulted unexpectedly.");
                    return new BackfillPlan(null); // null watermark ⇒ plan the full retention window
                });
            return;
        }

        // Dequeue and fold exactly ONE bounded slice.
        var slice = _backfillSlices.Dequeue();
        _backfillInFlight = true;
        var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;

        RunBackfillPass(slice.FromHourUtc, slice.ToHourUtc, cancellationToken).PipeTo(
            Self,
            success: () => BackfillSliceComplete.Instance,
            failure: ex =>
            {
                _logger.LogError(ex, "KPI rollup backfill slice faulted unexpectedly.");
                return BackfillSliceComplete.Instance;
            });
    }

    /// <summary>
    /// Enumerates <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
    /// contiguous <see cref="BackfillSliceWidth"/> windows (the last possibly shorter),
    /// oldest-first, and enqueues them onto <see cref="_backfillSlices"/>.
    /// </summary>
    private void EnqueueBackfillSlices(DateTime fromHourUtc, DateTime toHourUtc)
    {
        var sliceStart = fromHourUtc;
        while (sliceStart < toHourUtc)
        {
            var sliceEnd = sliceStart + BackfillSliceWidth;
            if (sliceEnd > toHourUtc)
            {
                sliceEnd = toHourUtc;
            }

            _backfillSlices.Enqueue((sliceStart, sliceEnd));
            sliceStart = sliceEnd;
        }
    }

    /// <summary>
    /// Plans the backfill from the resolved rollup watermark (actor thread). The window floor is
    /// the newer of the watermark (re-folding the watermark hour itself is a cheap idempotent
    /// safety margin) and the raw-retention floor; a null watermark (fresh install) keeps the full
    /// retention window. If the floor is already within
    /// <see cref="KpiHistoryOptions.RollupLookbackHours"/> of now the periodic fold's lookback
    /// covers everything from there forward, so the whole historical pass is skipped (the failover
    /// fast-path); otherwise the tail <c>[floor, TruncateToHour(now))</c> is enumerated into
    /// bounded slices and draining is self-armed. Either way the plan-pass guard is lowered here
    /// (arch-review 04 round 2, R1).
    /// </summary>
    private void HandleBackfillPlan(BackfillPlan plan)
    {
        var toHourUtc = TruncateToHour(DateTime.UtcNow);
        var retentionFloor = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
        // Shrink to the un-rolled tail; re-fold the watermark hour itself (idempotent) as a safety
        // margin. A null watermark (fresh install) keeps the full retention window.
        var floor = plan.Watermark is { } w && w > retentionFloor ? w : retentionFloor;

        if (floor >= toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours))
        {
            // Failover fast-path: rollups are already current — the periodic fold's lookback
            // window covers everything from the watermark forward, so the full historical pass is
            // skipped entirely (arch-review 04 round 2, R1).
            _backfillInFlight = false;
            _backfillDone = true;
            _logger.LogInformation(
                "KPI rollup backfill skipped — rollups current through {Watermark:o}.", plan.Watermark);
            return;
        }

        EnqueueBackfillSlices(floor, toHourUtc);
        _backfillInFlight = false;

        if (_backfillSlices.Count == 0)
        {
            // Degenerate window (no whole hours to fold) — nothing to backfill.
            _backfillDone = true;
            return;
        }

        _logger.LogInformation(
            "KPI rollup backfill starting — folding [{FromHour:o}, {ToHour:o}) across {SliceCount} "
            + "slice(s) of up to {SliceHours} h (watermark {Watermark:o}).",
            floor, toHourUtc, _backfillSlices.Count, BackfillSliceWidth.TotalHours, plan.Watermark);

        // Start draining via the timer (uniform path with the between-slice re-arm).
        Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
    }

    /// <summary>
    /// Resolves the newest folded rollup hour (the backfill watermark) off the actor thread in a
    /// fresh DI scope. Never faults — on cancellation or error it returns <c>null</c>, which the
    /// plan treats as "no rollups", keeping the full retention window (best-effort, mirrors the
    /// other passes).
    /// </summary>
    private async Task<DateTime?> RunBackfillPlanPass(CancellationToken cancellationToken)
    {
        try
        {
            await using var scope = _serviceProvider.CreateAsyncScope();
            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
            return await repository.GetLatestRollupHourAsync(cancellationToken);
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            // Shutdown interrupted the lookup; a fresh active node re-plans. Not a failure.
            return null;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "KPI rollup backfill watermark lookup failed; planning the full retention window.");
            return null;
        }
    }

    /// <summary>
    /// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
    /// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
    /// the hourly rollup table via the same idempotent upsert the periodic fold uses. The whole
    /// body is wrapped so the returned task never faults — best-effort observability must never
    /// disrupt startup. The fold method returns no count, so each slice is logged at Debug with its
    /// window and elapsed wall time (the queue-drained completion is logged at Information).
    /// </summary>
    private async Task RunBackfillPass(
        DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
    {
        var startedAt = DateTime.UtcNow;
        try
        {
            await using var scope = _serviceProvider.CreateAsyncScope();
            var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
            await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);

            _logger.LogDebug(
                "KPI rollup backfill slice folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
                fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            // Shutdown interrupted the backfill; a failover re-runs it (idempotent). Not a failure.
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "KPI rollup backfill failed unexpectedly.");
        }
    }

    /// <summary>
    /// Truncates a UTC instant to the start of its hour (minutes/seconds/ticks dropped),
    /// preserving <see cref="DateTimeKind.Utc"/>. Used to derive the fold window's whole-hour
    /// boundaries so the in-progress hour is excluded.
    /// </summary>
    private static DateTime TruncateToHour(DateTime utc) =>
        new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);

    /// <summary>Self-tick triggering a sampling pass across all registered sources.</summary>
    internal sealed class SampleTick
    {
        public static readonly SampleTick Instance = new();
        private SampleTick() { }
    }

    /// <summary>
    /// Piped-back completion of a sampling pass; lets the pass run off the actor thread and
    /// lowers the <c>_sampleInFlight</c> guard on the actor thread (fires on success and fault).
    /// </summary>
    internal sealed class SampleComplete
    {
        public static readonly SampleComplete Instance = new();
        private SampleComplete() { }
    }

    /// <summary>Self-tick triggering a retention purge sweep.</summary>
    internal sealed class PurgeTick
    {
        public static readonly PurgeTick Instance = new();
        private PurgeTick() { }
    }

    /// <summary>Piped-back completion of a purge sweep; lets the sweep run off the actor thread.</summary>
    internal sealed class PurgeComplete
    {
        public static readonly PurgeComplete Instance = new();
        private PurgeComplete() { }
    }

    /// <summary>Self-tick triggering an hourly rollup fold over the trailing lookback window.</summary>
    internal sealed class RollupTick
    {
        public static readonly RollupTick Instance = new();
        private RollupTick() { }
    }

    /// <summary>
    /// Piped-back completion of a rollup fold; lets the fold run off the actor thread and
    /// lowers the <c>_rollupInFlight</c> guard on the actor thread (fires on success and fault).
    /// </summary>
    internal sealed class RollupComplete
    {
        public static readonly RollupComplete Instance = new();
        private RollupComplete() { }
    }

    /// <summary>
    /// Self-tick triggering the one-shot backfill: on the first tick it enumerates the full
    /// raw-retention window into bounded slices, and each subsequent tick folds one slice. Armed
    /// once via a single timer in <see cref="PreStart"/> (and re-armed to drain the next slice or
    /// to defer past an in-flight periodic fold); the whole pass runs at most once per actor
    /// lifetime.
    /// </summary>
    internal sealed class BackfillTick
    {
        public static readonly BackfillTick Instance = new();
        private BackfillTick() { }
    }

    /// <summary>
    /// Piped-back result of the backfill plan pass: the newest folded rollup hour (the watermark),
    /// or <c>null</c> when no rollups exist. Consumed on the actor thread to skip or shrink the
    /// backfill to the un-rolled tail (arch-review 04 round 2, R1).
    /// </summary>
    internal sealed record BackfillPlan(DateTime? Watermark);

    /// <summary>
    /// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
    /// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard BETWEEN slices — then
    /// either drains the next slice (self-armed timer) or, when the queue is empty, latches
    /// <c>_backfillDone</c> (fires on success and fault).
    /// </summary>
    internal sealed class BackfillSliceComplete
    {
        public static readonly BackfillSliceComplete Instance = new();
        private BackfillSliceComplete() { }
    }
}