merge(r2): r2-plan04
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
BEGIN TRANSACTION;
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260713142234_AddSiteCallsTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
EXEC(N'CREATE INDEX [IX_SiteCalls_Terminal] ON [SiteCalls] ([TerminalAtUtc]) WHERE [TerminalAtUtc] IS NOT NULL');
|
||||
END;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260713142234_AddSiteCallsTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||
VALUES (N'20260713142234_AddSiteCallsTerminalIndex', N'10.0.7');
|
||||
END;
|
||||
|
||||
COMMIT;
|
||||
GO
|
||||
|
||||
@@ -88,9 +88,9 @@ A timer fires every `SampleInterval` (default 60s; an immediate first tick prime
|
||||
|
||||
**Best-effort, per-source isolation.** Each source call and the write are individually guarded. A throwing source is logged and its samples skipped for that tick; it never aborts the tick, the other sources, or the source component itself. This is the same `IEnumerable<>`-of-adapters decoupling pattern used by `INotificationDeliveryAdapter`.
|
||||
|
||||
**Hourly rollup tick.** A third timer (`kpi-rollup`) fires every `RollupInterval` (default 1h). On each tick the recorder opens a per-tick DI scope and calls `IKpiHistoryRepository.FoldHourlyRollupsAsync` over the window `[TruncateToHour(now) − RollupLookbackHours, TruncateToHour(now))` — an **exclusive** upper bound at the current hour start, so the in-progress hour is never folded. The fold groups raw `KpiSample` rows by `(series, hour)`, applies the per-metric aggregation intent (sum for rate metrics, last-value for gauges — see Query), and **idempotently upserts** each `(Source, Metric, Scope, ScopeKey, HourStartUtc)` row. Re-folding the trailing lookback window (default 3h) rather than only the last hour is the self-heal for a **singleton-failover-missed tick**: because the upsert overwrites the aggregate in place, re-running the same window produces identical values (no double-count), so any complete hour a missed tick skipped is refilled on the next fold. A `_rollupInFlight` guard (mirroring `_sampleInFlight`) skips a tick while a prior fold is still running so two idempotent upserts never race on overlapping rows.
|
||||
**Hourly rollup tick.** A third timer (`kpi-rollup`) fires every `RollupInterval` (default 1h). On each tick the recorder opens a per-tick DI scope and calls `IKpiHistoryRepository.FoldHourlyRollupsAsync` over the window `[TruncateToHour(now) − RollupLookbackHours, TruncateToHour(now))` — an **exclusive** upper bound at the current hour start, so the in-progress hour is never folded. The fold groups raw `KpiSample` rows by `(series, hour)`, applies the per-metric aggregation intent (sum for rate metrics, last-value for gauges — see Query), and **idempotently upserts** each `(Source, Metric, Scope, ScopeKey, HourStartUtc)` row. Re-folding the trailing lookback window (default 3h) rather than only the last hour is the self-heal for a **singleton-failover-missed tick**: because the upsert overwrites the aggregate in place, re-running the same window produces identical values (no double-count), so any complete hour a missed tick skipped is refilled on the next fold. A `_rollupInFlight` guard (mirroring `_sampleInFlight`) skips a tick while a prior fold is still running so two idempotent upserts never race on overlapping rows. During a **failover overlap** — the old singleton incarnation's in-flight fold and the new node's first fold both `Add`-ing the same `(series, hour)` row — the loser of the unique `IX_KpiRollupHourly_Series` upsert race discards its **whole fold pass** (a single `SaveChanges`), logged at Information and repaired by the next idempotent fold; when reading fold logs after a failover, the failure grain is the **pass, not the row** (arch-review 04 round 2, R5).
|
||||
|
||||
**One-shot backfill.** So the 30 d / 90 d charts are not blank until enough wall-clock passes, the recorder runs a **single** backfill on start (`kpi-rollup-backfill` timer) that folds the full retention window once, `[TruncateToHour(now) − RetentionDays, TruncateToHour(now))`, reusing the same `FoldHourlyRollupsAsync` path. A `_backfillDone` latch makes it at-most-once per actor lifetime, and it defers (re-arms) while a periodic fold is in flight; being an idempotent upsert, it is cheap and safe to re-run on a later failover restart.
|
||||
**One-shot backfill.** So the 30 d / 90 d charts are not blank until enough wall-clock passes, the recorder runs a backfill on start (`kpi-rollup-backfill` timer) that folds the retention window, reusing the same `FoldHourlyRollupsAsync` path. The `FoldHourlyRollupsAsync` repository call materializes its whole window in memory, so the backfill folds the window in **≤24 h slices, oldest-first** — one bounded slice per fold — rather than handing the fold the full 90-day window at once (which at fleet volume is tens of millions of rows in one tracked pass). Between slices it lowers its in-flight guard so periodic folds **interleave** instead of stalling behind the historical pass; a periodic fold and a backfill slice strictly alternate through the single `_rollupInFlight`/`_backfillInFlight` gate pair, so the two idempotent upserts never race on overlapping rows. On a **failover restart** the plan first consults the `IKpiHistoryRepository.GetLatestRollupHourAsync` watermark: if the newest folded hour is already within `RollupLookbackHours` of now (the common case) the whole historical pass is **skipped** — the periodic fold's lookback covers the tail — otherwise the window is **shrunk to the un-rolled tail** `[watermark, TruncateToHour(now))` (re-folding the watermark hour itself as an idempotent safety margin) instead of the full `RetentionDays` floor. A `_backfillDone` latch makes the whole pass at-most-once per actor lifetime, and it defers (re-arms) while a periodic fold is in flight. Being an idempotent upsert, it is safe (idempotent) to re-run on a later failover restart; the watermark fast-path makes the failover re-run cheap as well.
|
||||
|
||||
**Retention.** A daily purge timer (`PurgeInterval`, default 24h) now runs **both** purges: raw `KpiSample` rows older than `RetentionDays` (default 90) via `IKpiHistoryRepository.PurgeOlderThanAsync`, **and** hourly rollup rows older than `RollupRetentionDays` (default 365) via `IKpiHistoryRepository.PurgeRollupsOlderThanAsync` (a one-hour-sliced batched DELETE mirroring the raw purge). Rollups deliberately **outlive** raw samples so long-range trend charts have data after the raw rows are purged; the options validator enforces `RollupRetentionDays >= RetentionDays` so a window never falls into a hole where raw is purged but no rollup exists.
|
||||
|
||||
@@ -148,15 +148,17 @@ Reads `ICentralHealthAggregator.GetAllSiteStates()` (in-memory, no DB). Scope: *
|
||||
|
||||
### Bucketed query
|
||||
|
||||
`IKpiHistoryRepository.GetRawSeriesAsync(source, metric, scope, scopeKey, fromUtc, toUtc, …)` returns the raw points for one series over `[fromUtc, toUtc]`. `GetHourlySeriesAsync(…)` returns the same ascending `KpiSeriesPoint` shape from `KpiRollupHourly` (projecting `HourStartUtc` → the point timestamp, `Value`), so the bucketer is agnostic to the source. `KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, maxPoints)` then partitions the window into ≤ `maxPoints` time buckets and returns the **last value per bucket** as `KpiSeriesPoint(BucketStartUtc, Value)` — unchanged, and still applied on top of a rollup series (a 90 d rollup is ~2,160 points > the 200-point cap, so bucketing still thins it). The chart plots the single `Value` series; avg / min / max charting is still deferred, though the rollup's `MinValue` / `MaxValue` / `SampleCount` columns are now stored to unblock it.
|
||||
`IKpiHistoryRepository.GetRawSeriesAsync(source, metric, scope, scopeKey, fromUtc, toUtc, …)` returns the raw points for one series over `[fromUtc, toUtc]`. `GetHourlySeriesAsync(…)` returns the same ascending `KpiSeriesPoint` shape from `KpiRollupHourly` (projecting `HourStartUtc` → the point timestamp, `Value`), so the bucketer is agnostic to the source. `KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, maxPoints, aggregation)` then partitions the window into ≤ `maxPoints` time buckets and reduces each bucket **per-metric**: it keeps the **last value** for a **Gauge** series and **sums per bucket** for a **Rate** series, driven by the same `KpiMetricAggregationCatalog` the hourly fold consults. The previous unconditional last-value-per-bucket kept **1 of ~11 hourly sums** per 90 d bucket on the rollup path — the exact fold error the catalog's own doc warns about, re-introduced one layer up — and is fixed per arch-review 04 round 2, R3. The reduction is still applied on top of a rollup series (a 90 d rollup is ~2,160 points > the 200-point cap, so bucketing still thins it). The chart plots the single `Value` series; avg / min / max charting is still deferred, though the rollup's `MinValue` / `MaxValue` / `SampleCount` columns are now stored to unblock it.
|
||||
|
||||
### Raw-vs-rollup range routing
|
||||
|
||||
`GetRawSeriesAsync` and `GetHourlySeriesAsync` are selected by window width in the query service (below), governed by `RollupThresholdHours` (default 168 = 7 d). Windows **at or below** the threshold read raw samples (preserving intra-minute detail on the 24 h / 7 d charts); windows **strictly wider** than the threshold read the pre-aggregated hourly rollups (≈60× fewer rows scanned on the 30 d / 90 d charts). The boundary is strict — a window of exactly `RollupThresholdHours` stays on the raw path.
|
||||
|
||||
Rate charts keep **one unit (events per chart bucket)** on both sides of the boundary: the catalog-driven per-bucket sum reduction is applied to the raw-minute deltas and the hourly rollup sums alike, so widening 7 d → 30 d no longer changes a Rate chart's magnitude ~60×. Residual: an unbucketed just-over-threshold rollup series plots native per-hour sums vs. the at-threshold raw path's ~50-minute bucket sums (≤~1.2×) — accepted and documented, not hidden.
|
||||
|
||||
### Aggregation intent — gauge vs. rate
|
||||
|
||||
Not every metric folds the same way, and choosing wrong silently corrupts long-range totals. `KpiMetricAggregationCatalog.Resolve(source, metric)` classifies each `(Source, Metric)` pair as `KpiRollupAggregation.Gauge` (fold = **last-value-per-hour**) or `KpiRollupAggregation.Rate` (fold = **sum-per-hour**):
|
||||
Not every metric folds the same way, and choosing wrong silently corrupts long-range totals. `KpiMetricAggregationCatalog.Resolve(source, metric)` classifies each `(Source, Metric)` pair as `KpiRollupAggregation.Gauge` (fold = **last-value-per-hour**) or `KpiRollupAggregation.Rate` (fold = **sum-per-hour**). The catalog now has **two consumers**: the hourly rollup fold (which picks last-value vs. sum per hour) and the chart-time bucket reduction in `KpiSeriesBucketer` (which picks last-value vs. sum per chart bucket) — one classification keeps both layers consistent:
|
||||
|
||||
- **Gauge** (last-value) — instantaneous readings where a per-hour sum is meaningless: `queueDepth`, `buffered`, `stuck`/`stuckCount`, `parked*`, `oldestPendingAgeSeconds`, `connectionsUp`/`connectionsDown`, `deployedInstances`, and every other metric not explicitly listed as a rate. This is the **safe default**: an unknown / newly-added metric resolves to Gauge and never crashes the fold or invents a total.
|
||||
- **Rate** (sum-per-hour) — already-windowed per-interval deltas whose hour total is the sum of its minute samples: `NotificationOutbox/deliveredLastInterval`, `SiteCallAudit/failedLastInterval`, `SiteCallAudit/deliveredLastInterval`, and the per-report `SiteHealth` error counts `scriptErrors` / `alarmEvalErrors` / `deadLetters` / `eventLogWriteFailures`. Folding these as last-value would keep one minute's delta and discard the other 59, under-counting every long-range total.
|
||||
|
||||
@@ -60,6 +60,8 @@ public sealed class SiteAuditRetentionService : IHostedService, IDisposable
|
||||
}
|
||||
|
||||
private async Task RunLoopAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
// InitialDelay before the first purge — short so a daily-recycled node
|
||||
// still purges soon after start rather than waiting a full interval it may
|
||||
@@ -89,6 +91,14 @@ public sealed class SiteAuditRetentionService : IHostedService, IDisposable
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown landed mid-purge: SafePurgeAsync rethrows OCE by design so the purge
|
||||
// aborts promptly, but the loop task must complete CLEANLY — StopAsync hands
|
||||
// _loop straight to the host, and a canceled task there is shutdown-log noise
|
||||
// (arch-review 04 round 2, R7). Cancellation here IS the clean exit.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SafePurgeAsync(CancellationToken ct)
|
||||
{
|
||||
|
||||
@@ -51,6 +51,14 @@ public partial class KpiTrendChart
|
||||
/// The series to plot, assumed ordered ascending by
|
||||
/// <see cref="KpiSeriesPoint.BucketStartUtc"/>. <c>null</c> or empty renders
|
||||
/// the unavailable placeholder rather than an empty chart.
|
||||
/// <para>
|
||||
/// The per-point value's meaning depends on the metric's
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiMetricAggregationCatalog"/>
|
||||
/// classification: a Rate-classified series carries <b>per-bucket totals</b> (the
|
||||
/// bucketer sums each bucket's deltas), while a Gauge series carries
|
||||
/// last-value-per-bucket readings. The chart is agnostic to which — it plots the
|
||||
/// supplied values verbatim (arch-review 04 round 2, R3).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Parameter] public IReadOnlyList<KpiSeriesPoint>? Points { get; set; }
|
||||
|
||||
@@ -58,7 +66,10 @@ public partial class KpiTrendChart
|
||||
/// <c>data-test</c> slug. Required-ish; defaults to "Trend" if blank.</summary>
|
||||
[Parameter] public string Title { get; set; } = "Trend";
|
||||
|
||||
/// <summary>Optional unit suffix appended to value labels (e.g. "s").</summary>
|
||||
/// <summary>Optional unit suffix appended to value labels (e.g. "s"). For a
|
||||
/// Rate-classified metric the supplied text must state the per-bucket-total
|
||||
/// semantics (e.g. "delivered/bucket"), not a per-minute/per-interval reading,
|
||||
/// so the label matches the summed values in <see cref="Points"/> (R3).</summary>
|
||||
[Parameter] public string? Unit { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -76,12 +76,18 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
{
|
||||
var effectiveMax = maxPoints ?? _options.DefaultMaxSeriesPoints;
|
||||
|
||||
// Per-metric reduction intent: the same catalog that drives the hourly fold drives
|
||||
// the chart-time bucket reduction, so a Rate series is summed per bucket on BOTH the
|
||||
// raw-minute and hourly-rollup routes — one unit ("events per chart bucket") on both
|
||||
// sides of the RollupThresholdHours routing boundary (arch-review 04 round 2, R3).
|
||||
var aggregation = KpiMetricAggregationCatalog.Resolve(source, metric);
|
||||
|
||||
// Test-seam ctor: use the injected repository directly.
|
||||
if (_injectedRepository is not null)
|
||||
{
|
||||
var injectedSeries = await FetchSeriesAsync(
|
||||
_injectedRepository, source, metric, scope, scopeKey, fromUtc, toUtc, cancellationToken);
|
||||
return KpiSeriesBucketer.Bucket(injectedSeries, fromUtc, toUtc, effectiveMax);
|
||||
return KpiSeriesBucketer.Bucket(injectedSeries, fromUtc, toUtc, effectiveMax, aggregation);
|
||||
}
|
||||
|
||||
// Production: a fresh scope (and thus a fresh DbContext) per query so a
|
||||
@@ -90,7 +96,7 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
var repository = serviceScope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
var series = await FetchSeriesAsync(
|
||||
repository, source, metric, scope, scopeKey, fromUtc, toUtc, cancellationToken);
|
||||
return KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, effectiveMax);
|
||||
return KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, effectiveMax, aggregation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -103,6 +109,13 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
/// the caller's <see cref="KpiSeriesBucketer.Bucket"/> step is agnostic to the source
|
||||
/// (a 90 d rollup can still exceed the point ceiling, so bucketing still applies).
|
||||
/// The boundary is strict: exactly <c>RollupThresholdHours</c> stays on the raw path.
|
||||
/// <para>
|
||||
/// The routing boundary no longer changes a Rate chart's magnitude: both paths reduce to
|
||||
/// per-bucket sums via the catalog-driven aggregation applied in
|
||||
/// <see cref="GetSeriesAsync"/>. A residual ≤~1.2× native-resolution difference exists only
|
||||
/// for an unbucketed just-over-threshold rollup series (plotted at native per-hour sums) vs.
|
||||
/// an at-threshold bucketed raw series — documented, not hidden (arch-review 04 round 2, R3).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private Task<IReadOnlyList<KpiSeriesPoint>> FetchSeriesAsync(
|
||||
IKpiHistoryRepository repository,
|
||||
|
||||
@@ -109,6 +109,16 @@ public interface IKpiHistoryRepository
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the newest <c>KpiRollupHourly.HourStartUtc</c> across all series, or
|
||||
/// <c>null</c> when no rollups exist. The recorder's one-shot backfill consults
|
||||
/// this watermark to skip (or shrink to) the un-rolled tail instead of re-folding
|
||||
/// the whole raw-retention window on every failover (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task resolving to the newest folded hour start, or <c>null</c> when empty.</returns>
|
||||
Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Bulk-deletes hourly rollup rows whose
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi.KpiRollupHourly.HourStartUtc"/> is
|
||||
|
||||
@@ -42,10 +42,10 @@ public enum KpiRollupAggregation
|
||||
/// Keying by <c>(source, metric)</c> — not metric alone — is deliberate:
|
||||
/// <c>deliveredLastInterval</c> is emitted by both
|
||||
/// <see cref="KpiSources.NotificationOutbox"/> and <see cref="KpiSources.SiteCallAudit"/>,
|
||||
/// so a metric name is not globally unique. Every metric string is taken from the
|
||||
/// emitting source verbatim (shared <see cref="KpiMetrics"/> constants where they exist,
|
||||
/// otherwise the source's own private literal), so this catalog stays a faithful mirror
|
||||
/// of what actually lands in <c>KpiSample.Metric</c>.
|
||||
/// so a metric name is not globally unique. Every catalogued metric string is now a shared
|
||||
/// <see cref="KpiMetrics"/> constant — the emitting source consumes the same constant — so an
|
||||
/// emitter rename is a <strong>compile error</strong>, not a silent Gauge downgrade (R4), and
|
||||
/// this catalog stays a faithful mirror of what actually lands in <c>KpiSample.Metric</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The catalog stores only the <see cref="KpiRollupAggregation.Rate"/> pairs; anything
|
||||
@@ -57,12 +57,6 @@ public enum KpiRollupAggregation
|
||||
/// </remarks>
|
||||
public static class KpiMetricAggregationCatalog
|
||||
{
|
||||
// Metric literals for the sources that keep their names private (no shared
|
||||
// KpiMetrics constant exists for these). Charted metrics use the KpiMetrics catalog.
|
||||
private const string SiteCallAuditDeliveredLastInterval = "deliveredLastInterval";
|
||||
private const string SiteHealthAlarmEvalErrors = "alarmEvalErrors";
|
||||
private const string SiteHealthEventLogWriteFailures = "eventLogWriteFailures";
|
||||
|
||||
/// <summary>
|
||||
/// The <c>(source, metric)</c> pairs classified as <see cref="KpiRollupAggregation.Rate"/>
|
||||
/// (sum-per-hour). Every other emitted metric is a <see cref="KpiRollupAggregation.Gauge"/>.
|
||||
@@ -99,11 +93,11 @@ public static class KpiMetricAggregationCatalog
|
||||
{
|
||||
(KpiSources.NotificationOutbox, KpiMetrics.NotificationOutbox.DeliveredLastInterval),
|
||||
(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval),
|
||||
(KpiSources.SiteCallAudit, SiteCallAuditDeliveredLastInterval),
|
||||
(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.DeliveredLastInterval),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.ScriptErrors),
|
||||
(KpiSources.SiteHealth, SiteHealthAlarmEvalErrors),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.AlarmEvalErrors),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.DeadLetters),
|
||||
(KpiSources.SiteHealth, SiteHealthEventLogWriteFailures),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.EventLogWriteFailures),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -18,11 +18,16 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
/// <strong>not</strong> a rename — changing any value would orphan historical samples.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only the metrics a trend page actually charts are catalogued here. Sources may emit
|
||||
/// additional internal metrics (e.g. the Notification Outbox <c>stuckCount</c> /
|
||||
/// <c>oldestPendingAgeSeconds</c>, the Site Call Audit <c>deliveredLastInterval</c> /
|
||||
/// <c>stuck</c> / <c>oldestPendingAgeSeconds</c>, and the bulk of the Site Health catalog)
|
||||
/// that no page references; those stay private to their source until a page charts them.
|
||||
/// This catalog now carries two kinds of metric: those a trend page charts, <em>and</em>
|
||||
/// those the <see cref="KpiMetricAggregationCatalog"/> classifies for the hourly rollup fold
|
||||
/// (e.g. <c>SiteCallAudit/deliveredLastInterval</c>, <c>SiteHealth/alarmEvalErrors</c>,
|
||||
/// <c>SiteHealth/eventLogWriteFailures</c>) even if no page charts them yet. The C4/R4 lesson
|
||||
/// is that <strong>any metric named in two places needs one shared symbol</strong>: a private
|
||||
/// literal on the emitter plus a matching private literal in the catalog drifts silently on a
|
||||
/// rename. Sources may still keep genuinely single-use internal metrics (e.g. the Notification
|
||||
/// Outbox <c>stuckCount</c> / <c>oldestPendingAgeSeconds</c>, the Site Call Audit <c>stuck</c> /
|
||||
/// <c>oldestPendingAgeSeconds</c>, and the bulk of the Site Health catalog) private until a page
|
||||
/// or the catalog references them.
|
||||
/// Constants are grouped by source via nested static classes mirroring
|
||||
/// <see cref="KpiSources"/>.
|
||||
/// </para>
|
||||
@@ -65,6 +70,11 @@ public static class KpiMetrics
|
||||
|
||||
/// <summary>Cached calls that failed permanently in the last KPI interval.</summary>
|
||||
public const string FailedLastInterval = "failedLastInterval";
|
||||
|
||||
/// <summary>Cached calls delivered in the last KPI interval. Not charted by a trend
|
||||
/// page today, but named in the rollup aggregation catalog as a Rate metric — so it
|
||||
/// is a shared symbol to keep the emitter and catalog in lock-step (R4).</summary>
|
||||
public const string DeliveredLastInterval = "deliveredLastInterval";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -98,6 +108,16 @@ public static class KpiMetrics
|
||||
/// <summary>Script-execution error count reported by the site.</summary>
|
||||
public const string ScriptErrors = "scriptErrors";
|
||||
|
||||
/// <summary>Alarm-evaluation error count reported by the site. Named in the rollup
|
||||
/// aggregation catalog as a Rate metric — shared symbol so an emitter rename is a
|
||||
/// compile error, not a silent Gauge downgrade (R4).</summary>
|
||||
public const string AlarmEvalErrors = "alarmEvalErrors";
|
||||
|
||||
/// <summary>Event-log write-failure count reported by the site. Named in the rollup
|
||||
/// aggregation catalog as a Rate metric — shared symbol so an emitter rename is a
|
||||
/// compile error, not a silent Gauge downgrade (R4).</summary>
|
||||
public const string EventLogWriteFailures = "eventLogWriteFailures";
|
||||
|
||||
/// <summary>Summed store-and-forward buffer depth across all buffers.</summary>
|
||||
public const string SfBufferDepth = "sfBufferDepth";
|
||||
}
|
||||
|
||||
@@ -3,8 +3,14 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
/// <summary>
|
||||
/// Pure, deterministic downsampling helper for KPI series charting ("KPI History & Trends").
|
||||
/// Reduces a raw <see cref="KpiSeriesPoint"/> series to at most <c>maxPoints</c> points using
|
||||
/// last-value-per-bucket / gauge semantics — suitable for step/area charts where the most
|
||||
/// recent value in a window best represents that window.
|
||||
/// <em>per-metric</em> bucket reduction driven by <see cref="KpiRollupAggregation"/>:
|
||||
/// <see cref="KpiRollupAggregation.Gauge"/> (the default) keeps the <strong>last value</strong>
|
||||
/// in each bucket — suitable for step/area charts where the most recent reading best represents
|
||||
/// the window; <see cref="KpiRollupAggregation.Rate"/> <strong>sums per bucket</strong> — each
|
||||
/// raw point of a Rate series is a per-interval delta, so the bucket's true total is the sum of
|
||||
/// its deltas. Summing on both the raw-minute and hourly-rollup paths charts the same unit
|
||||
/// ("events per chart bucket") on both sides of the raw/rollup routing boundary, eliminating the
|
||||
/// ~60× magnitude jump at that boundary (arch-review 04 round 2, R3).
|
||||
/// </summary>
|
||||
public static class KpiSeriesBucketer
|
||||
{
|
||||
@@ -27,12 +33,20 @@ public static class KpiSeriesBucketer
|
||||
/// <param name="fromUtc">UTC start of the query window (inclusive).</param>
|
||||
/// <param name="toUtc">UTC end of the query window (inclusive on the right edge).</param>
|
||||
/// <param name="maxPoints">Maximum number of output points. Must be ≥ 2.</param>
|
||||
/// <param name="aggregation">
|
||||
/// How each bucket is reduced. <see cref="KpiRollupAggregation.Gauge"/> (default) keeps the
|
||||
/// last value in the bucket — the historical behavior. <see cref="KpiRollupAggregation.Rate"/>
|
||||
/// sums the values in the bucket — each raw point of a Rate series is a per-interval delta, so
|
||||
/// the bucket total is the sum of its deltas (arch-review 04 round 2, R3).
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// An <see cref="IReadOnlyList{T}"/> of at most <paramref name="maxPoints"/> bucketed points,
|
||||
/// ordered by <see cref="KpiSeriesPoint.BucketStartUtc"/> ascending.
|
||||
/// Returns <paramref name="raw"/> unchanged (same reference) when
|
||||
/// <c>raw.Count <= maxPoints</c>; callers must not mutate the underlying
|
||||
/// collection in that case, as it is the same object passed in.
|
||||
/// For <see cref="KpiRollupAggregation.Gauge"/> returns <paramref name="raw"/> unchanged
|
||||
/// (same reference) when <c>raw.Count <= maxPoints</c>; callers must not mutate the
|
||||
/// underlying collection in that case. <see cref="KpiRollupAggregation.Rate"/> always runs
|
||||
/// the bucketing pass — two deltas landing in one bucket must sum even in a short series —
|
||||
/// so it never returns the input reference.
|
||||
/// </returns>
|
||||
/// <exception cref="ArgumentOutOfRangeException">
|
||||
/// Thrown when <paramref name="maxPoints"/> < 2 or
|
||||
@@ -44,7 +58,8 @@ public static class KpiSeriesBucketer
|
||||
IReadOnlyList<KpiSeriesPoint> raw,
|
||||
DateTime fromUtc,
|
||||
DateTime toUtc,
|
||||
int maxPoints)
|
||||
int maxPoints,
|
||||
KpiRollupAggregation aggregation = KpiRollupAggregation.Gauge)
|
||||
{
|
||||
if (maxPoints < 2)
|
||||
throw new ArgumentOutOfRangeException(nameof(maxPoints),
|
||||
@@ -54,11 +69,15 @@ public static class KpiSeriesBucketer
|
||||
throw new ArgumentOutOfRangeException(nameof(toUtc),
|
||||
toUtc, "toUtc must be strictly greater than fromUtc.");
|
||||
|
||||
// Normal runtime case — empty or short series: return as-is.
|
||||
// Normal runtime case — empty series: return as-is.
|
||||
if (raw is null || raw.Count == 0)
|
||||
return Array.Empty<KpiSeriesPoint>();
|
||||
|
||||
if (raw.Count <= maxPoints)
|
||||
// Short-series early return is Gauge-only: for Rate, two deltas landing in one bucket
|
||||
// must still sum, so a short Rate series cannot be returned verbatim (arch-review 04
|
||||
// round 2, R3). For well-spaced short Rate series the per-point buckets make the sum an
|
||||
// identity, so the output still equals the input anyway.
|
||||
if (aggregation == KpiRollupAggregation.Gauge && raw.Count <= maxPoints)
|
||||
return raw;
|
||||
|
||||
// Divide the window into maxPoints equal-width buckets.
|
||||
@@ -67,12 +86,11 @@ public static class KpiSeriesBucketer
|
||||
double windowTicks = (double)(toUtc.Ticks - fromUtc.Ticks);
|
||||
double bucketWidthTicks = windowTicks / maxPoints;
|
||||
|
||||
// For each bucket, track the candidate point: the one with the
|
||||
// maximum BucketStartUtc (last value within the bucket).
|
||||
// We use a fixed-size array indexed by bucket number.
|
||||
// Nullable KpiSeriesPoint[] with 'hasValue' flags is fine since the
|
||||
// struct is small.
|
||||
// For each bucket, track the candidate point: for Gauge the one with the maximum
|
||||
// BucketStartUtc (last value within the bucket); for Rate a running sum of the bucket's
|
||||
// deltas. Fixed-size arrays indexed by bucket number.
|
||||
var best = new KpiSeriesPoint[maxPoints];
|
||||
var sums = new double[maxPoints];
|
||||
var occupied = new bool[maxPoints];
|
||||
|
||||
foreach (var point in raw)
|
||||
@@ -89,13 +107,19 @@ public static class KpiSeriesBucketer
|
||||
if (bucketIndex >= maxPoints)
|
||||
bucketIndex = maxPoints - 1;
|
||||
|
||||
if (aggregation == KpiRollupAggregation.Rate)
|
||||
{
|
||||
// Rate: accumulate the bucket's per-interval deltas.
|
||||
sums[bucketIndex] += point.Value;
|
||||
occupied[bucketIndex] = true;
|
||||
}
|
||||
// Keep the last point in iteration order within this bucket. Because the stored
|
||||
// candidate's BucketStartUtc is the bucket-START timestamp (not the raw point's
|
||||
// capture time), the comparison below is true for essentially any in-bucket point,
|
||||
// so each later-in-iteration point overwrites the previous one. For the ascending-
|
||||
// sorted input this method requires, last-in-iteration IS the largest-timestamp
|
||||
// point — i.e. last-value-per-bucket semantics.
|
||||
if (!occupied[bucketIndex] ||
|
||||
else if (!occupied[bucketIndex] ||
|
||||
point.BucketStartUtc > best[bucketIndex].BucketStartUtc)
|
||||
{
|
||||
best[bucketIndex] = new KpiSeriesPoint(
|
||||
@@ -109,8 +133,13 @@ public static class KpiSeriesBucketer
|
||||
var result = new List<KpiSeriesPoint>(maxPoints);
|
||||
for (int i = 0; i < maxPoints; i++)
|
||||
{
|
||||
if (occupied[i])
|
||||
result.Add(best[i]);
|
||||
if (!occupied[i])
|
||||
continue;
|
||||
|
||||
result.Add(aggregation == KpiRollupAggregation.Rate
|
||||
? new KpiSeriesPoint(
|
||||
fromUtc + TimeSpan.FromTicks((long)(i * bucketWidthTicks)), sums[i])
|
||||
: best[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
+9
@@ -97,5 +97,14 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
|
||||
.HasDatabaseName("IX_SiteCalls_NonTerminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NULL")
|
||||
.IncludeProperties(nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status));
|
||||
|
||||
// Terminal backs the daily retention purge's predicate (TerminalAtUtc IS NOT NULL
|
||||
// AND TerminalAtUtc < cutoff) and Task 11's MIN() slice anchor: filtered to the
|
||||
// terminal population so the purge seeks the expired tail instead of full-scanning
|
||||
// the 365-day table (arch-review 04 round 2, R6). Complements IX_SiteCalls_NonTerminal,
|
||||
// which is filtered to IS NULL and unusable for this predicate.
|
||||
builder.HasIndex(s => s.TerminalAtUtc)
|
||||
.HasDatabaseName("IX_SiteCalls_Terminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NOT NULL");
|
||||
}
|
||||
}
|
||||
|
||||
+2090
File diff suppressed because it is too large
Load Diff
+28
@@ -0,0 +1,28 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddSiteCallsTerminalIndex : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_SiteCalls_Terminal",
|
||||
table: "SiteCalls",
|
||||
column: "TerminalAtUtc",
|
||||
filter: "[TerminalAtUtc] IS NOT NULL");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropIndex(
|
||||
name: "IX_SiteCalls_Terminal",
|
||||
table: "SiteCalls");
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
@@ -167,6 +167,10 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
|
||||
SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("CreatedAtUtc"), new[] { "SourceSite", "SourceNode", "Status" });
|
||||
|
||||
b.HasIndex("TerminalAtUtc")
|
||||
.HasDatabaseName("IX_SiteCalls_Terminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NOT NULL");
|
||||
|
||||
b.HasIndex("SourceSite", "CreatedAtUtc")
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_SiteCalls_Source_Created");
|
||||
|
||||
+45
-1
@@ -1,4 +1,6 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
@@ -13,14 +15,22 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
{
|
||||
private readonly ScadaBridgeDbContext _context;
|
||||
private readonly ILogger<KpiHistoryRepository> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="KpiHistoryRepository"/> class.
|
||||
/// </summary>
|
||||
/// <param name="context">The EF Core database context.</param>
|
||||
public KpiHistoryRepository(ScadaBridgeDbContext context)
|
||||
/// <param name="logger">
|
||||
/// Optional logger; defaults to <see cref="NullLogger{T}"/> (mirrors
|
||||
/// <c>SiteCallAuditRepository</c> — MS.DI resolves <see cref="ILogger{T}"/> automatically,
|
||||
/// so no registration churn). Used to classify the self-healing failover fold-race.
|
||||
/// </param>
|
||||
public KpiHistoryRepository(
|
||||
ScadaBridgeDbContext context, ILogger<KpiHistoryRepository>? logger = null)
|
||||
{
|
||||
_context = context ?? throw new ArgumentNullException(nameof(context));
|
||||
_logger = logger ?? NullLogger<KpiHistoryRepository>.Instance;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -101,8 +111,14 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
// fetch it and group IN MEMORY. This deliberately avoids provider-specific
|
||||
// DATEADD/DATEPART hour-truncation translation (SQLite in tests, SQL Server in prod),
|
||||
// keeping the hour-bucket arithmetic identical across providers and trivially correct.
|
||||
// Projected, untracked fetch: the fold only reads these six fields and never
|
||||
// mutates a KpiSample. A tracked ToListAsync registered ~5k-90k read-only
|
||||
// entities in the change tracker per healthy 3h fold — all re-scanned by
|
||||
// DetectChanges on the final SaveChanges (arch-review 04 round 2, R2). A
|
||||
// projection is inherently untracked and materializes no entity at all.
|
||||
var samples = await _context.KpiSamples
|
||||
.Where(s => s.CapturedAtUtc >= from && s.CapturedAtUtc < to)
|
||||
.Select(s => new FoldSample(s.Source, s.Metric, s.Scope, s.ScopeKey, s.CapturedAtUtc, s.Value))
|
||||
.ToListAsync(cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
@@ -164,8 +180,23 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
catch (DbUpdateException ex)
|
||||
{
|
||||
// Failover-overlap race: the old singleton incarnation's in-flight fold and the
|
||||
// new node's first fold can both Add the same (series, hour) row; the unique
|
||||
// IX_KpiRollupHourly_Series then faults the loser's ENTIRE SaveChanges — the
|
||||
// failure grain is the PASS, not the row (arch-review 04 round 2, R5). The fold
|
||||
// only ever writes KpiRollupHourly rows, so any save fault here is a lost upsert
|
||||
// race or a transient the next idempotent re-fold repairs identically; classify
|
||||
// at Information instead of surfacing a scary error for a self-healing no-op.
|
||||
_logger.LogInformation(ex,
|
||||
"KPI rollup fold lost a failover-overlap upsert race; pass discarded, next fold self-heals.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
|
||||
@@ -186,6 +217,15 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Single MAX over the unique IX_KpiRollupHourly_Series population (the rollup
|
||||
// table is small — one row per series-hour). Null when no rollups exist.
|
||||
return await _context.KpiRollupHourly
|
||||
.MaxAsync(r => (DateTime?)r.HourStartUtc, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -212,4 +252,8 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
/// <summary>In-memory grouping key: one KPI series (four-tuple) within one UTC hour.</summary>
|
||||
private readonly record struct SeriesHourKey(
|
||||
string Source, string Metric, string Scope, string? ScopeKey, DateTime HourStartUtc);
|
||||
|
||||
/// <summary>Narrow, untracked projection of one KpiSample row for the fold (R2).</summary>
|
||||
private readonly record struct FoldSample(
|
||||
string Source, string Metric, string Scope, string? ScopeKey, DateTime CapturedAtUtc, double Value);
|
||||
}
|
||||
|
||||
+22
-2
@@ -246,9 +246,29 @@ OPTION (RECOMPILE);";
|
||||
/// <inheritdoc />
|
||||
public async Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
{
|
||||
return await _context.Database.ExecuteSqlInterpolatedAsync(
|
||||
$"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {olderThanUtc};",
|
||||
// Time-sliced batches (arch-review 04 round 2, R6 — the one maintenance DELETE
|
||||
// that missed round 1's batching pass): each DELETE covers at most one DAY of
|
||||
// terminal rows, capping the lock/log footprint per statement. Steady state
|
||||
// (daily purge, 365-day retention) is a single slice; only catch-up after an
|
||||
// outage runs several. One-day (not one-hour) slices are proportionate to
|
||||
// SiteCalls volume, which is far below KpiSample's. The MIN() anchor and the
|
||||
// DELETE predicate both seek IX_SiteCalls_Terminal (filtered IS NOT NULL).
|
||||
var total = 0;
|
||||
var floor = await _context.SiteCalls
|
||||
.Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc)
|
||||
.MinAsync(s => s.TerminalAtUtc, ct);
|
||||
while (floor is not null && floor < olderThanUtc)
|
||||
{
|
||||
var ceiling = floor.Value.AddDays(1) < olderThanUtc ? floor.Value.AddDays(1) : olderThanUtc;
|
||||
total += await _context.Database.ExecuteSqlInterpolatedAsync(
|
||||
$"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {ceiling};",
|
||||
ct);
|
||||
floor = await _context.SiteCalls
|
||||
.Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc)
|
||||
.MinAsync(s => s.TerminalAtUtc, ct);
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
// Terminal status string literals for the interval-throughput KPIs. The
|
||||
|
||||
@@ -40,7 +40,7 @@ public sealed class SiteHealthKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricConnectionsUp = "connectionsUp";
|
||||
private const string MetricConnectionsDown = KpiMetrics.SiteHealth.ConnectionsDown;
|
||||
private const string MetricScriptErrors = KpiMetrics.SiteHealth.ScriptErrors;
|
||||
private const string MetricAlarmEvalErrors = "alarmEvalErrors";
|
||||
private const string MetricAlarmEvalErrors = KpiMetrics.SiteHealth.AlarmEvalErrors;
|
||||
private const string MetricSfBufferDepth = KpiMetrics.SiteHealth.SfBufferDepth;
|
||||
private const string MetricDeadLetters = KpiMetrics.SiteHealth.DeadLetters;
|
||||
private const string MetricParkedMessages = "parkedMessages";
|
||||
@@ -48,7 +48,7 @@ public sealed class SiteHealthKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricEnabledInstances = "enabledInstances";
|
||||
private const string MetricDisabledInstances = "disabledInstances";
|
||||
private const string MetricAuditBacklogPending = "auditBacklogPending";
|
||||
private const string MetricEventLogWriteFailures = "eventLogWriteFailures";
|
||||
private const string MetricEventLogWriteFailures = KpiMetrics.SiteHealth.EventLogWriteFailures;
|
||||
|
||||
private readonly ICentralHealthAggregator _aggregator;
|
||||
|
||||
|
||||
@@ -31,7 +31,12 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
|
||||
/// series. A one-shot backfill timer additionally folds the whole raw-retention
|
||||
/// window of already-recorded samples once shortly after start, so 30 d/90 d
|
||||
/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
|
||||
/// same idempotent fold and runs at most once per actor lifetime.
|
||||
/// same idempotent fold and runs at most once per actor lifetime. The backfill is
|
||||
/// sliced into bounded 24 h fold windows (oldest-first) rather than one unbounded
|
||||
/// pass, and lowers its in-flight guard between slices so periodic folds interleave
|
||||
/// — periodic folds and backfill slices strictly alternate through the single
|
||||
/// <c>_rollupInFlight</c>/<c>_backfillInFlight</c> gate pair, so the two idempotent
|
||||
/// upserts never race on overlapping rows (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -118,11 +123,32 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
private bool _rollupInFlight;
|
||||
|
||||
/// <summary>
|
||||
/// In-flight guard for the one-shot backfill fold. Set true when the backfill fold is
|
||||
/// launched and cleared when its <see cref="BackfillComplete"/> arrives. While true, periodic
|
||||
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so the (possibly
|
||||
/// long) full-retention-window backfill never runs concurrently with a periodic lookback fold
|
||||
/// on the same upsert rows.
|
||||
/// Width of one backfill fold slice (24 h). The repository fold materializes its whole
|
||||
/// window in memory, so the one-shot backfill must never hand it the full raw-retention
|
||||
/// window (90 d ≈ tens of millions of rows at fleet volume — arch-review 04 round 2, R1).
|
||||
/// One day per fold bounds each pass to roughly what a steady-state day writes, and the
|
||||
/// guard is lowered between slices so periodic folds interleave instead of stalling
|
||||
/// behind the historical pass.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan BackfillSliceWidth = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// Queue of remaining backfill fold slices (each a bounded <see cref="BackfillSliceWidth"/>
|
||||
/// window), oldest-first. Built once when the backfill plan is enumerated and drained one
|
||||
/// slice per fold; a periodic <see cref="RollupTick"/> may claim the fold path between slices
|
||||
/// (the two idempotent upserts strictly alternate through the
|
||||
/// <see cref="_rollupInFlight"/>/<see cref="_backfillInFlight"/> gate pair, never racing on
|
||||
/// overlapping rows — arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
private readonly Queue<(DateTime FromHourUtc, DateTime ToHourUtc)> _backfillSlices = new();
|
||||
|
||||
/// <summary>
|
||||
/// In-flight guard for a single backfill fold slice. Set true when a slice fold is launched
|
||||
/// and cleared when its <see cref="BackfillSliceComplete"/> arrives. While true, periodic
|
||||
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so a backfill
|
||||
/// slice never runs concurrently with a periodic lookback fold on the same upsert rows; it is
|
||||
/// lowered BETWEEN slices so periodic folds interleave instead of stalling behind the whole
|
||||
/// historical pass.
|
||||
/// </summary>
|
||||
private bool _backfillInFlight;
|
||||
|
||||
@@ -160,10 +186,21 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
Receive<RollupTick>(_ => HandleRollupTick());
|
||||
Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
|
||||
Receive<BackfillTick>(_ => HandleBackfillTick());
|
||||
Receive<BackfillComplete>(_ =>
|
||||
Receive<BackfillPlan>(HandleBackfillPlan);
|
||||
Receive<BackfillSliceComplete>(_ =>
|
||||
{
|
||||
_backfillInFlight = false; // lower the in-flight guard (success or fault)
|
||||
_backfillDone = true; // latch: at most once per actor lifetime
|
||||
_backfillInFlight = false; // release the fold path BETWEEN slices (R1)
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
_backfillDone = true; // latch: at most once per actor lifetime (unchanged semantics)
|
||||
_logger.LogInformation("KPI rollup backfill completed — all slices folded.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Next slice via the timer, not inline: a RollupTick already queued in the mailbox
|
||||
// is processed first, so periodic folds interleave; the existing _rollupInFlight
|
||||
// deferral branch in HandleBackfillTick then re-arms the backfill behind it.
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -483,13 +520,15 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
|
||||
/// <summary>
|
||||
/// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
|
||||
/// has already completed (<see cref="_backfillDone"/>) or is in flight
|
||||
/// has already completed (<see cref="_backfillDone"/>) or a slice is in flight
|
||||
/// (<see cref="_backfillInFlight"/>) the tick is ignored. If a periodic fold currently holds
|
||||
/// the fold path (<see cref="_rollupInFlight"/>) the backfill defers by re-arming its single
|
||||
/// timer, so the full-window backfill and the periodic lookback fold never race on overlapping
|
||||
/// upsert rows. Otherwise it raises the backfill guard and folds the whole raw-retention
|
||||
/// window <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> off the actor
|
||||
/// thread, piping a <see cref="BackfillComplete"/> back to lower the guard and set the latch.
|
||||
/// timer, so a backfill slice and the periodic lookback fold never race on overlapping upsert
|
||||
/// rows. When no slice plan exists yet it enumerates the raw-retention window
|
||||
/// <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> into bounded
|
||||
/// <see cref="BackfillSliceWidth"/> windows (oldest-first) and self-arms to start draining;
|
||||
/// otherwise it dequeues ONE slice, raises the guard, and folds it off the actor thread,
|
||||
/// piping a <see cref="BackfillSliceComplete"/> back to release the guard between slices.
|
||||
/// </summary>
|
||||
private void HandleBackfillTick()
|
||||
{
|
||||
@@ -500,38 +539,152 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (_rollupInFlight)
|
||||
{
|
||||
// A periodic fold is in flight; defer the one-shot backfill briefly so the two
|
||||
// A periodic fold is in flight; defer the next backfill slice briefly so the two
|
||||
// idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
|
||||
return;
|
||||
}
|
||||
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
// No slice plan yet: resolve the rollup watermark off the actor thread, then plan on
|
||||
// the actor thread (Receive<BackfillPlan>). Raise the guard so periodic ticks defer
|
||||
// while planning is in flight; the plan handler lowers it. On a failover restart the
|
||||
// watermark lets the plan SKIP or SHRINK to the un-rolled tail instead of re-folding
|
||||
// the whole raw-retention window (arch-review 04 round 2, R1).
|
||||
_backfillInFlight = true;
|
||||
var toHourUtc = TruncateToHour(DateTime.UtcNow);
|
||||
var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
|
||||
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
|
||||
var planCancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) ({RetentionDays}-day window).",
|
||||
fromHourUtc, toHourUtc, _options.RetentionDays);
|
||||
|
||||
RunBackfillPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
|
||||
RunBackfillPlanPass(planCancellationToken).PipeTo(
|
||||
Self,
|
||||
success: () => BackfillComplete.Instance,
|
||||
success: watermark => new BackfillPlan(watermark),
|
||||
failure: ex =>
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill faulted unexpectedly.");
|
||||
return BackfillComplete.Instance;
|
||||
_logger.LogError(ex, "KPI rollup backfill plan pass faulted unexpectedly.");
|
||||
return new BackfillPlan(null); // null watermark ⇒ plan the full retention window
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Dequeue and fold exactly ONE bounded slice.
|
||||
var slice = _backfillSlices.Dequeue();
|
||||
_backfillInFlight = true;
|
||||
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
|
||||
|
||||
RunBackfillPass(slice.FromHourUtc, slice.ToHourUtc, cancellationToken).PipeTo(
|
||||
Self,
|
||||
success: () => BackfillSliceComplete.Instance,
|
||||
failure: ex =>
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill slice faulted unexpectedly.");
|
||||
return BackfillSliceComplete.Instance;
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the one-shot backfill fold: opens a DI scope, resolves the repository, and folds the
|
||||
/// Enumerates <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
|
||||
/// contiguous <see cref="BackfillSliceWidth"/> windows (the last possibly shorter),
|
||||
/// oldest-first, and enqueues them onto <see cref="_backfillSlices"/>.
|
||||
/// </summary>
|
||||
private void EnqueueBackfillSlices(DateTime fromHourUtc, DateTime toHourUtc)
|
||||
{
|
||||
var sliceStart = fromHourUtc;
|
||||
while (sliceStart < toHourUtc)
|
||||
{
|
||||
var sliceEnd = sliceStart + BackfillSliceWidth;
|
||||
if (sliceEnd > toHourUtc)
|
||||
{
|
||||
sliceEnd = toHourUtc;
|
||||
}
|
||||
|
||||
_backfillSlices.Enqueue((sliceStart, sliceEnd));
|
||||
sliceStart = sliceEnd;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Plans the backfill from the resolved rollup watermark (actor thread). The window floor is
|
||||
/// the newer of the watermark (re-folding the watermark hour itself is a cheap idempotent
|
||||
/// safety margin) and the raw-retention floor; a null watermark (fresh install) keeps the full
|
||||
/// retention window. If the floor is already within
|
||||
/// <see cref="KpiHistoryOptions.RollupLookbackHours"/> of now the periodic fold's lookback
|
||||
/// covers everything from there forward, so the whole historical pass is skipped (the failover
|
||||
/// fast-path); otherwise the tail <c>[floor, TruncateToHour(now))</c> is enumerated into
|
||||
/// bounded slices and draining is self-armed. Either way the plan-pass guard is lowered here
|
||||
/// (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
private void HandleBackfillPlan(BackfillPlan plan)
|
||||
{
|
||||
var toHourUtc = TruncateToHour(DateTime.UtcNow);
|
||||
var retentionFloor = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
|
||||
// Shrink to the un-rolled tail; re-fold the watermark hour itself (idempotent) as a safety
|
||||
// margin. A null watermark (fresh install) keeps the full retention window.
|
||||
var floor = plan.Watermark is { } w && w > retentionFloor ? w : retentionFloor;
|
||||
|
||||
if (floor >= toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours))
|
||||
{
|
||||
// Failover fast-path: rollups are already current — the periodic fold's lookback
|
||||
// window covers everything from the watermark forward, so the full historical pass is
|
||||
// skipped entirely (arch-review 04 round 2, R1).
|
||||
_backfillInFlight = false;
|
||||
_backfillDone = true;
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill skipped — rollups current through {Watermark:o}.", plan.Watermark);
|
||||
return;
|
||||
}
|
||||
|
||||
EnqueueBackfillSlices(floor, toHourUtc);
|
||||
_backfillInFlight = false;
|
||||
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
// Degenerate window (no whole hours to fold) — nothing to backfill.
|
||||
_backfillDone = true;
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill starting — folding [{FromHour:o}, {ToHour:o}) across {SliceCount} "
|
||||
+ "slice(s) of up to {SliceHours} h (watermark {Watermark:o}).",
|
||||
floor, toHourUtc, _backfillSlices.Count, BackfillSliceWidth.TotalHours, plan.Watermark);
|
||||
|
||||
// Start draining via the timer (uniform path with the between-slice re-arm).
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the newest folded rollup hour (the backfill watermark) off the actor thread in a
|
||||
/// fresh DI scope. Never faults — on cancellation or error it returns <c>null</c>, which the
|
||||
/// plan treats as "no rollups", keeping the full retention window (best-effort, mirrors the
|
||||
/// other passes).
|
||||
/// </summary>
|
||||
private async Task<DateTime?> RunBackfillPlanPass(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = _serviceProvider.CreateAsyncScope();
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
return await repository.GetLatestRollupHourAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutdown interrupted the lookup; a fresh active node re-plans. Not a failure.
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill watermark lookup failed; planning the full retention window.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
|
||||
/// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
|
||||
/// the hourly rollup table via the same idempotent upsert the periodic fold uses (Task 3). The
|
||||
/// whole body is wrapped so the returned task never faults — best-effort observability must
|
||||
/// never disrupt startup. The fold method returns no count, so completion is logged with the
|
||||
/// window and elapsed wall time only.
|
||||
/// the hourly rollup table via the same idempotent upsert the periodic fold uses. The whole
|
||||
/// body is wrapped so the returned task never faults — best-effort observability must never
|
||||
/// disrupt startup. The fold method returns no count, so each slice is logged at Debug with its
|
||||
/// window and elapsed wall time (the queue-drained completion is logged at Information).
|
||||
/// </summary>
|
||||
private async Task RunBackfillPass(
|
||||
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
|
||||
@@ -543,8 +696,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill completed — folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
|
||||
_logger.LogDebug(
|
||||
"KPI rollup backfill slice folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
|
||||
fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
@@ -614,9 +767,11 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Self-tick triggering the one-shot backfill fold over the full raw-retention window. Armed
|
||||
/// once via a single timer in <see cref="PreStart"/> (and re-armed only to defer past an
|
||||
/// in-flight periodic fold); runs at most once per actor lifetime.
|
||||
/// Self-tick triggering the one-shot backfill: on the first tick it enumerates the full
|
||||
/// raw-retention window into bounded slices, and each subsequent tick folds one slice. Armed
|
||||
/// once via a single timer in <see cref="PreStart"/> (and re-armed to drain the next slice or
|
||||
/// to defer past an in-flight periodic fold); the whole pass runs at most once per actor
|
||||
/// lifetime.
|
||||
/// </summary>
|
||||
internal sealed class BackfillTick
|
||||
{
|
||||
@@ -625,13 +780,21 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Piped-back completion of the one-shot backfill fold; lets the fold run off the actor thread
|
||||
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard and latches
|
||||
/// Piped-back result of the backfill plan pass: the newest folded rollup hour (the watermark),
|
||||
/// or <c>null</c> when no rollups exist. Consumed on the actor thread to skip or shrink the
|
||||
/// backfill to the un-rolled tail (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
internal sealed record BackfillPlan(DateTime? Watermark);
|
||||
|
||||
/// <summary>
|
||||
/// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
|
||||
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard BETWEEN slices — then
|
||||
/// either drains the next slice (self-armed timer) or, when the queue is empty, latches
|
||||
/// <c>_backfillDone</c> (fires on success and fault).
|
||||
/// </summary>
|
||||
internal sealed class BackfillComplete
|
||||
internal sealed class BackfillSliceComplete
|
||||
{
|
||||
public static readonly BackfillComplete Instance = new();
|
||||
private BackfillComplete() { }
|
||||
public static readonly BackfillSliceComplete Instance = new();
|
||||
private BackfillSliceComplete() { }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ public sealed class SiteCallAuditKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricBuffered = KpiMetrics.SiteCallAudit.Buffered;
|
||||
private const string MetricParked = KpiMetrics.SiteCallAudit.Parked;
|
||||
private const string MetricFailedLastInterval = KpiMetrics.SiteCallAudit.FailedLastInterval;
|
||||
private const string MetricDeliveredLastInterval = "deliveredLastInterval";
|
||||
private const string MetricDeliveredLastInterval = KpiMetrics.SiteCallAudit.DeliveredLastInterval;
|
||||
private const string MetricStuck = "stuck";
|
||||
private const string MetricOldestPendingAgeSeconds = "oldestPendingAgeSeconds";
|
||||
|
||||
|
||||
@@ -63,6 +63,29 @@ public class SiteAuditRetentionServiceTests
|
||||
Assert.True(queue.PurgeCalls.Count >= 2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopAsync_MidPurge_CompletesCleanly_WithoutSurfacingCancellation()
|
||||
{
|
||||
// PurgeExpiredAsync blocks until cancelled, so shutdown lands mid-purge: SafePurgeAsync
|
||||
// rethrows the OCE by design (abort the purge promptly), but the loop task must still
|
||||
// complete CLEANLY — StopAsync hands _loop straight to the host, and a canceled task
|
||||
// there is shutdown-log noise (arch-review 04 round 2, R7).
|
||||
var queue = new RecordingSiteAuditQueue { BlockUntilCancelled = true };
|
||||
var options = Options.Create(new SiteAuditRetentionOptions
|
||||
{
|
||||
RetentionDays = 7,
|
||||
PurgeInterval = TimeSpan.FromHours(24),
|
||||
InitialDelay = TimeSpan.Zero,
|
||||
});
|
||||
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
||||
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Must NOT throw TaskCanceledException back into the host's shutdown path.
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
@@ -86,7 +109,11 @@ public class SiteAuditRetentionServiceTests
|
||||
public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
|
||||
public bool ThrowOnFirstCall { get; init; }
|
||||
|
||||
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
/// <summary>When set, PurgeExpiredAsync records the call then blocks until the token
|
||||
/// cancels — simulating a shutdown that lands while a purge is in flight.</summary>
|
||||
public bool BlockUntilCancelled { get; init; }
|
||||
|
||||
public async Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
{
|
||||
var n = Interlocked.Increment(ref _calls);
|
||||
_purgeCalls.Enqueue(olderThanUtc);
|
||||
@@ -95,7 +122,12 @@ public class SiteAuditRetentionServiceTests
|
||||
throw new InvalidOperationException("Simulated transient purge failure.");
|
||||
}
|
||||
|
||||
return Task.FromResult(0);
|
||||
if (BlockUntilCancelled)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
|
||||
|
||||
@@ -330,4 +330,95 @@ public class KpiHistoryQueryServiceTests
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
||||
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
// ---- Task 6: catalog-driven aggregation at the bucketer boundary (R3) -----------
|
||||
|
||||
[Fact]
|
||||
public async Task GetSeriesAsync_RateMetric_SumsPerBucket_OnRollupPath()
|
||||
{
|
||||
// (SiteCallAudit, FailedLastInterval) is a catalog Rate pair. 400 hourly points of
|
||||
// value 1 over a 400 h (>168 h) window, maxPoints 200 → 2 points per 2 h bucket →
|
||||
// every output value is 2.0 (the summed pair), not 1.0 (last value).
|
||||
var from = new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var to = from.AddHours(400);
|
||||
var rollup = Enumerable.Range(0, 400)
|
||||
.Select(i => new KpiSeriesPoint(from.AddHours(i), 1d))
|
||||
.ToList();
|
||||
|
||||
var repo = Substitute.For<IKpiHistoryRepository>();
|
||||
repo.GetHourlySeriesAsync(
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
||||
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(rollup));
|
||||
|
||||
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 200));
|
||||
|
||||
var result = await sut.GetSeriesAsync(
|
||||
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
|
||||
"global", null, from, to);
|
||||
|
||||
Assert.Equal(200, result.Count);
|
||||
Assert.All(result, p => Assert.Equal(2.0, p.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetSeriesAsync_RateMetric_PreservesTotal_AcrossRoutingBoundary()
|
||||
{
|
||||
// Same total event count (50) served as per-minute deltas on the raw path
|
||||
// (window == threshold) and as hourly sums on the rollup path (window just over
|
||||
// threshold): the summed chart total must be equal on both routes — the boundary
|
||||
// is visually seamless for a Rate metric.
|
||||
var from = new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
var rawDeltas = Enumerable.Range(0, 10)
|
||||
.Select(i => new KpiSeriesPoint(from.AddMinutes(i * 10), 5d)).ToList(); // total 50
|
||||
var hourlySums = Enumerable.Range(0, 5)
|
||||
.Select(i => new KpiSeriesPoint(from.AddHours(i), 10d)).ToList(); // total 50
|
||||
|
||||
var repo = Substitute.For<IKpiHistoryRepository>();
|
||||
repo.GetRawSeriesAsync(
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
||||
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(rawDeltas));
|
||||
repo.GetHourlySeriesAsync(
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
||||
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(hourlySums));
|
||||
|
||||
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 200, rollupThresholdHours: 168));
|
||||
|
||||
var rawResult = await sut.GetSeriesAsync(
|
||||
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
|
||||
"global", null, from, from.AddHours(168)); // raw path
|
||||
var rollupResult = await sut.GetSeriesAsync(
|
||||
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
|
||||
"global", null, from, from.AddHours(169)); // rollup path
|
||||
|
||||
Assert.Equal(50d, rawResult.Sum(p => p.Value));
|
||||
Assert.Equal(50d, rollupResult.Sum(p => p.Value));
|
||||
Assert.Equal(rawResult.Sum(p => p.Value), rollupResult.Sum(p => p.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetSeriesAsync_GaugeMetric_KeepsLastValuePerBucket()
|
||||
{
|
||||
// queueDepth (uncatalogued for this source pairing → Gauge) keeps the existing
|
||||
// last-value-per-bucket output unchanged. 4 points, maxPoints 2, raw path.
|
||||
var repo = Substitute.For<IKpiHistoryRepository>();
|
||||
var raw = Series((0, 1d), (15, 2d), (25, 99d), (35, 5d));
|
||||
repo.GetRawSeriesAsync(
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
||||
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult(raw));
|
||||
|
||||
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 2));
|
||||
|
||||
var result = await sut.GetSeriesAsync(
|
||||
KpiSources.NotificationOutbox, KpiMetrics.NotificationOutbox.QueueDepth,
|
||||
"global", null, From, To);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
Assert.Equal(99d, result[0].Value); // last value in bucket 0, not a sum
|
||||
Assert.Equal(5d, result[1].Value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,4 +148,22 @@ public class KpiMetricAggregationTests
|
||||
KpiRollupAggregation.Rate,
|
||||
KpiMetricAggregationCatalog.Resolve(KpiSources.SiteCallAudit, "deliveredLastInterval"));
|
||||
}
|
||||
|
||||
// ---- Task 8: catalog metric literals promoted to shared KpiMetrics constants (R4) ----
|
||||
|
||||
[Fact]
|
||||
public void PromotedMetricConstants_LockHistoricalPersistedValues()
|
||||
{
|
||||
// Persisted data contract — these are symbol promotions, NOT renames.
|
||||
Assert.Equal("deliveredLastInterval", KpiMetrics.SiteCallAudit.DeliveredLastInterval);
|
||||
Assert.Equal("alarmEvalErrors", KpiMetrics.SiteHealth.AlarmEvalErrors);
|
||||
Assert.Equal("eventLogWriteFailures", KpiMetrics.SiteHealth.EventLogWriteFailures);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.DeliveredLastInterval)]
|
||||
[InlineData(KpiSources.SiteHealth, KpiMetrics.SiteHealth.AlarmEvalErrors)]
|
||||
[InlineData(KpiSources.SiteHealth, KpiMetrics.SiteHealth.EventLogWriteFailures)]
|
||||
public void Resolve_PromotedRatePairs_StillClassifyAsRate(string source, string metric)
|
||||
=> Assert.Equal(KpiRollupAggregation.Rate, KpiMetricAggregationCatalog.Resolve(source, metric));
|
||||
}
|
||||
|
||||
@@ -369,4 +369,97 @@ public class KpiSeriesBucketerTests
|
||||
Assert.Equal(4.0, result[0].Value);
|
||||
Assert.Equal(9.0, result[1].Value);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-metric reduction: Rate series sum per bucket (arch-review 04 round 2, R3)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Bucket_RateSeries_SumsPerBucket_InsteadOfLastValue()
|
||||
{
|
||||
// 6 points of value 10 across a 60-min / 2-bucket window → each bucket = 30 (sum),
|
||||
// not 10 (last value). Three deltas per bucket; last-value would keep 1 of 3.
|
||||
var raw = new[]
|
||||
{
|
||||
new KpiSeriesPoint(T(5), 10.0), // bucket 0: [0, 30)
|
||||
new KpiSeriesPoint(T(15), 10.0), // bucket 0
|
||||
new KpiSeriesPoint(T(25), 10.0), // bucket 0
|
||||
new KpiSeriesPoint(T(35), 10.0), // bucket 1: [30, 60]
|
||||
new KpiSeriesPoint(T(45), 10.0), // bucket 1
|
||||
new KpiSeriesPoint(T(55), 10.0), // bucket 1
|
||||
};
|
||||
|
||||
var result = KpiSeriesBucketer.Bucket(
|
||||
raw, T(0), T(60), maxPoints: 2, aggregation: KpiRollupAggregation.Rate);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
Assert.Equal(30.0, result[0].Value);
|
||||
Assert.Equal(30.0, result[1].Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Bucket_RateSeries_ShortSeries_StillSumsClusteredPoints()
|
||||
{
|
||||
// 3 points (< maxPoints 5) with two sharing a bucket → the shared bucket is their SUM.
|
||||
// Rate series skip the raw.Count <= maxPoints early return so totals stay truthful.
|
||||
// 60-min / 5-bucket window → 12 min each. T(2),T(5) → bucket 0 [0,12); T(30) → bucket 2 [24,36).
|
||||
var raw = new[]
|
||||
{
|
||||
new KpiSeriesPoint(T(2), 10.0),
|
||||
new KpiSeriesPoint(T(5), 10.0),
|
||||
new KpiSeriesPoint(T(30), 10.0),
|
||||
};
|
||||
|
||||
var result = KpiSeriesBucketer.Bucket(
|
||||
raw, T(0), T(60), maxPoints: 5, aggregation: KpiRollupAggregation.Rate);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
Assert.Equal(20.0, result[0].Value); // T(2)+T(5) summed
|
||||
Assert.Equal(10.0, result[1].Value); // T(30) alone
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Bucket_RateSeries_PreservesSeriesTotal()
|
||||
{
|
||||
// Strong invariant: sum(output values) == sum(in-window input values) for Rate.
|
||||
var raw = Enumerable
|
||||
.Range(0, 20)
|
||||
.Select(i => new KpiSeriesPoint(T(i * 3), (double)(i + 1)))
|
||||
.ToArray();
|
||||
|
||||
var result = KpiSeriesBucketer.Bucket(
|
||||
raw, T(0), T(60), maxPoints: 4, aggregation: KpiRollupAggregation.Rate);
|
||||
|
||||
var inWindowTotal = raw
|
||||
.Where(p => p.BucketStartUtc >= T(0) && p.BucketStartUtc <= T(60))
|
||||
.Sum(p => p.Value);
|
||||
Assert.Equal(inWindowTotal, result.Sum(p => p.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Bucket_DefaultAggregation_IsGauge_AndBehaviorUnchanged()
|
||||
{
|
||||
// Calling the existing 4-arg shape produces identical output to the explicit Gauge call.
|
||||
var raw = new[]
|
||||
{
|
||||
new KpiSeriesPoint(T(5), 1.0),
|
||||
new KpiSeriesPoint(T(15), 2.0),
|
||||
new KpiSeriesPoint(T(25), 99.0),
|
||||
new KpiSeriesPoint(T(35), 5.0),
|
||||
};
|
||||
|
||||
var fourArg = KpiSeriesBucketer.Bucket(raw, T(0), T(60), maxPoints: 2);
|
||||
var explicitGauge = KpiSeriesBucketer.Bucket(
|
||||
raw, T(0), T(60), maxPoints: 2, aggregation: KpiRollupAggregation.Gauge);
|
||||
|
||||
Assert.Equal(fourArg.Count, explicitGauge.Count);
|
||||
for (int i = 0; i < fourArg.Count; i++)
|
||||
{
|
||||
Assert.Equal(fourArg[i].BucketStartUtc, explicitGauge[i].BucketStartUtc);
|
||||
Assert.Equal(fourArg[i].Value, explicitGauge[i].Value);
|
||||
}
|
||||
// And the last-value semantics are unchanged.
|
||||
Assert.Equal(99.0, fourArg[0].Value);
|
||||
Assert.Equal(5.0, fourArg[1].Value);
|
||||
}
|
||||
}
|
||||
|
||||
+143
@@ -386,10 +386,153 @@ public class KpiHistoryRepositoryTests
|
||||
Assert.Equal(new[] { 3d, 4d }, remaining.Select(p => p.Value).ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FoldHourlyRollups_DoesNotTrack_KpiSampleEntities()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
var repo = new KpiHistoryRepository(ctx);
|
||||
await repo.RecordSamplesAsync(new[]
|
||||
{
|
||||
Sample("NotificationOutbox", "queueDepth", "Global", null, 5, Base.AddMinutes(10)),
|
||||
Sample("NotificationOutbox", "queueDepth", "Global", null, 7, Base.AddMinutes(50)),
|
||||
});
|
||||
ctx.ChangeTracker.Clear(); // isolate the fold's tracking behavior from the seed
|
||||
|
||||
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(1));
|
||||
|
||||
// The fold READS samples and WRITES rollups; the read must not register
|
||||
// thousands of read-only KpiSample entries for DetectChanges to re-scan
|
||||
// (arch-review 04 round 2, R2).
|
||||
Assert.Empty(ctx.ChangeTracker.Entries<KpiSample>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetLatestRollupHour_ReturnsNull_WhenNoRollupsExist()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
var repo = new KpiHistoryRepository(ctx);
|
||||
Assert.Null(await repo.GetLatestRollupHourAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetLatestRollupHour_ReturnsNewestHourStart_AcrossAllSeries()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
var repo = new KpiHistoryRepository(ctx);
|
||||
await repo.RecordSamplesAsync(new[]
|
||||
{
|
||||
Sample("NotificationOutbox", "queueDepth", "Global", null, 5, Base.AddMinutes(10)),
|
||||
Sample("SiteCallAudit", "buffered", "Global", null, 2, Base.AddHours(3).AddMinutes(10)),
|
||||
});
|
||||
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(4));
|
||||
|
||||
Assert.Equal(Base.AddHours(3), await repo.GetLatestRollupHourAsync());
|
||||
}
|
||||
|
||||
// ---- Task 9: failover fold-race failure grain is the pass, not the row (R5) ----
|
||||
|
||||
[Fact]
|
||||
public async Task Fold_LosingFailoverUpsertRace_DoesNotThrow_AndNextFoldSelfHeals()
|
||||
{
|
||||
// A single shared in-memory SQLite connection so a second context (the interceptor's
|
||||
// side-write, and the "next tick" fold) sees the same database.
|
||||
await using var connection = new Microsoft.Data.Sqlite.SqliteConnection("DataSource=:memory:");
|
||||
await connection.OpenAsync();
|
||||
await using (var schema = ContextOn(connection))
|
||||
{
|
||||
await schema.Database.EnsureCreatedAsync();
|
||||
}
|
||||
|
||||
// The old singleton incarnation's in-flight fold wins the (series, hour) upsert race:
|
||||
// the interceptor inserts the conflicting rollup row on the same connection just before
|
||||
// THIS fold's SaveChanges, so the unique IX_KpiRollupHourly_Series faults the loser's save.
|
||||
// A non-null ScopeKey (site-scoped series) is used deliberately: SQLite treats NULLs as
|
||||
// distinct in a UNIQUE index, so only a non-null key deterministically forces the conflict.
|
||||
var interceptor = new InsertConflictingRollupOnFirstSave(connection, () => new KpiRollupHourly
|
||||
{
|
||||
Source = "NotificationOutbox", Metric = "queueDepth", Scope = "Site", ScopeKey = "plant-a",
|
||||
HourStartUtc = Base, Value = -1, MinValue = -1, MaxValue = -1, SampleCount = 1,
|
||||
});
|
||||
|
||||
await using var ctx = ContextOn(connection, interceptor);
|
||||
var repo = new KpiHistoryRepository(ctx);
|
||||
await repo.RecordSamplesAsync(new[]
|
||||
{
|
||||
Sample("NotificationOutbox", "queueDepth", "Site", "plant-a", 5, Base.AddMinutes(10)),
|
||||
});
|
||||
|
||||
// Losing the race must NOT propagate DbUpdateException — the failure grain is the pass.
|
||||
var ex = await Record.ExceptionAsync(() => repo.FoldHourlyRollupsAsync(Base, Base.AddHours(1)));
|
||||
Assert.Null(ex);
|
||||
|
||||
// Next tick's fold (a fresh scope/context over the same DB) self-heals: it finds the
|
||||
// conflicting row and idempotently upserts it in place to the correct folded value.
|
||||
await using var ctx2 = ContextOn(connection);
|
||||
var repo2 = new KpiHistoryRepository(ctx2);
|
||||
await repo2.FoldHourlyRollupsAsync(Base, Base.AddHours(1));
|
||||
|
||||
var series = await repo2.GetHourlySeriesAsync(
|
||||
"NotificationOutbox", "queueDepth", "Site", "plant-a", Base, Base.AddHours(1));
|
||||
Assert.Single(series);
|
||||
Assert.Equal(5, series[0].Value); // the fold's own value overwrote the side-inserted -1
|
||||
}
|
||||
|
||||
// Local mirror of the repo's private hour-truncation, for cutoff assertions.
|
||||
private static DateTime TruncateHour(DateTime utc) =>
|
||||
new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Builds a SQLite test context bound to an externally-owned open connection
|
||||
/// (so multiple contexts share one in-memory database), with optional interceptors.</summary>
|
||||
private static ScadaBridgeDbContext ContextOn(DbConnection connection, params IInterceptor[] interceptors)
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.AddInterceptors(interceptors)
|
||||
.Options;
|
||||
return new SqliteTestDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SaveChanges interceptor that, exactly once and only when a fold is about to insert a
|
||||
/// <see cref="KpiRollupHourly"/>, inserts a conflicting (series, hour) rollup row through a
|
||||
/// second context on the same connection — deterministically reproducing the failover-overlap
|
||||
/// upsert race the losing fold must survive (arch-review 04 round 2, R5).
|
||||
/// </summary>
|
||||
private sealed class InsertConflictingRollupOnFirstSave : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor
|
||||
{
|
||||
private readonly DbConnection _connection;
|
||||
private readonly Func<KpiRollupHourly> _rowFactory;
|
||||
private bool _fired;
|
||||
|
||||
public InsertConflictingRollupOnFirstSave(DbConnection connection, Func<KpiRollupHourly> rowFactory)
|
||||
{
|
||||
_connection = connection;
|
||||
_rowFactory = rowFactory;
|
||||
}
|
||||
|
||||
public override async ValueTask<InterceptionResult<int>> SavingChangesAsync(
|
||||
DbContextEventData eventData, InterceptionResult<int> result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_fired && eventData.Context is not null &&
|
||||
eventData.Context.ChangeTracker.Entries<KpiRollupHourly>()
|
||||
.Any(e => e.State == EntityState.Added))
|
||||
{
|
||||
_fired = true;
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(_connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
await using var side = new SqliteTestDbContext(options);
|
||||
side.KpiRollupHourly.Add(_rowFactory());
|
||||
await side.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
return await base.SavingChangesAsync(eventData, result, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EF command interceptor that counts the DELETE statements actually issued
|
||||
/// to the store, so a test can prove the purge is sliced into multiple
|
||||
|
||||
+101
@@ -1,4 +1,6 @@
|
||||
using System.Data.Common;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
@@ -474,6 +476,69 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
||||
Assert.NotNull(await repo.GetAsync(recentTerminalId));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task PurgeTerminal_SlicesMultiDayBacklog_IntoBoundedDeletes()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var site = NewSiteId();
|
||||
var now = DateTime.UtcNow;
|
||||
var cutoff = now.AddDays(-1);
|
||||
|
||||
// Attach a DELETE-counting interceptor so we can prove the purge issues MORE THAN ONE
|
||||
// DELETE for a multi-day backlog (arch-review 04 round 2, R6 — time-sliced batching).
|
||||
var deleteCounter = new DeleteCountingCommandInterceptor();
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.AddInterceptors(deleteCounter)
|
||||
.Options;
|
||||
await using var context = new ScadaBridgeDbContext(options);
|
||||
var repo = new SiteCallAuditRepository(context);
|
||||
|
||||
// Three terminal rows on three DISTINCT days beyond the cutoff — a multi-day backlog.
|
||||
var oldIds = new List<TrackedOperationId>();
|
||||
foreach (var days in new[] { 3, 4, 5 })
|
||||
{
|
||||
var id = TrackedOperationId.New();
|
||||
oldIds.Add(id);
|
||||
var terminalAt = now.AddDays(-days);
|
||||
await repo.UpsertAsync(NewRow(
|
||||
id, sourceSite: site, status: "Delivered", retryCount: 0,
|
||||
createdAtUtc: terminalAt.AddMinutes(-1), updatedAtUtc: terminalAt,
|
||||
terminal: true, terminalAtUtc: terminalAt));
|
||||
}
|
||||
|
||||
// A fresh terminal row inside the keep window — must survive.
|
||||
var recentTerminalId = TrackedOperationId.New();
|
||||
await repo.UpsertAsync(NewRow(
|
||||
recentTerminalId, sourceSite: site, status: "Delivered", retryCount: 0,
|
||||
createdAtUtc: now.AddHours(-2), updatedAtUtc: now.AddHours(-1),
|
||||
terminal: true, terminalAtUtc: now.AddHours(-1)));
|
||||
|
||||
// An OLD non-terminal row — non-terminal rows are NEVER purged on age.
|
||||
var oldActiveId = TrackedOperationId.New();
|
||||
await repo.UpsertAsync(NewRow(
|
||||
oldActiveId, sourceSite: site, status: "Attempted", retryCount: 1,
|
||||
createdAtUtc: now.AddDays(-10), updatedAtUtc: now.AddDays(-10),
|
||||
terminal: false));
|
||||
|
||||
deleteCounter.Reset();
|
||||
var purged = await repo.PurgeTerminalAsync(cutoff);
|
||||
|
||||
// The three seeded backlog rows are gone; the fresh terminal + old non-terminal survive.
|
||||
Assert.True(purged >= 3, $"Expected at least the three backlog rows purged; got {purged}.");
|
||||
foreach (var id in oldIds)
|
||||
{
|
||||
Assert.Null(await repo.GetAsync(id));
|
||||
}
|
||||
Assert.NotNull(await repo.GetAsync(recentTerminalId));
|
||||
Assert.NotNull(await repo.GetAsync(oldActiveId));
|
||||
|
||||
// Slicing: a multi-day backlog is broken into MORE THAN ONE DELETE (one per day slice).
|
||||
Assert.True(deleteCounter.DeleteCount > 1,
|
||||
$"Expected the multi-day purge to slice into >1 DELETE; issued {deleteCounter.DeleteCount}.");
|
||||
}
|
||||
|
||||
// --- KPI snapshot tests -------------------------------------------------
|
||||
|
||||
[SkippableFact]
|
||||
@@ -824,4 +889,40 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
||||
Assert.Equal("Attempted", loaded.Status);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EF command interceptor that counts the DELETE statements actually issued to the store,
|
||||
/// so a test can prove the terminal purge slices a multi-day backlog into several windowed
|
||||
/// DELETEs rather than one year-scale statement. <c>ExecuteSqlInterpolatedAsync</c> routes
|
||||
/// through the async non-query path.
|
||||
/// </summary>
|
||||
private sealed class DeleteCountingCommandInterceptor : DbCommandInterceptor
|
||||
{
|
||||
public int DeleteCount { get; private set; }
|
||||
|
||||
public void Reset() => DeleteCount = 0;
|
||||
|
||||
private void CountIfDelete(DbCommand command)
|
||||
{
|
||||
if (command.CommandText.Contains("DELETE", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
DeleteCount++;
|
||||
}
|
||||
}
|
||||
|
||||
public override InterceptionResult<int> NonQueryExecuting(
|
||||
DbCommand command, CommandEventData eventData, InterceptionResult<int> result)
|
||||
{
|
||||
CountIfDelete(command);
|
||||
return base.NonQueryExecuting(command, eventData, result);
|
||||
}
|
||||
|
||||
public override ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(
|
||||
DbCommand command, CommandEventData eventData, InterceptionResult<int> result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
CountIfDelete(command);
|
||||
return base.NonQueryExecutingAsync(command, eventData, result, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,6 +94,22 @@ public class SchemaConfigurationTests : IDisposable
|
||||
new[] { nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status) },
|
||||
index.GetIncludeProperties());
|
||||
}
|
||||
|
||||
// arch-review 04 round 2 (R6): the daily terminal retention purge filters on
|
||||
// TerminalAtUtc IS NOT NULL AND TerminalAtUtc < cutoff. A filtered index over the
|
||||
// terminal population lets the purge (and Task 11's MIN() slice anchor) seek the
|
||||
// expired tail instead of full-scanning the 365-day table. Locked shape:
|
||||
// key = TerminalAtUtc, filter = [TerminalAtUtc] IS NOT NULL.
|
||||
[Fact]
|
||||
public void SiteCalls_Has_Filtered_Terminal_Index()
|
||||
{
|
||||
var model = _context.GetService<IDesignTimeModel>().Model;
|
||||
var entity = model.FindEntityType(typeof(SiteCall))!;
|
||||
var index = entity.GetIndexes().Single(i => i.GetDatabaseName() == "IX_SiteCalls_Terminal");
|
||||
|
||||
Assert.Equal(new[] { nameof(SiteCall.TerminalAtUtc) }, index.Properties.Select(p => p.Name).ToArray());
|
||||
Assert.Equal("[TerminalAtUtc] IS NOT NULL", index.GetFilter());
|
||||
}
|
||||
}
|
||||
|
||||
public class SplitQueryBehaviourTests : IDisposable
|
||||
|
||||
@@ -111,6 +111,9 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
private DateTime? _foldFromHourUtc;
|
||||
private DateTime? _foldToHourUtc;
|
||||
private int _foldCount;
|
||||
private readonly List<(DateTime FromHourUtc, DateTime ToHourUtc)> _foldWindows = new();
|
||||
private DateTime? _latestRollupHour;
|
||||
private int _latestRollupHourQueryCount;
|
||||
|
||||
public IReadOnlyList<KpiSample> Recorded
|
||||
{
|
||||
@@ -148,6 +151,12 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
get { lock (_gate) { return _foldCount; } }
|
||||
}
|
||||
|
||||
/// <summary>Every fold window handed to <see cref="FoldHourlyRollupsAsync"/>, in call order.</summary>
|
||||
public IReadOnlyList<(DateTime FromHourUtc, DateTime ToHourUtc)> FoldWindows
|
||||
{
|
||||
get { lock (_gate) { return _foldWindows.ToArray(); } }
|
||||
}
|
||||
|
||||
public Task RecordSamplesAsync(
|
||||
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -176,6 +185,7 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
{
|
||||
_foldFromHourUtc = fromHourUtc;
|
||||
_foldToHourUtc = toHourUtc;
|
||||
_foldWindows.Add((fromHourUtc, toHourUtc));
|
||||
_foldCount++;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
@@ -191,6 +201,28 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
lock (_gate) { _rollupPurgeCutoff = before; }
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Watermark the fake returns from <see cref="GetLatestRollupHourAsync"/> (settable).</summary>
|
||||
public DateTime? LatestRollupHour
|
||||
{
|
||||
get { lock (_gate) { return _latestRollupHour; } }
|
||||
set { lock (_gate) { _latestRollupHour = value; } }
|
||||
}
|
||||
|
||||
/// <summary>Number of times the backfill plan pass queried the rollup watermark.</summary>
|
||||
public int LatestRollupHourQueryCount
|
||||
{
|
||||
get { lock (_gate) { return _latestRollupHourQueryCount; } }
|
||||
}
|
||||
|
||||
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_latestRollupHourQueryCount++;
|
||||
return Task.FromResult(_latestRollupHour);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -237,6 +269,9 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
|
||||
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<DateTime?>(null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -283,6 +318,9 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
|
||||
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<DateTime?>(null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -340,6 +378,92 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
|
||||
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<DateTime?>(null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Repository fake for the backfill-slice interleaving test. Classifies each fold as a backfill
|
||||
/// slice (24 h window) or a periodic fold (the distinct <c>RollupLookbackHours</c>-wide window)
|
||||
/// by window width. Backfill slices complete immediately (counted); the FIRST periodic fold
|
||||
/// blocks in flight until released, so the test can observe that a periodic fold claimed the
|
||||
/// fold path between backfill slices while backfill slices remained.
|
||||
/// </summary>
|
||||
private sealed class InterleaveFoldRepository : IKpiHistoryRepository
|
||||
{
|
||||
private readonly TimeSpan _periodicWindowWidth;
|
||||
private readonly object _gate = new();
|
||||
private readonly SemaphoreSlim _firstPeriodicRelease = new(initialCount: 0);
|
||||
private int _backfillFoldCount;
|
||||
private int _periodicFoldCount;
|
||||
private int _backfillCountAtFirstPeriodic = -1;
|
||||
|
||||
public InterleaveFoldRepository(TimeSpan periodicWindowWidth) =>
|
||||
_periodicWindowWidth = periodicWindowWidth;
|
||||
|
||||
/// <summary>Number of backfill-slice folds (24 h windows) entered so far.</summary>
|
||||
public int BackfillFoldCount { get { lock (_gate) { return _backfillFoldCount; } } }
|
||||
|
||||
/// <summary>Number of periodic folds (the lookback-width window) entered so far.</summary>
|
||||
public int PeriodicFoldCount { get { lock (_gate) { return _periodicFoldCount; } } }
|
||||
|
||||
/// <summary>Backfill-slice count observed at the instant the first periodic fold entered.</summary>
|
||||
public int BackfillCountAtFirstPeriodic { get { lock (_gate) { return _backfillCountAtFirstPeriodic; } } }
|
||||
|
||||
/// <summary>Releases the first (held-in-flight) periodic fold so the deferred backfill resumes.</summary>
|
||||
public void ReleaseFirstPeriodic() => _firstPeriodicRelease.Release();
|
||||
|
||||
public Task RecordSamplesAsync(
|
||||
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task<IReadOnlyList<KpiSeriesPoint>> GetRawSeriesAsync(
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
|
||||
|
||||
public Task<int> PurgeOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(0);
|
||||
|
||||
public Task FoldHourlyRollupsAsync(
|
||||
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var isPeriodic = (toHourUtc - fromHourUtc) == _periodicWindowWidth;
|
||||
if (isPeriodic)
|
||||
{
|
||||
bool blockThis;
|
||||
lock (_gate)
|
||||
{
|
||||
_periodicFoldCount++;
|
||||
blockThis = _periodicFoldCount == 1;
|
||||
if (blockThis)
|
||||
{
|
||||
_backfillCountAtFirstPeriodic = _backfillFoldCount;
|
||||
}
|
||||
}
|
||||
|
||||
// Hold the first periodic fold in flight (on a threadpool thread — PipeTo runs the
|
||||
// pass off the actor thread) so the between-slice interleave state is observable.
|
||||
return blockThis
|
||||
? Task.Run(() => _firstPeriodicRelease.Wait(cancellationToken), cancellationToken)
|
||||
: Task.CompletedTask;
|
||||
}
|
||||
|
||||
lock (_gate) { _backfillFoldCount++; }
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
|
||||
|
||||
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<DateTime?>(null);
|
||||
}
|
||||
|
||||
private IServiceProvider BuildServiceProvider(
|
||||
@@ -671,13 +795,15 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
public void BackfillTick_FoldsFullRetentionWindow_Once()
|
||||
{
|
||||
// The one-shot backfill must fold the WHOLE raw-retention window once, so long-range
|
||||
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy: the fold's
|
||||
// upper bound is the current hour start (in-progress hour excluded) and its lower bound is
|
||||
// RetentionDays earlier. A second BackfillTick to the same actor must be a no-op (the
|
||||
// _backfillDone latch makes it run at most once per actor lifetime).
|
||||
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy. It now
|
||||
// does so in bounded <=24 h slices (R1) rather than a single unbounded fold: the union
|
||||
// of the slice windows must cover exactly [truncate(now) − RetentionDays, truncate(now)),
|
||||
// each slice must be <=24 h, and the slices must be contiguous and oldest-first. A second
|
||||
// BackfillTick to the same actor must be a no-op (the _backfillDone latch makes the whole
|
||||
// pass run at most once per actor lifetime).
|
||||
var repository = new RecordingRepository();
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
const int retentionDays = 90;
|
||||
const int retentionDays = 5;
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
@@ -691,52 +817,168 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
Assert.Equal(1, repository.FoldCount);
|
||||
Assert.NotNull(repository.FoldToHourUtc);
|
||||
Assert.NotNull(repository.FoldFromHourUtc);
|
||||
|
||||
var toHour = repository.FoldToHourUtc!.Value;
|
||||
var fromHour = repository.FoldFromHourUtc!.Value;
|
||||
|
||||
// toHour is the truncated current hour (in-progress hour excluded).
|
||||
Assert.Equal(DateTimeKind.Utc, toHour.Kind);
|
||||
Assert.Equal(0, toHour.Minute);
|
||||
Assert.Equal(0, toHour.Second);
|
||||
Assert.Equal(0, toHour.Millisecond);
|
||||
var windows = repository.FoldWindows;
|
||||
// RetentionDays = 5 → exactly 5 contiguous 24 h slices.
|
||||
Assert.Equal(retentionDays, windows.Count);
|
||||
|
||||
var expectedToHour = new DateTime(
|
||||
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
|
||||
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
|
||||
// Within one hour of "now truncated" (guards a rare hour-boundary crossing mid-test).
|
||||
Assert.True(
|
||||
Math.Abs((toHour - expectedToHour).TotalHours) <= 1.0,
|
||||
$"backfill toHour {toHour:o} should be the current hour start {expectedToHour:o}");
|
||||
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
|
||||
|
||||
// fromHour is exactly RetentionDays (the full raw-retention window) before toHour.
|
||||
Assert.Equal(toHour - TimeSpan.FromDays(retentionDays), fromHour);
|
||||
// Oldest-first, contiguous, each <=24 h, union == full retention window.
|
||||
Assert.Equal(expectedFromHour, windows[0].FromHourUtc);
|
||||
Assert.Equal(expectedToHour, windows[^1].ToHourUtc);
|
||||
for (var i = 0; i < windows.Count; i++)
|
||||
{
|
||||
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
|
||||
$"slice {i} width {windows[i].ToHourUtc - windows[i].FromHourUtc} exceeds 24 h");
|
||||
if (i > 0)
|
||||
{
|
||||
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
|
||||
}
|
||||
}
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// Second BackfillTick to the SAME actor: the once-only latch must suppress it, so the
|
||||
// fold count stays at 1. (No periodic RollupTick is sent, and the periodic/backfill
|
||||
// auto-timers don't fire within this sub-10 s window, so FoldCount is driven solely by
|
||||
// the two backfill ticks under test.)
|
||||
// Second BackfillTick to the SAME actor: the once-only latch must suppress the whole pass,
|
||||
// so no further slices are folded. (No periodic RollupTick is sent, and the periodic/backfill
|
||||
// auto-timers don't fire within this window, so the fold windows are driven solely by the
|
||||
// backfill under test.)
|
||||
var foldedCount = repository.FoldWindows.Count;
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
Thread.Sleep(300);
|
||||
Assert.Equal(1, repository.FoldCount);
|
||||
Assert.Equal(foldedCount, repository.FoldWindows.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backfill_SlicesRetentionWindow_IntoBoundedDayFolds_OldestFirst()
|
||||
{
|
||||
// RetentionDays = 3 → 3 fold calls, each window <=24 h, contiguous, oldest-first, whose
|
||||
// union is exactly [TruncateToHour(now) − 3 d, TruncateToHour(now)). Pre-R1 a single fold
|
||||
// call spanned the whole 3-day window (arch-review 04 round 2, R1).
|
||||
var repository = new RecordingRepository();
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
const int retentionDays = 3;
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
RollupInterval = TimeSpan.FromHours(1),
|
||||
RetentionDays = retentionDays,
|
||||
});
|
||||
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var windows = repository.FoldWindows;
|
||||
Assert.Equal(retentionDays, windows.Count); // 3 × 24 h slices
|
||||
|
||||
var expectedToHour = new DateTime(
|
||||
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
|
||||
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
|
||||
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
|
||||
|
||||
Assert.Equal(expectedFromHour, windows[0].FromHourUtc); // oldest first
|
||||
Assert.Equal(expectedToHour, windows[^1].ToHourUtc); // newest last
|
||||
for (var i = 0; i < windows.Count; i++)
|
||||
{
|
||||
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
|
||||
$"slice {i} exceeds 24 h");
|
||||
if (i > 0)
|
||||
{
|
||||
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
|
||||
}
|
||||
}
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PeriodicRollupTick_Interleaves_BetweenBackfillSlices()
|
||||
{
|
||||
// A periodic fold must be able to claim the fold path BETWEEN backfill slices: while the
|
||||
// historical backfill runs slice-by-slice, a RollupTick that lands in a between-slice gap
|
||||
// runs its (recent-window) fold and the remaining backfill slices defer behind it, then
|
||||
// ALL backfill slices still complete afterwards. Pre-R1 _backfillInFlight blocked every
|
||||
// periodic fold for the whole (single-fold) pass, so no interleaving was possible
|
||||
// (arch-review 04 round 2, R1).
|
||||
const int retentionDays = 8; // 8 backfill slices of 24 h
|
||||
const int lookbackHours = 2; // periodic fold window width — distinct from the 24 h slices
|
||||
var repository = new InterleaveFoldRepository(TimeSpan.FromHours(lookbackHours));
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
RollupInterval = TimeSpan.FromHours(24), // periodic auto-timer won't fire during the test
|
||||
RetentionDays = retentionDays,
|
||||
RollupLookbackHours = lookbackHours,
|
||||
});
|
||||
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
|
||||
// Continuously offer periodic ticks; one will win a between-slice gap and interleave.
|
||||
using var stopFlood = new CancellationTokenSource();
|
||||
var flood = Task.Run(() =>
|
||||
{
|
||||
while (!stopFlood.IsCancellationRequested)
|
||||
{
|
||||
actor.Tell(KpiHistoryRecorderActor.RollupTick.Instance);
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
// A periodic fold interleaved BEFORE the backfill finished all its slices.
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
Assert.True(repository.PeriodicFoldCount >= 1, "no periodic fold interleaved");
|
||||
Assert.True(
|
||||
repository.BackfillCountAtFirstPeriodic < retentionDays,
|
||||
"the periodic fold did not interleave before the backfill completed");
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(20));
|
||||
}
|
||||
finally
|
||||
{
|
||||
stopFlood.Cancel();
|
||||
await flood;
|
||||
}
|
||||
|
||||
// The first periodic fold is held in flight, deferring the remaining backfill slices;
|
||||
// release it so the deferred backfill resumes and ALL slices still complete.
|
||||
repository.ReleaseFirstPeriodic();
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(retentionDays, repository.BackfillFoldCount),
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BackfillInFlight_SkipsPeriodicRollupTick()
|
||||
{
|
||||
// While the (possibly long) full-window backfill fold is in flight, a periodic RollupTick
|
||||
// must be skipped so the two idempotent upserts never race on overlapping rows. With a
|
||||
// gated fold holding the backfill open, a RollupTick must NOT enter a second fold — the
|
||||
// fold count stays at 1 until the gate is released.
|
||||
// While a backfill slice fold is in flight, a periodic RollupTick must be skipped so the
|
||||
// two idempotent upserts never race on overlapping rows (unchanged semantics — a RollupTick
|
||||
// arriving while a *slice* is in flight is still skipped). With a gated fold holding the
|
||||
// single backfill slice open, a RollupTick must NOT enter a second fold — the fold count
|
||||
// stays at 1 until the gate is released. RetentionDays = 1 → exactly one 24 h backfill slice.
|
||||
var repository = new GatedFoldRepository();
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
var actor = CreateActor(sp);
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
RollupInterval = TimeSpan.FromHours(1),
|
||||
RetentionDays = 1,
|
||||
});
|
||||
|
||||
// Backfill tick: raises the backfill guard, enters the (gated, blocking) fold — held in flight.
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
@@ -753,7 +995,7 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
|
||||
// Release the gate so the backfill completes and its guard is lowered; a fresh periodic
|
||||
// tick must now run a new fold (guards correctly reset, not stuck). Re-send on each poll so
|
||||
// we don't race the asynchronous BackfillComplete that lowers the guard.
|
||||
// we don't race the asynchronous BackfillSliceComplete that lowers the guard.
|
||||
repository.Release();
|
||||
AwaitAssert(
|
||||
() =>
|
||||
@@ -764,4 +1006,85 @@ public class KpiHistoryRecorderActorTests : TestKit
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backfill_Skips_WhenRollupsAlreadyCurrent()
|
||||
{
|
||||
// Watermark within RollupLookbackHours of now (the common failover case): the backfill
|
||||
// must be skipped entirely — ZERO folds, the done-latch set — because the periodic fold's
|
||||
// lookback window already covers everything from the watermark forward (arch-review 04
|
||||
// round 2, R1).
|
||||
var repository = new RecordingRepository();
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
RollupInterval = TimeSpan.FromHours(1),
|
||||
RetentionDays = 90,
|
||||
RollupLookbackHours = 3,
|
||||
});
|
||||
|
||||
// Rollups current through one hour ago — well inside the 3 h lookback tail.
|
||||
var nowHour = new DateTime(
|
||||
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
|
||||
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
|
||||
repository.LatestRollupHour = nowHour - TimeSpan.FromHours(1);
|
||||
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
|
||||
// The plan pass must consult the watermark…
|
||||
AwaitAssert(
|
||||
() => Assert.True(repository.LatestRollupHourQueryCount >= 1),
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// …and then skip: no slices are ever folded.
|
||||
Thread.Sleep(300);
|
||||
Assert.Equal(0, repository.FoldCount);
|
||||
|
||||
// A second BackfillTick is a no-op (the done-latch is set).
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
Thread.Sleep(300);
|
||||
Assert.Equal(0, repository.FoldCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backfill_ShrinksToWatermark_InsteadOfRetentionFloor()
|
||||
{
|
||||
// RetentionDays = 90 but the watermark is only 2 days old: the first slice must start at
|
||||
// the WATERMARK hour (inclusive — re-folding the newest already-folded hour is a cheap
|
||||
// idempotent safety margin), NOT 90 days ago, and there must be <= 3 slices total
|
||||
// (arch-review 04 round 2, R1).
|
||||
var repository = new RecordingRepository();
|
||||
var sp = BuildServiceProvider(repository, new HealthySource());
|
||||
var actor = CreateActor(sp, new KpiHistoryOptions
|
||||
{
|
||||
SampleInterval = TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
RollupInterval = TimeSpan.FromHours(1),
|
||||
RetentionDays = 90,
|
||||
RollupLookbackHours = 3,
|
||||
});
|
||||
|
||||
var nowHour = new DateTime(
|
||||
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
|
||||
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
|
||||
var watermark = nowHour - TimeSpan.FromDays(2);
|
||||
repository.LatestRollupHour = watermark;
|
||||
|
||||
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var windows = repository.FoldWindows;
|
||||
Assert.NotEmpty(windows);
|
||||
// Shrunk to the un-rolled tail: first slice starts AT the watermark, not 90 d ago.
|
||||
Assert.Equal(watermark, windows[0].FromHourUtc);
|
||||
Assert.True(windows.Count <= 3, $"expected <= 3 slices from the 2-day tail, got {windows.Count}");
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user