feat(kpihistory): add hourly rollup fold + query + purge repository methods (plan #22 T3)
Add FoldHourlyRollupsAsync (in-memory grouped, per-metric gauge-last/rate-sum, idempotent upsert on the series+hour key with null-ScopeKey equality, exclusive upper hour bound so the in-progress hour is never folded), GetHourlySeriesAsync (same KpiSeriesPoint contract as GetRawSeriesAsync), and PurgeRollupsOlderThanAsync (one-hour-sliced batched DELETE mirroring PurgeOlderThanAsync). Adds 7 repo tests. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
@@ -50,4 +50,75 @@ public interface IKpiHistoryRepository
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task resolving to the number of rows deleted.</returns>
|
||||
Task<int> PurgeOlderThanAsync(DateTime before, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Folds the raw <c>KpiSample</c> rows in the window into the hourly
|
||||
/// <c>KpiRollupHourly</c> table — one row per
|
||||
/// (<c>Source</c>, <c>Metric</c>, <c>Scope</c>, <c>ScopeKey</c>) per UTC hour.
|
||||
/// The window is truncated to whole-hour boundaries: only the complete hours in
|
||||
/// <c>[truncate(fromHourUtc), truncate(toHourUtc))</c> are folded — the upper bound
|
||||
/// is <em>exclusive</em>, so a caller that passes the current hour's start as
|
||||
/// <paramref name="toHourUtc"/> never folds the in-progress (incomplete) hour.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// For each (series, hour) group the fold records <c>SampleCount</c> (raw rows in the
|
||||
/// group), <c>MinValue</c>/<c>MaxValue</c> (min/max raw value), and a <c>Value</c>
|
||||
/// whose aggregation is chosen per metric by
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiMetricAggregationCatalog.Resolve(string, string)"/>:
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiRollupAggregation.Gauge"/> stores the
|
||||
/// last (latest-timestamp) value in the hour; <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiRollupAggregation.Rate"/>
|
||||
/// stores the sum of values in the hour.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The write is an idempotent upsert keyed on
|
||||
/// (<c>Source</c>, <c>Metric</c>, <c>Scope</c>, <c>ScopeKey</c>, <c>HourStartUtc</c>)
|
||||
/// with <c>null</c> <c>ScopeKey</c> participating in equality: an existing row is updated
|
||||
/// in place, otherwise a new row is inserted. Re-running the fold over the same window
|
||||
/// yields identical rows (no duplicates, no double-counting), so a singleton-failover
|
||||
/// handover that missed a tick self-heals on the next re-fold. Metrics that are only
|
||||
/// conditionally emitted (missing in some minutes) are handled naturally by grouping.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="fromHourUtc">Inclusive lower bound of the fold window (UTC), truncated to the hour.</param>
|
||||
/// <param name="toHourUtc">Exclusive upper bound of the fold window (UTC), truncated to the hour; pass the current hour's start to exclude the in-progress hour.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous fold + upsert.</returns>
|
||||
Task FoldHourlyRollupsAsync(
|
||||
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the hourly rollup points for one series in <c>[fromUtc, toUtc]</c>, ordered by
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi.KpiRollupHourly.HourStartUtc"/>
|
||||
/// ascending. Same contract and return shape as
|
||||
/// <see cref="GetRawSeriesAsync"/> (each point carries the hour start plus the folded
|
||||
/// <c>Value</c>) so the downstream bucketer is agnostic to raw-vs-rollup source.
|
||||
/// </summary>
|
||||
/// <param name="source">Source identifier — a value from <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiSources"/>.</param>
|
||||
/// <param name="metric">Metric name from the source's catalog.</param>
|
||||
/// <param name="scope">Scope discriminator — a value from <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiScopes"/>.</param>
|
||||
/// <param name="scopeKey">Scope qualifier (site id / node name); <c>null</c> for the Global scope.</param>
|
||||
/// <param name="fromUtc">Inclusive lower bound of the time window (UTC).</param>
|
||||
/// <param name="toUtc">Inclusive upper bound of the time window (UTC).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>
|
||||
/// A task resolving to the hourly series points in ascending hour order; empty when the
|
||||
/// series has no rollups in the window.
|
||||
/// </returns>
|
||||
Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Bulk-deletes hourly rollup rows whose
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi.KpiRollupHourly.HourStartUtc"/> is
|
||||
/// strictly older than <paramref name="before"/> (rollup retention purge). Mirrors
|
||||
/// <see cref="PurgeOlderThanAsync"/>: the purge is sliced into one-hour DELETE batches
|
||||
/// so a large catch-up never runs as a single lock/log-heavy transaction
|
||||
/// (arch-review 04, P4).
|
||||
/// </summary>
|
||||
/// <param name="before">UTC cut-off; rollups whose hour start is before this instant are deleted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous purge.</returns>
|
||||
Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -82,4 +82,134 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task FoldHourlyRollupsAsync(
|
||||
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Truncate the window to whole-hour boundaries. The fold covers the complete hours
|
||||
// in [truncate(from), truncate(to)) — the upper bound is EXCLUSIVE, so a caller that
|
||||
// passes the current hour's start as toHourUtc never folds the in-progress hour.
|
||||
var from = TruncateToHour(fromHourUtc);
|
||||
var to = TruncateToHour(toHourUtc);
|
||||
if (to <= from)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// The caller passes a small trailing lookback (e.g. 3 h), so the window is bounded:
|
||||
// fetch it and group IN MEMORY. This deliberately avoids provider-specific
|
||||
// DATEADD/DATEPART hour-truncation translation (SQLite in tests, SQL Server in prod),
|
||||
// keeping the hour-bucket arithmetic identical across providers and trivially correct.
|
||||
var samples = await _context.KpiSamples
|
||||
.Where(s => s.CapturedAtUtc >= from && s.CapturedAtUtc < to)
|
||||
.ToListAsync(cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var groups = samples.GroupBy(s => new SeriesHourKey(
|
||||
s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var key = group.Key;
|
||||
|
||||
// Per-metric aggregation intent: Rate metrics sum the hour (last-value would
|
||||
// discard 59 of 60 intra-hour deltas); Gauge metrics keep the last (latest
|
||||
// timestamp) value in the hour. Min/Max/Count preserve the fold's fidelity.
|
||||
var aggregation = KpiMetricAggregationCatalog.Resolve(key.Source, key.Metric);
|
||||
var value = aggregation == KpiRollupAggregation.Rate
|
||||
? group.Sum(s => s.Value)
|
||||
: group.OrderByDescending(s => s.CapturedAtUtc).First().Value;
|
||||
var minValue = group.Min(s => s.Value);
|
||||
var maxValue = group.Max(s => s.Value);
|
||||
var sampleCount = group.Count();
|
||||
|
||||
// Idempotent upsert on the unique series+hour key. The ScopeKey == key.ScopeKey
|
||||
// comparison matches null against the Global-scope rows (IS NULL) exactly as the
|
||||
// UNIQUE IX_KpiRollupHourly_Series index treats a null key as participating.
|
||||
var existing = await _context.KpiRollupHourly.FirstOrDefaultAsync(
|
||||
r => r.Source == key.Source
|
||||
&& r.Metric == key.Metric
|
||||
&& r.Scope == key.Scope
|
||||
&& r.ScopeKey == key.ScopeKey
|
||||
&& r.HourStartUtc == key.HourStartUtc,
|
||||
cancellationToken);
|
||||
|
||||
if (existing is null)
|
||||
{
|
||||
_context.KpiRollupHourly.Add(new KpiRollupHourly
|
||||
{
|
||||
Source = key.Source,
|
||||
Metric = key.Metric,
|
||||
Scope = key.Scope,
|
||||
ScopeKey = key.ScopeKey,
|
||||
HourStartUtc = key.HourStartUtc,
|
||||
Value = value,
|
||||
MinValue = minValue,
|
||||
MaxValue = maxValue,
|
||||
SampleCount = sampleCount,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
// Re-fold overwrites the aggregate in place — a re-run over the same window
|
||||
// produces identical values (no double-count), so a missed tick self-heals.
|
||||
existing.Value = value;
|
||||
existing.MinValue = minValue;
|
||||
existing.MaxValue = maxValue;
|
||||
existing.SampleCount = sampleCount;
|
||||
}
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Same null-ScopeKey semantics as GetRawSeriesAsync: scopeKey == null translates to
|
||||
// "ScopeKey IS NULL", matching Global-scope rollups and excluding site/node-keyed ones.
|
||||
return await _context.KpiRollupHourly
|
||||
.Where(r => r.Source == source
|
||||
&& r.Metric == metric
|
||||
&& r.Scope == scope
|
||||
&& r.ScopeKey == scopeKey
|
||||
&& r.HourStartUtc >= fromUtc
|
||||
&& r.HourStartUtc <= toUtc)
|
||||
.OrderBy(r => r.HourStartUtc)
|
||||
.Select(r => new KpiSeriesPoint(r.HourStartUtc, r.Value))
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Mirror PurgeOlderThanAsync: one-hour-sliced batched DELETEs cap the lock/log
|
||||
// footprint per statement (arch-review 04, P4). Rollups are one-per-hour, so each
|
||||
// slice deletes a single hour bucket.
|
||||
var floor = await _context.KpiRollupHourly.Where(r => r.HourStartUtc < before)
|
||||
.MinAsync(r => (DateTime?)r.HourStartUtc, cancellationToken);
|
||||
while (floor is not null && floor < before)
|
||||
{
|
||||
var ceiling = floor.Value.AddHours(1) < before ? floor.Value.AddHours(1) : before;
|
||||
await _context.KpiRollupHourly
|
||||
.Where(r => r.HourStartUtc < ceiling)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
floor = await _context.KpiRollupHourly.Where(r => r.HourStartUtc < before)
|
||||
.MinAsync(r => (DateTime?)r.HourStartUtc, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Truncates a UTC timestamp to the start of its hour (minutes/seconds/ticks zeroed).</summary>
|
||||
private static DateTime TruncateToHour(DateTime utc) =>
|
||||
new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>In-memory grouping key: one KPI series (four-tuple) within one UTC hour.</summary>
|
||||
private readonly record struct SeriesHourKey(
|
||||
string Source, string Metric, string Scope, string? ScopeKey, DateTime HourStartUtc);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user