merge(r2): r2-plan04

This commit is contained in:
Joseph Doherty
2026-07-13 11:09:36 -04:00
26 changed files with 3427 additions and 142 deletions
@@ -63,6 +63,29 @@ public class SiteAuditRetentionServiceTests
Assert.True(queue.PurgeCalls.Count >= 2);
}
[Fact]
public async Task StopAsync_MidPurge_CompletesCleanly_WithoutSurfacingCancellation()
{
// PurgeExpiredAsync blocks until cancelled, so shutdown lands mid-purge: SafePurgeAsync
// rethrows the OCE by design (abort the purge promptly), but the loop task must still
// complete CLEANLY — StopAsync hands _loop straight to the host, and a canceled task
// there is shutdown-log noise (arch-review 04 round 2, R7).
var queue = new RecordingSiteAuditQueue { BlockUntilCancelled = true };
var options = Options.Create(new SiteAuditRetentionOptions
{
RetentionDays = 7,
PurgeInterval = TimeSpan.FromHours(24),
InitialDelay = TimeSpan.Zero,
});
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
await svc.StartAsync(CancellationToken.None);
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
// Must NOT throw TaskCanceledException back into the host's shutdown path.
await svc.StopAsync(CancellationToken.None);
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
var sw = Stopwatch.StartNew();
@@ -86,7 +109,11 @@ public class SiteAuditRetentionServiceTests
public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
public bool ThrowOnFirstCall { get; init; }
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
/// <summary>When set, PurgeExpiredAsync records the call then blocks until the token
/// cancels — simulating a shutdown that lands while a purge is in flight.</summary>
public bool BlockUntilCancelled { get; init; }
public async Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
{
var n = Interlocked.Increment(ref _calls);
_purgeCalls.Enqueue(olderThanUtc);
@@ -95,7 +122,12 @@ public class SiteAuditRetentionServiceTests
throw new InvalidOperationException("Simulated transient purge failure.");
}
return Task.FromResult(0);
if (BlockUntilCancelled)
{
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
}
return 0;
}
public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
@@ -330,4 +330,95 @@ public class KpiHistoryQueryServiceTests
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>());
}
// ---- Task 6: catalog-driven aggregation at the bucketer boundary (R3) -----------
[Fact]
public async Task GetSeriesAsync_RateMetric_SumsPerBucket_OnRollupPath()
{
// (SiteCallAudit, FailedLastInterval) is a catalog Rate pair. 400 hourly points of
// value 1 over a 400 h (>168 h) window, maxPoints 200 → 2 points per 2 h bucket →
// every output value is 2.0 (the summed pair), not 1.0 (last value).
var from = new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc);
var to = from.AddHours(400);
var rollup = Enumerable.Range(0, 400)
.Select(i => new KpiSeriesPoint(from.AddHours(i), 1d))
.ToList();
var repo = Substitute.For<IKpiHistoryRepository>();
repo.GetHourlySeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(rollup));
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 200));
var result = await sut.GetSeriesAsync(
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
"global", null, from, to);
Assert.Equal(200, result.Count);
Assert.All(result, p => Assert.Equal(2.0, p.Value));
}
[Fact]
public async Task GetSeriesAsync_RateMetric_PreservesTotal_AcrossRoutingBoundary()
{
// Same total event count (50) served as per-minute deltas on the raw path
// (window == threshold) and as hourly sums on the rollup path (window just over
// threshold): the summed chart total must be equal on both routes — the boundary
// is visually seamless for a Rate metric.
var from = new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc);
var rawDeltas = Enumerable.Range(0, 10)
.Select(i => new KpiSeriesPoint(from.AddMinutes(i * 10), 5d)).ToList(); // total 50
var hourlySums = Enumerable.Range(0, 5)
.Select(i => new KpiSeriesPoint(from.AddHours(i), 10d)).ToList(); // total 50
var repo = Substitute.For<IKpiHistoryRepository>();
repo.GetRawSeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(rawDeltas));
repo.GetHourlySeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(hourlySums));
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 200, rollupThresholdHours: 168));
var rawResult = await sut.GetSeriesAsync(
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
"global", null, from, from.AddHours(168)); // raw path
var rollupResult = await sut.GetSeriesAsync(
KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval,
"global", null, from, from.AddHours(169)); // rollup path
Assert.Equal(50d, rawResult.Sum(p => p.Value));
Assert.Equal(50d, rollupResult.Sum(p => p.Value));
Assert.Equal(rawResult.Sum(p => p.Value), rollupResult.Sum(p => p.Value));
}
[Fact]
public async Task GetSeriesAsync_GaugeMetric_KeepsLastValuePerBucket()
{
// queueDepth (uncatalogued for this source pairing → Gauge) keeps the existing
// last-value-per-bucket output unchanged. 4 points, maxPoints 2, raw path.
var repo = Substitute.For<IKpiHistoryRepository>();
var raw = Series((0, 1d), (15, 2d), (25, 99d), (35, 5d));
repo.GetRawSeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult(raw));
var sut = new KpiHistoryQueryService(repo, Options(defaultMax: 2));
var result = await sut.GetSeriesAsync(
KpiSources.NotificationOutbox, KpiMetrics.NotificationOutbox.QueueDepth,
"global", null, From, To);
Assert.Equal(2, result.Count);
Assert.Equal(99d, result[0].Value); // last value in bucket 0, not a sum
Assert.Equal(5d, result[1].Value);
}
}
@@ -148,4 +148,22 @@ public class KpiMetricAggregationTests
KpiRollupAggregation.Rate,
KpiMetricAggregationCatalog.Resolve(KpiSources.SiteCallAudit, "deliveredLastInterval"));
}
// ---- Task 8: catalog metric literals promoted to shared KpiMetrics constants (R4) ----
[Fact]
public void PromotedMetricConstants_LockHistoricalPersistedValues()
{
// Persisted data contract — these are symbol promotions, NOT renames.
Assert.Equal("deliveredLastInterval", KpiMetrics.SiteCallAudit.DeliveredLastInterval);
Assert.Equal("alarmEvalErrors", KpiMetrics.SiteHealth.AlarmEvalErrors);
Assert.Equal("eventLogWriteFailures", KpiMetrics.SiteHealth.EventLogWriteFailures);
}
[Theory]
[InlineData(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.DeliveredLastInterval)]
[InlineData(KpiSources.SiteHealth, KpiMetrics.SiteHealth.AlarmEvalErrors)]
[InlineData(KpiSources.SiteHealth, KpiMetrics.SiteHealth.EventLogWriteFailures)]
public void Resolve_PromotedRatePairs_StillClassifyAsRate(string source, string metric)
=> Assert.Equal(KpiRollupAggregation.Rate, KpiMetricAggregationCatalog.Resolve(source, metric));
}
@@ -369,4 +369,97 @@ public class KpiSeriesBucketerTests
Assert.Equal(4.0, result[0].Value);
Assert.Equal(9.0, result[1].Value);
}
// -----------------------------------------------------------------------
// Per-metric reduction: Rate series sum per bucket (arch-review 04 round 2, R3)
// -----------------------------------------------------------------------
[Fact]
public void Bucket_RateSeries_SumsPerBucket_InsteadOfLastValue()
{
// 6 points of value 10 across a 60-min / 2-bucket window → each bucket = 30 (sum),
// not 10 (last value). Three deltas per bucket; last-value would keep 1 of 3.
var raw = new[]
{
new KpiSeriesPoint(T(5), 10.0), // bucket 0: [0, 30)
new KpiSeriesPoint(T(15), 10.0), // bucket 0
new KpiSeriesPoint(T(25), 10.0), // bucket 0
new KpiSeriesPoint(T(35), 10.0), // bucket 1: [30, 60]
new KpiSeriesPoint(T(45), 10.0), // bucket 1
new KpiSeriesPoint(T(55), 10.0), // bucket 1
};
var result = KpiSeriesBucketer.Bucket(
raw, T(0), T(60), maxPoints: 2, aggregation: KpiRollupAggregation.Rate);
Assert.Equal(2, result.Count);
Assert.Equal(30.0, result[0].Value);
Assert.Equal(30.0, result[1].Value);
}
[Fact]
public void Bucket_RateSeries_ShortSeries_StillSumsClusteredPoints()
{
// 3 points (< maxPoints 5) with two sharing a bucket → the shared bucket is their SUM.
// Rate series skip the raw.Count <= maxPoints early return so totals stay truthful.
// 60-min / 5-bucket window → 12 min each. T(2),T(5) → bucket 0 [0,12); T(30) → bucket 2 [24,36).
var raw = new[]
{
new KpiSeriesPoint(T(2), 10.0),
new KpiSeriesPoint(T(5), 10.0),
new KpiSeriesPoint(T(30), 10.0),
};
var result = KpiSeriesBucketer.Bucket(
raw, T(0), T(60), maxPoints: 5, aggregation: KpiRollupAggregation.Rate);
Assert.Equal(2, result.Count);
Assert.Equal(20.0, result[0].Value); // T(2)+T(5) summed
Assert.Equal(10.0, result[1].Value); // T(30) alone
}
[Fact]
public void Bucket_RateSeries_PreservesSeriesTotal()
{
// Strong invariant: sum(output values) == sum(in-window input values) for Rate.
var raw = Enumerable
.Range(0, 20)
.Select(i => new KpiSeriesPoint(T(i * 3), (double)(i + 1)))
.ToArray();
var result = KpiSeriesBucketer.Bucket(
raw, T(0), T(60), maxPoints: 4, aggregation: KpiRollupAggregation.Rate);
var inWindowTotal = raw
.Where(p => p.BucketStartUtc >= T(0) && p.BucketStartUtc <= T(60))
.Sum(p => p.Value);
Assert.Equal(inWindowTotal, result.Sum(p => p.Value));
}
[Fact]
public void Bucket_DefaultAggregation_IsGauge_AndBehaviorUnchanged()
{
// Calling the existing 4-arg shape produces identical output to the explicit Gauge call.
var raw = new[]
{
new KpiSeriesPoint(T(5), 1.0),
new KpiSeriesPoint(T(15), 2.0),
new KpiSeriesPoint(T(25), 99.0),
new KpiSeriesPoint(T(35), 5.0),
};
var fourArg = KpiSeriesBucketer.Bucket(raw, T(0), T(60), maxPoints: 2);
var explicitGauge = KpiSeriesBucketer.Bucket(
raw, T(0), T(60), maxPoints: 2, aggregation: KpiRollupAggregation.Gauge);
Assert.Equal(fourArg.Count, explicitGauge.Count);
for (int i = 0; i < fourArg.Count; i++)
{
Assert.Equal(fourArg[i].BucketStartUtc, explicitGauge[i].BucketStartUtc);
Assert.Equal(fourArg[i].Value, explicitGauge[i].Value);
}
// And the last-value semantics are unchanged.
Assert.Equal(99.0, fourArg[0].Value);
Assert.Equal(5.0, fourArg[1].Value);
}
}
@@ -386,10 +386,153 @@ public class KpiHistoryRepositoryTests
Assert.Equal(new[] { 3d, 4d }, remaining.Select(p => p.Value).ToArray());
}
[Fact]
public async Task FoldHourlyRollups_DoesNotTrack_KpiSampleEntities()
{
await using var ctx = NewContext();
var repo = new KpiHistoryRepository(ctx);
await repo.RecordSamplesAsync(new[]
{
Sample("NotificationOutbox", "queueDepth", "Global", null, 5, Base.AddMinutes(10)),
Sample("NotificationOutbox", "queueDepth", "Global", null, 7, Base.AddMinutes(50)),
});
ctx.ChangeTracker.Clear(); // isolate the fold's tracking behavior from the seed
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(1));
// The fold READS samples and WRITES rollups; the read must not register
// thousands of read-only KpiSample entries for DetectChanges to re-scan
// (arch-review 04 round 2, R2).
Assert.Empty(ctx.ChangeTracker.Entries<KpiSample>());
}
[Fact]
public async Task GetLatestRollupHour_ReturnsNull_WhenNoRollupsExist()
{
await using var ctx = NewContext();
var repo = new KpiHistoryRepository(ctx);
Assert.Null(await repo.GetLatestRollupHourAsync());
}
[Fact]
public async Task GetLatestRollupHour_ReturnsNewestHourStart_AcrossAllSeries()
{
await using var ctx = NewContext();
var repo = new KpiHistoryRepository(ctx);
await repo.RecordSamplesAsync(new[]
{
Sample("NotificationOutbox", "queueDepth", "Global", null, 5, Base.AddMinutes(10)),
Sample("SiteCallAudit", "buffered", "Global", null, 2, Base.AddHours(3).AddMinutes(10)),
});
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(4));
Assert.Equal(Base.AddHours(3), await repo.GetLatestRollupHourAsync());
}
// ---- Task 9: failover fold-race failure grain is the pass, not the row (R5) ----
[Fact]
public async Task Fold_LosingFailoverUpsertRace_DoesNotThrow_AndNextFoldSelfHeals()
{
// A single shared in-memory SQLite connection so a second context (the interceptor's
// side-write, and the "next tick" fold) sees the same database.
await using var connection = new Microsoft.Data.Sqlite.SqliteConnection("DataSource=:memory:");
await connection.OpenAsync();
await using (var schema = ContextOn(connection))
{
await schema.Database.EnsureCreatedAsync();
}
// The old singleton incarnation's in-flight fold wins the (series, hour) upsert race:
// the interceptor inserts the conflicting rollup row on the same connection just before
// THIS fold's SaveChanges, so the unique IX_KpiRollupHourly_Series faults the loser's save.
// A non-null ScopeKey (site-scoped series) is used deliberately: SQLite treats NULLs as
// distinct in a UNIQUE index, so only a non-null key deterministically forces the conflict.
var interceptor = new InsertConflictingRollupOnFirstSave(connection, () => new KpiRollupHourly
{
Source = "NotificationOutbox", Metric = "queueDepth", Scope = "Site", ScopeKey = "plant-a",
HourStartUtc = Base, Value = -1, MinValue = -1, MaxValue = -1, SampleCount = 1,
});
await using var ctx = ContextOn(connection, interceptor);
var repo = new KpiHistoryRepository(ctx);
await repo.RecordSamplesAsync(new[]
{
Sample("NotificationOutbox", "queueDepth", "Site", "plant-a", 5, Base.AddMinutes(10)),
});
// Losing the race must NOT propagate DbUpdateException — the failure grain is the pass.
var ex = await Record.ExceptionAsync(() => repo.FoldHourlyRollupsAsync(Base, Base.AddHours(1)));
Assert.Null(ex);
// Next tick's fold (a fresh scope/context over the same DB) self-heals: it finds the
// conflicting row and idempotently upserts it in place to the correct folded value.
await using var ctx2 = ContextOn(connection);
var repo2 = new KpiHistoryRepository(ctx2);
await repo2.FoldHourlyRollupsAsync(Base, Base.AddHours(1));
var series = await repo2.GetHourlySeriesAsync(
"NotificationOutbox", "queueDepth", "Site", "plant-a", Base, Base.AddHours(1));
Assert.Single(series);
Assert.Equal(5, series[0].Value); // the fold's own value overwrote the side-inserted -1
}
// Local mirror of the repo's private hour-truncation, for cutoff assertions.
private static DateTime TruncateHour(DateTime utc) =>
new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
/// <summary>Builds a SQLite test context bound to an externally-owned open connection
/// (so multiple contexts share one in-memory database), with optional interceptors.</summary>
private static ScadaBridgeDbContext ContextOn(DbConnection connection, params IInterceptor[] interceptors)
{
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlite(connection)
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
.AddInterceptors(interceptors)
.Options;
return new SqliteTestDbContext(options);
}
/// <summary>
/// SaveChanges interceptor that, exactly once and only when a fold is about to insert a
/// <see cref="KpiRollupHourly"/>, inserts a conflicting (series, hour) rollup row through a
/// second context on the same connection — deterministically reproducing the failover-overlap
/// upsert race the losing fold must survive (arch-review 04 round 2, R5).
/// </summary>
private sealed class InsertConflictingRollupOnFirstSave : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor
{
private readonly DbConnection _connection;
private readonly Func<KpiRollupHourly> _rowFactory;
private bool _fired;
public InsertConflictingRollupOnFirstSave(DbConnection connection, Func<KpiRollupHourly> rowFactory)
{
_connection = connection;
_rowFactory = rowFactory;
}
public override async ValueTask<InterceptionResult<int>> SavingChangesAsync(
DbContextEventData eventData, InterceptionResult<int> result,
CancellationToken cancellationToken = default)
{
if (!_fired && eventData.Context is not null &&
eventData.Context.ChangeTracker.Entries<KpiRollupHourly>()
.Any(e => e.State == EntityState.Added))
{
_fired = true;
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlite(_connection)
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
.Options;
await using var side = new SqliteTestDbContext(options);
side.KpiRollupHourly.Add(_rowFactory());
await side.SaveChangesAsync(cancellationToken);
}
return await base.SavingChangesAsync(eventData, result, cancellationToken);
}
}
/// <summary>
/// EF command interceptor that counts the DELETE statements actually issued
/// to the store, so a test can prove the purge is sliced into multiple
@@ -1,4 +1,6 @@
using System.Data.Common;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
@@ -474,6 +476,69 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.NotNull(await repo.GetAsync(recentTerminalId));
}
[SkippableFact]
public async Task PurgeTerminal_SlicesMultiDayBacklog_IntoBoundedDeletes()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var site = NewSiteId();
var now = DateTime.UtcNow;
var cutoff = now.AddDays(-1);
// Attach a DELETE-counting interceptor so we can prove the purge issues MORE THAN ONE
// DELETE for a multi-day backlog (arch-review 04 round 2, R6 — time-sliced batching).
var deleteCounter = new DeleteCountingCommandInterceptor();
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.AddInterceptors(deleteCounter)
.Options;
await using var context = new ScadaBridgeDbContext(options);
var repo = new SiteCallAuditRepository(context);
// Three terminal rows on three DISTINCT days beyond the cutoff — a multi-day backlog.
var oldIds = new List<TrackedOperationId>();
foreach (var days in new[] { 3, 4, 5 })
{
var id = TrackedOperationId.New();
oldIds.Add(id);
var terminalAt = now.AddDays(-days);
await repo.UpsertAsync(NewRow(
id, sourceSite: site, status: "Delivered", retryCount: 0,
createdAtUtc: terminalAt.AddMinutes(-1), updatedAtUtc: terminalAt,
terminal: true, terminalAtUtc: terminalAt));
}
// A fresh terminal row inside the keep window — must survive.
var recentTerminalId = TrackedOperationId.New();
await repo.UpsertAsync(NewRow(
recentTerminalId, sourceSite: site, status: "Delivered", retryCount: 0,
createdAtUtc: now.AddHours(-2), updatedAtUtc: now.AddHours(-1),
terminal: true, terminalAtUtc: now.AddHours(-1)));
// An OLD non-terminal row — non-terminal rows are NEVER purged on age.
var oldActiveId = TrackedOperationId.New();
await repo.UpsertAsync(NewRow(
oldActiveId, sourceSite: site, status: "Attempted", retryCount: 1,
createdAtUtc: now.AddDays(-10), updatedAtUtc: now.AddDays(-10),
terminal: false));
deleteCounter.Reset();
var purged = await repo.PurgeTerminalAsync(cutoff);
// The three seeded backlog rows are gone; the fresh terminal + old non-terminal survive.
Assert.True(purged >= 3, $"Expected at least the three backlog rows purged; got {purged}.");
foreach (var id in oldIds)
{
Assert.Null(await repo.GetAsync(id));
}
Assert.NotNull(await repo.GetAsync(recentTerminalId));
Assert.NotNull(await repo.GetAsync(oldActiveId));
// Slicing: a multi-day backlog is broken into MORE THAN ONE DELETE (one per day slice).
Assert.True(deleteCounter.DeleteCount > 1,
$"Expected the multi-day purge to slice into >1 DELETE; issued {deleteCounter.DeleteCount}.");
}
// --- KPI snapshot tests -------------------------------------------------
[SkippableFact]
@@ -824,4 +889,40 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.Equal("Attempted", loaded.Status);
}
}
/// <summary>
/// EF command interceptor that counts the DELETE statements actually issued to the store,
/// so a test can prove the terminal purge slices a multi-day backlog into several windowed
/// DELETEs rather than one year-scale statement. <c>ExecuteSqlInterpolatedAsync</c> routes
/// through the async non-query path.
/// </summary>
private sealed class DeleteCountingCommandInterceptor : DbCommandInterceptor
{
public int DeleteCount { get; private set; }
public void Reset() => DeleteCount = 0;
private void CountIfDelete(DbCommand command)
{
if (command.CommandText.Contains("DELETE", StringComparison.OrdinalIgnoreCase))
{
DeleteCount++;
}
}
public override InterceptionResult<int> NonQueryExecuting(
DbCommand command, CommandEventData eventData, InterceptionResult<int> result)
{
CountIfDelete(command);
return base.NonQueryExecuting(command, eventData, result);
}
public override ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(
DbCommand command, CommandEventData eventData, InterceptionResult<int> result,
CancellationToken cancellationToken = default)
{
CountIfDelete(command);
return base.NonQueryExecutingAsync(command, eventData, result, cancellationToken);
}
}
}
@@ -94,6 +94,22 @@ public class SchemaConfigurationTests : IDisposable
new[] { nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status) },
index.GetIncludeProperties());
}
// arch-review 04 round 2 (R6): the daily terminal retention purge filters on
// TerminalAtUtc IS NOT NULL AND TerminalAtUtc < cutoff. A filtered index over the
// terminal population lets the purge (and Task 11's MIN() slice anchor) seek the
// expired tail instead of full-scanning the 365-day table. Locked shape:
// key = TerminalAtUtc, filter = [TerminalAtUtc] IS NOT NULL.
[Fact]
public void SiteCalls_Has_Filtered_Terminal_Index()
{
var model = _context.GetService<IDesignTimeModel>().Model;
var entity = model.FindEntityType(typeof(SiteCall))!;
var index = entity.GetIndexes().Single(i => i.GetDatabaseName() == "IX_SiteCalls_Terminal");
Assert.Equal(new[] { nameof(SiteCall.TerminalAtUtc) }, index.Properties.Select(p => p.Name).ToArray());
Assert.Equal("[TerminalAtUtc] IS NOT NULL", index.GetFilter());
}
}
public class SplitQueryBehaviourTests : IDisposable
@@ -111,6 +111,9 @@ public class KpiHistoryRecorderActorTests : TestKit
private DateTime? _foldFromHourUtc;
private DateTime? _foldToHourUtc;
private int _foldCount;
private readonly List<(DateTime FromHourUtc, DateTime ToHourUtc)> _foldWindows = new();
private DateTime? _latestRollupHour;
private int _latestRollupHourQueryCount;
public IReadOnlyList<KpiSample> Recorded
{
@@ -148,6 +151,12 @@ public class KpiHistoryRecorderActorTests : TestKit
get { lock (_gate) { return _foldCount; } }
}
/// <summary>Every fold window handed to <see cref="FoldHourlyRollupsAsync"/>, in call order.</summary>
public IReadOnlyList<(DateTime FromHourUtc, DateTime ToHourUtc)> FoldWindows
{
get { lock (_gate) { return _foldWindows.ToArray(); } }
}
public Task RecordSamplesAsync(
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default)
{
@@ -176,6 +185,7 @@ public class KpiHistoryRecorderActorTests : TestKit
{
_foldFromHourUtc = fromHourUtc;
_foldToHourUtc = toHourUtc;
_foldWindows.Add((fromHourUtc, toHourUtc));
_foldCount++;
}
return Task.CompletedTask;
@@ -191,6 +201,28 @@ public class KpiHistoryRecorderActorTests : TestKit
lock (_gate) { _rollupPurgeCutoff = before; }
return Task.CompletedTask;
}
/// <summary>Watermark the fake returns from <see cref="GetLatestRollupHourAsync"/> (settable).</summary>
public DateTime? LatestRollupHour
{
get { lock (_gate) { return _latestRollupHour; } }
set { lock (_gate) { _latestRollupHour = value; } }
}
/// <summary>Number of times the backfill plan pass queried the rollup watermark.</summary>
public int LatestRollupHourQueryCount
{
get { lock (_gate) { return _latestRollupHourQueryCount; } }
}
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default)
{
lock (_gate)
{
_latestRollupHourQueryCount++;
return Task.FromResult(_latestRollupHour);
}
}
}
/// <summary>
@@ -237,6 +269,9 @@ public class KpiHistoryRecorderActorTests : TestKit
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<DateTime?>(null);
}
/// <summary>
@@ -283,6 +318,9 @@ public class KpiHistoryRecorderActorTests : TestKit
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<DateTime?>(null);
}
/// <summary>
@@ -340,6 +378,92 @@ public class KpiHistoryRecorderActorTests : TestKit
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<DateTime?>(null);
}
/// <summary>
/// Repository fake for the backfill-slice interleaving test. Classifies each fold as a backfill
/// slice (24 h window) or a periodic fold (the distinct <c>RollupLookbackHours</c>-wide window)
/// by window width. Backfill slices complete immediately (counted); the FIRST periodic fold
/// blocks in flight until released, so the test can observe that a periodic fold claimed the
/// fold path between backfill slices while backfill slices remained.
/// </summary>
private sealed class InterleaveFoldRepository : IKpiHistoryRepository
{
private readonly TimeSpan _periodicWindowWidth;
private readonly object _gate = new();
private readonly SemaphoreSlim _firstPeriodicRelease = new(initialCount: 0);
private int _backfillFoldCount;
private int _periodicFoldCount;
private int _backfillCountAtFirstPeriodic = -1;
public InterleaveFoldRepository(TimeSpan periodicWindowWidth) =>
_periodicWindowWidth = periodicWindowWidth;
/// <summary>Number of backfill-slice folds (24 h windows) entered so far.</summary>
public int BackfillFoldCount { get { lock (_gate) { return _backfillFoldCount; } } }
/// <summary>Number of periodic folds (the lookback-width window) entered so far.</summary>
public int PeriodicFoldCount { get { lock (_gate) { return _periodicFoldCount; } } }
/// <summary>Backfill-slice count observed at the instant the first periodic fold entered.</summary>
public int BackfillCountAtFirstPeriodic { get { lock (_gate) { return _backfillCountAtFirstPeriodic; } } }
/// <summary>Releases the first (held-in-flight) periodic fold so the deferred backfill resumes.</summary>
public void ReleaseFirstPeriodic() => _firstPeriodicRelease.Release();
public Task RecordSamplesAsync(
IReadOnlyCollection<KpiSample> samples, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<IReadOnlyList<KpiSeriesPoint>> GetRawSeriesAsync(
string source, string metric, string scope, string? scopeKey,
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
public Task<int> PurgeOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task FoldHourlyRollupsAsync(
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken = default)
{
var isPeriodic = (toHourUtc - fromHourUtc) == _periodicWindowWidth;
if (isPeriodic)
{
bool blockThis;
lock (_gate)
{
_periodicFoldCount++;
blockThis = _periodicFoldCount == 1;
if (blockThis)
{
_backfillCountAtFirstPeriodic = _backfillFoldCount;
}
}
// Hold the first periodic fold in flight (on a threadpool thread — PipeTo runs the
// pass off the actor thread) so the between-slice interleave state is observable.
return blockThis
? Task.Run(() => _firstPeriodicRelease.Wait(cancellationToken), cancellationToken)
: Task.CompletedTask;
}
lock (_gate) { _backfillFoldCount++; }
return Task.CompletedTask;
}
public Task<IReadOnlyList<KpiSeriesPoint>> GetHourlySeriesAsync(
string source, string metric, string scope, string? scopeKey,
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(Array.Empty<KpiSeriesPoint>());
public Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<DateTime?>(null);
}
private IServiceProvider BuildServiceProvider(
@@ -671,13 +795,15 @@ public class KpiHistoryRecorderActorTests : TestKit
public void BackfillTick_FoldsFullRetentionWindow_Once()
{
// The one-shot backfill must fold the WHOLE raw-retention window once, so long-range
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy: the fold's
// upper bound is the current hour start (in-progress hour excluded) and its lower bound is
// RetentionDays earlier. A second BackfillTick to the same actor must be a no-op (the
// _backfillDone latch makes it run at most once per actor lifetime).
// (30 d/90 d) charts aren't blank until enough wall-clock passes after deploy. It now
// does so in bounded <=24 h slices (R1) rather than a single unbounded fold: the union
// of the slice windows must cover exactly [truncate(now) RetentionDays, truncate(now)),
// each slice must be <=24 h, and the slices must be contiguous and oldest-first. A second
// BackfillTick to the same actor must be a no-op (the _backfillDone latch makes the whole
// pass run at most once per actor lifetime).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
const int retentionDays = 90;
const int retentionDays = 5;
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
@@ -691,52 +817,168 @@ public class KpiHistoryRecorderActorTests : TestKit
AwaitAssert(
() =>
{
Assert.Equal(1, repository.FoldCount);
Assert.NotNull(repository.FoldToHourUtc);
Assert.NotNull(repository.FoldFromHourUtc);
var toHour = repository.FoldToHourUtc!.Value;
var fromHour = repository.FoldFromHourUtc!.Value;
// toHour is the truncated current hour (in-progress hour excluded).
Assert.Equal(DateTimeKind.Utc, toHour.Kind);
Assert.Equal(0, toHour.Minute);
Assert.Equal(0, toHour.Second);
Assert.Equal(0, toHour.Millisecond);
var windows = repository.FoldWindows;
// RetentionDays = 5 → exactly 5 contiguous 24 h slices.
Assert.Equal(retentionDays, windows.Count);
var expectedToHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
// Within one hour of "now truncated" (guards a rare hour-boundary crossing mid-test).
Assert.True(
Math.Abs((toHour - expectedToHour).TotalHours) <= 1.0,
$"backfill toHour {toHour:o} should be the current hour start {expectedToHour:o}");
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
// fromHour is exactly RetentionDays (the full raw-retention window) before toHour.
Assert.Equal(toHour - TimeSpan.FromDays(retentionDays), fromHour);
// Oldest-first, contiguous, each <=24 h, union == full retention window.
Assert.Equal(expectedFromHour, windows[0].FromHourUtc);
Assert.Equal(expectedToHour, windows[^1].ToHourUtc);
for (var i = 0; i < windows.Count; i++)
{
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
$"slice {i} width {windows[i].ToHourUtc - windows[i].FromHourUtc} exceeds 24 h");
if (i > 0)
{
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
}
}
},
duration: TimeSpan.FromSeconds(3),
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
// Second BackfillTick to the SAME actor: the once-only latch must suppress it, so the
// fold count stays at 1. (No periodic RollupTick is sent, and the periodic/backfill
// auto-timers don't fire within this sub-10 s window, so FoldCount is driven solely by
// the two backfill ticks under test.)
// Second BackfillTick to the SAME actor: the once-only latch must suppress the whole pass,
// so no further slices are folded. (No periodic RollupTick is sent, and the periodic/backfill
// auto-timers don't fire within this window, so the fold windows are driven solely by the
// backfill under test.)
var foldedCount = repository.FoldWindows.Count;
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
Thread.Sleep(300);
Assert.Equal(1, repository.FoldCount);
Assert.Equal(foldedCount, repository.FoldWindows.Count);
}
[Fact]
public void Backfill_SlicesRetentionWindow_IntoBoundedDayFolds_OldestFirst()
{
// RetentionDays = 3 → 3 fold calls, each window <=24 h, contiguous, oldest-first, whose
// union is exactly [TruncateToHour(now) 3 d, TruncateToHour(now)). Pre-R1 a single fold
// call spanned the whole 3-day window (arch-review 04 round 2, R1).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
const int retentionDays = 3;
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = retentionDays,
});
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
AwaitAssert(
() =>
{
var windows = repository.FoldWindows;
Assert.Equal(retentionDays, windows.Count); // 3 × 24 h slices
var expectedToHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
var expectedFromHour = expectedToHour - TimeSpan.FromDays(retentionDays);
Assert.Equal(expectedFromHour, windows[0].FromHourUtc); // oldest first
Assert.Equal(expectedToHour, windows[^1].ToHourUtc); // newest last
for (var i = 0; i < windows.Count; i++)
{
Assert.True(windows[i].ToHourUtc - windows[i].FromHourUtc <= TimeSpan.FromHours(24),
$"slice {i} exceeds 24 h");
if (i > 0)
{
Assert.Equal(windows[i - 1].ToHourUtc, windows[i].FromHourUtc); // contiguous
}
}
},
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
}
[Fact]
public async Task PeriodicRollupTick_Interleaves_BetweenBackfillSlices()
{
// A periodic fold must be able to claim the fold path BETWEEN backfill slices: while the
// historical backfill runs slice-by-slice, a RollupTick that lands in a between-slice gap
// runs its (recent-window) fold and the remaining backfill slices defer behind it, then
// ALL backfill slices still complete afterwards. Pre-R1 _backfillInFlight blocked every
// periodic fold for the whole (single-fold) pass, so no interleaving was possible
// (arch-review 04 round 2, R1).
const int retentionDays = 8; // 8 backfill slices of 24 h
const int lookbackHours = 2; // periodic fold window width — distinct from the 24 h slices
var repository = new InterleaveFoldRepository(TimeSpan.FromHours(lookbackHours));
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(24), // periodic auto-timer won't fire during the test
RetentionDays = retentionDays,
RollupLookbackHours = lookbackHours,
});
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
// Continuously offer periodic ticks; one will win a between-slice gap and interleave.
using var stopFlood = new CancellationTokenSource();
var flood = Task.Run(() =>
{
while (!stopFlood.IsCancellationRequested)
{
actor.Tell(KpiHistoryRecorderActor.RollupTick.Instance);
Thread.Sleep(1);
}
});
try
{
// A periodic fold interleaved BEFORE the backfill finished all its slices.
AwaitAssert(
() =>
{
Assert.True(repository.PeriodicFoldCount >= 1, "no periodic fold interleaved");
Assert.True(
repository.BackfillCountAtFirstPeriodic < retentionDays,
"the periodic fold did not interleave before the backfill completed");
},
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(20));
}
finally
{
stopFlood.Cancel();
await flood;
}
// The first periodic fold is held in flight, deferring the remaining backfill slices;
// release it so the deferred backfill resumes and ALL slices still complete.
repository.ReleaseFirstPeriodic();
AwaitAssert(
() => Assert.Equal(retentionDays, repository.BackfillFoldCount),
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(50));
}
[Fact]
public void BackfillInFlight_SkipsPeriodicRollupTick()
{
// While the (possibly long) full-window backfill fold is in flight, a periodic RollupTick
// must be skipped so the two idempotent upserts never race on overlapping rows. With a
// gated fold holding the backfill open, a RollupTick must NOT enter a second fold — the
// fold count stays at 1 until the gate is released.
// While a backfill slice fold is in flight, a periodic RollupTick must be skipped so the
// two idempotent upserts never race on overlapping rows (unchanged semantics — a RollupTick
// arriving while a *slice* is in flight is still skipped). With a gated fold holding the
// single backfill slice open, a RollupTick must NOT enter a second fold — the fold count
// stays at 1 until the gate is released. RetentionDays = 1 → exactly one 24 h backfill slice.
var repository = new GatedFoldRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp);
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = 1,
});
// Backfill tick: raises the backfill guard, enters the (gated, blocking) fold — held in flight.
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
@@ -753,7 +995,7 @@ public class KpiHistoryRecorderActorTests : TestKit
// Release the gate so the backfill completes and its guard is lowered; a fresh periodic
// tick must now run a new fold (guards correctly reset, not stuck). Re-send on each poll so
// we don't race the asynchronous BackfillComplete that lowers the guard.
// we don't race the asynchronous BackfillSliceComplete that lowers the guard.
repository.Release();
AwaitAssert(
() =>
@@ -764,4 +1006,85 @@ public class KpiHistoryRecorderActorTests : TestKit
duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50));
}
[Fact]
public void Backfill_Skips_WhenRollupsAlreadyCurrent()
{
// Watermark within RollupLookbackHours of now (the common failover case): the backfill
// must be skipped entirely — ZERO folds, the done-latch set — because the periodic fold's
// lookback window already covers everything from the watermark forward (arch-review 04
// round 2, R1).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = 90,
RollupLookbackHours = 3,
});
// Rollups current through one hour ago — well inside the 3 h lookback tail.
var nowHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
repository.LatestRollupHour = nowHour - TimeSpan.FromHours(1);
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
// The plan pass must consult the watermark…
AwaitAssert(
() => Assert.True(repository.LatestRollupHourQueryCount >= 1),
duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50));
// …and then skip: no slices are ever folded.
Thread.Sleep(300);
Assert.Equal(0, repository.FoldCount);
// A second BackfillTick is a no-op (the done-latch is set).
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
Thread.Sleep(300);
Assert.Equal(0, repository.FoldCount);
}
[Fact]
public void Backfill_ShrinksToWatermark_InsteadOfRetentionFloor()
{
// RetentionDays = 90 but the watermark is only 2 days old: the first slice must start at
// the WATERMARK hour (inclusive — re-folding the newest already-folded hour is a cheap
// idempotent safety margin), NOT 90 days ago, and there must be <= 3 slices total
// (arch-review 04 round 2, R1).
var repository = new RecordingRepository();
var sp = BuildServiceProvider(repository, new HealthySource());
var actor = CreateActor(sp, new KpiHistoryOptions
{
SampleInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromHours(1),
RollupInterval = TimeSpan.FromHours(1),
RetentionDays = 90,
RollupLookbackHours = 3,
});
var nowHour = new DateTime(
DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day,
DateTime.UtcNow.Hour, 0, 0, DateTimeKind.Utc);
var watermark = nowHour - TimeSpan.FromDays(2);
repository.LatestRollupHour = watermark;
actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance);
AwaitAssert(
() =>
{
var windows = repository.FoldWindows;
Assert.NotEmpty(windows);
// Shrunk to the un-rolled tail: first slice starts AT the watermark, not 90 d ago.
Assert.Equal(watermark, windows[0].FromHourUtc);
Assert.True(windows.Count <= 3, $"expected <= 3 slices from the 2-day tail, got {windows.Count}");
},
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
}
}