test(communication)+test(sitecallaudit): cap parallelism, delta/collection-serialize gauge asserts, replace sleep-then-assert, stop leaked 100ms tickers (arch-review 2026-08-19 P8-5 + P8-6, register rows 40/82/95/96/102 patterns)

This commit is contained in:
Joseph Doherty
2026-08-19 15:03:03 -04:00
parent 9a3f006721
commit d9d72f1bc5
7 changed files with 422 additions and 227 deletions
@@ -15,6 +15,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
/// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect /// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect
/// re-seed, and periodic reconcile authoritative-replace (drift correction). /// re-seed, and periodic reconcile authoritative-replace (drift correction).
/// </summary> /// </summary>
/// <remarks>
/// Serialized with <c>SiteAlarmLiveCacheTelemetryTests</c> (register row 102): every actor
/// this class spins up moves the process-global
/// <c>scadabridge.site.alarm_cache.aggregators.active</c> gauge via
/// <see cref="SiteAlarmAggregatorActor"/>'s PreStart/PostStop, which the telemetry test's
/// fixed-baseline assertions require to hold still (or only increase) between its capture
/// and its checks. Same collection ⇒ never concurrent.
/// </remarks>
[Collection("SiteAlarmAggregatorGauge")]
public class SiteAlarmAggregatorActorTests : TestKit public class SiteAlarmAggregatorActorTests : TestKit
{ {
private const string SiteId = "site-alpha"; private const string SiteId = "site-alpha";
@@ -188,6 +197,28 @@ public class SiteAlarmAggregatorActorTests : TestKit
private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) => private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) =>
rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef); rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef);
/// <summary>
/// Row-40 house pattern for an ABSENCE assertion: proves <paramref name="checks"/> keep
/// holding throughout <paramref name="window"/> by polling, rather than a single blind
/// Thread.Sleep(window) followed by one end-of-window sample. A bare sleep-then-assert
/// cannot distinguish "the mechanism correctly stayed quiet" from "the mechanism just
/// hadn't run yet when the one sample was taken", and it also can't catch a value that
/// flips wrong and then flips back before that single sample. This fails immediately on
/// the first tick any check goes false, and only returns once the full window has
/// elapsed with every check still true.
/// </summary>
private static void AssertHoldsFor(TimeSpan window, params (Func<bool> Holds, string Because)[] checks)
{
var deadline = DateTime.UtcNow + window;
while (true)
{
foreach (var (holds, because) in checks)
Assert.True(holds(), because);
if (DateTime.UtcNow >= deadline) return;
Thread.Sleep(20);
}
}
// ── Tests ─────────────────────────────────────────────────────────────────── // ── Tests ───────────────────────────────────────────────────────────────────
[Fact] [Fact]
@@ -226,8 +257,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed)); seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3)); AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
// Give a beat to ensure no late (dropped) delta sneaks in. // Prove no late (dropped) delta sneaks in over the window, not just at one sample.
Thread.Sleep(150); AssertHoldsFor(TimeSpan.FromMilliseconds(150),
(() => Find(sink.Latest, "PumpFault", "") is not null, "seed row must still be present"),
(() => Find(sink.Latest, "PumpFault", "")?.Priority == 500,
"seed value kept; older buffered delta must stay dropped"));
var latest = sink.Latest!; var latest = sink.Latest!;
var pumpFault = Find(latest, "PumpFault", ""); var pumpFault = Find(latest, "PumpFault", "");
Assert.NotNull(pumpFault); Assert.NotNull(pumpFault);
@@ -382,8 +416,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
seed.FaultNext(); seed.FaultNext();
// A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet). // A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet).
Thread.Sleep(200); AssertHoldsFor(TimeSpan.FromMilliseconds(200),
Assert.Equal(0, sink.Count); (() => sink.Count == 0, "a failed initial seed must not publish"));
// The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared // The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared
// on failure), completes, and the actor publishes → now live. // on failure), completes, and the actor publishes → now live.
@@ -464,8 +498,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in
// Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored. // Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored.
Thread.Sleep(400); AssertHoldsFor(TimeSpan.FromMilliseconds(400),
Assert.Equal(2, TotalSubs()); (() => TotalSubs() == 2, "a stale gen-1 error must not open a third stream"));
} }
// ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ── // ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
@@ -520,8 +554,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
// … and without refunding it either: the next error is the 4th, so it exceeds // … and without refunding it either: the next error is the 4th, so it exceeds
// MaxRetries and the stream is given up rather than reconnected. // MaxRetries and the stream is given up rather than reconnected.
factory.ClientFor(GrpcNodeB).Subs[2].OnError(new Exception("4")); factory.ClientFor(GrpcNodeB).Subs[2].OnError(new Exception("4"));
Thread.Sleep(400); AssertHoldsFor(TimeSpan.FromMilliseconds(400),
Assert.Equal(before, TotalSubs()); (() => TotalSubs() == before, "budget exhausted — the 4th error must not reconnect"));
} }
[Fact] [Fact]
@@ -539,9 +573,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
// A completion racing out of the dead gen-1 stream must not tear down the live // A completion racing out of the dead gen-1 stream must not tear down the live
// gen-2 stream (which would go deltaless until the next reconcile tick). // gen-2 stream (which would go deltaless until the next reconcile tick).
firstSub.OnCompleted(); firstSub.OnCompleted();
Thread.Sleep(300); AssertHoldsFor(TimeSpan.FromMilliseconds(300),
Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).Unsubscribed); (() => !factory.ClientFor(GrpcNodeB).Unsubscribed.Contains("corr-1"),
Assert.Single(factory.ClientFor(GrpcNodeB).Subs); "a late gen-1 completion must not unsubscribe the live gen-2 stream"),
(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1,
"the live gen-2 stream must not be torn down"));
} }
// ── R2 T10: live-delta publish coalescing (N6) ── // ── R2 T10: live-delta publish coalescing (N6) ──
@@ -621,15 +657,15 @@ public class SiteAlarmAggregatorActorTests : TestKit
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3")); factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5)); AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
Thread.Sleep(300); AssertHoldsFor(TimeSpan.FromMilliseconds(300),
Assert.Equal(1, seed.CallCount); // pre-fix: 4 (() => seed.CallCount == 1, "failed reconnect attempts must not re-seed (pre-fix: 4)"));
// The stream finally comes up - exactly ONE re-seed, on the connect. // The stream finally comes up - exactly ONE re-seed, on the connect.
factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!(); factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!();
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5)); AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
seed.CompleteNext(); seed.CompleteNext();
Thread.Sleep(300); AssertHoldsFor(TimeSpan.FromMilliseconds(300),
Assert.Equal(2, seed.CallCount); // and no second one (() => seed.CallCount == 2, "the successful connect must re-seed exactly once"));
} }
[Fact] [Fact]
@@ -644,8 +680,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
Thread.Sleep(200); Thread.Sleep(200);
actor.Tell(new RunReconcile()); // skipped - the initial seed covered this window actor.Tell(new RunReconcile()); // skipped - the initial seed covered this window
Thread.Sleep(300); AssertHoldsFor(TimeSpan.FromMilliseconds(300),
Assert.Equal(1, seed.CallCount); (() => seed.CallCount == 1, "a tick inside the just-seeded window must be skipped"));
actor.Tell(new RunReconcile()); // and the next one runs actor.Tell(new RunReconcile()); // and the next one runs
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3)); AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
@@ -668,8 +704,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3)); AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot
Thread.Sleep(300); AssertHoldsFor(TimeSpan.FromMilliseconds(300),
Assert.Equal(1, sink.Count); // no second publish (() => sink.Count == 1, "an unchanged reconcile snapshot must not republish"));
// A genuinely changed snapshot still publishes. // A genuinely changed snapshot still publishes.
actor.Tell(new RunReconcile()); actor.Tell(new RunReconcile());
@@ -733,8 +769,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
Thread.Sleep(150); Thread.Sleep(150);
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
Thread.Sleep(200); AssertHoldsFor(TimeSpan.FromMilliseconds(200),
Assert.Equal(1, seed.CallCount); (() => seed.CallCount == 1, "the tick right after the initial seed must be skipped"));
// From here on, with no reconnect in between, EVERY tick must fan out. // From here on, with no reconnect in between, EVERY tick must fan out.
for (var expected = 2; expected <= 5; expected++) for (var expected = 2; expected <= 5; expected++)
@@ -778,8 +814,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
// That re-seed covers this window: the next tick is skipped… // That re-seed covers this window: the next tick is skipped…
actor.Tell(new RunReconcile()); actor.Tell(new RunReconcile());
Thread.Sleep(250); AssertHoldsFor(TimeSpan.FromMilliseconds(250),
Assert.Equal(3, seed.CallCount); (() => seed.CallCount == 3, "the connect-driven re-seed must stand the next tick down"));
// …and the one after it fans out again. // …and the one after it fans out again.
actor.Tell(new RunReconcile()); actor.Tell(new RunReconcile());
@@ -87,12 +87,25 @@ public class PendingDeploymentPurgeActorTests : TestKit
// Fast interval so the self-scheduling timer fires repeatedly; the actor must // Fast interval so the self-scheduling timer fires repeatedly; the actor must
// swallow each per-tick exception and keep ticking (best-effort hygiene — a // swallow each per-tick exception and keep ticking (best-effort hygiene — a
// crash here would drop the singleton until failover). // crash here would drop the singleton until failover).
CreateActor(repo, TimeSpan.FromMilliseconds(100)); var actor = CreateActor(repo, TimeSpan.FromMilliseconds(100));
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(Volatile.Read(ref calls) >= 2, () => Assert.True(Volatile.Read(ref calls) >= 2,
$"expected >= 2 ticks despite failures, got {Volatile.Read(ref calls)}"), $"expected >= 2 ticks despite failures, got {Volatile.Read(ref calls)}"),
duration: TimeSpan.FromSeconds(5), duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50)); interval: TimeSpan.FromMilliseconds(50));
} }
finally
{
// Row-95/row-82 pattern: a 100ms self-scheduling ticker left running strands
// later tests in the same assembly (it keeps firing — and keeps calling the
// now-disposed repo/service-provider — for the rest of the process). Stop it
// deterministically before the test ends.
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
}
}
} }
@@ -11,6 +11,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// relative to a baseline so the process-wide static instruments are robust to any /// relative to a baseline so the process-wide static instruments are robust to any
/// parallel test interleaving. /// parallel test interleaving.
/// </summary> /// </summary>
/// <remarks>
/// Register row 102's shape: a fixed-baseline delta only survives contributors that hold
/// constant or increase — a parallel class's actors moving the SAME process-global gauge
/// and then STOPPING after this test captures its baseline makes <c>baseline + N</c>
/// unreachable forever. <see cref="SiteAlarmAggregatorActor"/> calls
/// <see cref="ScadaBridgeTelemetry.LiveAlarmAggregatorStarted"/>/
/// <see cref="ScadaBridgeTelemetry.LiveAlarmAggregatorStopped"/> from its own
/// PreStart/PostStop, so every actor <c>Grpc.SiteAlarmAggregatorActorTests</c> spins up
/// moves this exact gauge — serialized into the same collection (row-96 pattern) so the
/// two classes never interleave.
/// </remarks>
[Collection("SiteAlarmAggregatorGauge")]
public class SiteAlarmLiveCacheTelemetryTests public class SiteAlarmLiveCacheTelemetryTests
{ {
private static long ReadGauge(string instrumentName) private static long ReadGauge(string instrumentName)
@@ -22,6 +22,10 @@
<Using Include="Xunit" /> <Using Include="Xunit" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj" /> <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj" />
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" /> <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
@@ -0,0 +1,4 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"maxParallelThreads": 4
}
@@ -114,6 +114,20 @@ public class SiteCallAuditPurgeTests : TestKit
NullLogger<SiteCallAuditActor>.Instance))); NullLogger<SiteCallAuditActor>.Instance)));
} }
/// <summary>
/// Row-82/row-95 house pattern: every test in this file arms a
/// <c>PurgeIntervalOverride = 100ms</c> ticker. Left running past the test's own
/// assertions it keeps firing against the recording repo — and, worse, against a
/// scope-per-tick DI container whose lifetime the test no longer controls — for the
/// rest of the assembly's run, stranding later tests. Stop it deterministically.
/// </summary>
private void StopTicker(IActorRef actor)
{
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 1. PurgeTick_CallsPurgeTerminal_WithRetentionThreshold // 1. PurgeTick_CallsPurgeTerminal_WithRetentionThreshold
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
@@ -124,8 +138,9 @@ public class SiteCallAuditPurgeTests : TestKit
var repo = new RecordingRepo { RowsDeletedPerCall = 7 }; var repo = new RecordingRepo { RowsDeletedPerCall = 7 };
// Non-default retention (30 days) so the assertion isn't accidentally // Non-default retention (30 days) so the assertion isn't accidentally
// satisfied by the 365-day default. // satisfied by the 365-day default.
CreateActor(repo, FastPurgeOptions(retentionDays: 30)); var actor = CreateActor(repo, FastPurgeOptions(retentionDays: 30));
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(repo.PurgeThresholds.Count >= 1, () => Assert.True(repo.PurgeThresholds.Count >= 1,
$"expected >= 1 PurgeTerminalAsync call, got {repo.PurgeThresholds.Count}"), $"expected >= 1 PurgeTerminalAsync call, got {repo.PurgeThresholds.Count}"),
@@ -140,6 +155,11 @@ public class SiteCallAuditPurgeTests : TestKit
Math.Abs((threshold - expected).TotalMinutes) < 1.0, Math.Abs((threshold - expected).TotalMinutes) < 1.0,
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}"); $"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 2. PurgeTick_UsesDefaultRetention_365Days // 2. PurgeTick_UsesDefaultRetention_365Days
@@ -149,8 +169,9 @@ public class SiteCallAuditPurgeTests : TestKit
public void PurgeTick_DefaultRetention_Uses365DayThreshold() public void PurgeTick_DefaultRetention_Uses365DayThreshold()
{ {
var repo = new RecordingRepo(); var repo = new RecordingRepo();
CreateActor(repo, FastPurgeOptions()); // default 365 days var actor = CreateActor(repo, FastPurgeOptions()); // default 365 days
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(repo.PurgeThresholds.Count >= 1), () => Assert.True(repo.PurgeThresholds.Count >= 1),
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
@@ -162,6 +183,11 @@ public class SiteCallAuditPurgeTests : TestKit
Math.Abs((threshold - expected).TotalMinutes) < 1.0, Math.Abs((threshold - expected).TotalMinutes) < 1.0,
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}"); $"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 3. PurgeTick_RepoThrows_ActorStaysAlive_RetriesNextTick (continue-on-error) // 3. PurgeTick_RepoThrows_ActorStaysAlive_RetriesNextTick (continue-on-error)
@@ -171,8 +197,9 @@ public class SiteCallAuditPurgeTests : TestKit
public void PurgeTick_PurgeThrows_ActorStaysAlive_RetriesNextTick() public void PurgeTick_PurgeThrows_ActorStaysAlive_RetriesNextTick()
{ {
var repo = new PurgeThrowingRepo(); var repo = new PurgeThrowingRepo();
CreateActor(repo, FastPurgeOptions()); var actor = CreateActor(repo, FastPurgeOptions());
try
{
// The singleton must NOT die on a purge fault — a second tick must still // The singleton must NOT die on a purge fault — a second tick must still
// arrive (continue-on-error). Two purge calls prove the actor survived // arrive (continue-on-error). Two purge calls prove the actor survived
// the first throw and the timer kept ticking. // the first throw and the timer kept ticking.
@@ -182,6 +209,11 @@ public class SiteCallAuditPurgeTests : TestKit
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50)); interval: TimeSpan.FromMilliseconds(50));
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 4. SiteCallAudit-007: purge timer arms even when the reconciliation // 4. SiteCallAudit-007: purge timer arms even when the reconciliation
@@ -206,11 +238,12 @@ public class SiteCallAuditPurgeTests : TestKit
.BuildServiceProvider(); .BuildServiceProvider();
var options = FastPurgeOptions(retentionDays: 30); var options = FastPurgeOptions(retentionDays: 30);
Sys.ActorOf(Props.Create(() => new SiteCallAuditActor( var actor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
provider, provider,
options, options,
NullLogger<SiteCallAuditActor>.Instance))); NullLogger<SiteCallAuditActor>.Instance)));
try
{
// No reconciliation collaborators were registered, yet the purge tick // No reconciliation collaborators were registered, yet the purge tick
// must still fire on the production path. // must still fire on the production path.
AwaitAssert( AwaitAssert(
@@ -226,4 +259,9 @@ public class SiteCallAuditPurgeTests : TestKit
Math.Abs((threshold - expected).TotalMinutes) < 1.0, Math.Abs((threshold - expected).TotalMinutes) < 1.0,
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}"); $"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
} }
finally
{
StopTicker(actor);
}
}
} }
@@ -251,6 +251,20 @@ public class SiteCallAuditReconciliationTests : TestKit
NullLogger<SiteCallAuditActor>.Instance))); NullLogger<SiteCallAuditActor>.Instance)));
} }
/// <summary>
/// Row-82/row-95 house pattern: every actor built above arms a fast-repeating
/// reconciliation (and, via <see cref="SiteCallAuditActor"/>, purge) ticker — as fast
/// as 100ms in most of the tests below. Left running past the owning test it keeps
/// firing against test doubles the test has stopped controlling, stranding later
/// tests in the same assembly. Stop it deterministically before the test ends.
/// </summary>
private void StopTicker(IActorRef actor)
{
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 1. AbsentRow_PulledFromSite_IsUpserted // 1. AbsentRow_PulledFromSite_IsUpserted
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
@@ -267,8 +281,9 @@ public class SiteCallAuditReconciliationTests : TestKit
new PullSiteCallsResponse(new[] { row }, MoreAvailable: false)); new PullSiteCallsResponse(new[] { row }, MoreAvailable: false));
var repo = new RecordingRepo(); var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
AwaitAssert( AwaitAssert(
() => () =>
{ {
@@ -280,6 +295,11 @@ public class SiteCallAuditReconciliationTests : TestKit
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50)); interval: TimeSpan.FromMilliseconds(50));
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 2. Cursor_Advances_ToMaxUpdatedAtUtc_NoRePullOfOldRows // 2. Cursor_Advances_ToMaxUpdatedAtUtc_NoRePullOfOldRows
@@ -304,8 +324,9 @@ public class SiteCallAuditReconciliationTests : TestKit
new PullSiteCallsResponse(new[] { r1, r2, r3 }, MoreAvailable: false)); new PullSiteCallsResponse(new[] { r1, r2, r3 }, MoreAvailable: false));
var repo = new RecordingRepo(); var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(client.Calls.Count >= 2, () => Assert.True(client.Calls.Count >= 2,
$"need at least 2 pulls to assert cursor advancement, got {client.Calls.Count}"), $"need at least 2 pulls to assert cursor advancement, got {client.Calls.Count}"),
@@ -317,6 +338,11 @@ public class SiteCallAuditReconciliationTests : TestKit
// The batch size flows through from options. // The batch size flows through from options.
Assert.Equal(500, client.Calls[0].BatchSize); Assert.Equal(500, client.Calls[0].BatchSize);
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 3. OneSiteThrows_OtherSitesStillProcessed (failure isolation) // 3. OneSiteThrows_OtherSitesStillProcessed (failure isolation)
@@ -337,8 +363,9 @@ public class SiteCallAuditReconciliationTests : TestKit
.Script(siteB, new PullSiteCallsResponse(new[] { bRow }, MoreAvailable: false)); .Script(siteB, new PullSiteCallsResponse(new[] { bRow }, MoreAvailable: false));
var repo = new RecordingRepo(); var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
AwaitAssert( AwaitAssert(
() => () =>
{ {
@@ -351,6 +378,11 @@ public class SiteCallAuditReconciliationTests : TestKit
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50)); interval: TimeSpan.FromMilliseconds(50));
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 4. A composition root without the reconciliation collaborators does NOT // 4. A composition root without the reconciliation collaborators does NOT
@@ -369,7 +401,7 @@ public class SiteCallAuditReconciliationTests : TestKit
// this actor upserted nothing on its own. // this actor upserted nothing on its own.
var repo = new RecordingRepo(); var repo = new RecordingRepo();
var bareProvider = TestSupport.StubRepositoryProvider.Build(repo); var bareProvider = TestSupport.StubRepositoryProvider.Build(repo);
Sys.ActorOf(Props.Create(() => new SiteCallAuditActor( var bareActor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
bareProvider, bareProvider,
FastTickOptions(), FastTickOptions(),
NullLogger<SiteCallAuditActor>.Instance))); NullLogger<SiteCallAuditActor>.Instance)));
@@ -379,12 +411,14 @@ public class SiteCallAuditReconciliationTests : TestKit
// collaborator-less actor would have ticked too, had it been wired. // collaborator-less actor would have ticked too, had it been wired.
var liveClient = new ScriptedPullClient(); var liveClient = new ScriptedPullClient();
var liveRepo = new RecordingRepo(); var liveRepo = new RecordingRepo();
CreateActor( var liveActor = CreateActor(
new StaticEnumerator(new SiteEntry("siteX", "http://siteX:8083")), new StaticEnumerator(new SiteEntry("siteX", "http://siteX:8083")),
liveClient, liveClient,
liveRepo, liveRepo,
FastTickOptions()); FastTickOptions());
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(liveClient.Calls.Count >= 1), () => Assert.True(liveClient.Calls.Count >= 1),
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
@@ -394,6 +428,12 @@ public class SiteCallAuditReconciliationTests : TestKit
// with, so it upserts nothing on its own. // with, so it upserts nothing on its own.
Assert.Equal(0, repo.UpsertCallCount); Assert.Equal(0, repo.UpsertCallCount);
} }
finally
{
StopTicker(liveActor);
StopTicker(bareActor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 5. SiteCallAudit-009: MoreAvailable drives a within-tick continuation // 5. SiteCallAudit-009: MoreAvailable drives a within-tick continuation
@@ -431,8 +471,9 @@ public class SiteCallAuditReconciliationTests : TestKit
ReconciliationBatchSize = 2, ReconciliationBatchSize = 2,
}; };
CreateActor(sites, client, repo, options); var actor = CreateActor(sites, client, repo, options);
try
{
AwaitAssert( AwaitAssert(
() => () =>
{ {
@@ -452,6 +493,11 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc); Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
Assert.Equal(t2, client.Calls[1].SinceUtc); Assert.Equal(t2, client.Calls[1].SinceUtc);
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 6. SiteCallAudit-009: single-timestamp saturation pin does NOT spin — // 6. SiteCallAudit-009: single-timestamp saturation pin does NOT spin —
@@ -484,8 +530,9 @@ public class SiteCallAuditReconciliationTests : TestKit
ReconciliationBatchSize = 2, ReconciliationBatchSize = 2,
}; };
CreateActor(sites, client, repo, options); var actor = CreateActor(sites, client, repo, options);
try
{
AwaitAssert( AwaitAssert(
() => Assert.True(client.CallCount >= 1, "the first reconciliation tick should have pulled"), () => Assert.True(client.CallCount >= 1, "the first reconciliation tick should have pulled"),
duration: TimeSpan.FromSeconds(3), duration: TimeSpan.FromSeconds(3),
@@ -514,6 +561,11 @@ public class SiteCallAuditReconciliationTests : TestKit
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the " $"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)"); + $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 7. Task 16: a keyset-honouring site drains a single-timestamp burst // 7. Task 16: a keyset-honouring site drains a single-timestamp burst
@@ -546,8 +598,9 @@ public class SiteCallAuditReconciliationTests : TestKit
ReconciliationBatchSize = 2, ReconciliationBatchSize = 2,
}; };
CreateActor(sites, client, repo, options); var actor = CreateActor(sites, client, repo, options);
try
{
AwaitAssert( AwaitAssert(
() => () =>
{ {
@@ -565,6 +618,11 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.True(client.CallCount is >= 2 and < 10, Assert.True(client.CallCount is >= 2 and < 10,
$"expected the burst to drain in a couple of keyset pages, got {client.CallCount}"); $"expected the burst to drain in a couple of keyset pages, got {client.CallCount}");
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 8. Task 16: a LEGACY site that ignores after_id can never be drained, so // 8. Task 16: a LEGACY site that ignores after_id can never be drained, so
@@ -593,12 +651,18 @@ public class SiteCallAuditReconciliationTests : TestKit
ReconciliationBatchSize = 2, ReconciliationBatchSize = 2,
}; };
CreateActor(sites, client, repo, options); var actor = CreateActor(sites, client, repo, options);
try
{
var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5)); var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
Assert.Equal(siteId, evt.SiteId); Assert.Equal(siteId, evt.SiteId);
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true"); Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
} }
finally
{
StopTicker(actor);
}
}
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest, // 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
@@ -648,6 +712,8 @@ public class SiteCallAuditReconciliationTests : TestKit
var actor = CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
// Wait until the drain is genuinely in flight and blocked inside PullAsync. // Wait until the drain is genuinely in flight and blocked inside PullAsync.
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5)); await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
@@ -661,9 +727,15 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.True(reply.Accepted); Assert.True(reply.Accepted);
Assert.Equal(id, reply.TrackedOperationId); Assert.Equal(id, reply.TrackedOperationId);
}
// Let the drain finish so the actor shuts down cleanly. finally
{
// Unblock the drain FIRST so the actor is free to process the Stop rather
// than parked inside the blocked pull, then stop it deterministically
// rather than leaving its 100ms ticker running (row-95).
client.Release(); client.Release();
StopTicker(actor);
}
} }
[Fact] [Fact]
@@ -678,16 +750,25 @@ public class SiteCallAuditReconciliationTests : TestKit
var client = new CountingBlockingPullClient(); var client = new CountingBlockingPullClient();
var repo = new RecordingRepo(); var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5)); await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
// Several tick intervals elapse while the first pass is still blocked. // Several tick intervals elapse while the first pass is still blocked.
await Task.Delay(TimeSpan.FromMilliseconds(600)); await Task.Delay(TimeSpan.FromMilliseconds(600));
Assert.Equal(1, client.CallCount); Assert.Equal(1, client.CallCount);
}
finally
{
// Unblock the drain FIRST so the actor is free to process the Stop, then
// stop it deterministically rather than leaving its 100ms ticker running
// (row-95).
client.Release(); client.Release();
StopTicker(actor);
}
} }
/// <summary> /// <summary>
@@ -715,6 +796,8 @@ public class SiteCallAuditReconciliationTests : TestKit
var actor = CreateActor(sites, client, repo, FastTickOptions()); var actor = CreateActor(sites, client, repo, FastTickOptions());
try
{
// Once the pull has been served the drain is upserting; flood the mailbox // Once the pull has been served the drain is upserting; flood the mailbox
// with ingest commands so the two writers overlap in wall-clock time. // with ingest commands so the two writers overlap in wall-clock time.
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5)); await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
@@ -735,6 +818,11 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.Equal(1, repo.MaxObservedConcurrency); Assert.Equal(1, repo.MaxObservedConcurrency);
} }
finally
{
StopTicker(actor);
}
}
/// <summary> /// <summary>
/// Serves one page of rows on the first pull and nothing afterwards, so the /// Serves one page of rows on the first pull and nothing afterwards, so the