test(communication)+test(sitecallaudit): cap parallelism, delta/collection-serialize gauge asserts, replace sleep-then-assert, stop leaked 100ms tickers (arch-review 2026-08-19 P8-5 + P8-6, register rows 40/82/95/96/102 patterns)
This commit is contained in:
+59
-23
@@ -15,6 +15,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
|||||||
/// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect
|
/// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect
|
||||||
/// re-seed, and periodic reconcile authoritative-replace (drift correction).
|
/// re-seed, and periodic reconcile authoritative-replace (drift correction).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Serialized with <c>SiteAlarmLiveCacheTelemetryTests</c> (register row 102): every actor
|
||||||
|
/// this class spins up moves the process-global
|
||||||
|
/// <c>scadabridge.site.alarm_cache.aggregators.active</c> gauge via
|
||||||
|
/// <see cref="SiteAlarmAggregatorActor"/>'s PreStart/PostStop, which the telemetry test's
|
||||||
|
/// fixed-baseline assertions require to hold still (or only increase) between its capture
|
||||||
|
/// and its checks. Same collection ⇒ never concurrent.
|
||||||
|
/// </remarks>
|
||||||
|
[Collection("SiteAlarmAggregatorGauge")]
|
||||||
public class SiteAlarmAggregatorActorTests : TestKit
|
public class SiteAlarmAggregatorActorTests : TestKit
|
||||||
{
|
{
|
||||||
private const string SiteId = "site-alpha";
|
private const string SiteId = "site-alpha";
|
||||||
@@ -188,6 +197,28 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) =>
|
private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) =>
|
||||||
rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef);
|
rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Row-40 house pattern for an ABSENCE assertion: proves <paramref name="checks"/> keep
|
||||||
|
/// holding throughout <paramref name="window"/> by polling, rather than a single blind
|
||||||
|
/// Thread.Sleep(window) followed by one end-of-window sample. A bare sleep-then-assert
|
||||||
|
/// cannot distinguish "the mechanism correctly stayed quiet" from "the mechanism just
|
||||||
|
/// hadn't run yet when the one sample was taken", and it also can't catch a value that
|
||||||
|
/// flips wrong and then flips back before that single sample. This fails immediately on
|
||||||
|
/// the first tick any check goes false, and only returns once the full window has
|
||||||
|
/// elapsed with every check still true.
|
||||||
|
/// </summary>
|
||||||
|
private static void AssertHoldsFor(TimeSpan window, params (Func<bool> Holds, string Because)[] checks)
|
||||||
|
{
|
||||||
|
var deadline = DateTime.UtcNow + window;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
foreach (var (holds, because) in checks)
|
||||||
|
Assert.True(holds(), because);
|
||||||
|
if (DateTime.UtcNow >= deadline) return;
|
||||||
|
Thread.Sleep(20);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
// ── Tests ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -226,8 +257,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
||||||
|
|
||||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||||
// Give a beat to ensure no late (dropped) delta sneaks in.
|
// Prove no late (dropped) delta sneaks in over the window, not just at one sample.
|
||||||
Thread.Sleep(150);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(150),
|
||||||
|
(() => Find(sink.Latest, "PumpFault", "") is not null, "seed row must still be present"),
|
||||||
|
(() => Find(sink.Latest, "PumpFault", "")?.Priority == 500,
|
||||||
|
"seed value kept; older buffered delta must stay dropped"));
|
||||||
var latest = sink.Latest!;
|
var latest = sink.Latest!;
|
||||||
var pumpFault = Find(latest, "PumpFault", "");
|
var pumpFault = Find(latest, "PumpFault", "");
|
||||||
Assert.NotNull(pumpFault);
|
Assert.NotNull(pumpFault);
|
||||||
@@ -382,8 +416,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
seed.FaultNext();
|
seed.FaultNext();
|
||||||
|
|
||||||
// A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet).
|
// A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet).
|
||||||
Thread.Sleep(200);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(200),
|
||||||
Assert.Equal(0, sink.Count);
|
(() => sink.Count == 0, "a failed initial seed must not publish"));
|
||||||
|
|
||||||
// The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared
|
// The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared
|
||||||
// on failure), completes, and the actor publishes → now live.
|
// on failure), completes, and the actor publishes → now live.
|
||||||
@@ -464,8 +498,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
|
|
||||||
firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in
|
firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in
|
||||||
// Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored.
|
// Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored.
|
||||||
Thread.Sleep(400);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(400),
|
||||||
Assert.Equal(2, TotalSubs());
|
(() => TotalSubs() == 2, "a stale gen-1 error must not open a third stream"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
|
// ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
|
||||||
@@ -520,8 +554,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
// … and without refunding it either: the next error is the 4th, so it exceeds
|
// … and without refunding it either: the next error is the 4th, so it exceeds
|
||||||
// MaxRetries and the stream is given up rather than reconnected.
|
// MaxRetries and the stream is given up rather than reconnected.
|
||||||
factory.ClientFor(GrpcNodeB).Subs[2].OnError(new Exception("4"));
|
factory.ClientFor(GrpcNodeB).Subs[2].OnError(new Exception("4"));
|
||||||
Thread.Sleep(400);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(400),
|
||||||
Assert.Equal(before, TotalSubs());
|
(() => TotalSubs() == before, "budget exhausted — the 4th error must not reconnect"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -539,9 +573,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
// A completion racing out of the dead gen-1 stream must not tear down the live
|
// A completion racing out of the dead gen-1 stream must not tear down the live
|
||||||
// gen-2 stream (which would go deltaless until the next reconcile tick).
|
// gen-2 stream (which would go deltaless until the next reconcile tick).
|
||||||
firstSub.OnCompleted();
|
firstSub.OnCompleted();
|
||||||
Thread.Sleep(300);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(300),
|
||||||
Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).Unsubscribed);
|
(() => !factory.ClientFor(GrpcNodeB).Unsubscribed.Contains("corr-1"),
|
||||||
Assert.Single(factory.ClientFor(GrpcNodeB).Subs);
|
"a late gen-1 completion must not unsubscribe the live gen-2 stream"),
|
||||||
|
(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1,
|
||||||
|
"the live gen-2 stream must not be torn down"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── R2 T10: live-delta publish coalescing (N6) ──
|
// ── R2 T10: live-delta publish coalescing (N6) ──
|
||||||
@@ -621,15 +657,15 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
|
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
|
||||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
Thread.Sleep(300);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(300),
|
||||||
Assert.Equal(1, seed.CallCount); // pre-fix: 4
|
(() => seed.CallCount == 1, "failed reconnect attempts must not re-seed (pre-fix: 4)"));
|
||||||
|
|
||||||
// The stream finally comes up - exactly ONE re-seed, on the connect.
|
// The stream finally comes up - exactly ONE re-seed, on the connect.
|
||||||
factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!();
|
factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!();
|
||||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
|
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
|
||||||
seed.CompleteNext();
|
seed.CompleteNext();
|
||||||
Thread.Sleep(300);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(300),
|
||||||
Assert.Equal(2, seed.CallCount); // and no second one
|
(() => seed.CallCount == 2, "the successful connect must re-seed exactly once"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -644,8 +680,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
Thread.Sleep(200);
|
Thread.Sleep(200);
|
||||||
|
|
||||||
actor.Tell(new RunReconcile()); // skipped - the initial seed covered this window
|
actor.Tell(new RunReconcile()); // skipped - the initial seed covered this window
|
||||||
Thread.Sleep(300);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(300),
|
||||||
Assert.Equal(1, seed.CallCount);
|
(() => seed.CallCount == 1, "a tick inside the just-seeded window must be skipped"));
|
||||||
|
|
||||||
actor.Tell(new RunReconcile()); // and the next one runs
|
actor.Tell(new RunReconcile()); // and the next one runs
|
||||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||||
@@ -668,8 +704,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||||
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot
|
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot
|
||||||
|
|
||||||
Thread.Sleep(300);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(300),
|
||||||
Assert.Equal(1, sink.Count); // no second publish
|
(() => sink.Count == 1, "an unchanged reconcile snapshot must not republish"));
|
||||||
|
|
||||||
// A genuinely changed snapshot still publishes.
|
// A genuinely changed snapshot still publishes.
|
||||||
actor.Tell(new RunReconcile());
|
actor.Tell(new RunReconcile());
|
||||||
@@ -733,8 +769,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
Thread.Sleep(150);
|
Thread.Sleep(150);
|
||||||
|
|
||||||
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
|
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
|
||||||
Thread.Sleep(200);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(200),
|
||||||
Assert.Equal(1, seed.CallCount);
|
(() => seed.CallCount == 1, "the tick right after the initial seed must be skipped"));
|
||||||
|
|
||||||
// From here on, with no reconnect in between, EVERY tick must fan out.
|
// From here on, with no reconnect in between, EVERY tick must fan out.
|
||||||
for (var expected = 2; expected <= 5; expected++)
|
for (var expected = 2; expected <= 5; expected++)
|
||||||
@@ -778,8 +814,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
|||||||
|
|
||||||
// That re-seed covers this window: the next tick is skipped…
|
// That re-seed covers this window: the next tick is skipped…
|
||||||
actor.Tell(new RunReconcile());
|
actor.Tell(new RunReconcile());
|
||||||
Thread.Sleep(250);
|
AssertHoldsFor(TimeSpan.FromMilliseconds(250),
|
||||||
Assert.Equal(3, seed.CallCount);
|
(() => seed.CallCount == 3, "the connect-driven re-seed must stand the next tick down"));
|
||||||
|
|
||||||
// …and the one after it fans out again.
|
// …and the one after it fans out again.
|
||||||
actor.Tell(new RunReconcile());
|
actor.Tell(new RunReconcile());
|
||||||
|
|||||||
+14
-1
@@ -87,12 +87,25 @@ public class PendingDeploymentPurgeActorTests : TestKit
|
|||||||
// Fast interval so the self-scheduling timer fires repeatedly; the actor must
|
// Fast interval so the self-scheduling timer fires repeatedly; the actor must
|
||||||
// swallow each per-tick exception and keep ticking (best-effort hygiene — a
|
// swallow each per-tick exception and keep ticking (best-effort hygiene — a
|
||||||
// crash here would drop the singleton until failover).
|
// crash here would drop the singleton until failover).
|
||||||
CreateActor(repo, TimeSpan.FromMilliseconds(100));
|
var actor = CreateActor(repo, TimeSpan.FromMilliseconds(100));
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(Volatile.Read(ref calls) >= 2,
|
() => Assert.True(Volatile.Read(ref calls) >= 2,
|
||||||
$"expected >= 2 ticks despite failures, got {Volatile.Read(ref calls)}"),
|
$"expected >= 2 ticks despite failures, got {Volatile.Read(ref calls)}"),
|
||||||
duration: TimeSpan.FromSeconds(5),
|
duration: TimeSpan.FromSeconds(5),
|
||||||
interval: TimeSpan.FromMilliseconds(50));
|
interval: TimeSpan.FromMilliseconds(50));
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// Row-95/row-82 pattern: a 100ms self-scheduling ticker left running strands
|
||||||
|
// later tests in the same assembly (it keeps firing — and keeps calling the
|
||||||
|
// now-disposed repo/service-provider — for the rest of the process). Stop it
|
||||||
|
// deterministically before the test ends.
|
||||||
|
Watch(actor);
|
||||||
|
Sys.Stop(actor);
|
||||||
|
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
|||||||
/// relative to a baseline so the process-wide static instruments are robust to any
|
/// relative to a baseline so the process-wide static instruments are robust to any
|
||||||
/// parallel test interleaving.
|
/// parallel test interleaving.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Register row 102's shape: a fixed-baseline delta only survives contributors that hold
|
||||||
|
/// constant or increase — a parallel class's actors moving the SAME process-global gauge
|
||||||
|
/// and then STOPPING after this test captures its baseline makes <c>baseline + N</c>
|
||||||
|
/// unreachable forever. <see cref="SiteAlarmAggregatorActor"/> calls
|
||||||
|
/// <see cref="ScadaBridgeTelemetry.LiveAlarmAggregatorStarted"/>/
|
||||||
|
/// <see cref="ScadaBridgeTelemetry.LiveAlarmAggregatorStopped"/> from its own
|
||||||
|
/// PreStart/PostStop, so every actor <c>Grpc.SiteAlarmAggregatorActorTests</c> spins up
|
||||||
|
/// moves this exact gauge — serialized into the same collection (row-96 pattern) so the
|
||||||
|
/// two classes never interleave.
|
||||||
|
/// </remarks>
|
||||||
|
[Collection("SiteAlarmAggregatorGauge")]
|
||||||
public class SiteAlarmLiveCacheTelemetryTests
|
public class SiteAlarmLiveCacheTelemetryTests
|
||||||
{
|
{
|
||||||
private static long ReadGauge(string instrumentName)
|
private static long ReadGauge(string instrumentName)
|
||||||
|
|||||||
+4
@@ -22,6 +22,10 @@
|
|||||||
<Using Include="Xunit" />
|
<Using Include="Xunit" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj" />
|
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj" />
|
||||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
|
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
|
||||||
|
"maxParallelThreads": 4
|
||||||
|
}
|
||||||
@@ -114,6 +114,20 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
NullLogger<SiteCallAuditActor>.Instance)));
|
NullLogger<SiteCallAuditActor>.Instance)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Row-82/row-95 house pattern: every test in this file arms a
|
||||||
|
/// <c>PurgeIntervalOverride = 100ms</c> ticker. Left running past the test's own
|
||||||
|
/// assertions it keeps firing against the recording repo — and, worse, against a
|
||||||
|
/// scope-per-tick DI container whose lifetime the test no longer controls — for the
|
||||||
|
/// rest of the assembly's run, stranding later tests. Stop it deterministically.
|
||||||
|
/// </summary>
|
||||||
|
private void StopTicker(IActorRef actor)
|
||||||
|
{
|
||||||
|
Watch(actor);
|
||||||
|
Sys.Stop(actor);
|
||||||
|
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 1. PurgeTick_CallsPurgeTerminal_WithRetentionThreshold
|
// 1. PurgeTick_CallsPurgeTerminal_WithRetentionThreshold
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
@@ -124,8 +138,9 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
var repo = new RecordingRepo { RowsDeletedPerCall = 7 };
|
var repo = new RecordingRepo { RowsDeletedPerCall = 7 };
|
||||||
// Non-default retention (30 days) so the assertion isn't accidentally
|
// Non-default retention (30 days) so the assertion isn't accidentally
|
||||||
// satisfied by the 365-day default.
|
// satisfied by the 365-day default.
|
||||||
CreateActor(repo, FastPurgeOptions(retentionDays: 30));
|
var actor = CreateActor(repo, FastPurgeOptions(retentionDays: 30));
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(repo.PurgeThresholds.Count >= 1,
|
() => Assert.True(repo.PurgeThresholds.Count >= 1,
|
||||||
$"expected >= 1 PurgeTerminalAsync call, got {repo.PurgeThresholds.Count}"),
|
$"expected >= 1 PurgeTerminalAsync call, got {repo.PurgeThresholds.Count}"),
|
||||||
@@ -140,6 +155,11 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
||||||
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 2. PurgeTick_UsesDefaultRetention_365Days
|
// 2. PurgeTick_UsesDefaultRetention_365Days
|
||||||
@@ -149,8 +169,9 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
public void PurgeTick_DefaultRetention_Uses365DayThreshold()
|
public void PurgeTick_DefaultRetention_Uses365DayThreshold()
|
||||||
{
|
{
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
CreateActor(repo, FastPurgeOptions()); // default 365 days
|
var actor = CreateActor(repo, FastPurgeOptions()); // default 365 days
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(repo.PurgeThresholds.Count >= 1),
|
() => Assert.True(repo.PurgeThresholds.Count >= 1),
|
||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
@@ -162,6 +183,11 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
||||||
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 3. PurgeTick_RepoThrows_ActorStaysAlive_RetriesNextTick (continue-on-error)
|
// 3. PurgeTick_RepoThrows_ActorStaysAlive_RetriesNextTick (continue-on-error)
|
||||||
@@ -171,8 +197,9 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
public void PurgeTick_PurgeThrows_ActorStaysAlive_RetriesNextTick()
|
public void PurgeTick_PurgeThrows_ActorStaysAlive_RetriesNextTick()
|
||||||
{
|
{
|
||||||
var repo = new PurgeThrowingRepo();
|
var repo = new PurgeThrowingRepo();
|
||||||
CreateActor(repo, FastPurgeOptions());
|
var actor = CreateActor(repo, FastPurgeOptions());
|
||||||
|
try
|
||||||
|
{
|
||||||
// The singleton must NOT die on a purge fault — a second tick must still
|
// The singleton must NOT die on a purge fault — a second tick must still
|
||||||
// arrive (continue-on-error). Two purge calls prove the actor survived
|
// arrive (continue-on-error). Two purge calls prove the actor survived
|
||||||
// the first throw and the timer kept ticking.
|
// the first throw and the timer kept ticking.
|
||||||
@@ -182,6 +209,11 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
interval: TimeSpan.FromMilliseconds(50));
|
interval: TimeSpan.FromMilliseconds(50));
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 4. SiteCallAudit-007: purge timer arms even when the reconciliation
|
// 4. SiteCallAudit-007: purge timer arms even when the reconciliation
|
||||||
@@ -206,11 +238,12 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
.BuildServiceProvider();
|
.BuildServiceProvider();
|
||||||
|
|
||||||
var options = FastPurgeOptions(retentionDays: 30);
|
var options = FastPurgeOptions(retentionDays: 30);
|
||||||
Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
var actor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
||||||
provider,
|
provider,
|
||||||
options,
|
options,
|
||||||
NullLogger<SiteCallAuditActor>.Instance)));
|
NullLogger<SiteCallAuditActor>.Instance)));
|
||||||
|
try
|
||||||
|
{
|
||||||
// No reconciliation collaborators were registered, yet the purge tick
|
// No reconciliation collaborators were registered, yet the purge tick
|
||||||
// must still fire on the production path.
|
// must still fire on the production path.
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
@@ -226,4 +259,9 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
Math.Abs((threshold - expected).TotalMinutes) < 1.0,
|
||||||
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
$"purge threshold {threshold:o} should be within 1 minute of {expected:o}");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+108
-20
@@ -251,6 +251,20 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
NullLogger<SiteCallAuditActor>.Instance)));
|
NullLogger<SiteCallAuditActor>.Instance)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Row-82/row-95 house pattern: every actor built above arms a fast-repeating
|
||||||
|
/// reconciliation (and, via <see cref="SiteCallAuditActor"/>, purge) ticker — as fast
|
||||||
|
/// as 100ms in most of the tests below. Left running past the owning test it keeps
|
||||||
|
/// firing against test doubles the test has stopped controlling, stranding later
|
||||||
|
/// tests in the same assembly. Stop it deterministically before the test ends.
|
||||||
|
/// </summary>
|
||||||
|
private void StopTicker(IActorRef actor)
|
||||||
|
{
|
||||||
|
Watch(actor);
|
||||||
|
Sys.Stop(actor);
|
||||||
|
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 1. AbsentRow_PulledFromSite_IsUpserted
|
// 1. AbsentRow_PulledFromSite_IsUpserted
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
@@ -267,8 +281,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
new PullSiteCallsResponse(new[] { row }, MoreAvailable: false));
|
new PullSiteCallsResponse(new[] { row }, MoreAvailable: false));
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() =>
|
() =>
|
||||||
{
|
{
|
||||||
@@ -280,6 +295,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
interval: TimeSpan.FromMilliseconds(50));
|
interval: TimeSpan.FromMilliseconds(50));
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 2. Cursor_Advances_ToMaxUpdatedAtUtc_NoRePullOfOldRows
|
// 2. Cursor_Advances_ToMaxUpdatedAtUtc_NoRePullOfOldRows
|
||||||
@@ -304,8 +324,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
new PullSiteCallsResponse(new[] { r1, r2, r3 }, MoreAvailable: false));
|
new PullSiteCallsResponse(new[] { r1, r2, r3 }, MoreAvailable: false));
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(client.Calls.Count >= 2,
|
() => Assert.True(client.Calls.Count >= 2,
|
||||||
$"need at least 2 pulls to assert cursor advancement, got {client.Calls.Count}"),
|
$"need at least 2 pulls to assert cursor advancement, got {client.Calls.Count}"),
|
||||||
@@ -317,6 +338,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
// The batch size flows through from options.
|
// The batch size flows through from options.
|
||||||
Assert.Equal(500, client.Calls[0].BatchSize);
|
Assert.Equal(500, client.Calls[0].BatchSize);
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 3. OneSiteThrows_OtherSitesStillProcessed (failure isolation)
|
// 3. OneSiteThrows_OtherSitesStillProcessed (failure isolation)
|
||||||
@@ -337,8 +363,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
.Script(siteB, new PullSiteCallsResponse(new[] { bRow }, MoreAvailable: false));
|
.Script(siteB, new PullSiteCallsResponse(new[] { bRow }, MoreAvailable: false));
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() =>
|
() =>
|
||||||
{
|
{
|
||||||
@@ -351,6 +378,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
interval: TimeSpan.FromMilliseconds(50));
|
interval: TimeSpan.FromMilliseconds(50));
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 4. A composition root without the reconciliation collaborators does NOT
|
// 4. A composition root without the reconciliation collaborators does NOT
|
||||||
@@ -369,7 +401,7 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
// this actor upserted nothing on its own.
|
// this actor upserted nothing on its own.
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
var bareProvider = TestSupport.StubRepositoryProvider.Build(repo);
|
var bareProvider = TestSupport.StubRepositoryProvider.Build(repo);
|
||||||
Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
var bareActor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
||||||
bareProvider,
|
bareProvider,
|
||||||
FastTickOptions(),
|
FastTickOptions(),
|
||||||
NullLogger<SiteCallAuditActor>.Instance)));
|
NullLogger<SiteCallAuditActor>.Instance)));
|
||||||
@@ -379,12 +411,14 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
// collaborator-less actor would have ticked too, had it been wired.
|
// collaborator-less actor would have ticked too, had it been wired.
|
||||||
var liveClient = new ScriptedPullClient();
|
var liveClient = new ScriptedPullClient();
|
||||||
var liveRepo = new RecordingRepo();
|
var liveRepo = new RecordingRepo();
|
||||||
CreateActor(
|
var liveActor = CreateActor(
|
||||||
new StaticEnumerator(new SiteEntry("siteX", "http://siteX:8083")),
|
new StaticEnumerator(new SiteEntry("siteX", "http://siteX:8083")),
|
||||||
liveClient,
|
liveClient,
|
||||||
liveRepo,
|
liveRepo,
|
||||||
FastTickOptions());
|
FastTickOptions());
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(liveClient.Calls.Count >= 1),
|
() => Assert.True(liveClient.Calls.Count >= 1),
|
||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
@@ -394,6 +428,12 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
// with, so it upserts nothing on its own.
|
// with, so it upserts nothing on its own.
|
||||||
Assert.Equal(0, repo.UpsertCallCount);
|
Assert.Equal(0, repo.UpsertCallCount);
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(liveActor);
|
||||||
|
StopTicker(bareActor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 5. SiteCallAudit-009: MoreAvailable drives a within-tick continuation
|
// 5. SiteCallAudit-009: MoreAvailable drives a within-tick continuation
|
||||||
@@ -431,8 +471,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
ReconciliationBatchSize = 2,
|
ReconciliationBatchSize = 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
CreateActor(sites, client, repo, options);
|
var actor = CreateActor(sites, client, repo, options);
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() =>
|
() =>
|
||||||
{
|
{
|
||||||
@@ -452,6 +493,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
|
Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
|
||||||
Assert.Equal(t2, client.Calls[1].SinceUtc);
|
Assert.Equal(t2, client.Calls[1].SinceUtc);
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 6. SiteCallAudit-009: single-timestamp saturation pin does NOT spin —
|
// 6. SiteCallAudit-009: single-timestamp saturation pin does NOT spin —
|
||||||
@@ -484,8 +530,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
ReconciliationBatchSize = 2,
|
ReconciliationBatchSize = 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
CreateActor(sites, client, repo, options);
|
var actor = CreateActor(sites, client, repo, options);
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() => Assert.True(client.CallCount >= 1, "the first reconciliation tick should have pulled"),
|
() => Assert.True(client.CallCount >= 1, "the first reconciliation tick should have pulled"),
|
||||||
duration: TimeSpan.FromSeconds(3),
|
duration: TimeSpan.FromSeconds(3),
|
||||||
@@ -514,6 +561,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
|
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
|
||||||
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
|
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 7. Task 16: a keyset-honouring site drains a single-timestamp burst
|
// 7. Task 16: a keyset-honouring site drains a single-timestamp burst
|
||||||
@@ -546,8 +598,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
ReconciliationBatchSize = 2,
|
ReconciliationBatchSize = 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
CreateActor(sites, client, repo, options);
|
var actor = CreateActor(sites, client, repo, options);
|
||||||
|
try
|
||||||
|
{
|
||||||
AwaitAssert(
|
AwaitAssert(
|
||||||
() =>
|
() =>
|
||||||
{
|
{
|
||||||
@@ -565,6 +618,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
Assert.True(client.CallCount is >= 2 and < 10,
|
Assert.True(client.CallCount is >= 2 and < 10,
|
||||||
$"expected the burst to drain in a couple of keyset pages, got {client.CallCount}");
|
$"expected the burst to drain in a couple of keyset pages, got {client.CallCount}");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 8. Task 16: a LEGACY site that ignores after_id can never be drained, so
|
// 8. Task 16: a LEGACY site that ignores after_id can never be drained, so
|
||||||
@@ -593,12 +651,18 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
ReconciliationBatchSize = 2,
|
ReconciliationBatchSize = 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
CreateActor(sites, client, repo, options);
|
var actor = CreateActor(sites, client, repo, options);
|
||||||
|
try
|
||||||
|
{
|
||||||
var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
|
var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
|
||||||
Assert.Equal(siteId, evt.SiteId);
|
Assert.Equal(siteId, evt.SiteId);
|
||||||
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
|
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
|
||||||
@@ -648,6 +712,8 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
|
|
||||||
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
// Wait until the drain is genuinely in flight and blocked inside PullAsync.
|
// Wait until the drain is genuinely in flight and blocked inside PullAsync.
|
||||||
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
@@ -661,9 +727,15 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
|
|
||||||
Assert.True(reply.Accepted);
|
Assert.True(reply.Accepted);
|
||||||
Assert.Equal(id, reply.TrackedOperationId);
|
Assert.Equal(id, reply.TrackedOperationId);
|
||||||
|
}
|
||||||
// Let the drain finish so the actor shuts down cleanly.
|
finally
|
||||||
|
{
|
||||||
|
// Unblock the drain FIRST so the actor is free to process the Stop rather
|
||||||
|
// than parked inside the blocked pull, then stop it deterministically
|
||||||
|
// rather than leaving its 100ms ticker running (row-95).
|
||||||
client.Release();
|
client.Release();
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -678,16 +750,25 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
var client = new CountingBlockingPullClient();
|
var client = new CountingBlockingPullClient();
|
||||||
var repo = new RecordingRepo();
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
// Several tick intervals elapse while the first pass is still blocked.
|
// Several tick intervals elapse while the first pass is still blocked.
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(600));
|
await Task.Delay(TimeSpan.FromMilliseconds(600));
|
||||||
|
|
||||||
Assert.Equal(1, client.CallCount);
|
Assert.Equal(1, client.CallCount);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// Unblock the drain FIRST so the actor is free to process the Stop, then
|
||||||
|
// stop it deterministically rather than leaving its 100ms ticker running
|
||||||
|
// (row-95).
|
||||||
client.Release();
|
client.Release();
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -715,6 +796,8 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
|
|
||||||
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
// Once the pull has been served the drain is upserting; flood the mailbox
|
// Once the pull has been served the drain is upserting; flood the mailbox
|
||||||
// with ingest commands so the two writers overlap in wall-clock time.
|
// with ingest commands so the two writers overlap in wall-clock time.
|
||||||
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
@@ -735,6 +818,11 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
|
|
||||||
Assert.Equal(1, repo.MaxObservedConcurrency);
|
Assert.Equal(1, repo.MaxObservedConcurrency);
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
StopTicker(actor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Serves one page of rows on the first pull and nothing afterwards, so the
|
/// Serves one page of rows on the first pull and nothing afterwards, so the
|
||||||
|
|||||||
Reference in New Issue
Block a user