fix(comms): review findings — consumer-based debug orphan net, foreign-cancel triad, honest onConnected, served-row-exact retirement, full-rate reconcile
F1 (HIGH) DebugStreamBridgeActor: the 5-minute orphan net measured the MAILBOX (SetReceiveTimeout), and once stream events were correctly marked INotInfluenceReceiveTimeout nothing recurring reset it — the snapshot lands once and GrpcStreamStable once — so every healthy session self-terminated at ~6 min with a false "Site disconnected". Replaced with a periodic self-tick (ConsumerLivenessCheckInterval, 30s) over a consumer-last-seen stamp renewed only by DebugStreamConsumerAlive, which DebugStreamService Tells on a shared timer to every session still in its registry (holding a session there IS "a consumer is attached" — both the Blazor view and the SignalR hub release it on dispose/disconnect, and it works headless). Reverting the wrapper was rejected: it would restore the quiet-instance orphan bug. F2 (MED) SiteStreamGrpcClient: the RpcException(Cancelled) filter now requires cts.IsCancellationRequested. A peer-originated / channel-dispose Cancelled fired none of onError/onCompleted/onConnected, leaving SiteAlarmAggregatorActor with _streamDown=false forever (IsLive stuck true, reconcile reopen guard never fired). F3 (MED) SiteStreamGrpcClient: a header TIMEOUT is no longer reported as connected — that shape is exactly what an unreachable site produces, and it cleared _streamDown, consumed _seedOnConnect and launched a full snapshot fan-out at a dead site. AwaitHeadersAsync returns bool; the first received event is the fallback connected signal, fired at most once from headers OR first event. F4 (LOW-MED) SqliteAuditWriter.MarkReconciledUpToAsync: the blanket below-cursor UPDATE retired late-stamped inserts that were never served (then age-purged — silent loss). The flip is now bounded by insertion order: a Pending row retires only if its rowid is at or below the high-water mark of rows this instance has served from ReadPendingSinceAsync (clamped on purge, since SQLite reuses rowids); Forwarded rows are exempt (central ACKed them over the push path). At-least-once is unchanged. F5 (LOW) Documented the liveness dependency (a served row never covered by a later cursor stays Pending forever; PurgeExpiredAsync never purges Pending) in ISiteAuditQueue + Component-AuditLog.md, and added a cheap site-health signal: SiteAuditBacklogReporter logs a rate-limited warning when the existing oldest-pending metric exceeds 24h. F6 (MED) SiteAlarmAggregatorActor: _fanoutSinceLastTick was armed by the reconcile's OWN fan-out, so steady state ran fan-out→skip→fan-out→skip — one reconcile per 2x interval (120s), halving the not-reporting refresh and the alarm reconcile backstop. The skip is now armed only by connect/failover-driven seeds (initial, _seedOnConnect, and a re-seed queued behind one). Tests: Communication.Tests 691 passed (+13), AuditLog.Tests 382 passed (+5).
This commit is contained in:
Joseph Doherty
@@ -720,6 +720,99 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
||||
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConsecutiveReconcileTicks_WithNoReconnect_EachRunAFanout()
|
||||
{
|
||||
// The bug: a tick's OWN fan-out completion armed the skip flag, so steady state ran
|
||||
// fan-out → skip → fan-out → skip — one reconcile per TWO intervals, halving both the
|
||||
// not-reporting refresh and the alarm reconcile backstop. Nothing was being
|
||||
// duplicated: only a connect/failover-driven seed can make a tick redundant.
|
||||
var (actor, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // initial seed — this one DOES stand a tick down
|
||||
Thread.Sleep(150);
|
||||
|
||||
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
|
||||
Thread.Sleep(200);
|
||||
Assert.Equal(1, seed.CallCount);
|
||||
|
||||
// From here on, with no reconnect in between, EVERY tick must fan out.
|
||||
for (var expected = 2; expected <= 5; expected++)
|
||||
{
|
||||
actor.Tell(new RunReconcile());
|
||||
AwaitCondition(() => seed.CallCount == expected, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
|
||||
Assert.Equal(5, seed.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConnectDrivenReseed_StillStandsTheNextTickDown()
|
||||
{
|
||||
// The other half of the fix: the skip exists for the connect/failover seed, and that
|
||||
// suppression must survive. A reconnect's re-seed still makes the very next tick a
|
||||
// no-op, so a flapping stream cannot double the whole-site snapshot rate.
|
||||
var (actor, seed, _, factory) = CreateActor(
|
||||
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
actor.Tell(new RunReconcile()); // consumed by the initial seed
|
||||
actor.Tell(new RunReconcile()); // real tick fan-out
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
// Stream faults → reconnect on node B → connect-driven re-seed (call 3).
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("site gone"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
|
||||
AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
// That re-seed covers this window: the next tick is skipped…
|
||||
actor.Tell(new RunReconcile());
|
||||
Thread.Sleep(250);
|
||||
Assert.Equal(3, seed.CallCount);
|
||||
|
||||
// …and the one after it fans out again.
|
||||
actor.Tell(new RunReconcile());
|
||||
AwaitCondition(() => seed.CallCount == 4, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForeignCancelledStreamError_DropsLiveness_AndReopensOnTheOtherNode()
|
||||
{
|
||||
// End-to-end for the SiteStreamGrpcClient fix: a peer-originated
|
||||
// RpcException(Cancelled) now reaches onError instead of being swallowed. Before the
|
||||
// fix NONE of onError/onCompleted/onConnected fired, so the aggregator kept
|
||||
// _streamDown=false — IsLive stuck true and the reconcile tick's reopen guard, which
|
||||
// only fires when the stream is known down, never ran.
|
||||
var (_, seed, sink, factory) = CreateActor(
|
||||
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
|
||||
seed.CompleteNext();
|
||||
AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(
|
||||
new global::Grpc.Core.RpcException(new global::Grpc.Core.Status(
|
||||
global::Grpc.Core.StatusCode.Cancelled, "cancelled by peer")));
|
||||
|
||||
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
// …and the stream is reopened on the other node, where a connect earns one re-seed.
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailedSeed_IsRetried_WithBackoff_BeforeTheNextReconcile()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user