fix(comms): review findings — consumer-based debug orphan net, foreign-cancel triad, honest onConnected, served-row-exact retirement, full-rate reconcile
F1 (HIGH) DebugStreamBridgeActor: the 5-minute orphan net measured the MAILBOX (SetReceiveTimeout), and once stream events were correctly marked INotInfluenceReceiveTimeout nothing recurring reset it — the snapshot lands once and GrpcStreamStable once — so every healthy session self-terminated at ~6 min with a false "Site disconnected". Replaced with a periodic self-tick (ConsumerLivenessCheckInterval, 30s) over a consumer-last-seen stamp renewed only by DebugStreamConsumerAlive, which DebugStreamService Tells on a shared timer to every session still in its registry (holding a session there IS "a consumer is attached" — both the Blazor view and the SignalR hub release it on dispose/disconnect, and it works headless). Reverting the wrapper was rejected: it would restore the quiet-instance orphan bug. F2 (MED) SiteStreamGrpcClient: the RpcException(Cancelled) filter now requires cts.IsCancellationRequested. A peer-originated / channel-dispose Cancelled fired none of onError/onCompleted/onConnected, leaving SiteAlarmAggregatorActor with _streamDown=false forever (IsLive stuck true, reconcile reopen guard never fired). F3 (MED) SiteStreamGrpcClient: a header TIMEOUT is no longer reported as connected — that shape is exactly what an unreachable site produces, and it cleared _streamDown, consumed _seedOnConnect and launched a full snapshot fan-out at a dead site. AwaitHeadersAsync returns bool; the first received event is the fallback connected signal, fired at most once from headers OR first event. F4 (LOW-MED) SqliteAuditWriter.MarkReconciledUpToAsync: the blanket below-cursor UPDATE retired late-stamped inserts that were never served (then age-purged — silent loss). The flip is now bounded by insertion order: a Pending row retires only if its rowid is at or below the high-water mark of rows this instance has served from ReadPendingSinceAsync (clamped on purge, since SQLite reuses rowids); Forwarded rows are exempt (central ACKed them over the push path). At-least-once is unchanged. F5 (LOW) Documented the liveness dependency (a served row never covered by a later cursor stays Pending forever; PurgeExpiredAsync never purges Pending) in ISiteAuditQueue + Component-AuditLog.md, and added a cheap site-health signal: SiteAuditBacklogReporter logs a rate-limited warning when the existing oldest-pending metric exceeds 24h. F6 (MED) SiteAlarmAggregatorActor: _fanoutSinceLastTick was armed by the reconcile's OWN fan-out, so steady state ran fan-out→skip→fan-out→skip — one reconcile per 2x interval (120s), halving the not-reporting refresh and the alarm reconcile backstop. The skip is now armed only by connect/failover-driven seeds (initial, _seedOnConnect, and a re-seed queued behind one). Tests: Communication.Tests 691 passed (+13), AuditLog.Tests 382 passed (+5).
This commit is contained in:
Joseph Doherty
@@ -351,9 +351,170 @@ public class SiteStreamGrpcClientTests
|
||||
Assert.Equal(0, completed);
|
||||
}
|
||||
|
||||
// --- Foreign vs. own Cancelled (review F2) ---
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_ForeignCancelled_InvokesOnError()
|
||||
{
|
||||
// A Cancelled status we did NOT ask for — the PEER cancelled, or the channel was
|
||||
// disposed underneath us. It used to be swallowed by an unguarded
|
||||
// `when (ex.StatusCode == StatusCode.Cancelled)` filter, firing none of
|
||||
// onError/onCompleted/onConnected: the consuming aggregator kept _streamDown=false,
|
||||
// so IsLive stayed true forever and the reconcile tick's reopen guard never fired.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource(); // deliberately NOT cancelled
|
||||
Exception? error = null;
|
||||
var completed = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-foreign-cancel",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(
|
||||
new RpcException(new Status(StatusCode.Cancelled, "cancelled by peer")))),
|
||||
_ => { },
|
||||
ex => error = ex,
|
||||
() => completed++);
|
||||
|
||||
var rpc = Assert.IsType<RpcException>(error);
|
||||
Assert.Equal(StatusCode.Cancelled, rpc.StatusCode);
|
||||
Assert.Equal(0, completed);
|
||||
Assert.False(cts.IsCancellationRequested);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_OwnCancelledRpcException_InvokesNeitherCallback()
|
||||
{
|
||||
// The other side of the same filter: OUR cancellation surfacing as
|
||||
// RpcException(Cancelled) is a teardown, so neither callback fires.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
await cts.CancelAsync();
|
||||
Exception? error = null;
|
||||
var completed = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-own-cancel",
|
||||
cts,
|
||||
() => throw new RpcException(new Status(StatusCode.Cancelled, "we cancelled")),
|
||||
_ => { },
|
||||
ex => error = ex,
|
||||
() => completed++);
|
||||
|
||||
Assert.Null(error);
|
||||
Assert.Equal(0, completed);
|
||||
}
|
||||
|
||||
// --- onConnected is only ever raised on real proof of a live peer (review F3) ---
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeadersArrive_InvokesOnConnectedOnce()
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-headers",
|
||||
cts,
|
||||
() => FakeCall(
|
||||
new StubStreamReader(
|
||||
new SiteStreamEvent { CorrelationId = "corr-headers" },
|
||||
new SiteStreamEvent { CorrelationId = "corr-headers" }),
|
||||
Task.FromResult(new Metadata())),
|
||||
_ => { },
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
// Once — the two events must not re-raise it.
|
||||
Assert.Equal(1, connected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeaderTimeoutOnADeadSite_NeverReportsConnected()
|
||||
{
|
||||
// The bug: a header TIMEOUT used to be reported as connected. An unreachable/wedged
|
||||
// site produces exactly that shape, so the aggregator cleared _streamDown, consumed
|
||||
// its pending re-seed and fanned a full snapshot out at a site that never answered.
|
||||
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
var neverArrives = new TaskCompletionSource<Metadata>();
|
||||
|
||||
// No headers AND no events: the site is dead. The stream ends (status OK) with
|
||||
// no connected signal ever raised.
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-dead",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(), neverArrives.Task),
|
||||
_ => { },
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
Assert.Equal(0, connected);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeaderTimeoutThenEvent_ReportsConnectedOnceFromTheEvent()
|
||||
{
|
||||
// The case the timeout was originally added for — a peer that defers its headers
|
||||
// until the first message — is now covered by the event itself, which is real proof
|
||||
// of a live peer. Connected must precede the event's delivery and fire only once.
|
||||
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
var connectedBeforeFirstEvent = false;
|
||||
var events = 0;
|
||||
var neverArrives = new TaskCompletionSource<Metadata>();
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-late-headers",
|
||||
cts,
|
||||
() => FakeCall(
|
||||
new StubStreamReader(
|
||||
new SiteStreamEvent { CorrelationId = "corr-late-headers" },
|
||||
new SiteStreamEvent { CorrelationId = "corr-late-headers" }),
|
||||
neverArrives.Task),
|
||||
_ =>
|
||||
{
|
||||
if (events == 0) connectedBeforeFirstEvent = connected == 1;
|
||||
events++;
|
||||
},
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
Assert.Equal(2, events);
|
||||
Assert.Equal(1, connected);
|
||||
Assert.True(connectedBeforeFirstEvent);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
|
||||
}
|
||||
}
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
|
||||
FakeCall(reader, Task.FromResult(new Metadata()));
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(
|
||||
StubStreamReader reader, Task<Metadata> responseHeaders) =>
|
||||
new(reader,
|
||||
Task.FromResult(new Metadata()),
|
||||
responseHeaders,
|
||||
() => Status.DefaultSuccess,
|
||||
() => new Metadata(),
|
||||
() => { });
|
||||
|
||||
Reference in New Issue
Block a user