fix(comms): review findings — consumer-based debug orphan net, foreign-cancel triad, honest onConnected, served-row-exact retirement, full-rate reconcile
F1 (HIGH) DebugStreamBridgeActor: the 5-minute orphan net measured the MAILBOX (SetReceiveTimeout), and once stream events were correctly marked INotInfluenceReceiveTimeout nothing recurring reset it — the snapshot lands once and GrpcStreamStable once — so every healthy session self-terminated at ~6 min with a false "Site disconnected". Replaced with a periodic self-tick (ConsumerLivenessCheckInterval, 30s) over a consumer-last-seen stamp renewed only by DebugStreamConsumerAlive, which DebugStreamService Tells on a shared timer to every session still in its registry (holding a session there IS "a consumer is attached" — both the Blazor view and the SignalR hub release it on dispose/disconnect, and it works headless). Reverting the wrapper was rejected: it would restore the quiet-instance orphan bug. F2 (MED) SiteStreamGrpcClient: the RpcException(Cancelled) filter now requires cts.IsCancellationRequested. A peer-originated / channel-dispose Cancelled fired none of onError/onCompleted/onConnected, leaving SiteAlarmAggregatorActor with _streamDown=false forever (IsLive stuck true, reconcile reopen guard never fired). F3 (MED) SiteStreamGrpcClient: a header TIMEOUT is no longer reported as connected — that shape is exactly what an unreachable site produces, and it cleared _streamDown, consumed _seedOnConnect and launched a full snapshot fan-out at a dead site. AwaitHeadersAsync returns bool; the first received event is the fallback connected signal, fired at most once from headers OR first event. F4 (LOW-MED) SqliteAuditWriter.MarkReconciledUpToAsync: the blanket below-cursor UPDATE retired late-stamped inserts that were never served (then age-purged — silent loss). The flip is now bounded by insertion order: a Pending row retires only if its rowid is at or below the high-water mark of rows this instance has served from ReadPendingSinceAsync (clamped on purge, since SQLite reuses rowids); Forwarded rows are exempt (central ACKed them over the push path). At-least-once is unchanged. F5 (LOW) Documented the liveness dependency (a served row never covered by a later cursor stays Pending forever; PurgeExpiredAsync never purges Pending) in ISiteAuditQueue + Component-AuditLog.md, and added a cheap site-health signal: SiteAuditBacklogReporter logs a rate-limited warning when the existing oldest-pending metric exceeds 24h. F6 (MED) SiteAlarmAggregatorActor: _fanoutSinceLastTick was armed by the reconcile's OWN fan-out, so steady state ran fan-out→skip→fan-out→skip — one reconcile per 2x interval (120s), halving the not-reporting refresh and the alarm reconcile backstop. The skip is now armed only by connect/failover-driven seeds (initial, _seedOnConnect, and a re-seed queued behind one). Tests: Communication.Tests 691 passed (+13), AuditLog.Tests 382 passed (+5).
This commit is contained in:
Joseph Doherty
@@ -16,6 +16,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
/// <summary>
|
||||
/// Tests for DebugStreamService session lifecycle.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamBridgeActorTests</c>
|
||||
/// so the two classes never race on the actor's static test seams (see that class).
|
||||
/// </remarks>
|
||||
[Collection("DebugStreamStatics")]
|
||||
public class DebugStreamServiceTests : TestKit
|
||||
{
|
||||
[Fact]
|
||||
@@ -74,4 +79,89 @@ public class DebugStreamServiceTests : TestKit
|
||||
Assert.Contains("Site1.Pump01", ex.Message);
|
||||
Assert.NotNull(ex.InnerException);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AttachedSession_IsKeptAliveByTheServiceKeepalive_AndDiesOnceDetached()
|
||||
{
|
||||
// The consumer half of the orphan net: holding a session in DebugStreamService IS
|
||||
// "a consumer is attached", and the service's shared timer is what renews the bridge
|
||||
// actor's window. Without this wiring every healthy session self-terminated one
|
||||
// window after its snapshot (the old mailbox-based ReceiveTimeout had nothing
|
||||
// recurring to reset it) and the consumer was told "Site disconnected".
|
||||
var previousKeepalive = DebugStreamService.KeepaliveInterval;
|
||||
var previousIdle = DebugStreamBridgeActor.ConsumerIdleTimeout;
|
||||
var previousCheck = DebugStreamBridgeActor.ConsumerLivenessCheckInterval;
|
||||
DebugStreamService.KeepaliveInterval = TimeSpan.FromMilliseconds(50);
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var instance = new Instance("Site1.Pump01") { Id = 7, SiteId = 3 };
|
||||
var site = new Site("Site One", "site-1")
|
||||
{
|
||||
Id = 3,
|
||||
GrpcNodeAAddress = "http://localhost:5100",
|
||||
GrpcNodeBAddress = "http://localhost:5200"
|
||||
};
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstanceByIdAsync(7, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetSiteByIdAsync(3, Arg.Any<CancellationToken>()).Returns(site);
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => instanceRepo);
|
||||
services.AddScoped(_ => siteRepo);
|
||||
using var provider = services.BuildServiceProvider();
|
||||
|
||||
var commProbe = CreateTestProbe();
|
||||
var commService = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
commService.SetCommunicationActor(commProbe.Ref);
|
||||
|
||||
// Mock gRPC factory: the real one would dial localhost:5100, fail, and trip the
|
||||
// bridge actor's retry budget — a termination unrelated to the orphan net under
|
||||
// test. The mock keeps the stream "up" so the only thing that can end this
|
||||
// session is the consumer-liveness decision.
|
||||
using var grpcFactory = new Grpc.MockSiteStreamGrpcClientFactory(
|
||||
new Grpc.MockSiteStreamGrpcClient());
|
||||
using var service = new DebugStreamService(
|
||||
commService, provider, grpcFactory, NullLogger<DebugStreamService>.Instance);
|
||||
service.SetActorSystem(Sys);
|
||||
|
||||
var startTask = service.StartStreamAsync(
|
||||
instanceId: 7, onEvent: _ => { }, onTerminated: () => { });
|
||||
|
||||
commProbe.ExpectMsg<SiteEnvelope>(TimeSpan.FromSeconds(5));
|
||||
var bridgeActor = commProbe.LastSender;
|
||||
Watch(bridgeActor);
|
||||
|
||||
// Resolve the snapshot so the session is fully established.
|
||||
bridgeActor.Tell(new ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView.DebugViewSnapshot(
|
||||
"Site1.Pump01",
|
||||
new List<ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming.AttributeValueChanged>(),
|
||||
new List<ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming.AlarmStateChanged>(),
|
||||
DateTimeOffset.UtcNow));
|
||||
var session = await startTask;
|
||||
|
||||
// Several orphan windows with NO traffic of any kind — only the service's
|
||||
// keepalive timer. The session must survive.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1500));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
||||
Assert.False(bridgeActor.IsNobody());
|
||||
|
||||
// Detach the consumer: the session leaves the registry, keepalives stop, and the
|
||||
// actor is stopped (StopStream) — the orphan net is the backstop for the case
|
||||
// where that explicit stop never happens.
|
||||
service.StopStream(session.SessionId);
|
||||
ExpectTerminated(bridgeActor, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
finally
|
||||
{
|
||||
DebugStreamService.KeepaliveInterval = previousKeepalive;
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = previousIdle;
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = previousCheck;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+139
-9
@@ -11,6 +11,13 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
/// <summary>
|
||||
/// Tests for DebugStreamBridgeActor with gRPC streaming integration.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamServiceTests</c>:
|
||||
/// both tune the actor's static test seams (<c>SnapshotTimeout</c>, <c>ConsumerIdleTimeout</c>,
|
||||
/// …), and xUnit parallelizes distinct classes by default, which would let one class's
|
||||
/// try/finally restore clobber the other's window mid-test.
|
||||
/// </remarks>
|
||||
[Collection("DebugStreamStatics")]
|
||||
public class DebugStreamBridgeActorTests : TestKit
|
||||
{
|
||||
private const string SiteId = "site-alpha";
|
||||
@@ -1034,14 +1041,11 @@ public class DebugStreamBridgeActorTests : TestKit
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamEvents_AreWrapped_SoTheyDoNotResetTheOrphanReceiveTimeout()
|
||||
public void StreamEvents_StillReachTheConsumer_ThroughTheWrappedCallbackPath()
|
||||
{
|
||||
// Structural pin for the fix: the gRPC callback wraps every event in an envelope
|
||||
// marked INotInfluenceReceiveTimeout, so a busy site can no longer keep an
|
||||
// abandoned session alive indefinitely by feeding it events.
|
||||
Assert.True(typeof(Akka.Actor.INotInfluenceReceiveTimeout)
|
||||
.IsAssignableFrom(typeof(LiveDebugStreamEvent)));
|
||||
|
||||
// The gRPC callback wraps every event in LiveDebugStreamEvent; the wrapper must be
|
||||
// transparent to delivery (it exists only to keep the high-volume path explicit —
|
||||
// the orphan net keys off the consumer keepalive, not the mailbox).
|
||||
var ctx = CreateBridgeActor();
|
||||
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
||||
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
||||
@@ -1052,8 +1056,6 @@ public class DebugStreamBridgeActorTests : TestKit
|
||||
new List<AlarmStateChanged>(),
|
||||
DateTimeOffset.UtcNow));
|
||||
|
||||
// An event delivered through the real gRPC callback path still reaches the
|
||||
// consumer — the wrapper is transparent to delivery.
|
||||
var evt = new AttributeValueChanged(
|
||||
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
|
||||
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
|
||||
@@ -1063,6 +1065,134 @@ public class DebugStreamBridgeActorTests : TestKit
|
||||
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
|
||||
}, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
// ----- Orphan net: measures the CONSUMER, not the mailbox ----- //
|
||||
|
||||
[Fact]
|
||||
public void HealthySession_StreamingEvents_WithConsumerKeepalives_SurvivesWellPastTheOrphanWindow()
|
||||
{
|
||||
// THE regression this closes: with the orphan net armed off the mailbox
|
||||
// (SetReceiveTimeout) and stream events correctly excluded from it, nothing recurring
|
||||
// reset it — the snapshot lands once, GrpcStreamStable once — so a perfectly healthy
|
||||
// session self-terminated one window later and the consumer was told
|
||||
// "Site disconnected". Here the session streams events and receives the keepalive
|
||||
// DebugStreamService sends while it is attached; it must live through MANY windows.
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var ctx = CreateBridgeActor();
|
||||
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
||||
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
||||
Watch(ctx.BridgeActor);
|
||||
|
||||
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
||||
InstanceName,
|
||||
new List<AttributeValueChanged>(),
|
||||
new List<AlarmStateChanged>(),
|
||||
DateTimeOffset.UtcNow));
|
||||
|
||||
// ~5 orphan windows of pure stream traffic + consumer keepalives, and no other
|
||||
// mailbox activity whatsoever (no reconnects, no snapshots, no stop).
|
||||
var deadline = DateTime.UtcNow.AddSeconds(2);
|
||||
var delivered = 0;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
||||
InstanceName, "Modules.IO", "Temperature", 20.0 + delivered, "Good",
|
||||
DateTimeOffset.UtcNow));
|
||||
delivered++;
|
||||
// What DebugStreamService's shared timer does for an attached session.
|
||||
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive());
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
||||
Assert.False(ctx.TerminatedFlag[0]);
|
||||
|
||||
// Still serving: the events all arrived and the actor is alive.
|
||||
AwaitCondition(() =>
|
||||
{
|
||||
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Count() == delivered; }
|
||||
}, TimeSpan.FromSeconds(3));
|
||||
Assert.False(ctx.BridgeActor.IsNobody());
|
||||
}
|
||||
finally
|
||||
{
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OrphanedSession_ConsumerGone_StillTerminates_EvenWhileEventsKeepArriving()
|
||||
{
|
||||
// The other half of the contract: site chatter must NOT hold an abandoned session
|
||||
// open. No keepalive arrives (the consumer is gone), so the session terminates,
|
||||
// unsubscribes from the site and reports termination — while events stream in.
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var ctx = CreateBridgeActor();
|
||||
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
||||
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
||||
Watch(ctx.BridgeActor);
|
||||
|
||||
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
||||
InstanceName,
|
||||
new List<AttributeValueChanged>(),
|
||||
new List<AlarmStateChanged>(),
|
||||
DateTimeOffset.UtcNow));
|
||||
|
||||
// Keep the site chatty for longer than the orphan window — with no keepalive.
|
||||
var deadline = DateTime.UtcNow.AddSeconds(1);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
||||
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow));
|
||||
Thread.Sleep(50);
|
||||
}
|
||||
|
||||
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
||||
Assert.True(ctx.TerminatedFlag[0]);
|
||||
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
||||
}
|
||||
finally
|
||||
{
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConsumerKeepalive_RenewsTheWindow_AfterANearMiss()
|
||||
{
|
||||
// A single late keepalive is enough to save a session — the stamp is refreshed, not
|
||||
// a one-shot grace period.
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(500);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var ctx = CreateBridgeActor();
|
||||
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
||||
Watch(ctx.BridgeActor);
|
||||
|
||||
Thread.Sleep(350); // most of the window burnt
|
||||
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive()); // …then the consumer checks in
|
||||
Thread.Sleep(350); // past the ORIGINAL deadline
|
||||
|
||||
Assert.False(ctx.TerminatedFlag[0]);
|
||||
|
||||
// …and once the keepalives stop, it does terminate.
|
||||
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
finally
|
||||
{
|
||||
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
||||
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -720,6 +720,99 @@ public class SiteAlarmAggregatorActorTests : TestKit
|
||||
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConsecutiveReconcileTicks_WithNoReconnect_EachRunAFanout()
|
||||
{
|
||||
// The bug: a tick's OWN fan-out completion armed the skip flag, so steady state ran
|
||||
// fan-out → skip → fan-out → skip — one reconcile per TWO intervals, halving both the
|
||||
// not-reporting refresh and the alarm reconcile backstop. Nothing was being
|
||||
// duplicated: only a connect/failover-driven seed can make a tick redundant.
|
||||
var (actor, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // initial seed — this one DOES stand a tick down
|
||||
Thread.Sleep(150);
|
||||
|
||||
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
|
||||
Thread.Sleep(200);
|
||||
Assert.Equal(1, seed.CallCount);
|
||||
|
||||
// From here on, with no reconnect in between, EVERY tick must fan out.
|
||||
for (var expected = 2; expected <= 5; expected++)
|
||||
{
|
||||
actor.Tell(new RunReconcile());
|
||||
AwaitCondition(() => seed.CallCount == expected, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
|
||||
Assert.Equal(5, seed.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConnectDrivenReseed_StillStandsTheNextTickDown()
|
||||
{
|
||||
// The other half of the fix: the skip exists for the connect/failover seed, and that
|
||||
// suppression must survive. A reconnect's re-seed still makes the very next tick a
|
||||
// no-op, so a flapping stream cannot double the whole-site snapshot rate.
|
||||
var (actor, seed, _, factory) = CreateActor(
|
||||
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
actor.Tell(new RunReconcile()); // consumed by the initial seed
|
||||
actor.Tell(new RunReconcile()); // real tick fan-out
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
// Stream faults → reconnect on node B → connect-driven re-seed (call 3).
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("site gone"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
|
||||
AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
Thread.Sleep(150);
|
||||
|
||||
// That re-seed covers this window: the next tick is skipped…
|
||||
actor.Tell(new RunReconcile());
|
||||
Thread.Sleep(250);
|
||||
Assert.Equal(3, seed.CallCount);
|
||||
|
||||
// …and the one after it fans out again.
|
||||
actor.Tell(new RunReconcile());
|
||||
AwaitCondition(() => seed.CallCount == 4, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForeignCancelledStreamError_DropsLiveness_AndReopensOnTheOtherNode()
|
||||
{
|
||||
// End-to-end for the SiteStreamGrpcClient fix: a peer-originated
|
||||
// RpcException(Cancelled) now reaches onError instead of being swallowed. Before the
|
||||
// fix NONE of onError/onCompleted/onConnected fired, so the aggregator kept
|
||||
// _streamDown=false — IsLive stuck true and the reconcile tick's reopen guard, which
|
||||
// only fires when the stream is known down, never ran.
|
||||
var (_, seed, sink, factory) = CreateActor(
|
||||
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
|
||||
seed.CompleteNext();
|
||||
AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(
|
||||
new global::Grpc.Core.RpcException(new global::Grpc.Core.Status(
|
||||
global::Grpc.Core.StatusCode.Cancelled, "cancelled by peer")));
|
||||
|
||||
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
|
||||
// …and the stream is reopened on the other node, where a connect earns one re-seed.
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailedSeed_IsRetried_WithBackoff_BeforeTheNextReconcile()
|
||||
{
|
||||
|
||||
@@ -351,9 +351,170 @@ public class SiteStreamGrpcClientTests
|
||||
Assert.Equal(0, completed);
|
||||
}
|
||||
|
||||
// --- Foreign vs. own Cancelled (review F2) ---
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_ForeignCancelled_InvokesOnError()
|
||||
{
|
||||
// A Cancelled status we did NOT ask for — the PEER cancelled, or the channel was
|
||||
// disposed underneath us. It used to be swallowed by an unguarded
|
||||
// `when (ex.StatusCode == StatusCode.Cancelled)` filter, firing none of
|
||||
// onError/onCompleted/onConnected: the consuming aggregator kept _streamDown=false,
|
||||
// so IsLive stayed true forever and the reconcile tick's reopen guard never fired.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource(); // deliberately NOT cancelled
|
||||
Exception? error = null;
|
||||
var completed = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-foreign-cancel",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(
|
||||
new RpcException(new Status(StatusCode.Cancelled, "cancelled by peer")))),
|
||||
_ => { },
|
||||
ex => error = ex,
|
||||
() => completed++);
|
||||
|
||||
var rpc = Assert.IsType<RpcException>(error);
|
||||
Assert.Equal(StatusCode.Cancelled, rpc.StatusCode);
|
||||
Assert.Equal(0, completed);
|
||||
Assert.False(cts.IsCancellationRequested);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_OwnCancelledRpcException_InvokesNeitherCallback()
|
||||
{
|
||||
// The other side of the same filter: OUR cancellation surfacing as
|
||||
// RpcException(Cancelled) is a teardown, so neither callback fires.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
await cts.CancelAsync();
|
||||
Exception? error = null;
|
||||
var completed = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-own-cancel",
|
||||
cts,
|
||||
() => throw new RpcException(new Status(StatusCode.Cancelled, "we cancelled")),
|
||||
_ => { },
|
||||
ex => error = ex,
|
||||
() => completed++);
|
||||
|
||||
Assert.Null(error);
|
||||
Assert.Equal(0, completed);
|
||||
}
|
||||
|
||||
// --- onConnected is only ever raised on real proof of a live peer (review F3) ---
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeadersArrive_InvokesOnConnectedOnce()
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-headers",
|
||||
cts,
|
||||
() => FakeCall(
|
||||
new StubStreamReader(
|
||||
new SiteStreamEvent { CorrelationId = "corr-headers" },
|
||||
new SiteStreamEvent { CorrelationId = "corr-headers" }),
|
||||
Task.FromResult(new Metadata())),
|
||||
_ => { },
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
// Once — the two events must not re-raise it.
|
||||
Assert.Equal(1, connected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeaderTimeoutOnADeadSite_NeverReportsConnected()
|
||||
{
|
||||
// The bug: a header TIMEOUT used to be reported as connected. An unreachable/wedged
|
||||
// site produces exactly that shape, so the aggregator cleared _streamDown, consumed
|
||||
// its pending re-seed and fanned a full snapshot out at a site that never answered.
|
||||
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
var neverArrives = new TaskCompletionSource<Metadata>();
|
||||
|
||||
// No headers AND no events: the site is dead. The stream ends (status OK) with
|
||||
// no connected signal ever raised.
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-dead",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(), neverArrives.Task),
|
||||
_ => { },
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
Assert.Equal(0, connected);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_HeaderTimeoutThenEvent_ReportsConnectedOnceFromTheEvent()
|
||||
{
|
||||
// The case the timeout was originally added for — a peer that defers its headers
|
||||
// until the first message — is now covered by the event itself, which is real proof
|
||||
// of a live peer. Connected must precede the event's delivery and fire only once.
|
||||
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
|
||||
try
|
||||
{
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var connected = 0;
|
||||
var connectedBeforeFirstEvent = false;
|
||||
var events = 0;
|
||||
var neverArrives = new TaskCompletionSource<Metadata>();
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-late-headers",
|
||||
cts,
|
||||
() => FakeCall(
|
||||
new StubStreamReader(
|
||||
new SiteStreamEvent { CorrelationId = "corr-late-headers" },
|
||||
new SiteStreamEvent { CorrelationId = "corr-late-headers" }),
|
||||
neverArrives.Task),
|
||||
_ =>
|
||||
{
|
||||
if (events == 0) connectedBeforeFirstEvent = connected == 1;
|
||||
events++;
|
||||
},
|
||||
_ => { },
|
||||
() => { },
|
||||
() => connected++);
|
||||
|
||||
Assert.Equal(2, events);
|
||||
Assert.Equal(1, connected);
|
||||
Assert.True(connectedBeforeFirstEvent);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
|
||||
}
|
||||
}
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
|
||||
FakeCall(reader, Task.FromResult(new Metadata()));
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(
|
||||
StubStreamReader reader, Task<Metadata> responseHeaders) =>
|
||||
new(reader,
|
||||
Task.FromResult(new Metadata()),
|
||||
responseHeaders,
|
||||
() => Status.DefaultSuccess,
|
||||
() => new Metadata(),
|
||||
() => { });
|
||||
|
||||
@@ -243,6 +243,45 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
Assert.True(response.MoreAvailable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_RetiresBeforeItServes_SoThisBatchIsNeverSelfRetired()
|
||||
{
|
||||
// Ordering is load-bearing for the "only served rows retire" invariant. The queue
|
||||
// bounds the cursor flip by insertion order — the high-water mark of rows it has
|
||||
// SERVED — so the retire step must run BEFORE the read: retiring first can only ever
|
||||
// reach rows served by an EARLIER pull, never the ones this call is about to serve
|
||||
// (which would defeat at-least-once), and never a row inserted after them (the
|
||||
// late-stamped insert that used to be silently retired and then age-purged).
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)new[] { NewEvent() });
|
||||
|
||||
var server = CreateServer();
|
||||
server.SetSiteAuditQueue(queue);
|
||||
|
||||
var cursorTime = DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc);
|
||||
await server.PullAuditEvents(
|
||||
new PullAuditEventsRequest
|
||||
{
|
||||
SinceUtc = Timestamp.FromDateTime(cursorTime),
|
||||
BatchSize = 100,
|
||||
},
|
||||
NewContext());
|
||||
|
||||
Received.InOrder(() =>
|
||||
{
|
||||
queue.MarkReconciledUpToAsync(cursorTime, null, Arg.Any<CancellationToken>());
|
||||
queue.ReadPendingSinceAsync(cursorTime, 100, null, Arg.Any<CancellationToken>());
|
||||
});
|
||||
|
||||
// Serving still flips nothing by itself — the next cursor is the only receipt.
|
||||
await queue.DidNotReceive().MarkReconciledAsync(
|
||||
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
|
||||
await queue.DidNotReceive().MarkForwardedAsync(
|
||||
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_MarkReconciledUpToThrows_ResponseStillReturned()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user