dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'worktree-agent-a465fb3cd6ec48cd3' into arch-review-remediation

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 23:53:00 -04:00
parent 11abda1d1f fd5e023d08
commit a9ca51e008
16 changed files with 1192 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SiteAuditBacklogReporterCadenceTests.cs
+64
View File
@@ -1,4 +1,5 @@
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
@@ -115,4 +116,67 @@ public class SiteAuditBacklogReporterCadenceTests
Assert.Equal(TimeSpan.FromSeconds(3), reporter.RefreshInterval);
}
// ----- Stale-Pending signal (review F5) ----- //
[Fact]
public void StalePendingBacklog_IsWarned_ThenRateLimited()
{
// Pending rows are exempt from the retention purge by design, so a standing Pending
// backlog is the one site-store condition that never self-heals on age — it clears
// only when central acknowledges the rows. It is worth a log line, not just a number
// on the health report, and the warning must not spam every 30 s poll.
var logger = new CapturingLogger();
var reporter = new SiteAuditBacklogReporter(
Substitute.For<ISiteAuditQueue>(),
Substitute.For<ISiteHealthCollector>(),
logger,
TimeSpan.FromHours(1),
null);
var stale = DateTime.UtcNow - SiteAuditBacklogReporter.StalePendingThreshold - TimeSpan.FromHours(1);
reporter.WarnIfPendingIsStale(stale, pendingCount: 4321);
reporter.WarnIfPendingIsStale(stale, pendingCount: 4321); // same poll cycle-ish
var warning = Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning);
Assert.Contains("4321", warning.Message);
Assert.Contains("pending", warning.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void FreshOrEmptyPendingBacklog_IsNotWarned()
{
var logger = new CapturingLogger();
var reporter = new SiteAuditBacklogReporter(
Substitute.For<ISiteAuditQueue>(),
Substitute.For<ISiteHealthCollector>(),
logger,
TimeSpan.FromHours(1),
null);
reporter.WarnIfPendingIsStale(null, pendingCount: 0); // nothing pending
reporter.WarnIfPendingIsStale(DateTime.UtcNow.AddMinutes(-5), 12); // a normal drain lag
Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Warning);
}
/// <summary>Captures log entries so the stale-pending signal can be asserted.</summary>
private sealed class CapturingLogger : ILogger<SiteAuditBacklogReporter>
{
public List<(LogLevel Level, Exception? Exception, string Message)> Entries { get; } = new();
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
Entries.Add((logLevel, exception, formatter(state, exception)));
}
}
}
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SqliteAuditWriterWriteTests.cs
+97
View File
@@ -614,6 +614,10 @@ public class SqliteAuditWriterWriteTests
await writer.WriteAsync(older);
await writer.WriteAsync(boundary);
// Serve them first — retirement is bounded by what has actually been served, which
// is exactly the order the pull handler runs in (read, then next pull's cursor).
await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
var flipped = await writer.MarkReconciledUpToAsync(instant, afterId: null);
Assert.Equal(1, flipped);
@@ -648,6 +652,99 @@ public class SqliteAuditWriterWriteTests
second.Select(r => r.EventId).ToHashSet());
}
[Fact]
public async Task MarkReconciledUpToAsync_LateStampedInsertBelowTheCursor_IsNeverRetired()
{
// THE data-loss case. OccurredAtUtc is caller-stamped, so a row can be INSERTED
// after a batch was served yet carry a timestamp BELOW central's (by then advanced)
// cursor — a script that back-dates, a clock nudge, a queued write flushed late.
// The blanket cursor UPDATE retired exactly those rows: never served, never servable
// again (the keyset read has moved past them) and, being Reconciled, purged on age.
// The insertion-order (rowid) bound makes them unreachable by the flip.
var (writer, dataSource) = CreateWriter(
nameof(MarkReconciledUpToAsync_LateStampedInsertBelowTheCursor_IsNeverRetired));
await using var _w = writer;
var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var served1 = NewEvent(occurredAtUtc: t0);
var served2 = NewEvent(occurredAtUtc: t0.AddSeconds(20));
await writer.WriteAsync(served1);
await writer.WriteAsync(served2);
// Central pulls both and advances its cursor to the newest row.
var page = await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
Assert.Equal(2, page.Count);
var cursorTime = t0.AddSeconds(20);
var cursorId = served2.EventId.ToString();
// A row lands NOW carrying a timestamp between the two served rows.
var lateStamped = NewEvent(occurredAtUtc: t0.AddSeconds(10));
await writer.WriteAsync(lateStamped);
var flipped = await writer.MarkReconciledUpToAsync(cursorTime, cursorId);
// The two genuinely served rows retire; the late-stamped one does not.
Assert.Equal(2, flipped);
Assert.Equal(AuditForwardState.Reconciled.ToString(), ReadForwardState(dataSource, served1.EventId));
Assert.Equal(AuditForwardState.Reconciled.ToString(), ReadForwardState(dataSource, served2.EventId));
Assert.Equal(AuditForwardState.Pending.ToString(), ReadForwardState(dataSource, lateStamped.EventId));
// Still recoverable: a central that restarts (cursor resets to MinValue) re-serves it,
// and — being Pending — the retention purge can never drop it in the meantime.
var reread = await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
Assert.Equal(lateStamped.EventId, Assert.Single(reread).EventId);
}
[Fact]
public async Task MarkReconciledUpToAsync_ForwardedRowsRetire_EvenWithoutHavingBeenPulled()
{
// The bound applies to PENDING rows (nothing proves central saw them but the pull).
// A FORWARDED row was ACKED by central through the telemetry push path, so the cursor
// may retire it whether or not this node has ever served it in a pull — otherwise a
// site node that never serves a pull would accumulate acked rows forever.
var (writer, dataSource) = CreateWriter(
nameof(MarkReconciledUpToAsync_ForwardedRowsRetire_EvenWithoutHavingBeenPulled));
await using var _w = writer;
var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var pushed = NewEvent(occurredAtUtc: t0);
var neverShipped = NewEvent(occurredAtUtc: t0.AddSeconds(1));
await writer.WriteAsync(pushed);
await writer.WriteAsync(neverShipped);
// Central acked the first row over the telemetry drain — no pull involved.
await writer.MarkForwardedAsync(new[] { pushed.EventId });
var flipped = await writer.MarkReconciledUpToAsync(t0.AddSeconds(30), afterId: null);
Assert.Equal(1, flipped);
Assert.Equal(AuditForwardState.Reconciled.ToString(), ReadForwardState(dataSource, pushed.EventId));
Assert.Equal(AuditForwardState.Pending.ToString(), ReadForwardState(dataSource, neverShipped.EventId));
}
[Fact]
public async Task MarkReconciledUpToAsync_BeforeAnyPull_RetiresNothingPending()
{
// Bound state is per-process: after a site-node restart nothing has been served yet,
// so an incoming cursor retires no Pending row. Conservative in the safe direction —
// the rows stay servable and the next pull re-establishes the bound.
var (writer, dataSource) = CreateWriter(nameof(MarkReconciledUpToAsync_BeforeAnyPull_RetiresNothingPending));
await using var _w = writer;
var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var evt = NewEvent(occurredAtUtc: t0);
await writer.WriteAsync(evt);
var flipped = await writer.MarkReconciledUpToAsync(t0.AddHours(1), afterId: null);
Assert.Equal(0, flipped);
Assert.Equal(AuditForwardState.Pending.ToString(), ReadForwardState(dataSource, evt.EventId));
// …and the very next pull cycle retires it normally.
await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
Assert.Equal(1, await writer.MarkReconciledUpToAsync(t0.AddHours(1), afterId: null));
}
[Fact]
public async Task ReadPendingSinceAsync_InvalidBatchSize_Throws()
{
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/DebugStreamServiceTests.cs
+90
View File
@@ -16,6 +16,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// <summary>
/// Tests for DebugStreamService session lifecycle.
/// </summary>
/// <remarks>
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamBridgeActorTests</c>
/// so the two classes never race on the actor's static test seams (see that class).
/// </remarks>
[Collection("DebugStreamStatics")]
public class DebugStreamServiceTests : TestKit
{
[Fact]
@@ -74,4 +79,89 @@ public class DebugStreamServiceTests : TestKit
Assert.Contains("Site1.Pump01", ex.Message);
Assert.NotNull(ex.InnerException);
}
[Fact]
public async Task AttachedSession_IsKeptAliveByTheServiceKeepalive_AndDiesOnceDetached()
{
// The consumer half of the orphan net: holding a session in DebugStreamService IS
// "a consumer is attached", and the service's shared timer is what renews the bridge
// actor's window. Without this wiring every healthy session self-terminated one
// window after its snapshot (the old mailbox-based ReceiveTimeout had nothing
// recurring to reset it) and the consumer was told "Site disconnected".
var previousKeepalive = DebugStreamService.KeepaliveInterval;
var previousIdle = DebugStreamBridgeActor.ConsumerIdleTimeout;
var previousCheck = DebugStreamBridgeActor.ConsumerLivenessCheckInterval;
DebugStreamService.KeepaliveInterval = TimeSpan.FromMilliseconds(50);
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
try
{
var instance = new Instance("Site1.Pump01") { Id = 7, SiteId = 3 };
var site = new Site("Site One", "site-1")
{
Id = 3,
GrpcNodeAAddress = "http://localhost:5100",
GrpcNodeBAddress = "http://localhost:5200"
};
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
instanceRepo.GetInstanceByIdAsync(7, Arg.Any<CancellationToken>()).Returns(instance);
var siteRepo = Substitute.For<ISiteRepository>();
siteRepo.GetSiteByIdAsync(3, Arg.Any<CancellationToken>()).Returns(site);
var services = new ServiceCollection();
services.AddScoped(_ => instanceRepo);
services.AddScoped(_ => siteRepo);
using var provider = services.BuildServiceProvider();
var commProbe = CreateTestProbe();
var commService = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
commService.SetCommunicationActor(commProbe.Ref);
// Mock gRPC factory: the real one would dial localhost:5100, fail, and trip the
// bridge actor's retry budget — a termination unrelated to the orphan net under
// test. The mock keeps the stream "up" so the only thing that can end this
// session is the consumer-liveness decision.
using var grpcFactory = new Grpc.MockSiteStreamGrpcClientFactory(
new Grpc.MockSiteStreamGrpcClient());
using var service = new DebugStreamService(
commService, provider, grpcFactory, NullLogger<DebugStreamService>.Instance);
service.SetActorSystem(Sys);
var startTask = service.StartStreamAsync(
instanceId: 7, onEvent: _ => { }, onTerminated: () => { });
commProbe.ExpectMsg<SiteEnvelope>(TimeSpan.FromSeconds(5));
var bridgeActor = commProbe.LastSender;
Watch(bridgeActor);
// Resolve the snapshot so the session is fully established.
bridgeActor.Tell(new ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView.DebugViewSnapshot(
"Site1.Pump01",
new List<ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming.AttributeValueChanged>(),
new List<ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming.AlarmStateChanged>(),
DateTimeOffset.UtcNow));
var session = await startTask;
// Several orphan windows with NO traffic of any kind — only the service's
// keepalive timer. The session must survive.
await Task.Delay(TimeSpan.FromMilliseconds(1500));
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
Assert.False(bridgeActor.IsNobody());
// Detach the consumer: the session leaves the registry, keepalives stop, and the
// actor is stopped (StopStream) — the orphan net is the backstop for the case
// where that explicit stop never happens.
service.StopStream(session.SessionId);
ExpectTerminated(bridgeActor, TimeSpan.FromSeconds(3));
}
finally
{
DebugStreamService.KeepaliveInterval = previousKeepalive;
DebugStreamBridgeActor.ConsumerIdleTimeout = previousIdle;
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = previousCheck;
}
}
}
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/DebugStreamBridgeActorTests.cs
+139 -9
View File
@@ -11,6 +11,13 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
/// <summary>
/// Tests for DebugStreamBridgeActor with gRPC streaming integration.
/// </summary>
/// <remarks>
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamServiceTests</c>:
/// both tune the actor's static test seams (<c>SnapshotTimeout</c>, <c>ConsumerIdleTimeout</c>,
/// …), and xUnit parallelizes distinct classes by default, which would let one class's
/// try/finally restore clobber the other's window mid-test.
/// </remarks>
[Collection("DebugStreamStatics")]
public class DebugStreamBridgeActorTests : TestKit
{
private const string SiteId = "site-alpha";
@@ -1034,14 +1041,11 @@ public class DebugStreamBridgeActorTests : TestKit
}
[Fact]
public void StreamEvents_AreWrapped_SoTheyDoNotResetTheOrphanReceiveTimeout()
public void StreamEvents_StillReachTheConsumer_ThroughTheWrappedCallbackPath()
{
// Structural pin for the fix: the gRPC callback wraps every event in an envelope
// marked INotInfluenceReceiveTimeout, so a busy site can no longer keep an
// abandoned session alive indefinitely by feeding it events.
Assert.True(typeof(Akka.Actor.INotInfluenceReceiveTimeout)
.IsAssignableFrom(typeof(LiveDebugStreamEvent)));
// The gRPC callback wraps every event in LiveDebugStreamEvent; the wrapper must be
// transparent to delivery (it exists only to keep the high-volume path explicit —
// the orphan net keys off the consumer keepalive, not the mailbox).
var ctx = CreateBridgeActor();
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
@@ -1052,8 +1056,6 @@ public class DebugStreamBridgeActorTests : TestKit
new List<AlarmStateChanged>(),
DateTimeOffset.UtcNow));
// An event delivered through the real gRPC callback path still reaches the
// consumer — the wrapper is transparent to delivery.
var evt = new AttributeValueChanged(
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
@@ -1063,6 +1065,134 @@ public class DebugStreamBridgeActorTests : TestKit
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
}, TimeSpan.FromSeconds(3));
}
// ----- Orphan net: measures the CONSUMER, not the mailbox ----- //
[Fact]
public void HealthySession_StreamingEvents_WithConsumerKeepalives_SurvivesWellPastTheOrphanWindow()
{
// THE regression this closes: with the orphan net armed off the mailbox
// (SetReceiveTimeout) and stream events correctly excluded from it, nothing recurring
// reset it — the snapshot lands once, GrpcStreamStable once — so a perfectly healthy
// session self-terminated one window later and the consumer was told
// "Site disconnected". Here the session streams events and receives the keepalive
// DebugStreamService sends while it is attached; it must live through MANY windows.
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
try
{
var ctx = CreateBridgeActor();
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
Watch(ctx.BridgeActor);
ctx.BridgeActor.Tell(new DebugViewSnapshot(
InstanceName,
new List<AttributeValueChanged>(),
new List<AlarmStateChanged>(),
DateTimeOffset.UtcNow));
// ~5 orphan windows of pure stream traffic + consumer keepalives, and no other
// mailbox activity whatsoever (no reconnects, no snapshots, no stop).
var deadline = DateTime.UtcNow.AddSeconds(2);
var delivered = 0;
while (DateTime.UtcNow < deadline)
{
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
InstanceName, "Modules.IO", "Temperature", 20.0 + delivered, "Good",
DateTimeOffset.UtcNow));
delivered++;
// What DebugStreamService's shared timer does for an attached session.
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive());
Thread.Sleep(100);
}
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
Assert.False(ctx.TerminatedFlag[0]);
// Still serving: the events all arrived and the actor is alive.
AwaitCondition(() =>
{
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Count() == delivered; }
}, TimeSpan.FromSeconds(3));
Assert.False(ctx.BridgeActor.IsNobody());
}
finally
{
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
}
}
[Fact]
public void OrphanedSession_ConsumerGone_StillTerminates_EvenWhileEventsKeepArriving()
{
// The other half of the contract: site chatter must NOT hold an abandoned session
// open. No keepalive arrives (the consumer is gone), so the session terminates,
// unsubscribes from the site and reports termination — while events stream in.
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
try
{
var ctx = CreateBridgeActor();
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
Watch(ctx.BridgeActor);
ctx.BridgeActor.Tell(new DebugViewSnapshot(
InstanceName,
new List<AttributeValueChanged>(),
new List<AlarmStateChanged>(),
DateTimeOffset.UtcNow));
// Keep the site chatty for longer than the orphan window — with no keepalive.
var deadline = DateTime.UtcNow.AddSeconds(1);
while (DateTime.UtcNow < deadline)
{
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow));
Thread.Sleep(50);
}
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
Assert.True(ctx.TerminatedFlag[0]);
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
}
finally
{
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
}
}
[Fact]
public void ConsumerKeepalive_RenewsTheWindow_AfterANearMiss()
{
// A single late keepalive is enough to save a session — the stamp is refreshed, not
// a one-shot grace period.
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(500);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
try
{
var ctx = CreateBridgeActor();
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
Watch(ctx.BridgeActor);
Thread.Sleep(350); // most of the window burnt
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive()); // …then the consumer checks in
Thread.Sleep(350); // past the ORIGINAL deadline
Assert.False(ctx.TerminatedFlag[0]);
// …and once the keepalives stop, it does terminate.
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
}
finally
{
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
}
}
}
/// <summary>
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs
+93
View File
@@ -720,6 +720,99 @@ public class SiteAlarmAggregatorActorTests : TestKit
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
}
[Fact]
public void ConsecutiveReconcileTicks_WithNoReconnect_EachRunAFanout()
{
// The bug: a tick's OWN fan-out completion armed the skip flag, so steady state ran
// fan-out → skip → fan-out → skip — one reconcile per TWO intervals, halving both the
// not-reporting refresh and the alarm reconcile backstop. Nothing was being
// duplicated: only a connect/failover-driven seed can make a tick redundant.
var (actor, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
seed.CompleteNext(); // initial seed — this one DOES stand a tick down
Thread.Sleep(150);
actor.Tell(new RunReconcile()); // consumed by the initial seed's flag
Thread.Sleep(200);
Assert.Equal(1, seed.CallCount);
// From here on, with no reconnect in between, EVERY tick must fan out.
for (var expected = 2; expected <= 5; expected++)
{
actor.Tell(new RunReconcile());
AwaitCondition(() => seed.CallCount == expected, TimeSpan.FromSeconds(3));
seed.CompleteNext();
Thread.Sleep(100);
}
Assert.Equal(5, seed.CallCount);
}
[Fact]
public void ConnectDrivenReseed_StillStandsTheNextTickDown()
{
// The other half of the fix: the skip exists for the connect/failover seed, and that
// suppression must survive. A reconnect's re-seed still makes the very next tick a
// no-op, so a flapping stream cannot double the whole-site snapshot rate.
var (actor, seed, _, factory) = CreateActor(
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
seed.CompleteNext();
Thread.Sleep(150);
actor.Tell(new RunReconcile()); // consumed by the initial seed
actor.Tell(new RunReconcile()); // real tick fan-out
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
seed.CompleteNext();
Thread.Sleep(150);
// Stream faults → reconnect on node B → connect-driven re-seed (call 3).
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("site gone"));
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(3));
seed.CompleteNext();
Thread.Sleep(150);
// That re-seed covers this window: the next tick is skipped…
actor.Tell(new RunReconcile());
Thread.Sleep(250);
Assert.Equal(3, seed.CallCount);
// …and the one after it fans out again.
actor.Tell(new RunReconcile());
AwaitCondition(() => seed.CallCount == 4, TimeSpan.FromSeconds(3));
}
[Fact]
public void ForeignCancelledStreamError_DropsLiveness_AndReopensOnTheOtherNode()
{
// End-to-end for the SiteStreamGrpcClient fix: a peer-originated
// RpcException(Cancelled) now reaches onError instead of being swallowed. Before the
// fix NONE of onError/onCompleted/onConnected fired, so the aggregator kept
// _streamDown=false — IsLive stuck true and the reconcile tick's reopen guard, which
// only fires when the stream is known down, never ran.
var (_, seed, sink, factory) = CreateActor(
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
seed.CompleteNext();
AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(3));
factory.ClientFor(GrpcNodeA).Subs[0].OnError(
new global::Grpc.Core.RpcException(new global::Grpc.Core.Status(
global::Grpc.Core.StatusCode.Cancelled, "cancelled by peer")));
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
// …and the stream is reopened on the other node, where a connect earns one re-seed.
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
factory.ClientFor(GrpcNodeB).Subs[0].OnConnected!();
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
}
[Fact]
public void FailedSeed_IsRetried_WithBackoff_BeforeTheNextReconcile()
{
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteStreamGrpcClientTests.cs
+162 -1
View File
@@ -351,9 +351,170 @@ public class SiteStreamGrpcClientTests
Assert.Equal(0, completed);
}
// --- Foreign vs. own Cancelled (review F2) ---
[Fact]
public async Task ConsumeStream_ForeignCancelled_InvokesOnError()
{
// A Cancelled status we did NOT ask for — the PEER cancelled, or the channel was
// disposed underneath us. It used to be swallowed by an unguarded
// `when (ex.StatusCode == StatusCode.Cancelled)` filter, firing none of
// onError/onCompleted/onConnected: the consuming aggregator kept _streamDown=false,
// so IsLive stayed true forever and the reconcile tick's reopen guard never fired.
var client = SiteStreamGrpcClient.CreateForTesting();
var cts = new CancellationTokenSource(); // deliberately NOT cancelled
Exception? error = null;
var completed = 0;
await client.ConsumeStreamAsync(
"corr-foreign-cancel",
cts,
() => FakeCall(new StubStreamReader(
new RpcException(new Status(StatusCode.Cancelled, "cancelled by peer")))),
_ => { },
ex => error = ex,
() => completed++);
var rpc = Assert.IsType<RpcException>(error);
Assert.Equal(StatusCode.Cancelled, rpc.StatusCode);
Assert.Equal(0, completed);
Assert.False(cts.IsCancellationRequested);
}
[Fact]
public async Task ConsumeStream_OwnCancelledRpcException_InvokesNeitherCallback()
{
// The other side of the same filter: OUR cancellation surfacing as
// RpcException(Cancelled) is a teardown, so neither callback fires.
var client = SiteStreamGrpcClient.CreateForTesting();
var cts = new CancellationTokenSource();
await cts.CancelAsync();
Exception? error = null;
var completed = 0;
await client.ConsumeStreamAsync(
"corr-own-cancel",
cts,
() => throw new RpcException(new Status(StatusCode.Cancelled, "we cancelled")),
_ => { },
ex => error = ex,
() => completed++);
Assert.Null(error);
Assert.Equal(0, completed);
}
// --- onConnected is only ever raised on real proof of a live peer (review F3) ---
[Fact]
public async Task ConsumeStream_HeadersArrive_InvokesOnConnectedOnce()
{
var client = SiteStreamGrpcClient.CreateForTesting();
var cts = new CancellationTokenSource();
var connected = 0;
await client.ConsumeStreamAsync(
"corr-headers",
cts,
() => FakeCall(
new StubStreamReader(
new SiteStreamEvent { CorrelationId = "corr-headers" },
new SiteStreamEvent { CorrelationId = "corr-headers" }),
Task.FromResult(new Metadata())),
_ => { },
_ => { },
() => { },
() => connected++);
// Once — the two events must not re-raise it.
Assert.Equal(1, connected);
}
[Fact]
public async Task ConsumeStream_HeaderTimeoutOnADeadSite_NeverReportsConnected()
{
// The bug: a header TIMEOUT used to be reported as connected. An unreachable/wedged
// site produces exactly that shape, so the aggregator cleared _streamDown, consumed
// its pending re-seed and fanned a full snapshot out at a site that never answered.
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
try
{
var client = SiteStreamGrpcClient.CreateForTesting();
var cts = new CancellationTokenSource();
var connected = 0;
var neverArrives = new TaskCompletionSource<Metadata>();
// No headers AND no events: the site is dead. The stream ends (status OK) with
// no connected signal ever raised.
await client.ConsumeStreamAsync(
"corr-dead",
cts,
() => FakeCall(new StubStreamReader(), neverArrives.Task),
_ => { },
_ => { },
() => { },
() => connected++);
Assert.Equal(0, connected);
}
finally
{
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
}
}
[Fact]
public async Task ConsumeStream_HeaderTimeoutThenEvent_ReportsConnectedOnceFromTheEvent()
{
// The case the timeout was originally added for — a peer that defers its headers
// until the first message — is now covered by the event itself, which is real proof
// of a live peer. Connected must precede the event's delivery and fire only once.
var previous = SiteStreamGrpcClient.ConnectedHeaderTimeout;
SiteStreamGrpcClient.ConnectedHeaderTimeout = TimeSpan.FromMilliseconds(50);
try
{
var client = SiteStreamGrpcClient.CreateForTesting();
var cts = new CancellationTokenSource();
var connected = 0;
var connectedBeforeFirstEvent = false;
var events = 0;
var neverArrives = new TaskCompletionSource<Metadata>();
await client.ConsumeStreamAsync(
"corr-late-headers",
cts,
() => FakeCall(
new StubStreamReader(
new SiteStreamEvent { CorrelationId = "corr-late-headers" },
new SiteStreamEvent { CorrelationId = "corr-late-headers" }),
neverArrives.Task),
_ =>
{
if (events == 0) connectedBeforeFirstEvent = connected == 1;
events++;
},
_ => { },
() => { },
() => connected++);
Assert.Equal(2, events);
Assert.Equal(1, connected);
Assert.True(connectedBeforeFirstEvent);
}
finally
{
SiteStreamGrpcClient.ConnectedHeaderTimeout = previous;
}
}
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
FakeCall(reader, Task.FromResult(new Metadata()));
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(
StubStreamReader reader, Task<Metadata> responseHeaders) =>
new(reader,
Task.FromResult(new Metadata()),
responseHeaders,
() => Status.DefaultSuccess,
() => new Metadata(),
() => { });
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteStreamPullAuditEventsTests.cs
+39
View File
@@ -243,6 +243,45 @@ public class SiteStreamPullAuditEventsTests : TestKit
Assert.True(response.MoreAvailable);
}
[Fact]
public async Task PullAuditEvents_RetiresBeforeItServes_SoThisBatchIsNeverSelfRetired()
{
// Ordering is load-bearing for the "only served rows retire" invariant. The queue
// bounds the cursor flip by insertion order — the high-water mark of rows it has
// SERVED — so the retire step must run BEFORE the read: retiring first can only ever
// reach rows served by an EARLIER pull, never the ones this call is about to serve
// (which would defeat at-least-once), and never a row inserted after them (the
// late-stamped insert that used to be silently retired and then age-purged).
var queue = Substitute.For<ISiteAuditQueue>();
queue.ReadPendingSinceAsync(
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
.Returns((IReadOnlyList<AuditEvent>)new[] { NewEvent() });
var server = CreateServer();
server.SetSiteAuditQueue(queue);
var cursorTime = DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc);
await server.PullAuditEvents(
new PullAuditEventsRequest
{
SinceUtc = Timestamp.FromDateTime(cursorTime),
BatchSize = 100,
},
NewContext());
Received.InOrder(() =>
{
queue.MarkReconciledUpToAsync(cursorTime, null, Arg.Any<CancellationToken>());
queue.ReadPendingSinceAsync(cursorTime, 100, null, Arg.Any<CancellationToken>());
});
// Serving still flips nothing by itself — the next cursor is the only receipt.
await queue.DidNotReceive().MarkReconciledAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
await queue.DidNotReceive().MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task PullAuditEvents_MarkReconciledUpToThrows_ResponseStillReturned()
{