dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:10:19 -04:00
parent 1040dc0fcc
commit 2ce0ad7ed1
30 changed files with 1941 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs
+170 -12
View File
@@ -79,11 +79,14 @@ public class SiteAlarmAggregatorActorTests : TestKit
private readonly object _lock = new();
public List<IReadOnlyList<AlarmStateChanged>> Snapshots { get; } = new();
public void Publish(IReadOnlyList<AlarmStateChanged> snapshot)
public void Publish(IReadOnlyList<AlarmStateChanged> snapshot, bool streamLive)
{
lock (_lock) { Snapshots.Add(snapshot); }
lock (_lock) { Snapshots.Add(snapshot); StreamLive = streamLive; }
}
/// <summary>Liveness reported with the most recent publish.</summary>
public bool StreamLive { get; private set; }
public IReadOnlyList<AlarmStateChanged>? Latest
{
get { lock (_lock) { return Snapshots.Count == 0 ? null : Snapshots[^1]; } }
@@ -94,7 +97,7 @@ public class SiteAlarmAggregatorActorTests : TestKit
private sealed record SiteSub(
string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError,
Action OnCompleted, CancellationToken Ct);
Action OnCompleted, CancellationToken Ct, Action? OnConnected);
private sealed class MockSiteAlarmStreamClient : SiteStreamGrpcClient
{
@@ -107,11 +110,21 @@ public class SiteAlarmAggregatorActorTests : TestKit
public MockSiteAlarmStreamClient() : base() { }
/// <summary>When false the stream never reports connected — the site accepted the TCP
/// call but never answered, so the aggregator's connect-driven re-seed must not run.</summary>
public bool AutoConnect { get; set; } = true;
public override Task SubscribeSiteAsync(
string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError,
Action onCompleted, CancellationToken ct)
Action onCompleted, CancellationToken ct, Action? onConnected = null)
{
lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, onCompleted, ct)); }
lock (_lock)
{
_subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, onCompleted, ct, onConnected));
}
// A healthy site accepts the subscription immediately (it flushes response headers
// as soon as its relay is attached), which is the aggregator's connect signal.
if (AutoConnect) onConnected?.Invoke();
var tcs = new TaskCompletionSource();
ct.Register(() => tcs.TrySetResult());
return tcs.Task; // never completes until cancelled (simulates a live stream)
@@ -130,8 +143,11 @@ public class SiteAlarmAggregatorActorTests : TestKit
public MockSiteAlarmStreamClientFactory()
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance) { }
/// <summary>Applied to every client this factory hands out (set before the actor starts).</summary>
public bool AutoConnect { get; set; } = true;
public MockSiteAlarmStreamClient ClientFor(string endpoint) =>
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient());
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient { AutoConnect = AutoConnect });
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
=> ClientFor(grpcEndpoint);
@@ -141,18 +157,19 @@ public class SiteAlarmAggregatorActorTests : TestKit
}
private (IActorRef Actor, SeedStub Seed, PublishSink Sink, MockSiteAlarmStreamClientFactory Factory) CreateActor(
TimeSpan? reconcileInterval = null, TimeSpan? publishCoalesce = null)
TimeSpan? reconcileInterval = null, TimeSpan? publishCoalesce = null, bool autoConnect = true)
{
var seed = new SeedStub();
var sink = new PublishSink();
var factory = new MockSiteAlarmStreamClientFactory();
var factory = new MockSiteAlarmStreamClientFactory { AutoConnect = autoConnect };
var props = Props.Create(() => new SiteAlarmAggregatorActor(
SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
reconcileInterval ?? TimeSpan.FromMinutes(10),
publishCoalesce ?? TimeSpan.Zero,
TestReconnectDelay,
TestStabilityWindow));
TestStabilityWindow,
0.0)); // no reconcile jitter in tests — determinism
var actor = Sys.ActorOf(props);
return (actor, seed, sink, factory);
@@ -414,12 +431,16 @@ public class SiteAlarmAggregatorActorTests : TestKit
AwaitAssert(() => Assert.Equal(1, sink.Count));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
// The first tick after a fan-out is deliberately skipped (WP2.3: the initial seed
// already covered this window); the second one actually fans out.
actor.Tell(new RunReconcile()); // skipped — consumes the "already fanned out" flag
actor.Tell(new RunReconcile()); // reconcile fan-out now in flight (CallCount 2)
AwaitAssert(() => Assert.Equal(2, seed.CallCount));
// Stream error while the reconcile is in flight → the failover re-seed must not be
// silently skipped. Pre-fix: StartFanout no-ops and CallCount stays 2 until the next
// 60s reconcile tick.
// Stream error while the reconcile is in flight: the reconnect that follows
// reconnects (the mock accepts immediately) and its connect-driven re-seed must not
// be silently swallowed. Pre-fix: StartFanout no-ops and CallCount stays 2 until the
// next 60s reconcile tick.
factory.ClientFor(GrpcNodeA).Subs.Last().OnError(new Exception("stream fault"));
seed.CompleteNext(); // finish the in-flight reconcile
@@ -577,4 +598,141 @@ public class SiteAlarmAggregatorActorTests : TestKit
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Unsubscribed.Contains("corr-1"),
TimeSpan.FromSeconds(3));
}
// -- WP2.3: seed once per SUCCESSFUL (re)connect, and stream-truthful liveness --
[Fact]
public void ReconnectAttempts_DoNotEachReSeed_OnlyTheSuccessfulConnectDoes()
{
// Finding #10's per-attempt re-fan-out: every reconnect ATTEMPT used to kick a
// whole-site snapshot fan-out - against a site that is, by definition of the
// reconnect, unreachable. Now the re-seed is owed to the connect that succeeds.
var (_, seed, _, factory) = CreateActor(
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3)); // initial seed
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
seed.CompleteNext();
// Three failed attempts, each flipping the node - none of them may re-seed.
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("1"));
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
factory.ClientFor(GrpcNodeB).Subs[0].OnError(new Exception("2"));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
Thread.Sleep(300);
Assert.Equal(1, seed.CallCount); // pre-fix: 4
// The stream finally comes up - exactly ONE re-seed, on the connect.
factory.ClientFor(GrpcNodeB).Subs[1].OnConnected!();
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
seed.CompleteNext();
Thread.Sleep(300);
Assert.Equal(2, seed.CallCount); // and no second one
}
[Fact]
public void ReconcileTick_IsSkipped_WhenAFanoutAlreadyRanInTheWindow()
{
// The reconcile is a BACKSTOP, not an unconditional 60s whole-site snapshot: a
// window already covered by a seed costs nothing. Staleness stays bounded because
// the skip consumes the flag, so the next tick always fans out.
var (actor, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
seed.CompleteNext();
Thread.Sleep(200);
actor.Tell(new RunReconcile()); // skipped - the initial seed covered this window
Thread.Sleep(300);
Assert.Equal(1, seed.CallCount);
actor.Tell(new RunReconcile()); // and the next one runs
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
}
[Fact]
public void UnchangedReconcileSnapshot_DoesNotRepublish()
{
// A reconcile that finds nothing changed used to wake every viewer's render path
// once a minute regardless. It now publishes as a diff.
var (actor, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
var t = DateTimeOffset.UtcNow;
seed.CompleteNext(Alarm("PumpFault", "", 500, t));
AwaitAssert(() => Assert.Equal(1, sink.Count));
actor.Tell(new RunReconcile()); // skipped (seed covered the window)
actor.Tell(new RunReconcile()); // real fan-out
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // identical snapshot
Thread.Sleep(300);
Assert.Equal(1, sink.Count); // no second publish
// A genuinely changed snapshot still publishes.
actor.Tell(new RunReconcile());
actor.Tell(new RunReconcile());
AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(3));
seed.CompleteNext(Alarm("PumpFault", "", 900, t.AddSeconds(1)));
AwaitAssert(() => Assert.Equal(2, sink.Count));
}
[Fact]
public void StreamCompletion_DropsLiveness_AndTheReopenRestoresIt()
{
// WP2.3 carried residual: a completed (or given-up) stream kept reporting live
// until the next reconcile publish, so the page grafted a freezing snapshot over
// fresh poll data.
// Reconcile is held far away so the drop can be observed before the reopen
// restores liveness; the reopen is then driven explicitly.
var (actor, seed, sink, factory) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
seed.CompleteNext();
AwaitCondition(() => sink.Count >= 1 && sink.StreamLive, TimeSpan.FromSeconds(3));
// The site ends the stream cleanly (its 4h max lifetime).
factory.ClientFor(GrpcNodeA).Subs[0].OnCompleted();
// Liveness drops IMMEDIATELY - not at the next reconcile.
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
// The reconcile tick reopens; the connect restores liveness.
actor.Tell(new RunReconcile());
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(5));
}
[Fact]
public void StreamFault_DropsLiveness_Immediately()
{
var (_, seed, sink, factory) = CreateActor(
reconcileInterval: TimeSpan.FromMinutes(10), autoConnect: false);
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
factory.ClientFor(GrpcNodeA).Subs[0].OnConnected!();
seed.CompleteNext();
AwaitCondition(() => sink.StreamLive, TimeSpan.FromSeconds(3));
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("site gone"));
AwaitCondition(() => !sink.StreamLive, TimeSpan.FromSeconds(3));
}
[Fact]
public void FailedSeed_IsRetried_WithBackoff_BeforeTheNextReconcile()
{
// The seed leg had no backoff: a failing fan-out simply waited a full reconcile
// interval. Now it retries on its own timer (reconnectDelay, doubling).
var (_, seed, _, _) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
seed.FaultNext();
// TestReconnectDelay is 50 ms, so the first retry lands far inside the 10-minute
// reconcile interval - pre-fix nothing would run until that interval elapsed.
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
seed.FaultNext();
AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(5));
}
}