diff --git a/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs b/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs index 98cceee7..869d9886 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs @@ -99,6 +99,21 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers /// Ordered buffer of live deltas that arrived while a fan-out was in flight. Actor-thread only. private readonly List _buffer = new(); + /// + /// A failover re-seed was requested while a fan-out was already in flight; it must run + /// right after the in-flight one completes rather than being silently dropped (N7.1) — + /// the in-flight snapshot's read-time predates the stream death. Actor-thread only. + /// + private bool _reseedQueued; + + /// + /// Monotonic stream generation stamped on each opened gRPC stream and echoed back on its + /// error callback: a late error raced out of a previous (cancelled) stream carries a stale + /// generation and is ignored so it never burns retry budget or double-flips (N7.2). + /// Actor-thread only. + /// + private int _streamGeneration; + private const int BufferWarnThreshold = 10_000; private bool _bufferWarned; @@ -182,6 +197,15 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers // gRPC stream error — flip node + reconnect + re-seed. Receive(msg => { + // Ignore a late error raced out of a previous (cancelled) stream — the + // RpcException(Cancelled) filter at SiteStreamGrpcClient.cs covers the normal + // path, but a genuine socket fault can beat the cancel (N7.2). + if (msg.Generation != _streamGeneration) + { + _log.Debug("Ignoring stale gRPC error from stream generation {0} (current {1})", + msg.Generation, _streamGeneration); + return; + } _log.Warning("Site-alarm gRPC stream error for {0}: {1}", _siteIdentifier, msg.Exception.Message); HandleGrpcError(); }); @@ -266,7 +290,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers if (_stopped) return; if (_fanoutInFlight) { - // Already seeding/reconciling — don't stack a second fan-out. + // A failover re-seed requested mid-fan-out must run right after the in-flight + // one — its snapshot read-time predates the stream death, so skipping it would + // serve stale up to the next 60s reconcile (N7.1). An initial-seed collision + // never queues (there is only ever one). + if (!isInitial) _reseedQueued = true; return; } @@ -324,6 +352,13 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers Timers.Cancel(PublishTimerKey); _publishPending = false; Publish(); + + // A failover re-seed requested while this fan-out was in flight runs now (N7.1). + if (_reseedQueued) + { + _reseedQueued = false; + StartFanout(isInitial: false); + } } private void OnSeedFailed(SeedFailed msg) @@ -347,6 +382,13 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers _publishPending = false; Publish(); } + + // A failover re-seed requested while this fan-out was in flight runs now (N7.1). + if (_reseedQueued) + { + _reseedQueued = false; + StartFanout(isInitial: false); + } } /// @@ -459,6 +501,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), StabilityWindow); + var generation = ++_streamGeneration; var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint); var self = Self; var ct = _grpcCts.Token; @@ -468,7 +511,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers await client.SubscribeSiteAsync( _correlationId, alarm => self.Tell(alarm), - ex => self.Tell(new GrpcAlarmStreamError(ex)), + ex => self.Tell(new GrpcAlarmStreamError(ex, generation)), ct); }, ct); } @@ -564,8 +607,9 @@ internal sealed record RunReconcile; /// Internal: coalesced-publish tick — flush the dirty cache to viewers once (N6). internal sealed record PublishCoalesced; -/// Internal: site-alarm gRPC stream error occurred. -internal sealed record GrpcAlarmStreamError(Exception Exception); +/// Internal: site-alarm gRPC stream error occurred, stamped with the stream +/// generation it came from so a late error from a cancelled stream can be ignored (N7.2). +internal sealed record GrpcAlarmStreamError(Exception Exception, int Generation); /// Internal: reconnect the site-alarm gRPC stream (flip node). internal sealed record ReconnectAlarmStream; diff --git a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs index af3be444..e4e7b21d 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs @@ -397,6 +397,50 @@ public class SiteAlarmAggregatorActorTests : TestKit AwaitCondition(() => TotalSubs() > before, TimeSpan.FromSeconds(3)); } + // ── R2 T11: queued re-seed after reconnect + stream-generation stamp (N7.1/N7.2) ── + + [Fact] + public void ReseedRequestedDuringInFlightFanout_IsQueued_NotDropped() + { + var (actor, seed, sink, factory) = CreateActor(publishCoalesce: TimeSpan.Zero); + AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3)); + seed.CompleteNext(); // initial seed done (CallCount 1) + AwaitAssert(() => Assert.Equal(1, sink.Count)); + AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3)); + + actor.Tell(new RunReconcile()); // reconcile fan-out now in flight (CallCount 2) + AwaitAssert(() => Assert.Equal(2, seed.CallCount)); + + // Stream error while the reconcile is in flight → the failover re-seed must not be + // silently skipped. Pre-fix: StartFanout no-ops and CallCount stays 2 until the next + // 60s reconcile tick. + factory.ClientFor(GrpcNodeA).Subs.Last().OnError(new Exception("stream fault")); + + seed.CompleteNext(); // finish the in-flight reconcile + AwaitAssert(() => Assert.Equal(3, seed.CallCount)); // queued re-seed ran immediately after + } + + [Fact] + public void LateErrorFromAPreviousStreamGeneration_IsIgnored() + { + var (_, seed, _, factory) = CreateActor(publishCoalesce: TimeSpan.Zero); + AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3)); + seed.CompleteNext(); + AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3)); + + int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count; + + var firstSub = factory.ClientFor(GrpcNodeA).Subs.Single(); + firstSub.OnError(new Exception("real fault")); // gen 1 error → flip, reconnect + AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5)); // gen 2 stream open + AwaitAssert(() => Assert.Equal(2, TotalSubs())); + + firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in + // Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored. + Thread.Sleep(400); + Assert.Equal(2, TotalSubs()); + } + // ── R2 T10: live-delta publish coalescing (N6) ── [Fact]