diff --git a/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs b/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs
index 98cceee7..869d9886 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs
@@ -99,6 +99,21 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
/// Ordered buffer of live deltas that arrived while a fan-out was in flight. Actor-thread only.
private readonly List _buffer = new();
+ ///
+ /// A failover re-seed was requested while a fan-out was already in flight; it must run
+ /// right after the in-flight one completes rather than being silently dropped (N7.1) —
+ /// the in-flight snapshot's read-time predates the stream death. Actor-thread only.
+ ///
+ private bool _reseedQueued;
+
+ ///
+ /// Monotonic stream generation stamped on each opened gRPC stream and echoed back on its
+ /// error callback: a late error raced out of a previous (cancelled) stream carries a stale
+ /// generation and is ignored so it never burns retry budget or double-flips (N7.2).
+ /// Actor-thread only.
+ ///
+ private int _streamGeneration;
+
private const int BufferWarnThreshold = 10_000;
private bool _bufferWarned;
@@ -182,6 +197,15 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
// gRPC stream error — flip node + reconnect + re-seed.
Receive(msg =>
{
+ // Ignore a late error raced out of a previous (cancelled) stream — the
+ // RpcException(Cancelled) filter at SiteStreamGrpcClient.cs covers the normal
+ // path, but a genuine socket fault can beat the cancel (N7.2).
+ if (msg.Generation != _streamGeneration)
+ {
+ _log.Debug("Ignoring stale gRPC error from stream generation {0} (current {1})",
+ msg.Generation, _streamGeneration);
+ return;
+ }
_log.Warning("Site-alarm gRPC stream error for {0}: {1}", _siteIdentifier, msg.Exception.Message);
HandleGrpcError();
});
@@ -266,7 +290,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
if (_stopped) return;
if (_fanoutInFlight)
{
- // Already seeding/reconciling — don't stack a second fan-out.
+ // A failover re-seed requested mid-fan-out must run right after the in-flight
+ // one — its snapshot read-time predates the stream death, so skipping it would
+ // serve stale up to the next 60s reconcile (N7.1). An initial-seed collision
+ // never queues (there is only ever one).
+ if (!isInitial) _reseedQueued = true;
return;
}
@@ -324,6 +352,13 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
Timers.Cancel(PublishTimerKey);
_publishPending = false;
Publish();
+
+ // A failover re-seed requested while this fan-out was in flight runs now (N7.1).
+ if (_reseedQueued)
+ {
+ _reseedQueued = false;
+ StartFanout(isInitial: false);
+ }
}
private void OnSeedFailed(SeedFailed msg)
@@ -347,6 +382,13 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
_publishPending = false;
Publish();
}
+
+ // A failover re-seed requested while this fan-out was in flight runs now (N7.1).
+ if (_reseedQueued)
+ {
+ _reseedQueued = false;
+ StartFanout(isInitial: false);
+ }
}
///
@@ -459,6 +501,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), StabilityWindow);
+ var generation = ++_streamGeneration;
var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
var self = Self;
var ct = _grpcCts.Token;
@@ -468,7 +511,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
await client.SubscribeSiteAsync(
_correlationId,
alarm => self.Tell(alarm),
- ex => self.Tell(new GrpcAlarmStreamError(ex)),
+ ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
ct);
}, ct);
}
@@ -564,8 +607,9 @@ internal sealed record RunReconcile;
/// Internal: coalesced-publish tick — flush the dirty cache to viewers once (N6).
internal sealed record PublishCoalesced;
-/// Internal: site-alarm gRPC stream error occurred.
-internal sealed record GrpcAlarmStreamError(Exception Exception);
+/// Internal: site-alarm gRPC stream error occurred, stamped with the stream
+/// generation it came from so a late error from a cancelled stream can be ignored (N7.2).
+internal sealed record GrpcAlarmStreamError(Exception Exception, int Generation);
/// Internal: reconnect the site-alarm gRPC stream (flip node).
internal sealed record ReconnectAlarmStream;
diff --git a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs
index af3be444..e4e7b21d 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/SiteAlarmAggregatorActorTests.cs
@@ -397,6 +397,50 @@ public class SiteAlarmAggregatorActorTests : TestKit
AwaitCondition(() => TotalSubs() > before, TimeSpan.FromSeconds(3));
}
+ // ── R2 T11: queued re-seed after reconnect + stream-generation stamp (N7.1/N7.2) ──
+
+ [Fact]
+ public void ReseedRequestedDuringInFlightFanout_IsQueued_NotDropped()
+ {
+ var (actor, seed, sink, factory) = CreateActor(publishCoalesce: TimeSpan.Zero);
+ AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+ seed.CompleteNext(); // initial seed done (CallCount 1)
+ AwaitAssert(() => Assert.Equal(1, sink.Count));
+ AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+
+ actor.Tell(new RunReconcile()); // reconcile fan-out now in flight (CallCount 2)
+ AwaitAssert(() => Assert.Equal(2, seed.CallCount));
+
+ // Stream error while the reconcile is in flight → the failover re-seed must not be
+ // silently skipped. Pre-fix: StartFanout no-ops and CallCount stays 2 until the next
+ // 60s reconcile tick.
+ factory.ClientFor(GrpcNodeA).Subs.Last().OnError(new Exception("stream fault"));
+
+ seed.CompleteNext(); // finish the in-flight reconcile
+ AwaitAssert(() => Assert.Equal(3, seed.CallCount)); // queued re-seed ran immediately after
+ }
+
+ [Fact]
+ public void LateErrorFromAPreviousStreamGeneration_IsIgnored()
+ {
+ var (_, seed, _, factory) = CreateActor(publishCoalesce: TimeSpan.Zero);
+ AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+ seed.CompleteNext();
+ AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+
+ int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count;
+
+ var firstSub = factory.ClientFor(GrpcNodeA).Subs.Single();
+ firstSub.OnError(new Exception("real fault")); // gen 1 error → flip, reconnect
+ AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5)); // gen 2 stream open
+ AwaitAssert(() => Assert.Equal(2, TotalSubs()));
+
+ firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in
+ // Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored.
+ Thread.Sleep(400);
+ Assert.Equal(2, TotalSubs());
+ }
+
// ── R2 T10: live-delta publish coalescing (N6) ──
[Fact]