dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 19:57:08 -04:00
parent ee193cd2bb
commit 34a3f4bb69
9 changed files with 507 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/DebugStreamBridgeActor.cs
+81 -5
View File
@@ -63,6 +63,14 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
private bool _stopped;
private CancellationTokenSource? _grpcCts;
/// <summary>
/// Monotonic stream generation stamped on each opened gRPC stream and echoed back on its
/// error/completion callbacks: a late callback raced out of a previous (cancelled) stream
/// carries a stale generation and is ignored, so it can neither burn retry budget nor
/// open a duplicate stream. Mirrors <c>SiteAlarmAggregatorActor</c>. Actor-thread only.
/// </summary>
private int _streamGeneration;
/// <summary>
/// Phase flag. <see langword="false"/> until the initial
/// <see cref="DebugViewSnapshot"/> has been delivered and the pre-snapshot buffer
@@ -198,10 +206,33 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
// gRPC stream error — attempt reconnection
Receive<GrpcStreamError>(msg =>
{
// Ignore a late error raced out of a previous (cancelled) stream: it must not
// burn retry budget or flip the node a second time.
if (msg.Generation != _streamGeneration)
{
_log.Debug("Ignoring stale gRPC error from stream generation {0} (current {1})",
msg.Generation, _streamGeneration);
return;
}
_log.Warning("gRPC stream error for {0}: {1}", _instanceUniqueName, msg.Exception.Message);
HandleGrpcError();
});
// gRPC stream ended GRACEFULLY (server status OK) — the site's 4h max stream
// lifetime elapsing or a graceful site shutdown. Not a fault: reopen on the SAME
// node without spending retry budget. Without this the session went silently deaf.
Receive<GrpcStreamCompleted>(msg =>
{
if (_stopped) return;
if (msg.Generation != _streamGeneration)
{
_log.Debug("Ignoring stale gRPC completion from stream generation {0} (current {1})",
msg.Generation, _streamGeneration);
return;
}
HandleGrpcCompleted();
});
// Scheduled reconnection
Receive<ReconnectGrpcStream>(_ => OpenGrpcStream());
@@ -424,20 +455,57 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
// retry budget is recovered. Cancelled by HandleGrpcError.
Timers.StartSingleTimer(StabilityTimerKey, new GrpcStreamStable(), StabilityWindow);
var generation = ++_streamGeneration;
var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
var self = Self;
var ct = _grpcCts.Token;
// Launch as background task — onEvent and onError marshal back to actor via Tell
// Launch as background task — the callbacks marshal back to the actor via Tell.
// The task itself is observed below: a fault escaping SubscribeAsync would otherwise
// leave the session waiting on a stream that does not exist, exception unobserved.
Task.Run(async () =>
{
await client.SubscribeAsync(
_correlationId,
_instanceUniqueName,
evt => self.Tell(evt),
ex => self.Tell(new GrpcStreamError(ex)),
ex => self.Tell(new GrpcStreamError(ex, generation)),
() => self.Tell(new GrpcStreamCompleted(generation)),
ct);
}, ct);
}, ct).ContinueWith(t =>
{
if (t.IsFaulted)
self.Tell(new GrpcStreamError(t.Exception!.GetBaseException(), generation));
else if (t.IsCanceled && !ct.IsCancellationRequested)
self.Tell(new GrpcStreamCompleted(generation));
// RanToCompletion: SubscribeAsync already reported its own outcome.
}, TaskContinuationOptions.ExecuteSynchronously);
}
/// <summary>
/// Handles a graceful end of stream (server status OK). The stream simply expired or the
/// site shut down cleanly, so the retry budget is left untouched and the endpoint is not
/// flipped; the reopen is scheduled through the existing reconnect timer, which also
/// rate-limits a pathological site that keeps closing streams immediately.
/// </summary>
private void HandleGrpcCompleted()
{
// The stream is gone, so its armed stability timer must not later "recover" a
// budget that its successor has since spent.
Timers.Cancel(StabilityTimerKey);
_log.Info("gRPC stream for {0} completed gracefully (server end of stream); reopening",
_instanceUniqueName);
// Release the site-side relay for the finished stream before reopening, so the site
// is not left with a zombie relay actor for this correlation id.
_grpcCts?.Cancel();
_grpcCts?.Dispose();
_grpcCts = null;
_grpcFactory.TryGet(_siteIdentifier, _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress)
?.Unsubscribe(_correlationId);
Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectGrpcStream(), ReconnectDelay);
}
private void HandleGrpcError()
@@ -512,9 +580,17 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
public record StopDebugStream;
/// <summary>
/// Internal message indicating a gRPC stream error occurred.
/// Internal message indicating a gRPC stream error occurred, stamped with the stream
/// generation it came from so a late error out of a cancelled stream can be ignored.
/// </summary>
internal record GrpcStreamError(Exception Exception);
internal record GrpcStreamError(Exception Exception, int Generation);
/// <summary>
/// Internal message indicating the gRPC stream ended gracefully (server status OK — the
/// site's max stream lifetime elapsed, or the site shut down cleanly), stamped with its
/// stream generation.
/// </summary>
internal record GrpcStreamCompleted(int Generation);
/// <summary>
/// Internal message to trigger gRPC stream reconnection.
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs
+75 -6
View File
@@ -78,12 +78,23 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
private bool _stopped;
/// <summary>
/// True once the live gRPC stream has been given up (retry budget exhausted). Reconcile
/// snapshots keep serving in the meantime; the next reconcile tick self-heals the stream
/// by resetting the retry budget and reopening it, so a sustained site outage does not
/// permanently drop the live feed. Actor-thread only.
/// True while there is no live gRPC stream — either it was given up (retry budget
/// exhausted) or it ended gracefully (server status OK at the site's max stream
/// lifetime). Reconcile snapshots keep serving in the meantime; the next reconcile tick
/// self-heals the stream by resetting the retry budget and reopening it, so neither a
/// sustained site outage nor a routine 4h stream expiry permanently drops the live feed.
/// Actor-thread only.
/// </summary>
private bool _streamDown;
/// <summary>
/// Why the stream is down: <c>true</c> = the retry budget was exhausted, so the
/// self-healing reopen must also reset it; <c>false</c> = it ended gracefully, and the
/// budget — which a completion neither spends nor refunds — is carried across the reopen
/// untouched (a stream flapping between faults and clean closes must still trip
/// MaxRetries). Actor-thread only.
/// </summary>
private bool _retryBudgetExhausted;
private CancellationTokenSource? _grpcCts;
private CancellationTokenSource? _lifetimeCts;
@@ -218,6 +229,22 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
HandleGrpcError();
});
// gRPC stream ended GRACEFULLY (server status OK) — the site's 4h max stream
// lifetime elapsing or a graceful site shutdown. Not a fault: the stream is marked
// down so the reconcile tick reopens it (which also re-seeds), but the retry budget
// is untouched and the node is not flipped. Same generation fence as the error path.
Receive<GrpcAlarmStreamCompleted>(msg =>
{
if (_stopped) return;
if (msg.Generation != _streamGeneration)
{
_log.Debug("Ignoring stale gRPC completion from stream generation {0} (current {1})",
msg.Generation, _streamGeneration);
return;
}
HandleGrpcCompleted();
});
Receive<ReconnectAlarmStream>(_ => OpenGrpcStream());
// Owning service asks us to stop (last viewer left + linger elapsed).
@@ -279,7 +306,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
if (_streamDown)
{
_log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
_retryCount = 0;
if (_retryBudgetExhausted)
{
_retryBudgetExhausted = false;
_retryCount = 0;
}
// Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
OpenGrpcStream();
@@ -514,14 +545,26 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
var self = Self;
var ct = _grpcCts.Token;
// The subscription task itself is observed (below): a fault escaping
// SubscribeSiteAsync — or a Task.Run that never started because ct was already
// cancelled — would otherwise leave the actor waiting on a stream that does not
// exist, with the exception silently unobserved.
Task.Run(async () =>
{
await client.SubscribeSiteAsync(
_correlationId,
alarm => self.Tell(alarm),
ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
() => self.Tell(new GrpcAlarmStreamCompleted(generation)),
ct);
}, ct);
}, ct).ContinueWith(t =>
{
if (t.IsFaulted)
self.Tell(new GrpcAlarmStreamError(t.Exception!.GetBaseException(), generation));
else if (t.IsCanceled && !ct.IsCancellationRequested)
self.Tell(new GrpcAlarmStreamCompleted(generation));
// RanToCompletion: SubscribeSiteAsync already reported its own outcome.
}, TaskContinuationOptions.ExecuteSynchronously);
}
private void HandleGrpcError()
@@ -543,6 +586,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
"reconcile snapshots continue and the next reconcile tick will retry the stream",
_siteIdentifier, MaxRetries);
_streamDown = true;
_retryBudgetExhausted = true;
CleanupGrpc();
return;
}
@@ -569,6 +613,26 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectAlarmStream(), _reconnectDelay);
}
/// <summary>
/// Handles a graceful end of stream. The stream is torn down and left down for the
/// reconcile tick to reopen — deliberately NOT reopened inline, so a site that keeps
/// completing streams immediately can never spin this actor into a hot reconnect loop.
/// The retry budget is neither spent nor reset here (completion is not a fault), and the
/// endpoint is not flipped: the node that just closed a stream cleanly is healthy.
/// </summary>
private void HandleGrpcCompleted()
{
// The stream is gone, so its armed stability timer must not later "recover" a
// budget that its successor has since spent.
Timers.Cancel(StabilityTimerKey);
_log.Info("Site-alarm gRPC stream for {0} completed gracefully (server end of stream); " +
"reopening on the next reconcile tick", _siteIdentifier);
_streamDown = true;
CleanupGrpc();
}
private void CleanupGrpc()
{
_grpcCts?.Cancel();
@@ -619,6 +683,11 @@ internal sealed record PublishCoalesced;
/// generation it came from so a late error from a cancelled stream can be ignored (N7.2).</summary>
internal sealed record GrpcAlarmStreamError(Exception Exception, int Generation);
/// <summary>Internal: the site-alarm gRPC stream ended gracefully (server status OK — max
/// stream lifetime or site shutdown), stamped with its stream generation so a late completion
/// from a cancelled stream can be ignored.</summary>
internal sealed record GrpcAlarmStreamCompleted(int Generation);
/// <summary>Internal: reconnect the site-alarm gRPC stream (flip node).</summary>
internal sealed record ReconnectAlarmStream;