using Akka.Actor;
using Akka.Event;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
///
/// Long-lived (one per active debug session) actor on the central side. Debug sessions
/// are session-based and temporary — this actor holds no persisted state and does not
/// derive from an Akka.Persistence base class; its state does not survive a restart.
///
/// Stream-first lifecycle. To avoid losing any
/// / that occurs on
/// the site during the snapshot-build + network-transit window, the gRPC server-streaming
/// subscription is opened FIRST (in ), alongside the
/// SubscribeDebugViewRequest sent to the site via CentralCommunicationActor (with
/// THIS actor as the Sender). Live events that arrive before the
/// is delivered are buffered in arrival order .
/// When the snapshot arrives it is delivered to the consumer, then the buffer is flushed
/// in order, deduped against the snapshot (an event whose per-entity timestamp is
/// <= the snapshot's timestamp for the same entity is already reflected → dropped; a
/// strictly-newer event is delivered; an event for an entity absent from the snapshot is
/// delivered). After the flush the actor switches to pass-through: subsequent events go
/// straight to the consumer. A mid-session reconnect (after the snapshot) resumes
/// pass-through — the snapshot is a one-time thing.
///
/// Stream events are marshalled back to the actor via Self.Tell for thread safety; all
/// state (phase flag + buffer) is mutated only on the actor thread.
///
public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
{
private readonly ILoggingAdapter _log = Context.GetLogger();
private readonly string _siteIdentifier;
private readonly string _instanceUniqueName;
private readonly string _correlationId;
private readonly IActorRef _centralCommunicationActor;
private readonly Action _onEvent;
private readonly Action _onTerminated;
private readonly SiteStreamGrpcClientFactory _grpcFactory;
private readonly string _grpcNodeAAddress;
private readonly string _grpcNodeBAddress;
private const int MaxRetries = 3;
private const string ReconnectTimerKey = "grpc-reconnect";
private const string StabilityTimerKey = "grpc-stability";
/// Delay between gRPC reconnection attempts.
internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
///
/// How long a freshly-opened gRPC stream must stay up before its retry budget
/// is considered "recovered" and is reset to 0.
/// The retry count must NOT be reset by individual events —
/// a stream that connects, delivers one event, then fails repeatedly would
/// otherwise reconnect forever and never trip . Resetting
/// only after a stable interval bounds a flapping stream.
///
internal static TimeSpan StabilityWindow { get; set; } = TimeSpan.FromSeconds(60);
private int _retryCount;
private bool _useNodeA = true;
private bool _stopped;
private CancellationTokenSource? _grpcCts;
///
/// Phase flag. until the initial
/// has been delivered and the pre-snapshot buffer
/// flushed; thereafter (pass-through). Mutated only on the
/// actor thread. A reconnect does NOT touch this flag — a mid-session reconnect
/// (after the snapshot) therefore stays in pass-through, and a reconnect during the
/// buffering phase (before the snapshot) stays buffering.
///
private bool _snapshotDelivered;
///
/// Ordered buffer of live gRPC events ( /
/// ) that arrived before the snapshot was delivered.
/// Flushed (with per-entity dedup against the snapshot) when the snapshot arrives,
/// then never used again. Mutated only on the actor thread.
///
private readonly List _preSnapshotBuffer = new();
///
/// Defensive log threshold: if the pre-snapshot buffer grows past this many events
/// during a slow snapshot we log once (events are NOT dropped — the window is short).
///
private const int BufferWarnThreshold = 10_000;
private bool _bufferWarned;
/// Timer scheduler for reconnect and stability window timers.
public ITimerScheduler Timers { get; set; } = null!;
///
/// Initializes the debug stream bridge actor and registers message handlers.
///
/// Site identifier for targeting ClusterClient messages and logging.
/// Unique name of the instance whose debug stream is being bridged.
/// Correlation id for the debug session.
/// Actor used to forward ClusterClient messages to the site.
/// Callback invoked on each received debug event.
/// Callback invoked when the stream terminates.
/// Factory for creating gRPC streaming clients.
/// gRPC address of the site's node A.
/// gRPC address of the site's node B.
public DebugStreamBridgeActor(
string siteIdentifier,
string instanceUniqueName,
string correlationId,
IActorRef centralCommunicationActor,
Action onEvent,
Action onTerminated,
SiteStreamGrpcClientFactory grpcFactory,
string grpcNodeAAddress,
string grpcNodeBAddress)
{
_siteIdentifier = siteIdentifier;
_instanceUniqueName = instanceUniqueName;
_correlationId = correlationId;
_centralCommunicationActor = centralCommunicationActor;
_onEvent = onEvent;
_onTerminated = onTerminated;
_grpcFactory = grpcFactory;
_grpcNodeAAddress = grpcNodeAAddress;
_grpcNodeBAddress = grpcNodeBAddress;
// Initial snapshot response from the site (via ClusterClient).
// If the site reports InstanceNotFound=true the instance is not
// deployed there. Under the stream-first lifecycle the gRPC stream
// was already opened in PreStart, so the not-found path must tear it down
// (CleanupGrpc) rather than enter pass-through. Forward the snapshot (with
// InstanceNotFound=true) to _onEvent so DebugStreamService's TCS resolves and
// the caller can inspect the flag; then stop cleanly.
Receive(snapshot =>
{
if (_snapshotDelivered)
{
// Defensive: a duplicate / late snapshot after we have already moved to
// pass-through. The snapshot is a one-time thing — ignore replays so we
// never re-buffer or double-deliver.
_log.Debug("Ignoring duplicate DebugViewSnapshot for {0} (already delivered)",
_instanceUniqueName);
return;
}
if (snapshot.InstanceNotFound)
{
_log.Warning("Instance {0} is not deployed on site; terminating debug stream",
_instanceUniqueName);
// The stream-first subscription opened in PreStart is for a
// non-deployed instance — cancel it (and any buffered gap events are
// discarded with the actor). No pass-through.
// _stopped is set AFTER CleanupGrpc() to match the ordering in the
// DebugStreamTerminated and ReceiveTimeout handlers (cosmetic consistency).
CleanupGrpc();
_stopped = true;
_preSnapshotBuffer.Clear();
_onEvent(snapshot); // resolves the snapshot TCS with InstanceNotFound=true
// Note: after Context.Stop(Self) below the actor is dead. DebugStreamService
// inspects InitialSnapshot.InstanceNotFound and calls StopStream, which sends
// a StopDebugStream message. That Tell arrives after the actor has already
// stopped, producing a benign Akka dead-letter — expected and harmless.
Context.Stop(Self);
return;
}
_log.Info("Received initial snapshot for {0} ({1} attrs, {2} alarms); flushing {3} buffered event(s)",
_instanceUniqueName, snapshot.AttributeValues.Count, snapshot.AlarmStates.Count,
_preSnapshotBuffer.Count);
// Deliver the snapshot, then flush the gap-window buffer (deduped), then
// switch to pass-through. Order matters: snapshot first, buffered events next.
_onEvent(snapshot);
FlushBuffer(snapshot);
_snapshotDelivered = true;
});
// Domain events arriving via Self.Tell from gRPC callback.
// Receiving an event must NOT reset _retryCount — a
// flapping stream that delivers a single event between failures would
// otherwise never trip MaxRetries. The retry budget is recovered only by
// GrpcStreamStable (a stream that has stayed up for StabilityWindow).
// Before the snapshot has been delivered, BUFFER (in arrival order)
// rather than deliver — these may be gap-window events. After the snapshot has
// been flushed, pass through directly (same handler, phase-dependent behavior).
Receive(changed => HandleStreamEvent(changed));
Receive(changed => HandleStreamEvent(changed));
// Stream has been stably connected for StabilityWindow — recover the
// retry budget so a future transient fault gets a fresh set of retries.
Receive(_ =>
{
if (_stopped) return;
_retryCount = 0;
_log.Debug("gRPC stream for {0} stable, retry count reset", _instanceUniqueName);
});
// gRPC stream error — attempt reconnection
Receive(msg =>
{
_log.Warning("gRPC stream error for {0}: {1}", _instanceUniqueName, msg.Exception.Message);
HandleGrpcError();
});
// Scheduled reconnection
Receive(_ => OpenGrpcStream());
// Consumer requests stop
Receive(_ =>
{
_log.Info("Stopping debug stream for {0}", _instanceUniqueName);
CleanupGrpc();
SendUnsubscribe();
_stopped = true;
Context.Stop(Self);
});
// Site disconnected — CentralCommunicationActor notifies us
Receive(msg =>
{
if (_stopped) return; // Idempotent — gRPC error may arrive simultaneously
_log.Warning("Debug stream terminated for {0} (site {1} disconnected)", _instanceUniqueName, msg.SiteId);
CleanupGrpc();
_stopped = true;
_onTerminated();
Context.Stop(Self);
});
// Orphan safety net — if nobody stops us within 5 minutes, self-terminate
Context.SetReceiveTimeout(TimeSpan.FromMinutes(5));
Receive(_ =>
{
_log.Warning("Debug stream for {0} timed out (orphaned session), stopping", _instanceUniqueName);
CleanupGrpc();
SendUnsubscribe();
_stopped = true;
_onTerminated();
Context.Stop(Self);
});
}
///
/// Handles a live gRPC stream event ( or
/// ). Before the snapshot has been delivered the
/// event is appended to the ordered pre-snapshot buffer (gap-window capture); after
/// the snapshot+flush it is passed straight through to the consumer. Always runs on
/// the actor thread (events are marshalled in via Self.Tell), so the phase flag and
/// buffer are accessed without locking.
///
private void HandleStreamEvent(object evt)
{
if (_snapshotDelivered)
{
_onEvent(evt);
return;
}
_preSnapshotBuffer.Add(evt);
if (!_bufferWarned && _preSnapshotBuffer.Count > BufferWarnThreshold)
{
_bufferWarned = true;
_log.Warning(
"Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot; " +
"events are still retained (not dropped).",
_instanceUniqueName, BufferWarnThreshold);
}
}
///
/// Flushes the pre-snapshot buffer in arrival order, deduping each event against the
/// just-delivered snapshot.
///
/// Dedup rule. Identity is per-entity:
/// attributes by (InstanceUniqueName, AttributePath, AttributeName); alarms by
/// (InstanceUniqueName, AlarmName, SourceReference). For a buffered event whose entity
/// is present in the snapshot, the comparison is against that entity's snapshot
/// timestamp: a buffered timestamp <= the snapshot timestamp means the event is
/// already reflected in the snapshot → DROP; a strictly-newer (>) timestamp means
/// the event happened after the snapshot was built → DELIVER. The boundary is inclusive
/// on the snapshot side (equal timestamps are treated as duplicates) — the snapshot is
/// the authoritative point-in-time value, so an event at the exact same instant carries
/// no new information. A buffered event whose entity is NOT in the snapshot is a genuine
/// gap-window event → DELIVER.
///
///
private void FlushBuffer(DebugViewSnapshot snapshot)
{
if (_preSnapshotBuffer.Count == 0) return;
// Build per-entity "as-of" timestamps from the snapshot. If (defensively) the
// snapshot lists the same entity twice, keep the newest timestamp.
var attrAsOf = new Dictionary();
foreach (var a in snapshot.AttributeValues)
{
var key = AttributeKey(a);
if (!attrAsOf.TryGetValue(key, out var existing) || a.Timestamp > existing)
attrAsOf[key] = a.Timestamp;
}
var alarmAsOf = new Dictionary();
foreach (var al in snapshot.AlarmStates)
{
var key = AlarmKey(al);
if (!alarmAsOf.TryGetValue(key, out var existing) || al.Timestamp > existing)
alarmAsOf[key] = al.Timestamp;
}
var flushed = 0;
var dropped = 0;
foreach (var evt in _preSnapshotBuffer)
{
if (IsReflectedInSnapshot(evt, attrAsOf, alarmAsOf))
{
dropped++;
continue;
}
_onEvent(evt);
flushed++;
}
if (dropped > 0 || flushed > 0)
{
_log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot",
flushed, _instanceUniqueName, dropped);
}
_preSnapshotBuffer.Clear();
}
///
/// Returns when a buffered event is already reflected in the
/// snapshot (same entity, buffered timestamp <= snapshot timestamp) and must be
/// dropped; otherwise (deliver).
///
private static bool IsReflectedInSnapshot(
object evt,
IReadOnlyDictionary attrAsOf,
IReadOnlyDictionary alarmAsOf)
{
switch (evt)
{
case AttributeValueChanged a:
return attrAsOf.TryGetValue(AttributeKey(a), out var attrTs) && a.Timestamp <= attrTs;
case AlarmStateChanged al:
return alarmAsOf.TryGetValue(AlarmKey(al), out var alarmTs) && al.Timestamp <= alarmTs;
default:
// Unknown buffered type (should not happen — only attr/alarm are buffered):
// never treat as a duplicate.
return false;
}
}
///
/// Delimiter used to join identity components into a single dedup key. A NUL
/// control character cannot appear in an instance/attribute/alarm name, so
/// distinct identities never collide on a shared boundary (unlike a space, which
/// may legitimately occur within a name). Declared as an escaped char so the
/// source carries no raw NUL byte.
///
private const char KeyDelimiter = '\u0000';
///
/// Per-entity dedup key for an attribute change. Each nullable component is guarded
/// with ?? string.Empty so a null can never silently collide with another
/// key via (e.g. two entries with null AttributePath
/// would otherwise share a key with any entry whose AttributePath is the empty string).
///
private static string AttributeKey(AttributeValueChanged a) =>
string.Concat(
a.InstanceUniqueName ?? string.Empty, KeyDelimiter,
a.AttributePath ?? string.Empty, KeyDelimiter,
a.AttributeName ?? string.Empty);
///
/// Per-entity dedup key for an alarm change. Includes
/// so native per-condition alarms (which share an AlarmName but differ by source
/// reference) are not conflated; empty for computed alarms. Each nullable component is
/// guarded with ?? string.Empty to prevent silent key collisions.
///
private static string AlarmKey(AlarmStateChanged al) =>
string.Concat(
al.InstanceUniqueName ?? string.Empty, KeyDelimiter,
al.AlarmName ?? string.Empty, KeyDelimiter,
al.SourceReference ?? string.Empty);
///
protected override void PreStart()
{
_log.Info("Starting debug stream bridge for {0} on site {1}", _instanceUniqueName, _siteIdentifier);
// Stream-first: open the gRPC live-event subscription BEFORE (and
// alongside) requesting the snapshot, so events occurring during the
// snapshot-build + network-transit window are captured (buffered) and not lost.
OpenGrpcStream();
// Send subscribe request via CentralCommunicationActor for the initial snapshot.
var request = new SubscribeDebugViewRequest(_instanceUniqueName, _correlationId);
var envelope = new SiteEnvelope(_siteIdentifier, request);
_centralCommunicationActor.Tell(envelope, Self);
}
///
protected override void PostStop()
{
_grpcCts?.Cancel();
_grpcCts?.Dispose();
_grpcCts = null;
base.PostStop();
}
private void OpenGrpcStream()
{
if (_stopped) return;
var endpoint = _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress;
_log.Info("Opening gRPC stream for {0} to {1}", _instanceUniqueName, endpoint);
_grpcCts?.Cancel();
_grpcCts?.Dispose();
_grpcCts = new CancellationTokenSource();
// Arm the stability timer: if the stream stays up for StabilityWindow the
// retry budget is recovered. Cancelled by HandleGrpcError.
Timers.StartSingleTimer(StabilityTimerKey, new GrpcStreamStable(), StabilityWindow);
var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
var self = Self;
var ct = _grpcCts.Token;
// Launch as background task — onEvent and onError marshal back to actor via Tell
Task.Run(async () =>
{
await client.SubscribeAsync(
_correlationId,
_instanceUniqueName,
evt => self.Tell(evt),
ex => self.Tell(new GrpcStreamError(ex)),
ct);
}, ct);
}
private void HandleGrpcError()
{
if (_stopped) return;
// The stream failed before reaching the stability window — its retry
// budget is NOT recovered.
Timers.Cancel(StabilityTimerKey);
_retryCount++;
if (_retryCount > MaxRetries)
{
_log.Error("gRPC stream for {0} exceeded max retries ({1}), terminating", _instanceUniqueName, MaxRetries);
CleanupGrpc();
_stopped = true;
_onTerminated();
Context.Stop(Self);
return;
}
// Unsubscribe the failed stream on the *previous* endpoint before reconnecting.
// This cancels the local subscription CTS and -- where the channel is still
// alive -- propagates gRPC cancellation to the site so its SiteStreamGrpcServer
// stops the StreamRelayActor for this correlation ID, rather than leaving a
// zombie relay actor until TCP RST / keepalive eventually detects the loss.
var previousEndpoint = _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress;
// TryGet, not GetOrCreate: unsubscribing a failed stream must never open a
// fresh channel (and, with (site,endpoint) keying, must never touch another
// session's healthy channel). Absent client => the channel is already gone
// and the site-side relay will be reaped by keepalive/session-lifetime.
_grpcFactory.TryGet(_siteIdentifier, previousEndpoint)?.Unsubscribe(_correlationId);
// Flip to the other node
_useNodeA = !_useNodeA;
// First retry is immediate, subsequent retries use a short backoff
if (_retryCount == 1)
{
Self.Tell(new ReconnectGrpcStream());
}
else
{
Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectGrpcStream(), ReconnectDelay);
}
}
private void CleanupGrpc()
{
_grpcCts?.Cancel();
_grpcCts?.Dispose();
_grpcCts = null;
// TryGet, not GetOrCreate: teardown must never open a fresh channel just to
// unsubscribe. Absent client => nothing to cancel.
var endpoint = _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress;
_grpcFactory.TryGet(_siteIdentifier, endpoint)?.Unsubscribe(_correlationId);
}
private void SendUnsubscribe()
{
var request = new UnsubscribeDebugViewRequest(_instanceUniqueName, _correlationId);
var envelope = new SiteEnvelope(_siteIdentifier, request);
_centralCommunicationActor.Tell(envelope, Self);
}
}
///
/// Message sent to a DebugStreamBridgeActor to stop the debug stream session.
///
public record StopDebugStream;
///
/// Internal message indicating a gRPC stream error occurred.
///
internal record GrpcStreamError(Exception Exception);
///
/// Internal message to trigger gRPC stream reconnection.
///
internal record ReconnectGrpcStream;
///
/// Internal message indicating the current gRPC stream has been connected long
/// enough ( ) to be considered
/// stable, so the reconnect retry budget can be recovered.
///
internal record GrpcStreamStable;