using Akka.Actor; using Akka.Cluster; using Akka.Cluster.Tools.Client; using Akka.Event; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification; using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery; namespace ZB.MOM.WW.ScadaBridge.Communication.Actors; /// /// Site-side actor that receives messages from central via ClusterClient and routes /// them to the appropriate local actors. Also sends heartbeats and health reports /// to central via the registered ClusterClient. /// /// Routes all 8 message patterns to local handlers. /// public class SiteCommunicationActor : ReceiveActor, IWithTimers { private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly string _siteId; private readonly CommunicationOptions _options; /// /// Predicate that returns true when this node is /// the active member of the local site cluster (used to stamp /// ). Production builds default to /// the Akka leader check; tests inject a stub so they /// do not need a real cluster. /// private readonly Func _isActiveCheck; /// /// Reference to the local Deployment Manager singleton proxy. /// private readonly IActorRef _deploymentManagerProxy; /// /// ClusterClient reference for sending messages to the central cluster. /// Set via RegisterCentralClient message. /// private IActorRef? _centralClient; /// /// Local actor references for routing specific message patterns. /// Populated via registration messages. /// private IActorRef? _eventLogHandler; private IActorRef? _parkedMessageHandler; private IActorRef? _integrationHandler; private IActorRef? _artifactHandler; /// Akka timer scheduler injected by the framework via . public ITimerScheduler Timers { get; set; } = null!; /// Initializes the actor, wires all message pattern handlers, and schedules the periodic heartbeat. /// The site identifier included in outbound messages. /// Communication options including heartbeat interval and transport settings. /// Local reference to the Deployment Manager singleton proxy. /// /// Optional override returning true when this node /// is the active member of the site cluster. null uses the shared oldest-Up /// evaluator (production wiring passes the Host's singleton-host delegate); tests /// pass a stub so they do not need to load Akka.Cluster into the TestKit /// ActorSystem. /// public SiteCommunicationActor( string siteId, CommunicationOptions options, IActorRef deploymentManagerProxy, Func? isActiveCheck = null) { _siteId = siteId; _options = options; _deploymentManagerProxy = deploymentManagerProxy; _isActiveCheck = isActiveCheck ?? DefaultIsActiveCheck; // Registration Receive(msg => { _centralClient = msg.Client; _log.Info("Registered central ClusterClient"); }); Receive(HandleRegisterLocalHandler); // Pattern 1: Instance Deployment — forward to Deployment Manager Receive(msg => { _log.Debug("Routing RefreshDeploymentCommand for {0} to DeploymentManager", msg.InstanceUniqueName); _deploymentManagerProxy.Forward(msg); }); // Pattern 2: Lifecycle — forward to Deployment Manager Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); // Query-the-site-before-redeploy — forward to // the Deployment Manager, which owns the deployed-config store and // answers with the instance's currently-applied deployment identity. Receive(msg => _deploymentManagerProxy.Forward(msg)); // Pattern 3: Artifact Deployment — forward to artifact handler if registered Receive(msg => { if (_artifactHandler != null) _artifactHandler.Forward(msg); else { _log.Warning("No artifact handler registered, replying with failure"); Sender.Tell(new ArtifactDeploymentResponse( msg.DeploymentId, _siteId, false, "Artifact handler not available", DateTimeOffset.UtcNow)); } }); // Pattern 4: Integration Routing — forward to integration handler Receive(msg => { if (_integrationHandler != null) _integrationHandler.Forward(msg); else { Sender.Tell(new IntegrationCallResponse( msg.CorrelationId, _siteId, false, null, "Integration handler not available", DateTimeOffset.UtcNow)); } }); // Pattern 5: Debug View — forward to Deployment Manager (which routes to Instance Actor) Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); // Pattern 6a: Debug Snapshot (one-shot) — forward to Deployment Manager Receive(msg => _deploymentManagerProxy.Forward(msg)); // Inbound API Route.To() — forward to Deployment Manager for instance routing Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); // OPC UA Tag Browser (interactive design-time query) — forward to the // Deployment Manager singleton, which always lands on the active site // node. Routing to the site-local /user/dcl-manager directly is wrong // because the standby node has a dcl-manager too, but its // DataConnectionActor children (which own the live OPC UA sessions) // only exist on the singleton's node. The singleton then re-forwards // to its own /user/dcl-manager, which DOES have the connection. Receive(msg => _deploymentManagerProxy.Forward(msg)); // Test Bindings (interactive design-time read) — same routing rationale // as BrowseNodeCommand above: the singleton always lands on the // active site node, which is the node that owns the DataConnectionActor // children holding the live OPC UA sessions. Receive(msg => _deploymentManagerProxy.Forward(msg)); // OPC UA tag-picker address-space search and secured-write execute // — same singleton routing rationale as BrowseNodeCommand above: the // DataConnectionActor children that own the live OPC UA sessions exist only // on the singleton's (active) node, so these must hop through the Deployment // Manager proxy too. Without these forwards the commands dead-letter and the // central Ask times out. Forward preserves the central Ask sender so the // result routes straight back to the waiting Ask. Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); // OPC UA endpoint Verify — probes a (possibly unsaved) endpoint config // WITHOUT persisting it. The Deployment Manager singleton's dcl-manager runs // the probe directly (no existing connection required), so — like the // commands above — Verify routes through the singleton's active node. Receive(msg => _deploymentManagerProxy.Forward(msg)); // OPC UA server-certificate trust management — forward to the // Deployment Manager singleton, which owns the cross-node trust broadcast. // The trusted-peer PKI store is node-wide per site node, so a trust/remove // decision must reach BOTH nodes' CertStoreActor; the singleton broadcasts // to every site node (list answers from the singleton's own node). The // singleton always lands on the active node, the same routing rationale as // BrowseNodeCommand above. Forward preserves the central Ask sender so the // CertTrustResult routes straight back to the waiting Ask. Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); Receive(msg => _deploymentManagerProxy.Forward(msg)); // Pattern 7: Remote Queries Receive(msg => { if (_eventLogHandler != null) _eventLogHandler.Forward(msg); else { Sender.Tell(new EventLogQueryResponse( msg.CorrelationId, _siteId, [], null, false, false, "Event log handler not available", DateTimeOffset.UtcNow)); } }); Receive(msg => { if (_parkedMessageHandler != null) _parkedMessageHandler.Forward(msg); else { Sender.Tell(new ParkedMessageQueryResponse( msg.CorrelationId, _siteId, [], 0, msg.PageNumber, msg.PageSize, false, "Parked message handler not available", DateTimeOffset.UtcNow)); } }); Receive(msg => { if (_parkedMessageHandler != null) _parkedMessageHandler.Forward(msg); else { Sender.Tell(new ParkedMessageRetryResponse( msg.CorrelationId, false, "Parked message handler not available")); } }); Receive(msg => { if (_parkedMessageHandler != null) _parkedMessageHandler.Forward(msg); else { Sender.Tell(new ParkedMessageDiscardResponse( msg.CorrelationId, false, "Parked message handler not available")); } }); // Central→site Retry/Discard relay for parked cached // operations. SiteCallAuditActor relays these over the command/control // channel; the parked-message handler executes them against the local // S&F buffer and replies a ParkedOperationActionAck that routes back to // the relaying SiteCallAuditActor's Ask. Receive(msg => { if (_parkedMessageHandler != null) _parkedMessageHandler.Forward(msg); else { Sender.Tell(new ParkedOperationActionAck( msg.CorrelationId, Applied: false, "Parked message handler not available")); } }); Receive(msg => { if (_parkedMessageHandler != null) _parkedMessageHandler.Forward(msg); else { Sender.Tell(new ParkedOperationActionAck( msg.CorrelationId, Applied: false, "Parked message handler not available")); } }); // Notification Outbox: forward a buffered notification submitted by the site // Store-and-Forward Engine to the central cluster. The original Sender (the // S&F forwarder's Ask) is forwarded as the ClusterClient.Send sender so the // NotificationSubmitAck routes straight back to the waiting Ask, not here. Receive(msg => { if (_centralClient == null) { // No ClusterClient registered yet (e.g. central contact points not // configured, or registration not yet completed). A non-accepted ack // makes the S&F forwarder treat this as transient and retry later. _log.Warning( "Cannot forward NotificationSubmit {0} — no central ClusterClient registered", msg.NotificationId); Sender.Tell(new NotificationSubmitAck( msg.NotificationId, Accepted: false, Error: "Central ClusterClient not registered")); return; } _log.Debug("Forwarding NotificationSubmit {0} to central", msg.NotificationId); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); // Notification Outbox: forward a Notify.Status query to the central cluster. // The original Sender (the Notify helper's Ask) is forwarded as the // ClusterClient.Send sender so the NotificationStatusResponse routes straight // back to the waiting Ask, not here. Receive(msg => { if (_centralClient == null) { // No ClusterClient registered yet. Reply Found: false so Notify.Status // falls back to the site S&F buffer to decide Forwarding vs Unknown. _log.Warning( "Cannot forward NotificationStatusQuery {0} — no central ClusterClient registered", msg.NotificationId); Sender.Tell(new NotificationStatusResponse( msg.CorrelationId, Found: false, Status: "Unknown", RetryCount: 0, LastError: null, DeliveredAt: null)); return; } _log.Debug("Forwarding NotificationStatusQuery {0} to central", msg.NotificationId); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); // Audit Log: forward a batch of site-local audit events to the // central cluster. The site SiteAuditTelemetryActor drains its SQLite // Pending queue through the ClusterClientSiteAuditClient, which Asks // this actor; the original Sender (that Ask) is passed as the // ClusterClient.Send sender so the IngestAuditEventsReply routes // straight back to the waiting Ask, not here. Mirrors NotificationSubmit. Receive(msg => { if (_centralClient == null) { // No ClusterClient registered yet (e.g. central contact points // not configured, or registration not yet completed). Faulting // the Ask makes the SiteAuditTelemetryActor drain loop treat // this as transient and keep the rows Pending for the next tick. _log.Warning( "Cannot forward IngestAuditEventsCommand ({0} events) — no central ClusterClient registered", msg.Events.Count); Sender.Tell(new Status.Failure( new InvalidOperationException("Central ClusterClient not registered"))); return; } _log.Debug("Forwarding IngestAuditEventsCommand ({0} events) to central", msg.Events.Count); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); // Audit Log: forward a batch of combined cached-call telemetry // packets to the central cluster. Same forward + reply-routing pattern // as IngestAuditEventsCommand; central replies with an // IngestCachedTelemetryReply. Receive(msg => { if (_centralClient == null) { _log.Warning( "Cannot forward IngestCachedTelemetryCommand ({0} entries) — no central ClusterClient registered", msg.Entries.Count); Sender.Tell(new Status.Failure( new InvalidOperationException("Central ClusterClient not registered"))); return; } _log.Debug("Forwarding IngestCachedTelemetryCommand ({0} entries) to central", msg.Entries.Count); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); // Site startup reconciliation: forward the node's local-inventory // ReconcileSiteRequest to the central cluster. The original Sender (the // SiteReconciliationActor's Ask) is passed as the ClusterClient.Send sender so // the ReconcileSiteResponse routes straight back to the waiting Ask, not here. // Mirrors IngestAuditEventsCommand. Receive(msg => { if (_centralClient == null) { // No ClusterClient registered yet (e.g. central contact points not // configured, or registration not yet completed). Faulting the Ask makes // the SiteReconciliationActor treat the pass as best-effort-failed; it // logs a warning and retries reconcile on the next node startup. _log.Warning( "Cannot forward ReconcileSiteRequest for site {0} node {1} — no central ClusterClient registered", msg.SiteIdentifier, msg.NodeId); Sender.Tell(new Status.Failure( new InvalidOperationException("Central ClusterClient not registered"))); return; } _log.Debug( "Forwarding ReconcileSiteRequest for site {0} node {1} ({2} local instance(s)) to central", msg.SiteIdentifier, msg.NodeId, msg.LocalNameToRevisionHash.Count); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); // Internal: send heartbeat tick Receive(_ => SendHeartbeatToCentral()); // Internal: forward health report to central. The original Sender (the // AkkaHealthReportTransport's Ask) is forwarded as the ClusterClient.Send // sender so the central SiteHealthReportAck routes straight back to the // waiting Ask — making report delivery observable end-to-end (review 01 // [Medium]). Mirrors the NotificationSubmit ack pattern above. Receive(msg => { if (_centralClient == null) { // No ClusterClient registered yet. A non-accepted ack makes the // sender's counter-restore path treat this tick as a loss. _log.Warning( "Cannot forward SiteHealthReport #{0} — no central ClusterClient registered", msg.SequenceNumber); Sender.Tell(new SiteHealthReportAck( msg.SiteId, msg.SequenceNumber, Accepted: false, Error: "Central ClusterClient not registered")); return; } _centralClient.Tell( new ClusterClient.Send("/user/central-communication", msg), Sender); }); } /// protected override SupervisorStrategy SupervisorStrategy() { return new OneForOneStrategy( maxNrOfRetries: -1, withinTimeRange: Timeout.InfiniteTimeSpan, decider: Decider.From(ex => { _log.Warning(ex, "Child actor of SiteCommunicationActor faulted, resuming (state preserved)"); return Directive.Resume; })); } /// protected override void PreStart() { _log.Info("SiteCommunicationActor started for site {0}", _siteId); // Schedule periodic heartbeat to central. Uses the application heartbeat // cadence — distinct from the Akka.Remote transport failure-detector // interval — so the two can be tuned independently. Timers.StartPeriodicTimer( "heartbeat", new SendHeartbeat(), TimeSpan.FromSeconds(1), // initial delay _options.ApplicationHeartbeatInterval); } private void HandleRegisterLocalHandler(RegisterLocalHandler msg) { switch (msg.HandlerType) { case LocalHandlerType.EventLog: _eventLogHandler = msg.Handler; break; case LocalHandlerType.ParkedMessages: _parkedMessageHandler = msg.Handler; break; case LocalHandlerType.Integration: _integrationHandler = msg.Handler; break; case LocalHandlerType.Artifacts: _artifactHandler = msg.Handler; break; } _log.Info("Registered local handler for {0}", msg.HandlerType); } private void SendHeartbeatToCentral() { if (_centralClient == null) return; var hostname = Environment.MachineName; // Stamp HeartbeatMessage.IsActive with this node's // true active/standby role rather than hard-coding `true`. The field is // part of the wire contract (additive-only-evolution) so a future // central health dashboard can distinguish "active node down, standby // up" from "site fully offline" without a new message type. bool isActive; try { isActive = _isActiveCheck(); } catch (Exception ex) { // Defensive: never let a cluster-state read failure abort the // heartbeat itself (heartbeats are health signal — their absence is // already meaningful). Fall back to the safest non-claiming value: // standby. Logged at Debug because this path normally only fires // during ActorSystem warm-up. _log.Debug(ex, "Active-node check threw while sending heartbeat for site {0}; reporting IsActive=false", _siteId); isActive = false; } var heartbeat = new HeartbeatMessage( _siteId, hostname, IsActive: isActive, DateTimeOffset.UtcNow); _centralClient.Tell( new ClusterClient.Send("/user/central-communication", heartbeat), Self); } /// /// Default active-node check used when no override is supplied: oldest-Up member /// semantics via the shared — the /// same predicate as the S&F delivery gate and the replication resync authority /// (review 02 round 2, N1). Unscoped by role: a site cluster's members all carry /// the site role, so role scoping is a no-op here; production wiring passes the /// Host's role-scoped IClusterNodeProvider delegate anyway. Any other state /// (still joining, leaving) reports standby — safe-by-default. /// private bool DefaultIsActiveCheck() => ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(Cluster.Get(Context.System)); // ── Internal messages ── internal record SendHeartbeat; } /// /// Command to register a ClusterClient for communicating with the central cluster. /// public record RegisterCentralClient(IActorRef Client); /// /// Command to register a local actor as a handler for a specific message pattern. /// public record RegisterLocalHandler(LocalHandlerType HandlerType, IActorRef Handler); public enum LocalHandlerType { EventLog, ParkedMessages, Integration, Artifacts }