using System.Diagnostics;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
///
/// Per-node supervisor that receives from the admin-role
/// coordinator and applies the deployment locally. Three Become states:
///
///
/// - Bootstrapping — PreStart only. Reads for self;
/// chooses next state.
/// - Steady(rev) — caught up. Idempotent on same-rev dispatch (immediate ApplyAck.Applied).
/// New rev → transitions to Applying.
/// - Applying(id) — applying a delta. Buffers further dispatches.
/// - Stale — ConfigDb unreachable on bootstrap. Background reconnect loop tries to advance.
///
///
/// Children (DriverInstance/VirtualTag/etc.) are fully wired: DispatchDeployment drives
/// which runs the spawn-plan via ,
/// maintains the FullName→NodeId live-value routing map, and forwards results to the
/// OPC UA publish actor via ForwardToMux.
///
public sealed class DriverHostActor : ReceiveActor, IWithTimers
{
public const string DeploymentsTopic = "deployments";
public const string DeploymentAcksTopic = "deployment-acks";
public const string DriverControlTopic = ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin.DriverControlTopic.Name;
public static readonly TimeSpan ReconnectInterval = TimeSpan.FromSeconds(30);
/// Publishing interval handed to each driver's SubscribeBulk pass after an apply.
private static readonly TimeSpan SubscriptionPublishingInterval = TimeSpan.FromSeconds(1);
///
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
/// someone reads the table during an incident.
///
private const string SingleClusterCacheKey = "__single";
///
/// Node-local cache of applied deployment artifacts, or null when this node has none
/// (admin-only graphs, and tests that do not exercise it).
///
private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
///
/// Per-cluster mesh Phase 3. When (ConfigSource:Mode =
/// FetchAndCache) this node obtains its config by fetching the artifact bytes from central
/// over gRPC and applying them from the LocalDb cache, reading NO config from central SQL.
/// When (default Direct) every path reads central SQL as before.
///
private readonly bool _fetchAndCacheMode;
///
/// Node-side gRPC artifact fetcher, non-null only under .
///
private readonly IDeploymentArtifactFetcher? _artifactFetcher;
///
/// True once this node has booted a cached artifact because central SQL was unreachable.
///
///
/// Surfaced on because a node running from cache looks
/// completely healthy from the outside — it serves a full address space with live values.
/// The difference is that its configuration is frozen at whatever the cache held, and no
/// deployment can reach it. Without an explicit signal that is invisible until someone
/// wonders why a deploy "succeeded" everywhere but did not take effect here.
///
private bool _isRunningFromCache;
/// Latches after the first no-op ack-write on a DB-less node so the explanatory Debug line
/// is logged once, not on every apply state transition.
private bool _loggedNoConfigDbAck;
///
/// Config database context factory, or on a driver-only node (per-cluster
/// mesh Phase 4). A driver-only node has NO ConfigDb — LocalDb is its config store and central is
/// the ack system of record — so every ConfigDb-touching path here is guarded on this being
/// non-null. Non-null on a fused admin+driver node and in the Direct-mode/EF test harnesses.
///
private readonly IDbContextFactory? _dbFactory;
///
/// Scripted-alarm condition-state store handed to the ScriptedAlarm engine, or
/// when none was wired (per-cluster mesh Phase 4). On a driver-role node
/// this is the replicated LocalDb store (LocalDbAlarmConditionStateStore) so condition
/// state survives without central SQL; when null the actor skips spawning the alarm host (the
/// ConfigDb-backed EF fallback was retired in Phase 4 Task 9).
///
private readonly IAlarmStateStore? _alarmStateStore;
/// Optional node-local live-telemetry hub (Phase 5): each Primary-gated native-alarm
/// alerts transition is also emitted here, and it is threaded into the spawned VirtualTag +
/// ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.
private readonly ITelemetryLocalHub? _telemetryHub;
private readonly CommonsNodeId _localNode;
private readonly IActorRef? _ackRouter;
private readonly IDriverFactory _driverFactory;
/// Builds the per-driver-instance attached to each
/// spawned . Resolved from DI like ;
/// defaults to (pass-through) when the fused Host
/// hasn't bound the real resilience factory.
private readonly IDriverCapabilityInvokerFactory _invokerFactory;
private readonly IReadOnlySet _localRoles;
private readonly IActorRef? _dependencyMux;
private readonly IActorRef? _opcUaPublishActor;
private readonly IDriverHealthPublisher _healthPublisher;
private readonly IVirtualTagEvaluator _virtualTagEvaluator;
private readonly IHistoryWriter _historyWriter;
private readonly IActorRef? _virtualTagHostOverride;
private readonly ScriptRootLogger? _scriptRootLogger;
private readonly IActorRef? _scriptedAlarmHostOverride;
private readonly ILoggingAdapter _log = Context.GetLogger();
/// The single VirtualTag-host child that spawns/reconciles Equipment-namespace
/// VirtualTagActors and bridges their results onto the OPC UA publish actor. Spawned in
/// when an OPC UA publish actor is wired; receives
/// from .
private IActorRef? _virtualTagHost;
/// The single ScriptedAlarm-host child that owns the per-node
/// , feeds it live tag values, and bridges its emissions onto the
/// OPC UA publish actor + the cluster alerts topic. Spawned in
/// alongside the VirtualTag host (requires both an OPC UA publish actor and a
/// ); receives
/// from .
private IActorRef? _scriptedAlarmHost;
private RevisionHash? _currentRevision;
private DeploymentId? _applyingDeploymentId;
private readonly Dictionary _children = new(StringComparer.Ordinal);
/// Per-driver children — one for each spawned driver
/// that implements (e.g. the Calculation pseudo-driver). Each registers
/// the driver's declared dependency refs on the per-node and forwards value
/// changes into the driver. Spawned in , torn down with the driver in
/// , and re-registered on every apply (refs change when tags/scripts
/// change). Empty when no mux is wired (the dev/None path) or no dependency-consuming driver is hosted.
private readonly Dictionary _depConsumerAdapters = new(StringComparer.Ordinal);
// Monotonic counter feeding the child actor-name suffix (see ActorNameFor / SpawnChild). Single-
// threaded actor, so a plain increment is safe; it only ever grows, guaranteeing a unique name per
// spawn so a restart's respawn never collides with the still-terminating old child.
private long _childSpawnGeneration;
/// A materialised NodeId together with the v3 it lives in, so the
/// driver value fan-out posts each update to the sink with the right namespace. The same driver ref fans
/// to its raw node () and to every referencing UNS node
/// ().
private readonly record struct NodeRealmRef(string NodeId, AddressSpaceRealm Realm);
///
/// Driver live-value routing map: (DriverInstanceId, RawPath) → materialised NodeId(s) (realm-tagged).
/// Rebuilt every apply by from the composition's RawTags
/// ∪ UnsReferenceVariables, and resolved in so a driver value
/// published by wire-ref RawPath fans to its raw node AND every referencing UNS node with identical
/// value/quality/timestamp — the single-source-fan-out (no independent buffer, so no drift). A set
/// because one driver ref can back the raw node plus N UNS references; the per-apply rebuild dedups.
///
private readonly Dictionary<(string DriverInstanceId, string RawPath), HashSet> _nodeIdByDriverRef = new();
///
/// Inverse of : (Realm, BARE s= id) → (DriverInstanceId,
/// RawPath). v3 Batch 4: keyed by BOTH the raw NodeId () AND every
/// referencing UNS NodeId () — all mapping to the same driver ref
/// (the single value source). Rebuilt every apply by from the
/// composition's RawTags ∪ UnsReferenceVariables, and resolved by
/// so an inbound operator write to EITHER NodeId is forwarded to the
/// owning driver child as a write of its wire-ref RawPath.
/// The realm is part of the key (Wave B review H1): a raw s=<RawPath> and a UNS
/// s=<Area/Line/Equip/Eff> can collide as bare strings (folder/driver/device names and
/// Area/Line/Equip names are independent), so a bare-only key would let a colliding raw + UNS pair route
/// to the WRONG driver ref (last-writer-wins). The node manager's write hook passes the full
/// ns-qualified id PLUS the realm it resolved (RealmOf);
/// normalises the id to the bare form and looks up (realm, bareId), preserving exactly the
/// namespace disambiguation the ns-qualified id carries.
///
private readonly Dictionary<(AddressSpaceRealm Realm, string NodeId), (string DriverInstanceId, string RawPath)> _driverRefByNodeId = new();
/// (DriverInstanceId, RawPath = alarm ConditionId) → materialised condition NodeId(s) (realm-tagged).
/// v3 Batch 4: a native alarm is a single condition at the RawPath (Raw realm); WP4 adds the referencing
/// equipment notifier NodeIds. Resolves a native alarm transition to the materialised Part 9 condition
/// node(s). Alarm tags are conditions, not value variables, so they are kept OUT of the value maps +
/// value-subscription set.
private readonly Dictionary<(string DriverInstanceId, string RawPath), HashSet> _alarmNodeIdByDriverRef = new();
///
/// Inverse of : folder-scoped condition NodeId →
/// (DriverInstanceId, FullName = alarm ConditionId/AlarmFullReference). Built in the SAME apply
/// pass from the alarm-bearing EquipmentTags, and resolved by
/// so an inbound OPC UA acknowledge of a native condition (resolved to the condition's NodeId by the
/// node manager) is forwarded to the owning driver child as an acknowledge of its wire-ref
/// FullName. Each condition NodeId maps to exactly one driver ref (a condition is backed by a
/// single driver alarm), so this is a flat 1:1 map (the forward map fans out 1:N because one ref can
/// back several conditions on identical machines).
///
private readonly Dictionary _driverRefByAlarmNodeId =
new(StringComparer.Ordinal);
/// Condition NodeId → (EquipmentId, tag Name, OPC UA alarm type, HistorizeToAveva) for building
/// the AlarmTransitionEvent fan-out. Built in the same PushDesiredSubscriptions alarm branch.
/// HistorizeToAveva (bool?, null ⇒ historize) is the native per-condition opt-out parsed from
/// TagConfig.alarm.historizeToAveva; it is threaded onto the transition so the
/// HistorianAdapterActor's is not false gate suppresses the durable AVEVA row only on an
/// explicit false (mirroring the scripted-alarm opt-out).
private readonly Dictionary ReferencingEquipmentPaths)> _alarmMetaByNodeId =
new(StringComparer.Ordinal);
/// Derives a full Part 9 condition snapshot from each native alarm transition delta,
/// tracking per-condition-NodeId prior state. 'd on every apply alongside the
/// value maps so stale condition state never leaks across redeploys.
private readonly NativeAlarmProjector _nativeAlarmProjector = new();
/// In-flight sends, keyed by correlation, so
/// can advance the cached spec only once the child confirms it adopted
/// the config (#516). Bounded by the number of driver children with a delta in flight.
private readonly Dictionary _pendingDelta = new();
/// The composition from the most-recent apply (set at the END of
/// ). Null until the first apply.
private AddressSpaceComposition? _lastComposition;
///
/// Cached local from the latest
/// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary
/// data-plane gates (, ,
/// ) resolve this through : a KNOWN
/// role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster
/// stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a
/// snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window).
///
private RedundancyRole? _localRole;
/// Test seam (archreview 03/S4): overrides the count of Up driver-role cluster members
/// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a
/// non-cluster ActorRefProvider). See .
private readonly Func? _driverMemberCountProvider;
/// Where the Primary-gate decision is published for consumers that live outside a mailbox —
/// today, the alarm store-and-forward drain. Null on nodes that do not wire one.
private readonly IRedundancyRoleView? _redundancyRoleView;
///
/// Host of this node's LocalDb replication partner — the only node that holds a copy of this
/// node's alarm queue, and therefore the only node it may ever stand down in favour of.
/// null when this node dials nobody (unpaired, or the listening half of a pair).
///
private readonly string? _replicationPeerHost;
/// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
/// identity mismatch (03/S5) would otherwise log on every snapshot.
private bool _warnedSnapshotMissingLocalNode;
/// Cached cluster DistributedPubSub mediator, resolved once in (on the
/// actor thread) and reused for the Primary-gated native-alarm alerts fan-out in
/// instead of re-resolving it per-publish. Mirrors
/// ScriptedAlarmHostActor._mediator.
private IActorRef _mediator = null!;
///
/// Routes an inbound operator write (resolved from the OPC UA node-manager side) to the
/// owning driver child. gates on the local node being the
/// driver Primary, resolves the reverse map, and Asks the child a
/// carrying the driver-side .
///
/// The folder-scoped equipment-variable NodeId the operator wrote to.
/// The value to write (the driver coerces it to the attribute's data type).
public sealed record RouteNodeWrite(string NodeId, object? Value, AddressSpaceRealm Realm);
/// Reply to : the outcome of forwarding the write to the driver
/// (or a gate/lookup failure that never reached the driver).
/// True when the driver accepted the write.
/// Failure detail when is false; null on success.
public sealed record NodeWriteResult(bool Success, string? Reason);
///
/// Routes an inbound native-condition acknowledge to the owning driver child. The host wires the
/// OPC UA node manager's NativeAlarmAckRouter to Tell this in when a client Acknowledges a
/// NATIVE Part 9 condition; applies the same Primary gate the
/// inbound write path uses, resolves the inverse map, and Tells
/// the owning a
/// carrying the principal — fire-and-forget (the Part 9 ack already committed the local condition
/// state). Deliberately decoupled from the OpcUaServer NativeAlarmAck type: the host maps
/// NativeAlarmAck → this at the wiring boundary so Runtime does not depend on the OPC UA layer.
///
/// The folder-scoped condition NodeId the operator acknowledged.
/// Operator-supplied comment forwarded to the upstream alarm system; null when none.
/// The authenticated principal performing the acknowledge.
public sealed record RouteNativeAlarmAck(string ConditionNodeId, string? Comment, string OperatorUser);
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
// Convenience accessors for sites that don't need the full spec.
/// Gets the driver type name from the child's spec.
public string DriverType => Spec.DriverType;
/// Gets the last applied driver configuration JSON from the child's spec.
public string LastConfigJson => Spec.DriverConfig;
/// Gets the per-instance ResilienceConfig JSON the child's invoker was built with (null = tier defaults).
public string? ResilienceConfig => Spec.ResilienceConfig;
}
/// Gets or sets the timer scheduler for this actor's config-db retry and rediscovery timers.
public ITimerScheduler Timers { get; set; } = null!;
public sealed class RetryConfigDbConnection
{
public static readonly RetryConfigDbConnection Instance = new();
private RetryConfigDbConnection() { }
}
/// Creates props for a DriverHostActor with the specified dependencies.
/// Database context factory for configuration database access.
/// The local cluster node identifier.
/// Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// MeshTransport:Mode = ClusterClient the node's NodeCommunicationActor, which relays
/// across the boundary. Null publishes on the deployment-acks DPS topic.
/// Optional driver factory; defaults to null factory if not provided.
/// Optional set of roles assigned to the local node.
/// Optional actor reference for dependency multiplexing.
/// Optional actor reference for OPC UA publishing.
/// Optional driver-health publisher; defaults to
/// so test harnesses and smoke fixtures don't need to wire it.
/// Optional evaluator handed to the spawned
/// 's children; defaults to
/// (the dev/Mac path where no expression is evaluated). Production passes the DI-resolved
/// Roslyn evaluator.
/// Optional sink handed to the spawned
/// for VirtualTag results whose plan opted into Historize=true; defaults to
/// (the durable AVEVA sink is infra-gated, so no live-data historian
/// write RPC exists). A deployment that binds a real in DI overrides it.
/// Test seam: when supplied, this actor is used as the
/// VirtualTag host instead of spawning a real child, so tests
/// can intercept the message. Null in
/// production (the real host is spawned).
/// Retained for constructor-signature stability; unused since Phase 4
/// Task 9 retired the EfAlarmConditionStateStore fallback that consumed it. Defaults to null.
/// Optional root script logger required to spawn the ScriptedAlarm
/// host (the engine + its script logging hang off it). When null the ScriptedAlarm host is left
/// unspawned — the graceful dev/None-deployment path.
/// Test seam: when supplied, this actor is used as the
/// ScriptedAlarm host instead of spawning a real child, so
/// tests can intercept the message. Null
/// in production (the real host is spawned).
/// Optional Phase 6.1 resilience-invoker factory used to attach an
/// to each spawned driver; defaults to
/// (pass-through) when not supplied.
/// Per-cluster mesh Phase 4: scripted-alarm condition-state store. On a
/// driver-role node this is the replicated LocalDb store, so condition state persists with no
/// ConfigDb; null skips spawning the alarm host (the ConfigDb-backed EF fallback was retired in
/// Phase 4 Task 9). Defaults to null.
/// Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
/// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
/// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).
/// The Akka.NET used to spawn this actor.
public static Props Props(
IDbContextFactory? dbFactory,
CommonsNodeId localNode,
IActorRef? ackRouter = null,
IDriverFactory? driverFactory = null,
IReadOnlySet? localRoles = null,
IActorRef? dependencyMux = null,
IActorRef? opcUaPublishActor = null,
IDriverHealthPublisher? healthPublisher = null,
IVirtualTagEvaluator? virtualTagEvaluator = null,
IHistoryWriter? historyWriter = null,
IActorRef? virtualTagHostOverride = null,
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null,
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
IAlarmStateStore? alarmStateStore = null,
ITelemetryLocalHub? telemetryHub = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
// the wrong dependency at runtime. New parameters go LAST in all three places — this
// signature, the constructor's, and this call — and nothing else moves.
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, ackRouter, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub));
/// Initializes a new DriverHostActor with the specified dependencies.
/// Database context factory for configuration database access.
/// The local cluster node identifier.
/// Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// MeshTransport:Mode = ClusterClient the node's NodeCommunicationActor, which relays
/// across the boundary. Null publishes on the deployment-acks DPS topic.
/// Optional driver factory; defaults to null factory if not provided.
/// Optional set of roles assigned to the local node.
/// Optional actor reference for dependency multiplexing.
/// Optional actor reference for OPC UA publishing.
/// Optional driver-health publisher; defaults to .
/// Optional evaluator handed to the VirtualTag host's children;
/// defaults to .
/// Optional sink handed to the spawned
/// for historized VirtualTag results; defaults to .
/// Test seam: when supplied, used as the VirtualTag host
/// instead of spawning a real child.
/// Retained for constructor-signature stability; unused since Phase 4
/// Task 9 retired the EfAlarmConditionStateStore fallback that consumed it. Defaults to null.
/// Optional root script logger required to spawn the ScriptedAlarm
/// host; when null the host is left unspawned.
/// Test seam: when supplied, used as the ScriptedAlarm host
/// instead of spawning a real child.
/// Phase 6.1 resilience-invoker factory used to attach an
/// to each spawned driver; defaults to
/// (pass-through) when null.
/// Test seam (archreview 03/S4): overrides the count of Up
/// driver-role cluster members the Primary gate reads while the role is unknown. When null the
/// default reads Cluster.Get(Context.System).State.Members (0 on a non-cluster ActorRefProvider).
/// Optional node-local cache of applied deployment artifacts.
/// When supplied, each successful apply stores its artifact so the node can boot from its
/// last-known-good configuration while central SQL is unreachable. Null on admin-only nodes and in
/// tests that do not exercise the cache — caching is then simply skipped.
/// Per-cluster mesh Phase 4: scripted-alarm condition-state store. On a
/// driver-role node this is the replicated LocalDb store; null skips spawning the alarm host (the
/// ConfigDb-backed EF fallback was retired in Phase 4 Task 9).
/// Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
/// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
/// VirtualTag + ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.
public DriverHostActor(
IDbContextFactory? dbFactory,
CommonsNodeId localNode,
IActorRef? ackRouter,
IDriverFactory? driverFactory = null,
IReadOnlySet? localRoles = null,
IActorRef? dependencyMux = null,
IActorRef? opcUaPublishActor = null,
IDriverHealthPublisher? healthPublisher = null,
IVirtualTagEvaluator? virtualTagEvaluator = null,
IHistoryWriter? historyWriter = null,
IActorRef? virtualTagHostOverride = null,
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null,
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
IAlarmStateStore? alarmStateStore = null,
ITelemetryLocalHub? telemetryHub = null)
{
_telemetryHub = telemetryHub;
_deploymentArtifactCache = deploymentArtifactCache;
_redundancyRoleView = redundancyRoleView;
_replicationPeerHost = replicationPeerHost;
_fetchAndCacheMode = fetchAndCacheMode;
_artifactFetcher = artifactFetcher;
_alarmStateStore = alarmStateStore;
_dbFactory = dbFactory;
_localNode = localNode;
_ackRouter = ackRouter;
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
_driverMemberCountProvider = driverMemberCountProvider;
_localRoles = localRoles ?? new HashSet(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
_healthPublisher = healthPublisher ?? NullDriverHealthPublisher.Instance;
_virtualTagEvaluator = virtualTagEvaluator ?? NullVirtualTagEvaluator.Instance;
_historyWriter = historyWriter ?? NullHistoryWriter.Instance;
_virtualTagHostOverride = virtualTagHostOverride;
_scriptRootLogger = scriptRootLogger;
_scriptedAlarmHostOverride = scriptedAlarmHostOverride;
// Default behavior is Steady — PreStart may flip to Stale or replay an orphan apply.
Become(Steady);
}
///
protected override void PreStart()
{
// Resolve the cluster DPS mediator once, on the actor thread, and reuse it for every subscribe +
// the Primary-gated native-alarm alerts fan-out in ForwardNativeAlarm (mirrors ScriptedAlarmHostActor).
_mediator = DistributedPubSub.Get(Context.System).Mediator;
// Subscribe to deployments topic so the coordinator's broadcast lands here.
_mediator.Tell(new Subscribe(DeploymentsTopic, Self));
// Subscribe to driver-control topic so AdminUI Reconnect/Restart commands land here.
_mediator.Tell(new Subscribe(DriverControlTopic, Self));
// Subscribe to the redundancy-state topic so cluster role changes cache this node's role — the
// inbound-write gate in HandleRouteNodeWrite reuses the SAME signal the scripted-alarm emit gate
// uses so only the Primary services operator writes (the secondary keeps state warm for failover).
_mediator.Tell(
new Subscribe(ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RedundancyStateTopic, Self));
// Per-cluster mesh Phase 2 dark switch. Under MeshTransport:Mode = ClusterClient these same
// commands arrive from NodeCommunicationActor on the node-local EventStream instead of the
// mesh-wide DPS topic. Subscribing to BOTH unconditionally is deliberate: exactly one of the
// two ever publishes, so the transport flag lives entirely on the publishing side and the
// switch can be flipped (or reverted) without touching any subscriber.
Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment));
Context.System.EventStream.Subscribe(Self, typeof(RestartDriver));
Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver));
// Spawn the VirtualTag host BEFORE Bootstrap so the bootstrap-restore path (which routes
// through PushDesiredSubscriptions and Tells ApplyVirtualTags) has a live host to target.
SpawnVirtualTagHost();
// Same rationale for the ScriptedAlarm host — the bootstrap-restore path Tells it
// ApplyScriptedAlarms through the same PushDesiredSubscriptions pass.
SpawnScriptedAlarmHost();
Bootstrap();
}
///
/// Spawns the single child that owns the Equipment-namespace
/// VirtualTagActors and bridges their results onto the OPC UA publish actor. A test-supplied
/// virtualTagHostOverride short-circuits the spawn so a probe can intercept
/// . The real host requires a non-null
/// (its ctor throws otherwise), so when no publish actor is
/// wired (legacy ControlPlane test harnesses with no OPC UA sink) the host is left null and
/// ApplyVirtualTags becomes a no-op — VirtualTags can't have anywhere to publish without it.
///
private void SpawnVirtualTagHost()
{
if (_virtualTagHostOverride is not null)
{
_virtualTagHost = _virtualTagHostOverride;
return;
}
if (_opcUaPublishActor is null)
{
_log.Debug("DriverHost {Node}: no OPC UA publish actor wired; skipping VirtualTag host spawn", _localNode);
return;
}
_virtualTagHost = Context.ActorOf(
VirtualTagHostActor.Props(_opcUaPublishActor, _dependencyMux, _virtualTagEvaluator, _historyWriter, _telemetryHub),
"virtual-tag-host");
}
///
/// Spawns the single child that owns the per-node
/// , feeds it live tag values from the dependency mux, and
/// bridges its emissions onto the OPC UA publish actor + the cluster alerts topic. A
/// test-supplied scriptedAlarmHostOverride short-circuits the spawn so a probe can
/// intercept . The real host needs
/// both a non-null (the emission sink) and a non-null
/// (the engine + its script logging hang off it); when either
/// is missing (legacy ControlPlane test harnesses, dev/None deployments) the host is left
/// null and ApplyScriptedAlarms becomes a no-op. The engine is built around a fresh
/// + the wired (the
/// replicated LocalDbAlarmConditionStateStore on a driver-role node); when no store was
/// wired the host is skipped. The host (spawned as a child) owns + disposes the engine in its
/// PostStop, so it stops with the driver host.
///
private void SpawnScriptedAlarmHost()
{
if (_scriptedAlarmHostOverride is not null)
{
_scriptedAlarmHost = _scriptedAlarmHostOverride;
return;
}
if (_opcUaPublishActor is null || _scriptRootLogger is null)
{
_log.Debug(
"DriverHost {Node}: skipping ScriptedAlarm host spawn (no publish actor / root logger)",
_localNode);
return;
}
// Per-cluster mesh Phase 4: condition state lives in the wired store — the replicated LocalDb
// store on a driver-role node, which persists with no ConfigDb. The ConfigDb-backed EF fallback
// was retired (Task 9); when no store was wired there is nowhere to persist condition state, so
// we skip the alarm host outright rather than run the engine against a store it can never save to
// (mirrors the _opcUaPublishActor-null skip above).
if (_alarmStateStore is null)
{
_log.Debug(
"DriverHost {Node}: skipping ScriptedAlarm host spawn (no condition-state store)",
_localNode);
return;
}
var store = _alarmStateStore;
var upstream = new DependencyMuxTagUpstreamSource();
var engine = new ScriptedAlarmEngine(
upstream, store, new ScriptLoggerFactory(_scriptRootLogger.Logger), _scriptRootLogger.Logger);
_scriptedAlarmHost = Context.ActorOf(
ScriptedAlarmHostActor.Props(
_opcUaPublishActor, _dependencyMux, upstream, engine, _localNode, telemetryHub: _telemetryHub),
"scripted-alarm-host");
}
private void Bootstrap()
{
// Per-cluster mesh Phase 3: a FetchAndCache node NEVER reads central SQL for config, at boot
// or otherwise. It restores its served state from the LocalDb pointer instead.
if (_fetchAndCacheMode)
{
BootstrapFromCache();
return;
}
// Belt-and-suspenders (Phase 4): every ConfigDb read below is on the Direct path only — a
// FetchAndCache node already returned via BootstrapFromCache above, and only a FetchAndCache node
// runs DB-less. If a null factory ever reaches a Direct path it is a misconfiguration; enter
// Steady with no revision rather than NRE, and let the first dispatch drive the apply.
if (_dbFactory is null)
{
_log.Info("DriverHost {Node}: no ConfigDb on the Direct bootstrap path; entering Steady (no revision)", _localNode);
Become(Steady);
return;
}
// Read the most-recent NodeDeploymentState for this node; if it's Applied, jump
// to Steady with that revision. If Applying (orphan from a crash), discard and replay.
// If the DB is unreachable, fall back to Stale and start the reconnect loop.
try
{
using var db = _dbFactory.CreateDbContext();
var latest = db.NodeDeploymentStates
.Where(s => s.NodeId == _localNode.Value)
.OrderByDescending(s => s.StartedAtUtc)
.Select(s => new { s.DeploymentId, s.Status, s.StartedAtUtc })
.FirstOrDefault();
if (latest is null)
{
_log.Info("DriverHost {Node}: no prior deployments; entering Steady (no revision)", _localNode);
Become(Steady);
return;
}
var deployment = db.Deployments
.AsNoTracking()
.FirstOrDefault(d => d.DeploymentId == latest.DeploymentId);
var revision = deployment is null
? (RevisionHash?)null
: RevisionHash.Parse(deployment.RevisionHash);
switch (latest.Status)
{
case NodeDeploymentStatus.Applied:
_currentRevision = revision;
_log.Info("DriverHost {Node}: recovered Applied state at rev {Rev}", _localNode, revision);
Become(Steady);
// The revision is recovered but the in-memory driver children + OPC UA address
// space were lost on restart. Re-spawn + re-materialise + re-subscribe from the
// applied deployment so a restarted/rebuilt node restores its served state instead
// of waiting for a config change (whose identical-config revision would no-op).
RestoreApplied(new DeploymentId(latest.DeploymentId), revision);
break;
case NodeDeploymentStatus.Applying:
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying",
_localNode, latest.DeploymentId);
if (revision is not null) ApplyAndAck(new DeploymentId(latest.DeploymentId), revision.Value, CorrelationId.NewId());
else Become(Steady);
break;
case NodeDeploymentStatus.Failed:
default:
_log.Info("DriverHost {Node}: prior deployment {Id} failed; entering Steady at last known rev",
_localNode, latest.DeploymentId);
Become(Steady);
break;
}
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: ConfigDb unreachable on bootstrap; entering Stale", _localNode);
// Central is unreachable, so try the node-local cache before giving up. On a hit this
// node serves its last-known-good configuration through the outage instead of coming up
// with an empty address space. On a miss, behaviour is exactly what it always was.
if (!TryBootFromCache())
Become(Stale);
}
}
///
/// Per-cluster mesh Phase 3 boot for FetchAndCache mode. Restores served state from the
/// LocalDb pointer without any central-SQL read.
///
///
///
/// Distinct from (the Direct-mode SQL-unreachable fallback):
/// reading from the cache is normal operation here, not a degraded state, so
/// stays — the node can still
/// fetch new deployments. An empty or unreadable cache lands Steady-with-no-revision (the
/// first dispatch fetches), never Stale: Stale means "central SQL is down, retry it",
/// and FetchAndCache has no config SQL read to recover.
///
///
private void BootstrapFromCache()
{
try
{
var cached = _deploymentArtifactCache?.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
if (cached is null)
{
_log.Info(
"DriverHost {Node}: FetchAndCache boot — the LocalDb cache is empty; entering Steady " +
"(no revision). The first dispatch will fetch this node's configuration from central.",
_localNode);
Become(Steady);
return;
}
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
_currentRevision = revision;
Become(Steady);
// Restore drivers + address space + subscriptions from the cached bytes — no SQL read, no
// re-ack (the deployment is already Applied). Same in-hand-bytes apply as the Direct-mode
// cache boot, but this is the steady-state config source, not an outage fallback.
ApplyCachedArtifact(deploymentId, cached.Artifact);
_log.Info(
"DriverHost {Node}: FetchAndCache boot — restored served state for deployment {Id} " +
"(rev {Rev}) from the LocalDb pointer.",
_localNode, deploymentId, revision);
}
catch (Exception ex)
{
_log.Warning(ex,
"DriverHost {Node}: FetchAndCache boot — failed to read the LocalDb pointer; entering " +
"Steady (no revision). The next dispatch will fetch.",
_localNode);
Become(Steady);
}
}
///
/// Last-resort boot path: apply the artifact cached by a previous successful deploy (or
/// replicated from this node's pair peer) when central SQL cannot be reached.
///
///
/// when a cached artifact was applied and the actor is now Steady;
/// when the caller should fall through to Stale.
///
///
///
/// The read is unkeyed. The cache is keyed by ClusterId so a pair shares one
/// entry, but ClusterId is only derivable from an artifact you already hold or from the
/// central DB — and at this seam we have neither. So the newest pointer row wins, which
/// is correct in every real topology because a node belongs to one cluster and its peer
/// replicates that same cluster's row. A node re-homed between clusters is the only
/// ambiguous case, and the cache logs a warning naming both.
///
///
/// This does not make the node current. _currentRevision is set from the
/// cached artifact, so a subsequent dispatch of that same revision correctly no-ops,
/// while any NEW revision still requires central — the cache holds the past, not the
/// future. Dispatch handling is unchanged.
///
///
/// Never throws: a fault here must degrade to Stale, which is exactly where the node
/// would have been without a cache at all.
///
///
private bool TryBootFromCache()
{
if (_deploymentArtifactCache is null)
return false;
try
{
var cached = _deploymentArtifactCache.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
if (cached is null)
{
_log.Info(
"DriverHost {Node}: no cached deployment available; entering Stale with no configuration.",
_localNode);
return false;
}
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
// Steady, not Stale: this node is serving a real configuration. The retry-db timer that
// Stale would have started does not run here, so recovery rides on the next dispatch —
// matching how a normally-booted node behaves.
_currentRevision = revision;
Become(Steady);
ApplyCachedArtifact(deploymentId, cached.Artifact);
_isRunningFromCache = true;
_log.Warning(
"DriverHost {Node}: RUNNING FROM CACHE — central ConfigDb is unreachable, so this node " +
"booted deployment {Id} (rev {Rev}) cached at {CachedAtUtc:o} from its local database. " +
"Configuration changes cannot be applied until the ConfigDb is reachable again.",
_localNode, deploymentId, revision, cached.AppliedAtUtc);
return true;
}
catch (Exception ex)
{
_log.Error(ex,
"DriverHost {Node}: failed to boot from the local deployment cache; falling back to Stale.",
_localNode);
return false;
}
}
///
/// Applies an artifact already in hand — no ConfigDb read, no ACK.
///
///
/// Mirrors , but sources the artifact from the cache rather than
/// re-reading it from a database that is by definition unreachable here. It also skips
/// UpsertNodeDeploymentState and SendAck for the same reason: both write to or
/// depend on central.
///
private void ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
{
var correlation = CorrelationId.NewId();
ReconcileDriversFromBlob(deploymentId, blob);
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
// definition — and the rebuild would no-op, leaving OPC UA clients browsing an empty server
// while the drivers below poll happily. (Found on the docker-dev live gate.)
_opcUaPublishActor?.Tell(
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, blob));
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
}
private void Steady()
{
Receive(HandleDispatchFromSteady);
Receive(HandleGetDiagnostics);
Receive(ForwardToMux);
Receive(ForwardNativeAlarm);
Receive(OnDriverConnectivityChanged);
Receive(HandleDeltaApplied);
Receive(HandleApplyResult);
Receive(HandleRestartDriver);
Receive(HandleReconnectDriver);
Receive(HandleRouteNodeWrite);
Receive(HandleRouteNativeAlarmAck);
Receive(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive(_ => { });
Receive(_ => { /* PubSub ack */ });
}
private void Applying()
{
Receive(msg =>
{
if (_applyingDeploymentId is not null && msg.DeploymentId == _applyingDeploymentId.Value)
{
_log.Debug("DriverHost {Node}: duplicate DispatchDeployment for in-flight {Id}; ignoring",
_localNode, msg.DeploymentId);
return;
}
_log.Info("DriverHost {Node}: dispatch for {Id} received while still applying {Cur}; deferring",
_localNode, msg.DeploymentId, _applyingDeploymentId);
Self.Forward(msg); // re-deliver after we transition back
});
// The FetchAndCache fetch pipes its result back here (BeginFetchAndCacheApply ran in Steady,
// then Become(Applying)); completing the apply transitions back to Steady.
Receive(HandleFetchedForApply);
Receive(HandleGetDiagnostics);
Receive(ForwardToMux);
Receive(ForwardNativeAlarm);
Receive(OnDriverConnectivityChanged);
Receive(HandleDeltaApplied);
Receive(HandleApplyResult);
Receive(HandleRestartDriver);
Receive(HandleReconnectDriver);
Receive(HandleRouteNodeWrite);
Receive(HandleRouteNativeAlarmAck);
Receive(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive(_ => { });
Receive(_ => { /* PubSub ack */ });
}
private void ForwardToMux(DriverInstanceActor.AttributeValuePublished msg)
{
// Pass driver-published values to the dependency mux when one is wired (VirtualTag inputs,
// keyed by FullReference). Without a mux, VirtualTagActor evaluation can't fire — that's the
// dev/Mac path (no virtual tags registered); production binds the mux via the RuntimeActors
// extension. KEEP this unchanged — VirtualTag inputs are still keyed by FullReference.
_dependencyMux?.Tell(msg);
if (_opcUaPublishActor is null) return;
// Route the value to the OPC UA sink at the variable's ACTUAL NodeId. Equipment-tag variables
// are materialised with folder-scoped NodeIds (EquipmentId/FolderPath/Name), while the driver
// publishes keyed by its wire-ref FullName (FullReference). The _nodeIdByDriverRef map — built
// each apply from the composition's EquipmentTags — resolves (DriverInstanceId, FullName) to
// the folder-scoped NodeId(s) the materialiser placed the variable(s) at, so the value lands
// instead of leaving the variable at BadWaitingForInitialData. One driver ref can back several
// equipment variables (identical machines sharing a register), hence the fan-out.
if (_nodeIdByDriverRef.TryGetValue((msg.DriverInstanceId, msg.FullReference), out var nodeIds))
{
// v3 Batch 4 single-source fan-out: one driver publish for a RawPath lands on the raw NodeId AND
// every referencing UNS NodeId, each with its realm, carrying IDENTICAL value/quality/timestamp.
foreach (var n in nodeIds)
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AttributeValueUpdate(
n.NodeId, msg.Value, msg.Quality, msg.TimestampUtc, n.Realm));
}
else
{
_log.Debug("DriverHost {Node}: no bound NodeId for ({Driver},{Ref}) — value dropped",
_localNode, msg.DriverInstanceId, msg.FullReference);
}
}
///
/// Routes a native alarm transition (published by a driver child as
/// ) to its materialised Part 9 condition
/// node(s). The alarm path analogue of : the transition's ConditionId
/// (the dotted alarm full-reference, which equals the authored equipment-tag FullName — NOT the bare
/// SourceNodeId owning object) is resolved by the map —
/// built each apply from the alarm-bearing EquipmentTags — to the folder-scoped condition NodeId(s)
/// the materialiser placed the condition(s) at.
/// For each node the projects the transition delta into a full
/// AlarmConditionSnapshot, then this Tells
/// — the SAME message scripted alarms use, so it routes through WriteAlarmCondition. An
/// unknown ref is Debug-logged and dropped (mirrors the value drop).
///
///
/// Each transition is ALSO published as an to the cluster
/// alerts topic — the single historization + live /alerts fan-out path, exactly as
/// scripted alarms do (ScriptedAlarmHostActor.OnEngineEmission). The OPC UA condition
/// write above stays UNGATED (a Secondary keeps its address space warm for failover), but the
/// cluster-wide alerts publish is Primary-gated on the same redundancy
/// signal the inbound-write gate uses — only the Primary publishes the single fleet-wide copy.
///
///
///
/// #477 — a child driver's connectivity transition. Annotates the source-data Quality of EVERY native
/// alarm condition the driver owns: comms lost → , restored →
/// . This is the ONLY signal for a comms-lost native source, because a
/// disconnected driver emits no alarm transitions — the alarm feed goes silent, so without this a
/// comms-lost condition would keep reporting the accidentally-Good default forever.
///
/// UNGATED by redundancy role (like the condition write in ): a
/// Secondary keeps its address space — including condition quality — warm for failover. Quality is
/// a pure annotation: WriteAlarmQuality touches ONLY the condition's Quality, never its
/// Active/Acked/Retain, and fires a Part 9 event only on a real quality-bucket change. No cluster
/// alerts row is published here — driver comms health has its own status surface
/// (); a row per condition would be alarm-fatigue.
///
///
private void OnDriverConnectivityChanged(DriverInstanceActor.ConnectivityChanged msg)
{
if (_opcUaPublishActor is null) return;
var quality = msg.Connected ? OpcUaQuality.Good : OpcUaQuality.Bad;
var ts = DateTime.UtcNow;
var annotated = 0;
// Fan out to every condition this driver owns. _alarmNodeIdByDriverRef is keyed by
// (DriverInstanceId, RawPath); one driver ref can back several condition NodeIds (identical machines).
foreach (var ((driverId, _), nodeIds) in _alarmNodeIdByDriverRef)
{
if (driverId != msg.DriverInstanceId) continue;
foreach (var n in nodeIds)
{
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmQualityUpdate(
n.NodeId, quality, ts, n.Realm));
annotated++;
}
}
if (annotated > 0)
_log.Debug("DriverHost {Node}: driver {Driver} {State} — annotated {Count} native condition(s) {Quality}",
_localNode, msg.DriverInstanceId, msg.Connected ? "connected" : "disconnected", annotated, quality);
}
private void ForwardNativeAlarm(DriverInstanceActor.AttributeAlarmPublished msg)
{
if (_opcUaPublishActor is null) return;
// Resolve on ConditionId, NOT SourceNodeId: for Galaxy the dotted alarm full-reference — which
// equals the authored equipment-tag FullName the map is keyed by — is carried in ConditionId
// (AlarmFullReference), while SourceNodeId is the bare owning object (SourceObjectReference).
if (!_alarmNodeIdByDriverRef.TryGetValue((msg.DriverInstanceId, msg.Args.ConditionId), out var nodeIds))
{
_log.Debug("DriverHost {Node}: no alarm condition for ({Driver},{Ref}) — transition dropped",
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
return;
}
// PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write
// below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary
// publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver
// cluster while the role is unknown (default-deny closes the dual-primary boot window that would
// otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied.
var serviceAlertsAsPrimary = ShouldServiceAsPrimary();
if (!serviceAlertsAsPrimary)
{
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair("site", "alarm-emit"),
new KeyValuePair("reason", PrimaryGateDenyReason()));
_log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary",
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
}
foreach (var n in nodeIds)
{
var nodeId = n.NodeId;
var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args);
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate(
nodeId, snapshot, msg.Args.SourceTimestampUtc, n.Realm));
// Only the Primary publishes the cluster-wide alerts transition (see the gate above).
if (!serviceAlertsAsPrimary) continue;
var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null,
ReferencingEquipmentPaths: (IReadOnlyList)Array.Empty());
var alertEvent = new AlarmTransitionEvent(
AlarmId: nodeId,
EquipmentPath: meta.EquipmentId,
AlarmName: meta.Name,
TransitionKind: ToEventKind(msg.Args.Kind),
// The projector mapped the four-bucket AlarmSeverity onto the OPC UA 1..1000 scale already;
// reuse its ushort so the condition node + the alerts row agree on severity.
Severity: snapshot.Severity,
Message: msg.Args.Message,
// Native transitions are device-driven; an operator comment only rides along on an
// acknowledge from the upstream alarm system. "device" marks a non-operator origin.
User: msg.Args.OperatorComment is null ? string.Empty : "device",
TimestampUtc: msg.Args.SourceTimestampUtc,
AlarmTypeName: meta.AlarmType,
Comment: msg.Args.OperatorComment,
// Per-condition opt-out parsed from TagConfig.alarm.historizeToAveva (bool?, null ⇒ absent ⇒
// historize). The HistorianAdapterActor gate (historizeToAveva is not false) historizes null +
// true and suppresses the durable AVEVA row only on an explicit false — the same posture as the
// scripted-alarm opt-out. null here rides through unchanged (the gate treats it as default-on).
HistorizeToAveva: meta.HistorizeToAveva,
// WP4: the Area/Line/Equipment UNS paths that reference this raw condition — the /alerts row
// renders them as the equipment list (empty for a raw condition no equipment references yet).
ReferencingEquipmentPaths: meta.ReferencingEquipmentPaths);
_mediator.Tell(new Publish(ScriptedAlarmHostActor.AlertsTopic, alertEvent));
// Phase 5: fan the same transition into the node-local live-telemetry hub (no-op until a gRPC
// client subscribes). The DPS publish above is unchanged — a strictly additive tap, riding the
// same Primary gate (only reached when serviceAlertsAsPrimary).
_telemetryHub?.Emit(new TelemetryItem.Alarm(alertEvent));
}
}
/// Maps a native onto the canonical alarm event-kind
/// vocabulary scripted alarms emit (the EmissionKind names) so a native row renders with the
/// correct chip on the /alerts page and historizes into the same EventKind column as
/// scripted alarms. An unmapped/unknown transition surfaces as Activated (visible) rather than
/// a grey unknown label.
private static string ToEventKind(AlarmTransitionKind kind) => kind switch
{
AlarmTransitionKind.Raise or AlarmTransitionKind.Retrigger => "Activated",
AlarmTransitionKind.Clear => "Cleared",
AlarmTransitionKind.Acknowledge => "Acknowledged",
_ => "Activated",
};
///
/// Routes an inbound operator write (Asked from the OPC UA node-manager side) to the
/// owning driver child. Order matters:
///
/// - PRIMARY gate FIRST — reuses the same redundancy signal the
/// scripted-alarm emit gate uses. Only the Primary services writes (default-allow while the
/// role is unknown, so single-node deploys + the boot window never reject). A Secondary/Detached
/// node keeps its address space + driver state warm for failover but must NOT push writes to the
/// shared field device.
/// - Resolve the reverse map to the owning
/// (DriverInstanceId, FullName).
/// - Resolve the running driver child.
/// - Ask the child a bounded of the driver-side
/// FullName and pipe the translated
/// result back to the asker.
///
/// Every branch replies the asker a exactly once.
///
private void HandleRouteNodeWrite(RouteNodeWrite msg)
{
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN
// Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a
// multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is
// still serviced. The rejection rides the established optimistic-write self-correction surface (the
// node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued.
if (!ShouldServiceAsPrimary())
{
var reason = _localRole is null ? "not primary (role unknown)" : "not primary";
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair("site", "write"),
new KeyValuePair("reason", PrimaryGateDenyReason()));
Sender.Tell(new NodeWriteResult(false, reason));
return;
}
// v3 Batch 4 (review H1): the node manager's write hook passes the FULL ns-qualified NodeId string
// (node.NodeId.ToString(), e.g. "ns=3;s=") PLUS the realm it resolved from the namespace index.
// The routing map is keyed by (realm, BARE s= identifier). Normalise the id to its bare form and look
// up (realm, bareId) — the realm disambiguates a raw RawPath from a UNS path that happen to share a
// bare string, so a write always routes to its OWN driver ref (never a colliding sibling's).
var bareNodeId = BareNodeId(msg.NodeId);
if (!_driverRefByNodeId.TryGetValue((msg.Realm, bareNodeId), out var target))
{
Sender.Tell(new NodeWriteResult(false, $"no driver mapping for node {msg.NodeId} ({msg.Realm})"));
return;
}
if (!_children.TryGetValue(target.DriverInstanceId, out var entry))
{
Sender.Tell(new NodeWriteResult(false, "driver not running"));
return;
}
// Ask the child a bounded write — DriverInstanceActor.HandleWriteAsync already bounds the backend
// call to 5s, so the 8s Ask is a safety net for an actor that never replies. Capture Sender before
// the continuation: it runs off the actor thread, where raw Sender is unsafe to read.
var replyTo = Sender;
entry.Actor.Ask(
new DriverInstanceActor.WriteAttribute(target.RawPath, msg.Value!), TimeSpan.FromSeconds(8))
.ContinueWith(
t => t.IsCompletedSuccessfully
? new NodeWriteResult(t.Result.Success, t.Result.Reason)
: new NodeWriteResult(false, "write timeout"),
CancellationToken.None,
TaskContinuationOptions.None,
TaskScheduler.Default)
.PipeTo(replyTo);
}
///
/// Normalise an OPC UA NodeId string to its BARE s= identifier. The node manager's inbound-write
/// hook passes the full ns-qualified form (NodeId.ToString() == "ns=<N>;s=<id>");
/// the routing maps are keyed by the bare id (the RawPath / UNS path). The SDK format always prefixes
/// "ns=N;s=" for a string identifier in a non-zero namespace, so the first ";s=" is the
/// delimiter (an id may itself contain ";s=" later — IndexOf takes the FIRST, which is the
/// namespace delimiter). A bare id (no prefix) passes through unchanged.
///
/// The (possibly ns-qualified) NodeId string.
/// The bare s= identifier.
internal static string BareNodeId(string nodeId)
{
if (string.IsNullOrEmpty(nodeId)) return nodeId;
var i = nodeId.IndexOf(";s=", StringComparison.Ordinal);
if (i >= 0) return nodeId[(i + 3)..];
if (nodeId.StartsWith("s=", StringComparison.Ordinal)) return nodeId[2..];
return nodeId;
}
///
/// Routes an inbound native-condition acknowledge (the host Tells this from the OPC UA node-manager
/// side when a client Acknowledges a NATIVE Part 9 condition) to the owning driver child. Mirrors
/// 's order + gating, but the ack is fire-and-forget (no reply):
/// the OPC UA Part 9 acknowledge already committed the local condition state and the driver's
/// returns no per-condition status.
///
/// - PRIMARY gate FIRST — reuses the same redundancy signal as the
/// inbound-write gate. Only the Primary pushes the ack to the shared upstream alarm system
/// (default-allow while the role is unknown). A Secondary/Detached node keeps its condition
/// state warm for failover but must NOT push the command. Drop (debug-log) on non-primary.
/// - Resolve the inverse map to the owning
/// (DriverInstanceId, FullName = ConditionId); an unmapped node is debug-logged + dropped
/// (no throw) — mirrors 's unknown-ref drop.
/// - Resolve the running driver child and Tell it a
/// carrying the wire-ref FullName + principal.
///
///
private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg)
{
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as
// the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role
// on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream
// alarm system is a swallowed failure worth surfacing).
if (!ShouldServiceAsPrimary())
{
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair("site", "ack"),
new KeyValuePair("reason", PrimaryGateDenyReason()));
if (_localRole is null)
{
_log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)",
_localNode, msg.ConditionNodeId);
}
else
{
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
_localNode, msg.ConditionNodeId);
}
return;
}
if (!_driverRefByAlarmNodeId.TryGetValue(msg.ConditionNodeId, out var target))
{
_log.Debug("DriverHost {Node}: no driver mapping for alarm condition node {Cond} — ack dropped",
_localNode, msg.ConditionNodeId);
return;
}
if (!_children.TryGetValue(target.DriverInstanceId, out var entry))
{
_log.Debug("DriverHost {Node}: driver {Driver} not running for alarm ack of {Cond} — dropped",
_localNode, target.DriverInstanceId, msg.ConditionNodeId);
return;
}
// Fire-and-forget: the OPC UA Part 9 ack already committed the local condition state, and the
// driver's AcknowledgeAsync surfaces no per-condition status, so there is nothing to reply. The
// driver correlates on ConditionId (= the authored alarm FullName the inverse map keyed on).
entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.RawPath, msg.Comment, msg.OperatorUser));
}
/// Whether this node should service a Primary-only data-plane operation right now — the single
/// gate decision (archreview 03/S4), delegating to with the live driver
/// member count.
/// true to service as Primary; false to deny.
private bool ShouldServiceAsPrimary() =>
PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount());
/// Count of Up cluster members carrying the driver role. Uses the injected test seam when
/// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster
/// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture.
/// The number of Up driver-role members, or 0 when the cluster extension is unavailable.
private int DriverMemberCount()
{
if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
try
{
return Akka.Cluster.Cluster.Get(Context.System).State.Members
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
}
catch (Exception)
{
return 0; // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture
}
}
/// Host part of a host:port node id.
private static string HostOf(NodeId nodeId)
{
var text = nodeId.ToString();
var colon = text.LastIndexOf(':');
return colon < 0 ? text : text[..colon];
}
/// The Primary-gate deny reason tag for the denial meter (secondary|detached|role-unknown).
private string PrimaryGateDenyReason() => _localRole switch
{
RedundancyRole.Secondary => "secondary",
RedundancyRole.Detached => "detached",
_ => "role-unknown",
};
/// Caches this node's from a cluster redundancy snapshot so the
/// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves
/// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// ScriptedAlarmHostActor.OnRedundancyStateChanged / OpcUaPublishActor.
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
if (local is not null)
{
_localRole = local.Role;
}
else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
{
// S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
// Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
// identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
_warnedSnapshotMissingLocalNode = true;
_log.Warning(
"DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
_localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
}
// Publish on EVERY snapshot, including ones that leave the cached role untouched, so a role
// that changes without this node being named still reaches the drain within one heartbeat.
//
// NOTE the different policy: the drain gate keys on the role ALONE and opens when it is
// unknown, where the data-plane gate above resolves an unknown role by member count and
// denies. Publishing ShouldServiceAsPrimary() here suspended the drain on every node of a
// multi-driver cluster that had no redundancy configured — see ShouldDrainAlarmHistory.
// Defer ONLY to the node that holds a copy of our rows. The redundancy election is
// cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop
// draining — that Primary is usually in another pair entirely, and cannot deliver our events.
var peerIsPrimary =
_replicationPeerHost is not null
&& msg.Nodes.Any(n => n.Role == RedundancyRole.Primary
&& HostOf(n.NodeId) == _replicationPeerHost);
_redundancyRoleView?.Publish(
PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary));
}
private void Stale()
{
Receive(_ =>
{
_log.Warning("DriverHost {Node}: ignoring DispatchDeployment while Stale (DB unreachable)", _localNode);
});
Receive(HandleGetDiagnostics);
Receive(_ => TryRecoverFromStale());
Receive(HandleRestartDriver);
Receive(HandleReconnectDriver);
Receive(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive(_ => { });
// An inbound operator write can't be serviced while the config DB is unreachable — fast-fail so the
// node-manager's bounded Ask gets an immediate clear status instead of dead-lettering into a timeout.
Receive(_ =>
Sender.Tell(new NodeWriteResult(false, "driver host stale (config DB unreachable)")));
// An inbound native-condition ack can't be serviced while Stale either (the alarm inverse map is
// empty until an apply runs). The ack is fire-and-forget (no reply), so just drop it with a log —
// the local OPC UA condition state already committed on the Part 9 ack.
Receive(msg =>
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Node2} while Stale (config DB unreachable)",
_localNode, msg.ConditionNodeId));
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive(_ => { });
// A late DeltaApplied (an apply completed just before the DB went Stale) — re-register the driver's
// mux adapter anyway; it simply re-reads the driver's current refs (harmless, no DB access).
Receive(HandleDeltaApplied);
Receive(HandleApplyResult);
Receive(_ => { /* PubSub ack */ });
Timers.StartPeriodicTimer("retry-db", RetryConfigDbConnection.Instance, ReconnectInterval);
}
private void HandleGetDiagnostics(GetDiagnostics msg)
{
var drivers = _children
.Select(kv => new DriverInstanceDiagnostics(
DriverInstanceId: Guid.Empty,
Name: kv.Key,
State: kv.Value.Stubbed ? "Stubbed" : "Spawned",
ConnectedDevices: 0,
FaultedDevices: 0,
LastChangeUtc: DateTime.UtcNow))
.ToArray();
var snapshot = new NodeDiagnosticsSnapshot(
NodeId: _localNode,
CurrentRevision: _currentRevision,
Drivers: drivers,
AsOfUtc: DateTime.UtcNow,
RunningFromCache: _isRunningFromCache);
Sender.Tell(snapshot);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
{
if (_currentRevision is { } cur && cur == msg.RevisionHash)
{
// Idempotent — already at this rev. Ack and stay Steady.
_log.Debug("DriverHost {Node}: dispatch {Id} matches current rev {Rev}; immediate ACK",
_localNode, msg.DeploymentId, msg.RevisionHash);
SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.CorrelationId);
return;
}
if (_fetchAndCacheMode)
{
BeginFetchAndCacheApply(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
return;
}
ApplyAndAck(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
}
///
/// Per-cluster mesh Phase 3 (FetchAndCache). Kicks off a gRPC artifact fetch and pipes
/// the result back to as a — the fetch is I/O
/// and MUST NOT block the mailbox for the fetch deadline, so it never runs as an awaited call
/// inside a receive. The apply itself completes synchronously on the actor thread in
/// , mirroring .
///
private void BeginFetchAndCacheApply(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
{
_applyingDeploymentId = deploymentId;
Become(Applying);
// Persist Applying row (idempotent on PK) — same crash-boundary marker as ApplyAndAck.
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applying, failureReason: null);
if (_artifactFetcher is null)
{
// Misconfiguration: FetchAndCache with no fetcher wired. Fail the apply rather than fetch
// nothing forever; the node keeps serving last-known-good.
const string reason = "FetchAndCache mode is set but no artifact fetcher is configured on this node";
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Failed, reason);
SendAck(deploymentId, ApplyAckOutcome.Failed, reason, correlation);
_log.Error("DriverHost {Node}: cannot apply {Id} — {Reason}", _localNode, deploymentId, reason);
_applyingDeploymentId = null;
Become(Steady);
return;
}
_log.Debug("DriverHost {Node}: fetching artifact for {Id} (rev {Rev}) from central over gRPC",
_localNode, deploymentId, revision);
// A faulted fetch task maps to null bytes — the #485 apply-failure path, exactly like an
// all-endpoints-down fetch — so a fault can never escape as an unhandled actor message.
_artifactFetcher.FetchAsync(deploymentId.ToString(), revision.Value, CancellationToken.None)
.PipeTo(
Self,
success: bytes => new FetchedForApply(deploymentId, revision, correlation, bytes),
failure: _ => new FetchedForApply(deploymentId, revision, correlation, Bytes: null));
}
///
/// Completes a FetchAndCache apply once the fetched bytes are in hand. Null bytes are an
/// apply FAILURE (keep last-known-good, do not advance the revision — #485); non-null bytes are
/// applied from hand (no ConfigDb read) and cached.
///
private void HandleFetchedForApply(FetchedForApply msg)
{
// Ignore a result for anything but the in-flight apply (a stale/duplicate pipe-back).
if (_applyingDeploymentId is null || msg.DeploymentId != _applyingDeploymentId.Value)
{
_log.Debug("DriverHost {Node}: dropping stale fetch result for {Id} (in-flight={Cur})",
_localNode, msg.DeploymentId, _applyingDeploymentId);
return;
}
using var span = OtOpcUaTelemetry.StartDeployApplySpan(msg.DeploymentId.ToString());
span?.SetTag("otopcua.node_id", _localNode.ToString());
span?.SetTag("otopcua.revision", msg.Revision.ToString());
span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
var sw = Stopwatch.StartNew();
try
{
// #485: null bytes = "no answer" (every endpoint unreachable / NotFound / SHA mismatch).
// Fail the apply and stay on the current revision so a re-dispatch actually retries; keep
// serving the last-known-good address space, drivers and subscriptions throughout.
var appliedBlob = ReconcileDriversFromBlob(msg.DeploymentId, msg.Bytes);
if (appliedBlob is null)
{
const string reason =
"the deployment artifact could not be fetched from central (all endpoints unreachable, NotFound, or SHA-256 mismatch); nothing was applied";
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, reason);
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, reason, msg.Correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, reason);
_log.Error(
"DriverHost {Node}: FetchAndCache apply of {Id} FAILED — {Reason}. Staying on revision {Rev} and continuing to serve it; re-dispatch once central is reachable",
_localNode, msg.DeploymentId, reason, _currentRevision);
return;
}
_currentRevision = msg.Revision;
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.Correlation);
// Rebuild the address space + push subscriptions FROM the fetched bytes so OpcUaPublishActor
// never reads central SQL (the whole point of FetchAndCache).
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(
msg.Correlation, msg.DeploymentId, appliedBlob));
PushDesiredSubscriptionsFromArtifact(msg.DeploymentId, appliedBlob);
// Cache the fetched bytes as this node's last-known-good (idempotent store; skips empty).
CacheAppliedArtifact(msg.DeploymentId, msg.Revision, appliedBlob);
_isRunningFromCache = false;
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "ack"));
_log.Info("DriverHost {Node}: applied fetched deployment {Id} (rev {Rev}, children={Count})",
_localNode, msg.DeploymentId, msg.Revision, _children.Count);
}
catch (Exception ex)
{
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, ex.Message);
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, ex.Message, msg.Correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, ex.Message);
_log.Error(ex, "DriverHost {Node}: FetchAndCache apply of {Id} failed", _localNode, msg.DeploymentId);
}
finally
{
OtOpcUaTelemetry.DeploymentApplyDurationSec.Record(sw.Elapsed.TotalSeconds);
_applyingDeploymentId = null;
Become(Steady);
}
}
/// Pipe-back carrier for a FetchAndCache fetch result (null bytes = no answer).
private sealed record FetchedForApply(
DeploymentId DeploymentId, RevisionHash Revision, CorrelationId Correlation, byte[]? Bytes);
private void ApplyAndAck(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
{
_applyingDeploymentId = deploymentId;
Become(Applying);
using var span = OtOpcUaTelemetry.StartDeployApplySpan(deploymentId.ToString());
span?.SetTag("otopcua.node_id", _localNode.ToString());
span?.SetTag("otopcua.revision", revision.ToString());
span?.SetTag("otopcua.correlation_id", correlation.ToString());
var sw = Stopwatch.StartNew();
// Persist Applying row (idempotent on PK).
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applying, failureReason: null);
try
{
var appliedBlob = ReconcileDrivers(deploymentId);
// Issue #486 — a null blob means the artifact could not be read (unreachable ConfigDb, or the
// empty bytes a missing row yields), so NOTHING was applied. Reporting Applied here used to
// advance _currentRevision too, and HandleDispatchFromSteady short-circuits on a revision
// match — so the node claimed a configuration it never applied AND could never be healed by
// re-dispatching that revision. Failing the apply keeps the revision where it was, which is
// what makes the retry land instead of being waved through. The node keeps serving its
// last-known-good address space, drivers and subscriptions throughout (issue #485) — this is
// about telling the truth, not about tearing anything down.
if (appliedBlob is null)
{
const string reason =
"the deployment artifact could not be read (ConfigDb unreachable, or the row is missing/empty); nothing was applied";
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Failed, reason);
SendAck(deploymentId, ApplyAckOutcome.Failed, reason, correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, reason);
_log.Error(
"DriverHost {Node}: apply of {Id} FAILED — {Reason}. Staying on revision {Rev} and continuing to serve it; re-dispatch once the ConfigDb is readable",
_localNode, deploymentId, reason, _currentRevision);
return;
}
_currentRevision = revision;
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null);
SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
// Trigger the OPC UA address-space rebuild so the local SDK reflects the new
// composition. The publish actor handles the load-compose-diff-apply pipeline; we
// just forward the same correlation id so the audit trail joins up.
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
// SubscribeBulk pass: hand each driver its desired tag references so live values flow into
// the just-rebuilt address space instead of staying BadWaitingForInitialData.
PushDesiredSubscriptions(deploymentId);
CacheAppliedArtifact(deploymentId, revision, appliedBlob);
// Reaching here means the artifact came from central, so the node is no longer serving a
// cache-sourced configuration. Note this only clears on a real apply: a dispatch of the
// revision already booted from cache short-circuits in HandleDispatchFromSteady without
// touching the ConfigDb, and the flag correctly stays set.
_isRunningFromCache = false;
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "ack"));
_log.Info("DriverHost {Node}: applied deployment {Id} (rev {Rev}, children={Count})",
_localNode, deploymentId, revision, _children.Count);
}
catch (Exception ex)
{
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Failed, ex.Message);
SendAck(deploymentId, ApplyAckOutcome.Failed, ex.Message, correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, ex.Message);
_log.Error(ex, "DriverHost {Node}: apply of {Id} failed", _localNode, deploymentId);
}
finally
{
OtOpcUaTelemetry.DeploymentApplyDurationSec.Record(sw.Elapsed.TotalSeconds);
_applyingDeploymentId = null;
Become(Steady);
}
}
///
/// Read the deployment artifact + reconcile the set of running
/// children. Spawn missing, ApplyDelta on config change, stop removed/disabled drivers.
/// When the artifact blob is empty (legacy ControlPlane tests, smoke fixtures) the reconcile is
/// SKIPPED outright — see the issue #485 guard below — and when the configured
/// can't materialise any of the requested types this is a no-op.
///
///
/// The artifact blob that was reconciled, or when it could not be
/// loaded at all.
///
///
/// Returning the blob rather than swallowing it is load-bearing for the artifact cache. This
/// method catches its own DB failures, logs a warning and returns WITHOUT rethrowing — so
/// proceeds to its success path, ACKs Applied and logs success
/// having spawned zero drivers. A cache write that trusted "the apply succeeded" would then
/// persist an empty artifact as this node's last-known-good configuration, and the node would
/// later boot from it into an empty address space. The caller distinguishes the two cases by
/// this return value.
///
private byte[]? ReconcileDrivers(DeploymentId deploymentId)
{
// Phase 4 belt-and-suspenders: this ConfigDb read is Direct-only (the FetchAndCache apply uses
// ReconcileDriversFromBlob with bytes in hand). A null factory can't read, so return null — the
// caller treats that as "artifact could not be read" (#486) and keeps last-known-good.
if (_dbFactory is null)
{
_log.Warning("DriverHost {Node}: no ConfigDb to load artifact for {Id}; skipping reconcile", _localNode, deploymentId);
return null;
}
byte[] blob;
try
{
using var db = _dbFactory.CreateDbContext();
blob = db.Deployments.AsNoTracking()
.Where(d => d.DeploymentId == deploymentId.Value)
.Select(d => d.ArtifactBlob)
.FirstOrDefault() ?? Array.Empty();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to load artifact for {Id}; skipping reconcile",
_localNode, deploymentId);
return null;
}
return ReconcileDriversFromBlob(deploymentId, blob);
}
///
/// The blob-processing half of , split out so the
/// FetchAndCache apply path (which already holds the fetched bytes) and the
/// boot-from-cache path can reconcile without a ConfigDb read.
///
/// The reconciled blob, or when it was empty (#485).
private byte[]? ReconcileDriversFromBlob(DeploymentId deploymentId, byte[]? blob)
{
blob ??= Array.Empty();
// Issue #485 — no bytes is no answer, not "a configuration with no drivers". Parsing zero bytes
// yields zero specs, and DriverSpawnPlanner then plans EVERY running child for StopChild: a missing
// deployment row (or a half-written one) silently takes this node's entire field I/O down while the
// apply still ACKs Applied. Nothing legitimate produces a zero-length blob — deleting the last
// driver still deploys a JSON document with empty arrays — so treat it exactly like the load
// failure above and leave the running children alone.
if (blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: artifact for {Id} read back empty; skipping reconcile and keeping the {Count} running driver(s)",
_localNode, deploymentId, _children.Count);
return null;
}
var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
var snapshots = _children.ToDictionary(
kv => kv.Key,
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
StringComparer.Ordinal);
var plan = DriverSpawnPlanner.Compute(snapshots, specs);
foreach (var id in plan.ToStop) StopChild(id);
foreach (var spec in plan.ToApplyDelta) ApplyChildDelta(spec);
foreach (var spec in plan.ToSpawn) SpawnChild(spec);
// v3 (WP7): a dependency-consumer driver's declared dependency refs change when its tags/scripts are
// edited — the adapter must re-register the NEW set on the mux. Newly-spawned adapters register in
// PreStart; an ApplyDelta'd driver's adapter is re-registered when the child reports it finished
// reinitialising (DriverInstanceActor.DeltaApplied → HandleDeltaApplied). That notification is
// sequenced AFTER ReinitializeAsync rebuilds DependencyRefs — a synchronous re-register HERE would
// re-read the STALE ref set (ApplyDelta runs asynchronously on the child), so it is deliberately NOT
// done inline.
return blob;
}
///
/// Store a successfully applied artifact in the node-local cache, so this node can boot from
/// it while central SQL is unreachable.
///
///
///
/// Never throws. By the time this runs the deployment is already recorded Applied
/// in central SQL and an Applied ACK has been sent to the coordinator. An exception
/// escaping here would unwind into 's catch and send a second,
/// contradictory Failed ACK for a deployment the fleet already believes is live.
///
///
/// An empty or unloadable blob is skipped, not cached. See
/// : a DB failure there degrades silently, so "the apply
/// succeeded" does not imply "a real configuration was applied". Caching an empty
/// artifact would make it this node's last-known-good and boot it into an empty address
/// space during the next outage — strictly worse than having no cache at all.
///
///
/// Runs synchronously on the actor thread. That matches every other DB call on this path
/// (all of which already block), and the alternative — piping the result back as a
/// self-message — would need a handler registered in Steady, Applying AND Stale or it
/// dead-letters after the Become(Steady) in the enclosing finally.
///
///
private void CacheAppliedArtifact(DeploymentId deploymentId, RevisionHash revision, byte[]? blob)
{
if (_deploymentArtifactCache is null)
return;
if (blob is null || blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: not caching deployment {Id} — its artifact was empty or could not " +
"be loaded, so it is not a usable last-known-good configuration.",
_localNode, deploymentId);
return;
}
try
{
// The node's ClusterId is carried inside the artifact (ClusterNode rows), not in config
// or on IClusterRoleInfo. A single-cluster or unscoped artifact has none, so it shares
// one sentinel key — the pair still converges on a single pointer row either way.
var scope = DeploymentArtifact.ResolveClusterScope(blob, _localNode.Value);
var clusterId = scope.Mode == ClusterFilterMode.ScopeTo && !string.IsNullOrWhiteSpace(scope.ClusterId)
? scope.ClusterId
: SingleClusterCacheKey;
_deploymentArtifactCache
.StoreAsync(clusterId, deploymentId.ToString(), revision.Value, blob)
.GetAwaiter()
.GetResult();
_log.Debug("DriverHost {Node}: cached deployment {Id} (rev {Rev}, cluster {ClusterId}, {Bytes} bytes)",
_localNode, deploymentId, revision, clusterId, blob.Length);
}
catch (Exception ex)
{
_log.Error(ex,
"DriverHost {Node}: failed to cache applied deployment {Id}. The apply itself succeeded " +
"and is unaffected; this node simply has no local fallback for that revision.",
_localNode, deploymentId);
}
}
///
/// Restore the served state for an already-applied deployment after a process restart.
/// recovers from NodeDeploymentState,
/// but the driver children and OPC UA address space are in-memory and gone after a restart —
/// so without this a restarted node serves an empty address space until the next config
/// change (and an identical-config redeploy no-ops on the unchanged revision). Re-spawns
/// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk.
/// No re-ack: the deployment is already Applied.
///
private void RestoreApplied(DeploymentId deploymentId, RevisionHash? revision)
{
var correlation = CorrelationId.NewId();
try
{
var appliedBlob = ReconcileDrivers(deploymentId);
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, appliedBlob));
PushDesiredSubscriptions(deploymentId);
// Populate the node-local cache from the artifact this restored node is now serving. The
// cache invariant is "holds what the node currently serves"; without this only a FRESH
// apply (ApplyAndAck) ever wrote it, so a node whose cache was lost — a wiped/fresh volume,
// a disk failure — would recover its served state here yet stay cache-less, unable to
// boot-from-cache on the NEXT central outage until some future new deploy happened to land.
// Replication does not heal that gap either: a peer's already-acked rows are pruned from its
// oplog, so a fully-wiped node is never back-filled. Re-caching on restore is what closes
// it. (Found on the docker-dev live gate.)
if (revision is { } rev)
CacheAppliedArtifact(deploymentId, rev, appliedBlob);
_log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to restore served state for {Id} on bootstrap", _localNode, deploymentId);
}
}
///
/// SubscribeBulk pass. After an apply, read the deployment's Equipment-namespace tags,
/// group their driver-side FullName references by driver instance, and hand each running driver
/// child its desired subscription set via .
/// The child retains the set and (re)subscribes on every Connected entry, so values stream into
/// the OPC UA sink and resume after reconnects. Drivers with no configured tags get an empty set
/// (which clears any stale subscription from a previous deployment).
///
private void PushDesiredSubscriptions(DeploymentId deploymentId)
{
// Phase 4 belt-and-suspenders: Direct-only read (the FetchAndCache path calls
// PushDesiredSubscriptionsFromArtifact with bytes in hand). No ConfigDb ⇒ nothing to load.
if (_dbFactory is null)
{
_log.Warning("DriverHost {Node}: no ConfigDb to load artifact for SubscribeBulk ({Id}); skipping", _localNode, deploymentId);
return;
}
byte[] blob;
try
{
using var db = _dbFactory.CreateDbContext();
blob = db.Deployments.AsNoTracking()
.Where(d => d.DeploymentId == deploymentId.Value)
.Select(d => d.ArtifactBlob)
.FirstOrDefault() ?? Array.Empty();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to load artifact for SubscribeBulk ({Id})", _localNode, deploymentId);
return;
}
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
}
///
/// The artifact-processing half of , split out so the
/// boot-from-cache path can drive it with bytes already in hand.
///
///
/// Separated because the cache path runs precisely when the ConfigDb read above cannot
/// succeed — re-reading the artifact there would fail by definition.
///
private void PushDesiredSubscriptionsFromArtifact(DeploymentId deploymentId, byte[] blob)
{
// Issue #485 — the same "no bytes is no answer" rule as ReconcileDrivers. An empty artifact parses
// to a composition with no raw tags, which hands every child an EMPTY desired set — and an empty set
// drops the live subscription handle rather than re-subscribing. Reachable independently of the
// reconcile guard: this method does its OWN ConfigDb read, so the row can go missing between the two.
if (blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: artifact for {Id} read back empty; skipping SubscribeBulk and keeping the live subscriptions",
_localNode, deploymentId);
return;
}
AddressSpaceComposition composition;
try
{
composition = DeploymentArtifact.ParseComposition(blob, _localNode.Value);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to parse composition for SubscribeBulk ({Id})", _localNode, deploymentId);
return;
}
// v3 Batch 4: the driver subscribes + publishes by RawPath (the wire-ref == RawPath == RawTagPlan.NodeId
// per the v3 identity contract). Value-subscription set = the non-alarm raw tags' RawPaths; alarm-bearing
// raw tags are Part 9 conditions, fed via the native alarm event stream (ForwardNativeAlarm), so they are
// excluded from the value set.
var refsByDriver = composition.RawTags
.Where(t => t.Alarm is null)
.GroupBy(t => t.DriverInstanceId, StringComparer.Ordinal)
.ToDictionary(
g => g.Key,
g => (IReadOnlyList)g.Select(t => t.NodeId)
.Distinct(StringComparer.Ordinal)
.ToArray(),
StringComparer.Ordinal);
// Native-alarm subscription set: the alarm-bearing raw tags' RawPaths (= the driver's ConditionId).
// An IAlarmSource driver suppresses OnAlarmEvent until at least one alarm subscription exists, so the
// instance actor must SubscribeAlarmsAsync these refs to un-gate the feed. Routing stays by ConditionId
// in ForwardNativeAlarm; this set just opens (and scopes) the subscription.
var alarmRefsByDriver = composition.RawTags
.Where(t => t.Alarm is not null)
.GroupBy(t => t.DriverInstanceId, StringComparer.Ordinal)
.ToDictionary(
g => g.Key,
g => (IReadOnlyList)g.Select(t => t.NodeId)
.Distinct(StringComparer.Ordinal)
.ToArray(),
StringComparer.Ordinal);
// Referencing UNS nodes by backing RawPath — each raw value tag fans out to its own raw NodeId PLUS
// every UNS reference that projects it (dual-NodeId registration against the SAME driver ref).
var unsRefsByRawPath = new Dictionary>(StringComparer.Ordinal);
foreach (var v in composition.UnsReferenceVariables)
{
if (!unsRefsByRawPath.TryGetValue(v.BackingRawPath, out var list))
unsRefsByRawPath[v.BackingRawPath] = list = new List();
list.Add(v);
}
// Rebuild the driver live-value + write routing maps from the RawTags ∪ UnsReferenceVariables pass.
// Clear-and-repopulate every apply so renames/removals/re-points are reflected.
_nodeIdByDriverRef.Clear();
_driverRefByNodeId.Clear();
_alarmNodeIdByDriverRef.Clear();
_driverRefByAlarmNodeId.Clear();
_alarmMetaByNodeId.Clear();
_nativeAlarmProjector.Clear();
foreach (var t in composition.RawTags)
{
var key = (t.DriverInstanceId, t.NodeId); // (DriverInstanceId, RawPath) — RawPath IS the wire-ref
if (t.Alarm is not null)
{
// Native alarm → a single Part 9 condition at the RawPath (Raw realm). WP4 adds the referencing
// equipment notifier NodeIds; WP3 wires the single raw condition.
if (!_alarmNodeIdByDriverRef.TryGetValue(key, out var aset))
_alarmNodeIdByDriverRef[key] = aset = new HashSet();
aset.Add(new NodeRealmRef(t.NodeId, AddressSpaceRealm.Raw));
// Inverse 1:1 map for the inbound native-condition ack path (keyed by BARE condition NodeId).
_driverRefByAlarmNodeId[t.NodeId] = key;
// Per-condition metadata for the alerts fan-out. WP4 refines /alerts identity to the referencing
// equipment paths; WP3 keys the display off the RawPath. The referencing-equipment paths (the
// Area/Line/Equipment UNS folder paths carried on the RawTagPlan) ride onto every transition so
// the /alerts row lists which equipment reference this raw condition.
_alarmMetaByNodeId[t.NodeId] = (t.NodeId, t.Name, t.Alarm.AlarmType, t.Alarm.HistorizeToAveva, t.ReferencingEquipmentPaths);
continue;
}
// Value tag: register the RAW NodeId (Raw realm) AND every referencing UNS NodeId (Uns realm)
// against the SAME driver ref — the single value source fans to all of them.
if (!_nodeIdByDriverRef.TryGetValue(key, out var set))
_nodeIdByDriverRef[key] = set = new HashSet();
set.Add(new NodeRealmRef(t.NodeId, AddressSpaceRealm.Raw));
_driverRefByNodeId[(AddressSpaceRealm.Raw, t.NodeId)] = key;
if (unsRefsByRawPath.TryGetValue(t.NodeId, out var refs))
{
foreach (var v in refs)
{
set.Add(new NodeRealmRef(v.NodeId, AddressSpaceRealm.Uns));
// A UNS write resolves to the same driver ref — keyed under the Uns realm so it can never
// collide with a raw NodeId that shares its bare string.
_driverRefByNodeId[(AddressSpaceRealm.Uns, v.NodeId)] = key;
}
}
}
// One authored-only push (value refs + alarm refs from the maps built above), shared by the bulk loop AND
// the dropped-driver fallback so the two CANNOT drift: the fallback's correctness depends on sending the
// SAME payload the bulk loop would have, so it's a shared helper (structural), not a comment-maintained
// invariant. An EMPTY set is valid — the child's Connected handler routes it to Unsubscribe (dropping a
// stale handle) rather than a spurious subscribe. Returns the value-ref count for the bulk-loop log total.
int SendAuthoredOnly(IActorRef actor, string driverId)
{
var refs = refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty();
var alarmRefs = alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty();
actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(refs, SubscriptionPublishingInterval, alarmRefs));
return refs.Count;
}
var total = 0;
foreach (var (driverId, entry) in _children)
{
total += SendAuthoredOnly(entry.Actor, driverId);
}
if (total > 0)
{
_log.Info("DriverHost {Node}: SubscribeBulk pushed {Refs} references across {Drivers} driver(s)",
_localNode, total, refsByDriver.Count);
}
// Hand the Equipment-namespace VirtualTags to the host so it spawns/reconciles a
// VirtualTagActor per plan and streams their evaluated values back onto the just-rebuilt
// address space. Runs on BOTH the fresh-apply path (ApplyAndAck) and the bootstrap-restore
// path (RestoreApplied) because both call this method, so one send covers both.
// NOTE: the Stale-recovery path (TryRecoverFromStale) does NOT call PushDesiredSubscriptions,
// so — like drivers — VirtualTags remain empty after a Stale recovery until the next
// deployment dispatch. This is intentional and consistent with driver recovery: the Stale
// path only restores the revision marker + NodeDeploymentState; a subsequent dispatch
// (or a redeploy from AdminUI) triggers the full apply + subscribe pass.
_virtualTagHost?.Tell(new VirtualTagHostActor.ApplyVirtualTags(composition.EquipmentVirtualTags));
if (composition.EquipmentVirtualTags.Count > 0)
{
_log.Info("DriverHost {Node}: applied {Count} Equipment VirtualTag(s) to the VirtualTag host",
_localNode, composition.EquipmentVirtualTags.Count);
}
// Same pass for Equipment-namespace ScriptedAlarms: hand the plans to the host so it
// (re)loads its engine + re-registers mux interest for the union of dependency refs. Covers
// both the fresh-apply and bootstrap-restore paths (both call this method); the Stale-recovery
// path deliberately does not, matching driver + VirtualTag recovery.
_scriptedAlarmHost?.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(composition.EquipmentScriptedAlarms));
if (composition.EquipmentScriptedAlarms.Count > 0)
{
_log.Info("DriverHost {Node}: applied {Count} Equipment ScriptedAlarm(s) to the ScriptedAlarm host",
_localNode, composition.EquipmentScriptedAlarms.Count);
}
// Cache the applied composition LAST. Set here (not in ApplyAndAck) so both the fresh-apply and
// bootstrap-restore paths — which both route through this method — leave a current composition.
_lastComposition = composition;
}
private void SpawnChild(DriverInstanceSpec spec)
{
var stub = DriverInstanceActor.ShouldStub(spec.DriverType, _localRoles);
IDriver? driver = null;
if (!stub)
{
try { driver = _driverFactory.TryCreate(spec.DriverType, spec.DriverInstanceId, spec.DriverConfig); }
catch (Exception ex)
{
// A factory throw is a CONFIG error, not a connectivity one — TryCreate is pure parsing;
// device I/O happens later in InitializeAsync. Logged at Error because the node silently
// degrades to a stub that answers nothing, and since #516 routed DriverConfig changes
// through a respawn this path is now reachable by an ordinary operator edit rather than
// only by a brand-new driver. It still does NOT fail the deployment — making it do so is
// a deliberate follow-up, since it would let one malformed driver block a fleet deploy.
_log.Error(ex,
"DriverHost {Node}: factory for {Type} REJECTED the config for {Id} — the driver is " +
"stubbed and will serve nothing until the config is fixed and redeployed",
_localNode, spec.DriverType, spec.DriverInstanceId);
}
if (driver is null)
{
_log.Warning(
"DriverHost {Node}: no factory for driver type {Type} (instance {Id}); falling back to stub",
_localNode, spec.DriverType, spec.DriverInstanceId);
stub = true;
}
}
IActorRef child;
// Prefer the real ClusterId from the deployment artifact; fall back to the local node
// identity for pre-PR artifacts that don't carry it yet (older deploys persisted before
// ClusterId was added to DriverInstanceSpec).
var clusterId = !string.IsNullOrEmpty(spec.ClusterId) ? spec.ClusterId : _localNode.Value;
// A fresh generation-suffixed name on EVERY spawn so a respawn can never collide with a child
// still tearing down: Akka frees a stopped actor's name only after it FULLY terminates (async),
// but HandleRestartDriver stops + respawns within one (synchronous) message handler — the old
// child is still registered, so reusing the base name throws InvalidActorNameException
// ("actor name is not unique"). Children are tracked by the _children dict (by IActorRef), never
// by path, so the suffix is invisible to every caller.
var actorName = ActorNameFor(spec.DriverInstanceId, _childSpawnGeneration++);
if (stub)
{
child = Context.ActorOf(
DriverInstanceActor.Props(
new StubbedDriver(spec.DriverInstanceId, spec.DriverType),
reconnectInterval: null,
startStubbed: true,
healthPublisher: _healthPublisher,
clusterId: clusterId),
actorName);
}
else
{
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
// route through the retry/breaker/telemetry pipeline. The tier defaults are layered
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
// no-op invoker on nodes without the real resilience factory bound.
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
child = Context.ActorOf(
DriverInstanceActor.Props(
driver!,
healthPublisher: _healthPublisher,
clusterId: clusterId,
invoker: invoker),
actorName);
child.Tell(new DriverInstanceActor.InitializeRequested(spec.DriverConfig));
}
_children[spec.DriverInstanceId] = new ChildEntry(child, spec, stub);
// v3 (WP7): if the (real, non-stub) driver consumes other tags' live values, spawn a mux-adapter
// child that registers its dependency refs on the per-node dependency mux and forwards changes
// into the driver. Requires a wired mux (dev/None deployments have none — then calc inputs simply
// can't flow, matching the VirtualTag dev path).
if (!stub && driver is IDependencyConsumer consumer && _dependencyMux is not null)
{
var adapter = Context.ActorOf(
ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags.DependencyConsumerMuxAdapter.Props(consumer, _dependencyMux),
"depmux-" + actorName);
_depConsumerAdapters[spec.DriverInstanceId] = adapter;
_log.Info("DriverHost {Node}: wired dependency-consumer mux adapter for {Id} ({Count} ref(s))",
_localNode, spec.DriverInstanceId, consumer.DependencyRefs.Count);
}
_log.Info("DriverHost {Node}: spawned {Type} driver {Id} (stub={Stub})",
_localNode, spec.DriverType, spec.DriverInstanceId, stub);
}
/// Stops + forgets the dependency-consumer mux adapter for a driver (if any). Called wherever a
/// driver child is stopped/respawned so a stale adapter never keeps a dead driver's refs registered.
private void StopDependencyAdapter(string driverInstanceId)
{
if (_depConsumerAdapters.Remove(driverInstanceId, out var adapter))
Context.Stop(adapter);
}
///
/// Sends an in-place config delta to a running child.
/// Unreachable from the reconcile path since #516 — a DriverConfig change is now a
/// stop + respawn (), so ToApplyDelta is always empty.
/// Kept because the seam is still exercised by DriverInstanceActor's own mid-connect config
/// adoption.
/// Two seals were removed here. This used to overwrite the cached
/// Spec SYNCHRONOUSLY, before the child had even dequeued the message — so the host
/// immediately believed the new config was live, the NEXT reconcile computed no delta against it,
/// and any drift was sealed permanently. It also Telld with no Receive<ApplyResult>
/// handler registered, so a FAILED reinit — including Galaxy's deliberate
/// NotSupportedException — dead-lettered and was never surfaced.
///
private void ApplyChildDelta(DriverInstanceSpec spec)
{
if (!_children.TryGetValue(spec.DriverInstanceId, out var entry)) return;
var correlation = CorrelationId.NewId();
_pendingDelta[correlation] = spec;
entry.Actor.Tell(new DriverInstanceActor.ApplyDelta(spec.DriverConfig, correlation), Self);
_log.Debug("DriverHost {Node}: ApplyDelta queued for {Id}", _localNode, spec.DriverInstanceId);
}
///
/// A child's reply to . On success the cached Spec is advanced —
/// only now, when the child has actually adopted the config, so a failed reinit leaves the host
/// believing the OLD config is live and the next reconcile re-attempts the change instead of
/// silently treating the drift as applied. A failure is logged at Error rather than swallowed.
///
private void HandleApplyResult(DriverInstanceActor.ApplyResult msg)
{
if (msg.Success)
{
if (_pendingDelta.Remove(msg.Correlation, out var spec)
&& _children.TryGetValue(spec.DriverInstanceId, out var entry))
{
// A delta can change Name, Enabled, ClusterId etc. alongside the config.
_children[spec.DriverInstanceId] = entry with { Spec = spec };
}
return;
}
_pendingDelta.Remove(msg.Correlation, out var failed);
_log.Error(
"DriverHost {Node}: driver {Id} REJECTED an in-place config change ({Reason}) — the host keeps " +
"the previous config so the next reconcile re-attempts it",
_localNode, failed?.DriverInstanceId ?? "", msg.Reason ?? "no reason given");
}
///
/// A driver child finished applying an in-place (its
/// completed). Re-register THAT driver's dependency-consumer
/// mux adapter so a changed dependency-ref set (calc tags/scripts edited during the redeploy) is
/// adopted on the mux. This runs AFTER reinit — closing the ordering hazard where a synchronous
/// re-register at ApplyDelta dispatch would re-read the driver's stale
/// . A Tell (no blocking); a no-op for a driver with no
/// adapter (not an , or no mux wired on this node).
///
private void HandleDeltaApplied(DriverInstanceActor.DeltaApplied msg)
{
if (_depConsumerAdapters.TryGetValue(msg.DriverInstanceId, out var adapter))
adapter.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags.DependencyConsumerMuxAdapter.ReRegister());
}
private void StopChild(string driverInstanceId)
{
if (!_children.TryGetValue(driverInstanceId, out var entry)) return;
StopDependencyAdapter(driverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}
private static string ActorNameFor(string driverInstanceId, long generation)
{
// Akka actor names cannot contain '/', ':', or whitespace. Mangle defensively. The monotonic
// generation suffix guarantees a never-before-used name on every spawn (see SpawnChild) so a
// restart's respawn can never collide with the still-terminating predecessor.
var chars = driverInstanceId.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' or '.' ? c : '_').ToArray();
return "drv-" + new string(chars) + "-g" + generation;
}
///
/// Minimal placeholder driver used when no factory is registered for a driver type or when
/// returns true.
/// is started with startStubbed:true so the driver methods on this object never run.
///
private sealed class StubbedDriver : IDriver
{
///
public string DriverInstanceId { get; }
///
public string DriverType { get; }
/// Initializes a new stubbed driver with the specified ID and type.
/// The driver instance identifier.
/// The driver type name.
public StubbedDriver(string id, string type) { DriverInstanceId = id; DriverType = type; }
///
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
///
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
///
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
///
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);
///
public long GetMemoryFootprint() => 0;
///
public Task FlushOptionalCachesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
private void HandleRestartDriver(RestartDriver msg)
{
// DPS broadcast — only act if this node hosts the requested instance.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry))
return;
_log.Info("DriverHost {Node}: restarting driver {Id} by request of {User}",
_localNode, msg.DriverInstanceId, msg.ActorByUserName);
// Stop the existing child actor — DriverInstanceActor.PostStop calls ShutdownAsync.
StopDependencyAdapter(msg.DriverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(msg.DriverInstanceId);
// Respawn from the same spec the last reconcile used — preserves RowId, Name, ClusterId.
SpawnChild(entry.Spec);
}
private void HandleReconnectDriver(ReconnectDriver msg)
{
// DPS broadcast — only act if this node hosts the requested instance.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry))
return;
_log.Info("DriverHost {Node}: reconnecting driver {Id} by request of {User}",
_localNode, msg.DriverInstanceId, msg.ActorByUserName);
// Tell the child to drop its transport and re-enter the Reconnecting state.
entry.Actor.Tell(new DriverInstanceActor.ForceReconnect());
}
private void TryRecoverFromStale()
{
// Defensive: FetchAndCache never enters Stale (BootstrapFromCache always Becomes Steady, so
// the retry-db timer that drives this is never started), and its recovery must NOT read
// central SQL. A DB-less node (null factory, Phase 4) likewise has nothing to recover from. If
// we somehow land here, leave Steady rather than re-reading Deployments.
if (_fetchAndCacheMode || _dbFactory is null)
{
Timers.Cancel("retry-db");
Become(Steady);
return;
}
try
{
using var db = _dbFactory.CreateDbContext();
var latestSealed = db.Deployments
.AsNoTracking()
.Where(d => d.Status == DeploymentStatus.Sealed)
.OrderByDescending(d => d.SealedAtUtc)
.Select(d => new { d.DeploymentId, d.RevisionHash })
.FirstOrDefault();
_log.Info("DriverHost {Node}: ConfigDb back; recovering from Stale", _localNode);
Timers.Cancel("retry-db");
if (latestSealed is not null)
{
_currentRevision = RevisionHash.Parse(latestSealed.RevisionHash);
UpsertNodeDeploymentState(new DeploymentId(latestSealed.DeploymentId),
NodeDeploymentStatus.Applied, failureReason: null);
}
Become(Steady);
}
catch (Exception ex)
{
_log.Debug(ex, "DriverHost {Node}: still Stale; will retry in {Interval}", _localNode, ReconnectInterval);
}
}
private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason)
{
// Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of
// record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every
// ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is
// redundant and simply no-ops here with no loss of information. Logged once to keep the reason
// discoverable without spamming every apply transition.
if (_dbFactory is null)
{
if (!_loggedNoConfigDbAck)
{
_loggedNoConfigDbAck = true;
_log.Debug(
"DriverHost {Node}: no ConfigDb — NodeDeploymentState written by central from the ApplyAck",
_localNode);
}
return;
}
try
{
using var db = _dbFactory.CreateDbContext();
var existing = db.NodeDeploymentStates.FirstOrDefault(
x => x.NodeId == _localNode.Value && x.DeploymentId == deploymentId.Value);
if (existing is null)
{
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = _localNode.Value,
DeploymentId = deploymentId.Value,
Status = status,
FailureReason = failureReason,
AppliedAtUtc = status == NodeDeploymentStatus.Applied ? DateTime.UtcNow : null,
});
}
else
{
existing.Status = status;
existing.FailureReason = failureReason;
if (status == NodeDeploymentStatus.Applied) existing.AppliedAtUtc = DateTime.UtcNow;
}
db.SaveChanges();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to upsert NodeDeploymentState for {Id}", _localNode, deploymentId);
}
}
private void SendAck(DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation)
{
var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation);
if (_ackRouter is not null)
{
// Either a direct coordinator handle (tests) or, under
// MeshTransport:Mode = ClusterClient, the NodeCommunicationActor that relays this
// across the boundary. Renamed from _coordinatorOverride in per-cluster mesh Phase 2:
// under ClusterClient mode this ref is emphatically NOT a coordinator, and the old
// name would send the next reader looking for one.
_ackRouter.Tell(ack);
}
else
{
// No router — publish on the dedicated ACK topic. The coordinator singleton subscribes
// there in PreStart so the ACK reaches whichever admin node hosts it without an
// actor-path lookup.
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentAcksTopic, ack));
}
}
}