dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
88ef1ea9087feb34244f790077a6aa2d0eeabf38
lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
T
Joseph Doherty 1ea1da8236 refactor(runtime): delete the dead in-place ApplyDelta seam; config change = respawn only (#665) 
Since #516 a DriverConfig change is stop + respawn, so DriverSpawnPlan.ToApplyDelta
was always empty and the in-place path had no producer. #663 and #664 were both
latent bugs in it. This deletes it:
- host: ApplyChildDelta, _pendingDelta, HandleApplyResult and HandleDeltaApplied
- plan: DriverSpawnPlan.ToApplyDelta
- actor: the ApplyDelta, ApplyResult and DeltaApplied messages, the Connected
  reinit handler and the mid-connect AdoptConfigDuringInit
- tests: the actor tests that drove the seam

IDriver.ReinitializeAsync and each driver's implementation and unit tests stay.
Comments and docs that cited AdoptConfigDuringInit as the reason a driver must
release prior state at init now cite the IDriver contract and overlapping
retries, which still apply.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-09 11:23:16 -04:00

3826 lines
232 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Diagnostics;
using System.Text.Json;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
/// <summary>
/// Per-node supervisor that receives <see cref="DispatchDeployment"/> from the admin-role
/// coordinator and applies the deployment locally. Three Become states:
///
/// <list type="bullet">
/// <item><c>Bootstrapping</c> — PreStart only. Reads <see cref="NodeDeploymentState"/> for self;
/// chooses next state.</item>
/// <item><c>Steady(rev)</c> — caught up. Idempotent on same-rev dispatch (immediate <c>ApplyAck.Applied</c>).
/// New rev → transitions to <c>Applying</c>.</item>
/// <item><c>Applying(id)</c> — applying a delta. Buffers further dispatches.</item>
/// <item><c>Stale</c> — ConfigDb unreachable on bootstrap. Background reconnect loop tries to advance.</item>
/// </list>
///
/// Children (DriverInstance/VirtualTag/etc.) are fully wired: <c>DispatchDeployment</c> drives
/// <see cref="PushDesiredSubscriptions"/> which runs the spawn-plan via <see cref="IDriverFactory"/>,
/// maintains the <c>FullName→NodeId</c> live-value routing map, and forwards results to the
/// OPC UA publish actor via <c>ForwardBatchToMux</c> — one whole driver production burst per message,
/// and the ONLY value ingress from driver children (<see cref="DriverInstanceActor"/> wraps even a
/// single data change in an <c>AttributeValueBatchPublished</c>). <see cref="FanCommsLossToDependencyMux"/>
/// fabricates a batch of the same shape straight to the dependency mux, so the host neither receives nor
/// sends the per-value <c>AttributeValuePublished</c> any more.
/// </summary>
public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedStash
{
public const string DeploymentsTopic = "deployments";
public const string DeploymentAcksTopic = "deployment-acks";
public const string DriverControlTopic = ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin.DriverControlTopic.Name;
public static readonly TimeSpan ReconnectInterval = TimeSpan.FromSeconds(30);
/// <summary>Publishing interval handed to each driver's SubscribeBulk pass after an apply.</summary>
private static readonly TimeSpan SubscriptionPublishingInterval = TimeSpan.FromSeconds(1);
/// <summary>Timer key for the <see cref="Applying"/>-state wedge watchdog. Started by
/// <see cref="BeginFetchAndCacheApply"/>, cancelled by <see cref="BecomeSteady"/>.</summary>
private const string ApplyWedgeTimerKey = "apply-wedged";
/// <summary>
/// How long the host may sit in <see cref="Applying"/> before it says so.
/// </summary>
/// <remarks>
/// Five minutes ≈ 2× the worst configured artifact-fetch deadline, so an ordinary slow fetch
/// never trips it and a genuinely wedged one always does. It is not a timeout: nothing is
/// cancelled or failed, because the apply may still land and the coordinator's own apply
/// deadline is the authority on when a deployment is dead. The watchdog exists purely so a host
/// stuck in <see cref="Applying"/> is VISIBLE — the pre-stash forward-spin used to make that
/// unmissable (a flood of log lines), and taking the spin away accidentally took the only alarm
/// with it.
/// </remarks>
public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5);
/// <summary>
/// Gitea #601 — how long an Applied ack waits for <c>OpcUaPublishActor</c>'s address-space apply
/// outcome (<see cref="OpcUa.OpcUaPublishActor.RebuildCompleted"/>) before it is sent anyway.
/// </summary>
/// <remarks>
/// Derived from central's apply deadline (<see cref="DeployTimings.ApplyDeadline"/>, 2 min — the same
/// constant <c>ConfigPublishCoordinator.DefaultApplyDeadline</c> reads) minus
/// <see cref="DeployTimings.ApplyOutcomeSafetyMargin"/> (15 s) = 105 s: long enough that a
/// slow-but-clean rebuild of a large address space is not labelled degraded, short enough that the ack
/// still lands before central times the deployment out. An ack sent on this timeout is marked
/// <b>degraded</b> with a reason starting <see cref="OutcomeNotReportedReasonPrefix"/>, never clean —
/// the revision did advance, but nobody has confirmed the address space matches it.
/// </remarks>
public static readonly TimeSpan DefaultApplyOutcomeTimeout = DeployTimings.ApplyOutcomeTimeout;
/// <summary>Gitea #601 — the reason prefix of a degraded ack sent because the outcome TIMED OUT, so an
/// operator can tell "not reported in time" from a real sink failure.</summary>
public const string OutcomeNotReportedReasonPrefix = "address-space apply outcome not reported in time";
/// <summary>Minimum interval between two supervision reprovisions of the SAME driver child (archreview
/// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the
/// reprovision hook makes every restart do ShutdownAsync + InitializeAsync + SubscribeAsync — so an
/// un-limited hook turns a driver that faults on every value into a reconnect storm against its device.
/// Matched to <see cref="DriverInstanceActor.DefaultReconnectInterval"/>: the reprovision drives the same
/// connect work, so it should not run more often than an ordinary reconnect attempt.</summary>
public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10);
/// <summary>
/// How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs still
/// latched <see cref="OpcUaStatus.BadNoCommunication"/>. Sized so an ordinary poll driver has had
/// several poll cycles — and a slow one at least one — to re-publish before anything is said; the
/// audit's job is to name a driver that will NEVER re-publish, not to race a slow first poll.
/// </summary>
public static readonly TimeSpan DefaultCommsRecoveryAuditDelay = TimeSpan.FromSeconds(30);
/// <summary>
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
/// someone reads the table during an incident.
/// </summary>
private const string SingleClusterCacheKey = "__single";
/// <summary>
/// Node-local cache of applied deployment artifacts, or null when this node has none
/// (admin-only graphs, and tests that do not exercise it).
/// </summary>
private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
/// <summary>
/// Per-cluster mesh Phase 3. When <see langword="true"/> (<c>ConfigSource:Mode =
/// FetchAndCache</c>) this node obtains its config by fetching the artifact bytes from central
/// over gRPC and applying them from the LocalDb cache, reading NO config from central SQL.
/// When <see langword="false"/> (default <c>Direct</c>) every path reads central SQL as before.
/// </summary>
private readonly bool _fetchAndCacheMode;
/// <summary>
/// Node-side gRPC artifact fetcher, non-null only under <see cref="_fetchAndCacheMode"/>.
/// </summary>
private readonly IDeploymentArtifactFetcher? _artifactFetcher;
/// <summary>
/// True once this node has booted a cached artifact because central SQL was unreachable.
/// </summary>
/// <remarks>
/// Surfaced on <see cref="NodeDiagnosticsSnapshot"/> because a node running from cache looks
/// completely healthy from the outside — it serves a full address space with live values.
/// The difference is that its configuration is frozen at whatever the cache held, and no
/// deployment can reach it. Without an explicit signal that is invisible until someone
/// wonders why a deploy "succeeded" everywhere but did not take effect here.
/// </remarks>
private bool _isRunningFromCache;
/// <summary>Latches after the first no-op ack-write on a DB-less node so the explanatory Debug line
/// is logged once, not on every apply state transition.</summary>
private bool _loggedNoConfigDbAck;
/// <summary>
/// Config database context factory, or <see langword="null"/> on a driver-only node (per-cluster
/// mesh Phase 4). A driver-only node has NO ConfigDb — LocalDb is its config store and central is
/// the ack system of record — so every ConfigDb-touching path here is guarded on this being
/// non-null. Non-null on a fused admin+driver node and in the Direct-mode/EF test harnesses.
/// </summary>
private readonly IDbContextFactory<OtOpcUaConfigDbContext>? _dbFactory;
/// <summary>
/// Scripted-alarm condition-state store handed to the ScriptedAlarm engine, or
/// <see langword="null"/> when none was wired (per-cluster mesh Phase 4). On a driver-role node
/// this is the replicated LocalDb store (<c>LocalDbAlarmConditionStateStore</c>) so condition
/// state survives without central SQL; when null the actor skips spawning the alarm host (the
/// ConfigDb-backed EF fallback was retired in Phase 4 Task 9).
/// </summary>
private readonly IAlarmStateStore? _alarmStateStore;
/// <summary>Optional node-local live-telemetry hub (Phase 5): each Primary-gated native-alarm
/// <c>alerts</c> transition is also emitted here, and it is threaded into the spawned VirtualTag +
/// ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.</summary>
private readonly ITelemetryLocalHub? _telemetryHub;
private readonly CommonsNodeId _localNode;
private readonly IActorRef? _ackRouter;
private readonly IDriverFactory _driverFactory;
/// <summary>Builds the per-driver-instance <see cref="IDriverCapabilityInvoker"/> attached to each
/// spawned <see cref="DriverInstanceActor"/>. Resolved from DI like <see cref="_driverFactory"/>;
/// defaults to <see cref="NullDriverCapabilityInvokerFactory"/> (pass-through) when the fused Host
/// hasn't bound the real resilience factory.</summary>
private readonly IDriverCapabilityInvokerFactory _invokerFactory;
private readonly IReadOnlySet<string> _localRoles;
private readonly IActorRef? _dependencyMux;
private readonly IActorRef? _opcUaPublishActor;
private readonly IDriverHealthPublisher _healthPublisher;
private readonly IVirtualTagEvaluator _virtualTagEvaluator;
private readonly IHistoryWriter _historyWriter;
private readonly IActorRef? _virtualTagHostOverride;
private readonly ScriptRootLogger? _scriptRootLogger;
private readonly IActorRef? _scriptedAlarmHostOverride;
private readonly ILoggingAdapter _log = Context.GetLogger();
/// <summary>
/// Holds dispatches that arrive for a DIFFERENT deployment while one is being applied, until
/// the apply transitions back to <c>Steady</c>. Assigned by the ActorCell after construction —
/// it is <see langword="null"/> inside the constructor, so nothing there may touch it.
/// </summary>
/// <remarks>
/// The predecessor was <c>Self.Forward(msg)</c>, which put the message back at the tail of this
/// actor's OWN mailbox: with nothing else queued, the host spun dequeue → log → enqueue for the
/// whole apply (a <c>FetchAndCache</c> fetch window is seconds of network I/O), burning a core
/// and emitting a log line per iteration — measured at ~570k lines in 1.5 s. The stash holds the
/// message OUTSIDE the mailbox, so the host simply idles until there is something real to do.
/// </remarks>
public IStash Stash { get; set; } = null!;
/// <summary>
/// The set of deployment ids currently sitting in <see cref="Stash"/>, so the
/// <see cref="Applying"/> state stashes each distinct deployment AT MOST ONCE. Cleared in
/// <see cref="BecomeSteady"/>, which is the same moment <see cref="IStash.UnstashAll"/> empties
/// the stash itself — a dispatch that is re-stashed by the NEXT apply re-adds itself naturally.
/// </summary>
/// <remarks>
/// <para>
/// <b>It bounds the stash by DISTINCT deployments.</b> A coordinator that re-dispatches on a
/// timer (or several coordinators racing) can emit the same deployment many times inside one
/// apply window; without this, every copy is stashed and every copy is re-delivered on the
/// way back to <see cref="Steady"/> — an unbounded queue of messages that will all
/// short-circuit as duplicates anyway.
/// </para>
/// <para>
/// <b>It also moots an Akka stash-equality question — which was raised in review as a
/// restart hazard and, MEASURED, is not one on the pinned version.</b> The string
/// <c>"Can't stash the same message {0} more than once"</c> (the
/// <c>IllegalActorStateException</c> that <c>AbstractStash.Stash()</c> is documented to
/// throw when an envelope equals the last one stashed) IS present verbatim in the shipped
/// <c>Akka.dll</c> 1.5.62. But a throwaway probe actor on this exact package — stashing one
/// message instance twice in a row, then two value-equal instances, both with and without an
/// explicit sender — accepted all four stashes and never threw. So the guard is not on this
/// path in 1.5.62, and the dedup set is NOT fixing a live crash. What it does is make the
/// question moot: since <see cref="DispatchDeployment"/> is a <c>sealed record</c> (value
/// equality), a future Akka that does enforce the check would restart this host in the
/// middle of a deployment, and nothing here would have to change to stay safe.
/// </para>
/// </remarks>
private readonly HashSet<DeploymentId> _deferredDispatches = new();
/// <summary>The single VirtualTag-host child that spawns/reconciles Equipment-namespace
/// VirtualTagActors and bridges their results onto the OPC UA publish actor. Spawned in
/// <see cref="PreStart"/> when an OPC UA publish actor is wired; receives
/// <see cref="VirtualTagHostActor.ApplyVirtualTags"/> from <see cref="PushDesiredSubscriptions"/>.</summary>
private IActorRef? _virtualTagHost;
/// <summary>The single ScriptedAlarm-host child that owns the per-node
/// <see cref="ScriptedAlarmEngine"/>, feeds it live tag values, and bridges its emissions onto the
/// OPC UA publish actor + the cluster <c>alerts</c> topic. Spawned in <see cref="PreStart"/>
/// alongside the VirtualTag host (requires both an OPC UA publish actor and a
/// <see cref="ScriptRootLogger"/>); receives <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/>
/// from <see cref="PushDesiredSubscriptions"/>.</summary>
private IActorRef? _scriptedAlarmHost;
private RevisionHash? _currentRevision;
/// <summary>
/// Gitea #601 — the last address-space apply of <see cref="_currentRevision"/> reported DEGRADED (or no
/// outcome in time). A re-dispatch of that same revision then RE-APPLIES instead of taking the
/// idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied
/// for an address space known to be partial, and the re-apply is what repairs it (a degraded
/// non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing
/// nodes). Reset by every clean outcome and by every boot path that restores a revision.
/// </summary>
private bool _currentRevisionDegraded;
/// <summary>Gitea #601 — Applied acks held until the address-space apply reports its outcome, keyed by
/// the apply's correlation id (echoed back in <see cref="OpcUa.OpcUaPublishActor.RebuildCompleted"/>).</summary>
private readonly Dictionary<CorrelationId, PendingAck> _pendingAcks = new();
private DeploymentId? _applyingDeploymentId;
private readonly Dictionary<string, ChildEntry> _children = new(StringComparer.Ordinal);
/// <summary>Per-driver <see cref="DependencyConsumerMuxAdapter"/> children — one for each spawned driver
/// that implements <see cref="IDependencyConsumer"/> (e.g. the Calculation pseudo-driver). Each registers
/// the driver's declared dependency refs on the per-node <see cref="_dependencyMux"/> and forwards value
/// changes into the driver. Spawned in <see cref="SpawnChild"/>, torn down with the driver in
/// <see cref="StopDependencyAdapter"/>, and re-registered on every apply (refs change when tags/scripts
/// change). Empty when no mux is wired (the dev/None path) or no dependency-consuming driver is hosted.</summary>
private readonly Dictionary<string, IActorRef> _depConsumerAdapters = new(StringComparer.Ordinal);
// Monotonic counter feeding the child actor-name suffix (see ActorNameFor / SpawnChild). Single-
// threaded actor, so a plain increment is safe; it only ever grows, guaranteeing a unique name per
// spawn so a restart's respawn never collides with the still-terminating old child.
private long _childSpawnGeneration;
/// <summary>A materialised NodeId together with the v3 <see cref="AddressSpaceRealm"/> it lives in, so the
/// driver value fan-out posts each update to the sink with the right namespace. The same driver ref fans
/// to its raw node (<see cref="AddressSpaceRealm.Raw"/>) and to every referencing UNS node
/// (<see cref="AddressSpaceRealm.Uns"/>).</summary>
private readonly record struct NodeRealmRef(string NodeId, AddressSpaceRealm Realm);
/// <summary>
/// Driver live-value routing map: <c>(DriverInstanceId, RawPath) → materialised NodeId(s) (realm-tagged)</c>.
/// Rebuilt every apply by <see cref="PushDesiredSubscriptions"/> from the composition's <c>RawTags</c>
/// ∪ <c>UnsReferenceVariables</c>, and resolved in <see cref="ForwardBatchToMux"/> so a driver value
/// published by wire-ref RawPath fans to its raw node AND every referencing UNS node with identical
/// value/quality/timestamp — the single-source-fan-out (no independent buffer, so no drift). A set
/// because one driver ref can back the raw node plus N UNS references; the per-apply rebuild dedups.
/// </summary>
private readonly Dictionary<(string DriverInstanceId, string RawPath), HashSet<NodeRealmRef>> _nodeIdByDriverRef = new();
/// <summary>
/// Inverse of <see cref="_nodeIdByDriverRef"/>: <c>(Realm, BARE <c>s=</c> id) → (DriverInstanceId,
/// RawPath)</c>. v3 Batch 4: keyed by BOTH the raw NodeId (<see cref="AddressSpaceRealm.Raw"/>) AND every
/// referencing UNS NodeId (<see cref="AddressSpaceRealm.Uns"/>) — all mapping to the same driver ref
/// (the single value source). Rebuilt every apply by <see cref="PushDesiredSubscriptions"/> from the
/// composition's <c>RawTags</c> ∪ <c>UnsReferenceVariables</c>, and resolved by
/// <see cref="HandleRouteNodeWrite"/> so an inbound operator write to EITHER NodeId is forwarded to the
/// owning driver child as a write of its wire-ref RawPath.
/// <para><b>The realm is part of the key</b> (Wave B review H1): a raw <c>s=&lt;RawPath&gt;</c> and a UNS
/// <c>s=&lt;Area/Line/Equip/Eff&gt;</c> can collide as bare strings (folder/driver/device names and
/// Area/Line/Equip names are independent), so a bare-only key would let a colliding raw + UNS pair route
/// to the WRONG driver ref (last-writer-wins). The node manager's write hook passes the full
/// ns-qualified id PLUS the realm it resolved (<c>RealmOf</c>); <see cref="HandleRouteNodeWrite"/>
/// normalises the id to the bare form and looks up <c>(realm, bareId)</c>, preserving exactly the
/// namespace disambiguation the ns-qualified id carries.</para>
/// </summary>
private readonly Dictionary<(AddressSpaceRealm Realm, string NodeId), (string DriverInstanceId, string RawPath)> _driverRefByNodeId = new();
/// <summary>(DriverInstanceId, RawPath = alarm ConditionId) → materialised condition NodeId(s) (realm-tagged).
/// v3 Batch 4: a native alarm is a single condition at the RawPath (Raw realm); WP4 adds the referencing
/// equipment notifier NodeIds. Resolves a native alarm transition to the materialised Part 9 condition
/// node(s). Alarm tags are conditions, not value variables, so they are kept OUT of the value maps +
/// value-subscription set.</summary>
private readonly Dictionary<(string DriverInstanceId, string RawPath), HashSet<NodeRealmRef>> _alarmNodeIdByDriverRef = new();
/// <summary>
/// Secondary index over <see cref="_nodeIdByDriverRef"/>, grouping its entries by
/// <c>DriverInstanceId</c>: <c>DriverInstanceId → [(RawPath, the SAME node set)]</c>. Rebuilt whole in
/// the same <see cref="ApplyDesiredSubscriptions"/> pass that rebuilds the primary — never maintained
/// incrementally, so the two cannot drift.
/// <para>
/// It exists because the connectivity handlers are the only readers that need <i>one driver's</i>
/// entries, and the primary is keyed by the composite <c>(DriverInstanceId, RawPath)</c> — so
/// answering "which refs does this driver own?" meant scanning every entry, i.e. the whole node's
/// address space, for a single driver's disconnect. On a 50k-tag node a driver flap walked 50k
/// keys to find its own few thousand.
/// </para>
/// <para>
/// The node set is held BY REFERENCE, not copied: the entry is captured when the primary's set is
/// created, so any node later added to that set (a UNS reference registered in the same pass) is
/// visible here without a second bookkeeping step.
/// </para>
/// </summary>
private readonly Dictionary<string, List<(string RawPath, HashSet<NodeRealmRef> Nodes)>> _valueRefsByDriver =
new(StringComparer.Ordinal);
/// <summary>The alarm-side twin of <see cref="_valueRefsByDriver"/>: a secondary index over
/// <see cref="_alarmNodeIdByDriverRef"/> grouping its entries by <c>DriverInstanceId</c>, so the
/// connectivity handler annotates one driver's conditions without scanning every condition on the node.
/// Rebuilt whole in the same pass, and likewise holding each condition-node set by reference.</summary>
private readonly Dictionary<string, List<(string RawPath, HashSet<NodeRealmRef> Nodes)>> _alarmRefsByDriver =
new(StringComparer.Ordinal);
/// <summary>
/// Inverse of <see cref="_alarmNodeIdByDriverRef"/>: <c>folder-scoped condition NodeId →
/// (DriverInstanceId, FullName = alarm ConditionId/AlarmFullReference)</c>. Built in the SAME apply
/// pass from the alarm-bearing EquipmentTags, and resolved by <see cref="HandleRouteNativeAlarmAck"/>
/// so an inbound OPC UA acknowledge of a native condition (resolved to the condition's NodeId by the
/// node manager) is forwarded to the owning driver child as an acknowledge of its wire-ref
/// <c>FullName</c>. Each condition NodeId maps to exactly one driver ref (a condition is backed by a
/// single driver alarm), so this is a flat 1:1 map (the forward map fans out 1:N because one ref can
/// back several conditions on identical machines).
/// </summary>
private readonly Dictionary<string, (string DriverInstanceId, string RawPath)> _driverRefByAlarmNodeId =
new(StringComparer.Ordinal);
/// <summary>Condition NodeId → (EquipmentId, tag Name, OPC UA alarm type, HistorizeToAveva) for building
/// the AlarmTransitionEvent fan-out. Built in the same PushDesiredSubscriptions alarm branch.
/// HistorizeToAveva (bool?, null ⇒ historize) is the native per-condition opt-out parsed from
/// <c>TagConfig.alarm.historizeToAveva</c>; it is threaded onto the transition so the
/// HistorianAdapterActor's <c>is not false</c> gate suppresses the durable AVEVA row only on an
/// explicit false (mirroring the scripted-alarm opt-out).</summary>
private readonly Dictionary<string, (string EquipmentId, string Name, string AlarmType, bool? HistorizeToAveva, IReadOnlyList<string> ReferencingEquipmentPaths)> _alarmMetaByNodeId =
new(StringComparer.Ordinal);
/// <summary>Derives a full Part 9 condition snapshot from each native alarm transition delta,
/// tracking per-condition-NodeId prior state. <see cref="Clear"/>'d on every apply alongside the
/// value maps so stale condition state never leaks across redeploys.</summary>
private readonly NativeAlarmProjector _nativeAlarmProjector = new();
/// <summary>One driver child's desired subscription payload — exactly the
/// <see cref="DriverInstanceActor.SetDesiredSubscriptions"/> the apply's bulk push sent it.</summary>
/// <param name="ValueRefs">The non-alarm raw tags' RawPaths this driver should keep subscribed.</param>
/// <param name="AlarmRefs">The alarm-bearing raw tags' RawPaths that un-gate this driver's
/// <see cref="IAlarmSource"/> feed.</param>
private readonly record struct DesiredSubscriptionSet(
IReadOnlyList<string> ValueRefs,
IReadOnlyList<string> AlarmRefs);
/// <summary>Per-driver desired subscription sets from the most-recent apply. THE authority: the bulk
/// push, <see cref="SpawnChild"/> and the supervision-reprovision handler all send from here, so a
/// child obtained by ANY route carries the same set. Cleared and repopulated whole on every apply,
/// alongside the routing maps, so renames/removals can never leave a stale entry. Empty until the
/// first apply — a child spawned before any apply is handed nothing, which is what it should have.
/// <para>It is only ever rebuilt by a SUCCESSFUL apply, so after a failed one it deliberately lingers on
/// the last-good composition — matching the address space, drivers and subscriptions that also keep
/// serving through a failed apply (#485/#486). A respawn or reprovision in that window therefore hands
/// the child the last-known-good set, not an empty one.</para></summary>
private readonly Dictionary<string, DesiredSubscriptionSet> _desiredByDriver = new(StringComparer.Ordinal);
/// <summary>
/// Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
/// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary
/// data-plane gates (<see cref="HandleRouteNodeWrite"/>, <see cref="HandleRouteNativeAlarmAck"/>,
/// <see cref="ForwardNativeAlarm"/>) resolve this through <see cref="PrimaryGatePolicy"/>: a KNOWN
/// role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster
/// stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a
/// snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window).
/// </summary>
private RedundancyRole? _localRole;
/// <summary>Test seam (archreview 03/S4): overrides the count of Up <c>driver</c>-role cluster members
/// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a
/// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
private readonly Func<int>? _driverMemberCountProvider;
/// <summary>Where the Primary-gate decision is published for consumers that live outside a mailbox —
/// today, the alarm store-and-forward drain. Null on nodes that do not wire one.</summary>
private readonly IRedundancyRoleView? _redundancyRoleView;
/// <summary>
/// Where the Primary <b>data-plane</b> verdict is published for drivers that must know it
/// directly (Gitea #511). Null on nodes that wire no redundancy manager.
/// </summary>
/// <remarks>
/// Note the different verdict from <see cref="_redundancyRoleView"/>: that one carries
/// <see cref="PrimaryGatePolicy.ShouldDrainAlarmHistory"/>, which OPENS on an unknown role. A
/// Sparkplug host-id claim is an exclusive assertion on a shared broker, so it takes the
/// data-plane verdict, which closes. Publishing the drain verdict here would let both nodes of
/// a pair claim the host id during every boot window.
/// </remarks>
private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
/// <summary>
/// Gitea #537: where the continuous-historization <b>capture</b> verdict is published for the
/// recorder, which taps the mux fan-out and so needs the decision off its own mailbox. Null on
/// nodes that wire no redundancy manager.
/// </summary>
private readonly IHistorizationRoleView? _historizationRoleView;
/// <summary>
/// Tests whether a node named in a redundancy snapshot serves this node's address space — i.e.
/// is a per-cluster mesh partner recording the very same historized refs. Null (the default)
/// means the caller could not establish it, which never stands this node down. See
/// <see cref="PrimaryGatePolicy.ShouldRecordHistorizedValues"/> for why proving the sharing
/// matters rather than trusting "somebody is Primary".
/// </summary>
private readonly Func<CommonsNodeId, bool>? _sharesHistorizationScopeWithLocal;
/// <summary>Gitea #601 — how long an Applied ack waits for the address-space outcome (see
/// <see cref="DefaultApplyOutcomeTimeout"/>).</summary>
private readonly TimeSpan _applyOutcomeTimeout;
/// <summary>This host's <see cref="Applying"/>-wedge watchdog delay (see
/// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
private readonly TimeSpan _applyWedgeTimeout;
/// <summary>How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs
/// still latched <see cref="OpcUaStatus.BadNoCommunication"/> (see <see cref="CommsRecoveryAudit"/>).</summary>
private readonly TimeSpan _commsRecoveryAuditDelay;
/// <summary>
/// Archreview <b>03/S15</b> — RawPaths this host has stamped <c>BadNoCommunication</c> on the value
/// nodes of, per driver, and has not yet seen a value for. Set by
/// <see cref="FanCommsLossToValueNodes"/>, pruned by <see cref="ForwardBatchToMux"/> as each ref
/// re-publishes, and read once by <see cref="CommsRecoveryAudit"/> after the driver reconnects.
/// <para><b>It is a tripwire, not a mechanism.</b> Nothing depends on it being correct for the data
/// plane to behave: the host publishes nothing Good on reconnect either way. What it buys is that a
/// driver which reconnects and then fails to re-publish — the pre-STAB-18 MQTT shape, and what any
/// future push driver without an initial push will do — produces a Warning and a metered count
/// instead of a subscription that silently reports Bad forever.</para>
/// <para>Keyed by RawPath, so a raw node and its UNS mirrors count once: the operator is being told
/// how many <i>tags</i> did not come back, not how many address-space nodes they project onto. The
/// entry is removed as soon as it empties, so the steady-state cost in
/// <see cref="ForwardBatchToMux"/> is one dictionary miss per burst.</para>
/// </summary>
private readonly Dictionary<string, HashSet<string>> _commsLatched = new(StringComparer.Ordinal);
/// <summary>Per-driver rate limit for the supervision-reprovision hook (see
/// <see cref="DefaultReprovisionBackoff"/>).</summary>
private readonly TimeSpan _reprovisionBackoff;
/// <summary>When each driver was last reprovisioned after a supervision restart, so a crash loop is
/// paced rather than amplified. Bounded by the number of driver children; an entry for a driver that is
/// later removed simply stops being consulted (the handler resolves <c>_children</c> first).</summary>
private readonly Dictionary<string, DateTime> _lastReprovisionUtc = new(StringComparer.Ordinal);
/// <summary>
/// Host of this node's LocalDb replication partner — the only node that holds a copy of this
/// node's alarm queue, and therefore the only node it may ever stand down in favour of.
/// <c>null</c> when this node dials nobody (unpaired, or the listening half of a pair).
/// </summary>
private readonly string? _replicationPeerHost;
/// <summary>
/// Review R6 MINOR-4: identifies the alarm-queue-sharing peer when <see cref="_replicationPeerHost"/> is
/// unknown — the passive (listening) half of a pair, which replicates the same rows but dials nobody. Before
/// this the passive half had no peer to defer to, so as Secondary it kept draining alongside its Primary
/// partner and every alarm event reached the historian twice. Null ⇒ no peer known ⇒ never stand down.
/// </summary>
private readonly Func<CommonsNodeId, bool>? _sharesAlarmQueueWithLocal;
/// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
/// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
private bool _warnedSnapshotMissingLocalNode;
/// <summary>Cached cluster DistributedPubSub mediator, resolved once in <see cref="PreStart"/> (on the
/// actor thread) and reused for the Primary-gated native-alarm <c>alerts</c> fan-out in
/// <see cref="ForwardNativeAlarm"/> instead of re-resolving it per-publish. Mirrors
/// <c>ScriptedAlarmHostActor._mediator</c>.</summary>
private IActorRef _mediator = null!;
/// <summary>
/// Routes an inbound operator write (resolved from the OPC UA node-manager side) to the
/// owning driver child. <see cref="HandleRouteNodeWrite"/> gates on the local node being the
/// driver Primary, resolves the <see cref="_driverRefByNodeId"/> reverse map, and Asks the child a
/// <see cref="DriverInstanceActor.WriteAttribute"/> carrying the driver-side <see cref="FullName"/>.
/// </summary>
/// <param name="NodeId">The folder-scoped equipment-variable NodeId the operator wrote to.</param>
/// <param name="Value">The value to write (the driver coerces it to the attribute's data type).</param>
public sealed record RouteNodeWrite(string NodeId, object? Value, AddressSpaceRealm Realm);
/// <summary>Reply to <see cref="RouteNodeWrite"/>: the outcome of forwarding the write to the driver
/// (or a gate/lookup failure that never reached the driver).</summary>
/// <param name="Success">True when the driver accepted the write.</param>
/// <param name="Reason">Failure detail when <paramref name="Success"/> is false; null on success.</param>
public sealed record NodeWriteResult(bool Success, string? Reason);
/// <summary>
/// Routes an inbound native-condition acknowledge to the owning driver child. The host wires the
/// OPC UA node manager's <c>NativeAlarmAckRouter</c> to Tell this in when a client Acknowledges a
/// NATIVE Part 9 condition; <see cref="HandleRouteNativeAlarmAck"/> applies the same Primary gate the
/// inbound write path uses, resolves the <see cref="_driverRefByAlarmNodeId"/> inverse map, and Tells
/// the owning <see cref="DriverInstanceActor"/> a <see cref="DriverInstanceActor.RouteAlarmAck"/>
/// carrying the principal — fire-and-forget (the Part 9 ack already committed the local condition
/// state). Deliberately decoupled from the OpcUaServer <c>NativeAlarmAck</c> type: the host maps
/// NativeAlarmAck → this at the wiring boundary so Runtime does not depend on the OPC UA layer.
/// </summary>
/// <param name="ConditionNodeId">The folder-scoped condition NodeId the operator acknowledged.</param>
/// <param name="Comment">Operator-supplied comment forwarded to the upstream alarm system; null when none.</param>
/// <param name="OperatorUser">The authenticated principal performing the acknowledge.</param>
public sealed record RouteNativeAlarmAck(string ConditionNodeId, string? Comment, string OperatorUser);
/// <summary>Self-sent by the 03/S15 audit timer, <see cref="_commsRecoveryAuditDelay"/> after a driver
/// reconnected: report any of its refs still latched <c>BadNoCommunication</c>. See
/// <see cref="HandleCommsRecoveryAudit"/>.</summary>
/// <param name="DriverInstanceId">The driver whose recovery is being audited.</param>
public sealed record CommsRecoveryAudit(string DriverInstanceId);
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
// Convenience accessors for sites that don't need the full spec.
/// <summary>Gets the driver type name from the child's spec.</summary>
public string DriverType => Spec.DriverType;
/// <summary>Gets the last applied driver configuration JSON from the child's spec.</summary>
public string LastConfigJson => Spec.DriverConfig;
/// <summary>Gets the per-instance ResilienceConfig JSON the child's invoker was built with (null = tier defaults).</summary>
public string? ResilienceConfig => Spec.ResilienceConfig;
}
/// <summary>Gets or sets the timer scheduler for this actor's config-db retry and rediscovery timers.</summary>
public ITimerScheduler Timers { get; set; } = null!;
public sealed class RetryConfigDbConnection
{
public static readonly RetryConfigDbConnection Instance = new();
private RetryConfigDbConnection() { }
}
/// <summary>Creates props for a DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
/// <param name="localNode">The local cluster node identifier.</param>
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// <c>MeshTransport:Mode = Grpc</c> the node's <c>NodeCommunicationActor</c>, which relays the ack
/// across the mesh boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
/// <param name="opcUaPublishActor">Optional actor reference for OPC UA publishing.</param>
/// <param name="healthPublisher">Optional driver-health publisher; defaults to <see cref="NullDriverHealthPublisher"/>
/// so test harnesses and smoke fixtures don't need to wire it.</param>
/// <param name="virtualTagEvaluator">Optional evaluator handed to the spawned
/// <see cref="VirtualTagHostActor"/>'s children; defaults to <see cref="NullVirtualTagEvaluator"/>
/// (the dev/Mac path where no expression is evaluated). Production passes the DI-resolved
/// Roslyn evaluator.</param>
/// <param name="historyWriter">Optional sink handed to the spawned <see cref="VirtualTagHostActor"/>
/// for VirtualTag results whose plan opted into <c>Historize=true</c>; defaults to
/// <see cref="NullHistoryWriter"/> (the durable AVEVA sink is infra-gated, so no live-data historian
/// write RPC exists). A deployment that binds a real <see cref="IHistoryWriter"/> in DI overrides it.</param>
/// <param name="virtualTagHostOverride">Test seam: when supplied, this actor is used as the
/// VirtualTag host instead of spawning a real <see cref="VirtualTagHostActor"/> child, so tests
/// can intercept the <see cref="VirtualTagHostActor.ApplyVirtualTags"/> message. Null in
/// production (the real host is spawned).</param>
/// <param name="loggerFactory">Retained for constructor-signature stability; unused since Phase 4
/// Task 9 retired the <c>EfAlarmConditionStateStore</c> fallback that consumed it. Defaults to null.</param>
/// <param name="scriptRootLogger">Optional root script logger required to spawn the ScriptedAlarm
/// host (the engine + its script logging hang off it). When null the ScriptedAlarm host is left
/// unspawned — the graceful dev/None-deployment path.</param>
/// <param name="scriptedAlarmHostOverride">Test seam: when supplied, this actor is used as the
/// ScriptedAlarm host instead of spawning a real <see cref="ScriptedAlarmHostActor"/> child, so
/// tests can intercept the <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/> message. Null
/// in production (the real host is spawned).</param>
/// <param name="invokerFactory">Optional Phase 6.1 resilience-invoker factory used to attach an
/// <see cref="IDriverCapabilityInvoker"/> to each spawned driver; defaults to
/// <see cref="NullDriverCapabilityInvokerFactory"/> (pass-through) when not supplied.</param>
/// <param name="alarmStateStore">Per-cluster mesh Phase 4: scripted-alarm condition-state store. On a
/// driver-role node this is the replicated LocalDb store, so condition state persists with no
/// ConfigDb; null skips spawning the alarm host (the ConfigDb-backed EF fallback was retired in
/// Phase 4 Task 9). Defaults to null.</param>
/// <param name="telemetryHub">Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
/// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
/// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).</param>
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; tests shorten it.</param>
/// <param name="applyOutcomeTimeout">Gitea #601: how long an Applied ack waits for the address-space
/// apply outcome before it is sent anyway, marked degraded ("outcome unknown"). Defaults to
/// <see cref="DefaultApplyOutcomeTimeout"/>; tests shorten it.</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
IActorRef? ackRouter = null,
IDriverFactory? driverFactory = null,
IReadOnlySet<string>? localRoles = null,
IActorRef? dependencyMux = null,
IActorRef? opcUaPublishActor = null,
IDriverHealthPublisher? healthPublisher = null,
IVirtualTagEvaluator? virtualTagEvaluator = null,
IHistoryWriter? historyWriter = null,
IActorRef? virtualTagHostOverride = null,
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null,
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
IAlarmStateStore? alarmStateStore = null,
ITelemetryLocalHub? telemetryHub = null,
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null,
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
TimeSpan? applyOutcomeTimeout = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
// the wrong dependency at runtime. New parameters go LAST in all three places — this
// signature, the constructor's, and this call — and nothing else moves.
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, ackRouter, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
/// <param name="localNode">The local cluster node identifier.</param>
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// <c>MeshTransport:Mode = Grpc</c> the node's <c>NodeCommunicationActor</c>, which relays the ack
/// across the mesh boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
/// <param name="opcUaPublishActor">Optional actor reference for OPC UA publishing.</param>
/// <param name="healthPublisher">Optional driver-health publisher; defaults to <see cref="NullDriverHealthPublisher"/>.</param>
/// <param name="virtualTagEvaluator">Optional evaluator handed to the VirtualTag host's children;
/// defaults to <see cref="NullVirtualTagEvaluator"/>.</param>
/// <param name="historyWriter">Optional sink handed to the spawned <see cref="VirtualTagHostActor"/>
/// for historized VirtualTag results; defaults to <see cref="NullHistoryWriter"/>.</param>
/// <param name="virtualTagHostOverride">Test seam: when supplied, used as the VirtualTag host
/// instead of spawning a real <see cref="VirtualTagHostActor"/> child.</param>
/// <param name="loggerFactory">Retained for constructor-signature stability; unused since Phase 4
/// Task 9 retired the <c>EfAlarmConditionStateStore</c> fallback that consumed it. Defaults to null.</param>
/// <param name="scriptRootLogger">Optional root script logger required to spawn the ScriptedAlarm
/// host; when null the host is left unspawned.</param>
/// <param name="scriptedAlarmHostOverride">Test seam: when supplied, used as the ScriptedAlarm host
/// instead of spawning a real <see cref="ScriptedAlarmHostActor"/> child.</param>
/// <param name="invokerFactory">Phase 6.1 resilience-invoker factory used to attach an
/// <see cref="IDriverCapabilityInvoker"/> to each spawned driver; defaults to
/// <see cref="NullDriverCapabilityInvokerFactory"/> (pass-through) when null.</param>
/// <param name="driverMemberCountProvider">Test seam (archreview 03/S4): overrides the count of Up
/// <c>driver</c>-role cluster members the Primary gate reads while the role is unknown. When null the
/// default reads <c>Cluster.Get(Context.System).State.Members</c> (0 on a non-cluster ActorRefProvider).</param>
/// <param name="deploymentArtifactCache">Optional node-local cache of applied deployment artifacts.
/// When supplied, each successful apply stores its artifact so the node can boot from its
/// last-known-good configuration while central SQL is unreachable. Null on admin-only nodes and in
/// tests that do not exercise the cache — caching is then simply skipped.</param>
/// <param name="alarmStateStore">Per-cluster mesh Phase 4: scripted-alarm condition-state store. On a
/// driver-role node this is the replicated LocalDb store; null skips spawning the alarm host (the
/// ConfigDb-backed EF fallback was retired in Phase 4 Task 9).</param>
/// <param name="telemetryHub">Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
/// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
/// VirtualTag + ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.</param>
/// <param name="primaryRoleSource">Gitea #511: where the Primary <i>data-plane</i> verdict is published
/// for drivers that assert a standing exclusive identity on a shared external system — today the MQTT
/// driver's Sparkplug primary-host STATE claim. Deliberately a separate parameter from
/// <paramref name="redundancyRoleView"/> even though the production object implements both: a test that
/// supplies only a role-view double must not silently also satisfy this, or its primary-role assertions
/// pass without the publish ever happening. Null on nodes that wire no redundancy manager.</param>
/// <param name="historizationRoleView">Gitea #537: where the continuous-historization <i>capture</i>
/// verdict is published, for the recorder that taps the mux fan-out. A third distinct parameter for
/// the same reason <paramref name="primaryRoleSource"/> is a second — the production object
/// implements all three, and a test double supplied for one must not silently satisfy the others.
/// Null on nodes that wire no redundancy manager.</param>
/// <param name="sharesHistorizationScopeWithLocal">Tests whether a node named in a redundancy
/// snapshot serves this node's address space and therefore records the same historized refs. Null
/// (the default) means the caller could not establish it, and this node never stands down — see
/// <see cref="PrimaryGatePolicy.ShouldRecordHistorizedValues"/>.</param>
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>; a test shortens
/// it so the watchdog is observable in a test's lifetime.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
/// <param name="sharesAlarmQueueWithLocal">Review R6 MINOR-4: tests whether a node named in a redundancy
/// snapshot holds this node's replicated alarm queue, for a node that knows no
/// <paramref name="replicationPeerHost"/> — the PASSIVE half of a LocalDb pair (ApiKey + SyncListenPort, no
/// PeerAddress), whose partner is its per-cluster mesh partner. Ignored when
/// <paramref name="replicationPeerHost"/> is set. Null ⇒ no such peer is known and this node never stands its
/// drain down.</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
IActorRef? ackRouter,
IDriverFactory? driverFactory = null,
IReadOnlySet<string>? localRoles = null,
IActorRef? dependencyMux = null,
IActorRef? opcUaPublishActor = null,
IDriverHealthPublisher? healthPublisher = null,
IVirtualTagEvaluator? virtualTagEvaluator = null,
IHistoryWriter? historyWriter = null,
IActorRef? virtualTagHostOverride = null,
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null,
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
IAlarmStateStore? alarmStateStore = null,
ITelemetryLocalHub? telemetryHub = null,
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null,
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
TimeSpan? applyOutcomeTimeout = null)
{
_applyOutcomeTimeout = applyOutcomeTimeout ?? DefaultApplyOutcomeTimeout;
_sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
_historizationRoleView = historizationRoleView;
_sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
_telemetryHub = telemetryHub;
_deploymentArtifactCache = deploymentArtifactCache;
_redundancyRoleView = redundancyRoleView;
_primaryRoleSource = primaryRoleSource;
_replicationPeerHost = replicationPeerHost;
_fetchAndCacheMode = fetchAndCacheMode;
_artifactFetcher = artifactFetcher;
_alarmStateStore = alarmStateStore;
_dbFactory = dbFactory;
_localNode = localNode;
_ackRouter = ackRouter;
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
_driverMemberCountProvider = driverMemberCountProvider;
_localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
_healthPublisher = healthPublisher ?? NullDriverHealthPublisher.Instance;
_virtualTagEvaluator = virtualTagEvaluator ?? NullVirtualTagEvaluator.Instance;
_historyWriter = historyWriter ?? NullHistoryWriter.Instance;
_virtualTagHostOverride = virtualTagHostOverride;
_scriptRootLogger = scriptRootLogger;
_scriptedAlarmHostOverride = scriptedAlarmHostOverride;
// Default behavior is Steady — PreStart may flip to Stale or replay an orphan apply.
// The ONE Become(Steady) that is not BecomeSteady(): Stash is assigned by the ActorCell after
// the constructor returns, so it is null here — and nothing can have been stashed yet anyway.
Become(Steady);
}
/// <inheritdoc />
protected override void PreStart()
{
// Resolve the cluster DPS mediator once, on the actor thread, and reuse it for every subscribe +
// the Primary-gated native-alarm alerts fan-out in ForwardNativeAlarm (mirrors ScriptedAlarmHostActor).
_mediator = DistributedPubSub.Get(Context.System).Mediator;
// Subscribe to deployments topic so the coordinator's broadcast lands here.
_mediator.Tell(new Subscribe(DeploymentsTopic, Self));
// Subscribe to driver-control topic so AdminUI Reconnect/Restart commands land here.
_mediator.Tell(new Subscribe(DriverControlTopic, Self));
// Subscribe to the redundancy-state topic so cluster role changes cache this node's role — the
// inbound-write gate in HandleRouteNodeWrite reuses the SAME signal the scripted-alarm emit gate
// uses so only the Primary services operator writes (the secondary keeps state warm for failover).
_mediator.Tell(
new Subscribe(ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RedundancyStateTopic, Self));
// Per-cluster mesh Phase 2. Under MeshTransport:Mode = Grpc these same commands arrive from
// NodeCommunicationActor on the node-local EventStream instead of the mesh-wide DPS topic.
// Subscribing to BOTH unconditionally is deliberate: exactly one of the two ever publishes, so
// the transport choice lives entirely on the publishing side and the mode can be flipped (or
// reverted) without touching any subscriber.
Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment));
Context.System.EventStream.Subscribe(Self, typeof(RestartDriver));
Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver));
// Spawn the VirtualTag host BEFORE Bootstrap so the bootstrap-restore path (which routes
// through PushDesiredSubscriptions and Tells ApplyVirtualTags) has a live host to target.
SpawnVirtualTagHost();
// Same rationale for the ScriptedAlarm host — the bootstrap-restore path Tells it
// ApplyScriptedAlarms through the same PushDesiredSubscriptions pass.
SpawnScriptedAlarmHost();
StartPrimaryVerdictPublishing();
Bootstrap();
}
/// <summary>
/// Review R1 m2 (Gitea #597): the published Primary data-plane verdict (<see cref="IDriverPrimaryRoleSource"/>
/// — the native-ack gate and the Sparkplug primary-host claim) must mirror <see cref="ShouldServiceAsPrimary"/>
/// from the moment this host starts, not only from the first redundancy snapshot. Before this, the view sat
/// on its single-node seed (allow) through the whole boot window, so on a pair BOTH nodes accepted a native
/// ack while this host — which evaluates the member count live — dropped the upstream route on the
/// non-Primary: the acked → unacked revert #597 targets. So publish once now, and again whenever cluster
/// membership changes (a driver peer coming Up turns an unknown role from allow into deny before any
/// snapshot names this node). A non-cluster ActorRefProvider (legacy/test harnesses) just keeps the
/// initial publish.
/// <para>
/// ⚠️ <b>It MIRRORS the live gate; it does not deny from host start.</b> The host is spawned at
/// ActorSystem start, before this node has joined, when its view of <c>Cluster.State.Members</c> is
/// empty — a member count of 0, which the single-node posture ALLOWS. So the verdict published here is
/// allow, and it flips to deny only once this node's own cluster state shows a second Up driver member:
/// normally seconds, up to <c>Cluster:BootstrapGuard:PartnerProbeSeconds</c> (25 s) under the
/// bootstrap guard. In that pre-join window the Sparkplug primary-host claim can be held on both nodes of
/// a pair. Native ack is NOT exposed by it: the live route gate reads the same count and also allows, so
/// the view and the route still agree and the acked → unacked revert cannot occur (review R6 MINOR-2).
/// Denying until self is Up was considered and NOT done: it would change the live data-plane gate (the
/// two must stay one decision) and so delay writes and acks on a genuinely single-node cluster during its
/// own join, and this actor has no reliable way to know that a partner is configured.
/// </para>
/// </summary>
private void StartPrimaryVerdictPublishing()
{
if (_primaryRoleSource is null) return;
PublishPrimaryVerdict();
try
{
Akka.Cluster.Cluster.Get(Context.System).Subscribe(
Self, Akka.Cluster.ClusterEvent.SubscriptionInitialStateMode.InitialStateAsSnapshot,
typeof(Akka.Cluster.ClusterEvent.IMemberEvent));
_subscribedToMemberEvents = true;
}
catch (Exception)
{
// Non-cluster ActorRefProvider — no membership to follow; DriverMemberCount() reads 0 there too.
}
}
/// <summary>A cluster member changed status: re-publish the Primary verdict now and once more shortly after.
/// The deferred pass covers the cluster read-view (what <see cref="DriverMemberCount"/> reads) updating on its
/// own subscriber AFTER this event was delivered here — the two are not ordered.</summary>
private void OnDriverMembershipChanged(Akka.Cluster.ClusterEvent.IMemberEvent _)
{
PublishPrimaryVerdictIfChanged();
// One keyed timer: a burst of member events collapses into a single deferred pass.
Timers.StartSingleTimer("republish-primary-verdict", RepublishPrimaryVerdict.Instance, MembershipRepublishDelay);
}
/// <summary>Publishes <see cref="ShouldServiceAsPrimary"/> to the primary-role source (no-op when none is wired).
/// The snapshot path publishes on every snapshot, as the other verdicts do.</summary>
private void PublishPrimaryVerdict()
{
if (_primaryRoleSource is null) return;
var verdict = ShouldServiceAsPrimary();
_lastPublishedPrimaryVerdict = verdict;
_primaryRoleSource.PublishPrimaryRole(verdict);
}
/// <summary>The membership path publishes only a CHANGED verdict — member events are frequent and carry no
/// role information, so republishing an unchanged verdict on each would be noise.</summary>
private void PublishPrimaryVerdictIfChanged()
{
if (_primaryRoleSource is null || ShouldServiceAsPrimary() == _lastPublishedPrimaryVerdict) return;
PublishPrimaryVerdict();
}
/// <inheritdoc />
protected override void PostStop()
{
if (_subscribedToMemberEvents)
{
try { Akka.Cluster.Cluster.Get(Context.System).Unsubscribe(Self); }
catch (Exception) { /* system shutting down */ }
}
base.PostStop();
}
/// <summary>Self-message: re-evaluate and re-publish the Primary verdict (see <see cref="OnDriverMembershipChanged"/>).</summary>
internal sealed class RepublishPrimaryVerdict
{
/// <summary>The shared instance.</summary>
public static readonly RepublishPrimaryVerdict Instance = new();
private RepublishPrimaryVerdict() { }
}
/// <summary>How long after a membership event the Primary verdict is re-published a second time.</summary>
private static readonly TimeSpan MembershipRepublishDelay = TimeSpan.FromSeconds(1);
private bool _subscribedToMemberEvents;
private bool? _lastPublishedPrimaryVerdict;
/// <summary>
/// Spawns the single <see cref="VirtualTagHostActor"/> child that owns the Equipment-namespace
/// VirtualTagActors and bridges their results onto the OPC UA publish actor. A test-supplied
/// <c>virtualTagHostOverride</c> short-circuits the spawn so a probe can intercept
/// <see cref="VirtualTagHostActor.ApplyVirtualTags"/>. The real host requires a non-null
/// <see cref="_opcUaPublishActor"/> (its ctor throws otherwise), so when no publish actor is
/// wired (legacy ControlPlane test harnesses with no OPC UA sink) the host is left null and
/// ApplyVirtualTags becomes a no-op — VirtualTags can't have anywhere to publish without it.
/// </summary>
private void SpawnVirtualTagHost()
{
if (_virtualTagHostOverride is not null)
{
_virtualTagHost = _virtualTagHostOverride;
return;
}
if (_opcUaPublishActor is null)
{
_log.Debug("DriverHost {Node}: no OPC UA publish actor wired; skipping VirtualTag host spawn", _localNode);
return;
}
_virtualTagHost = Context.ActorOf(
VirtualTagHostActor.Props(_opcUaPublishActor, _dependencyMux, _virtualTagEvaluator, _historyWriter, _telemetryHub),
"virtual-tag-host");
}
/// <summary>
/// Spawns the single <see cref="ScriptedAlarmHostActor"/> child that owns the per-node
/// <see cref="ScriptedAlarmEngine"/>, feeds it live tag values from the dependency mux, and
/// bridges its emissions onto the OPC UA publish actor + the cluster <c>alerts</c> topic. A
/// test-supplied <c>scriptedAlarmHostOverride</c> short-circuits the spawn so a probe can
/// intercept <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/>. The real host needs
/// both a non-null <see cref="_opcUaPublishActor"/> (the emission sink) and a non-null
/// <see cref="_scriptRootLogger"/> (the engine + its script logging hang off it); when either
/// is missing (legacy ControlPlane test harnesses, dev/None deployments) the host is left
/// null and ApplyScriptedAlarms becomes a no-op. The engine is built around a fresh
/// <see cref="DependencyMuxTagUpstreamSource"/> + the wired <see cref="IAlarmStateStore"/> (the
/// replicated <c>LocalDbAlarmConditionStateStore</c> on a driver-role node); when no store was
/// wired the host is skipped. The host (spawned as a child) owns + disposes the engine in its
/// PostStop, so it stops with the driver host.
/// </summary>
private void SpawnScriptedAlarmHost()
{
if (_scriptedAlarmHostOverride is not null)
{
_scriptedAlarmHost = _scriptedAlarmHostOverride;
return;
}
if (_opcUaPublishActor is null || _scriptRootLogger is null)
{
_log.Debug(
"DriverHost {Node}: skipping ScriptedAlarm host spawn (no publish actor / root logger)",
_localNode);
return;
}
// Per-cluster mesh Phase 4: condition state lives in the wired store — the replicated LocalDb
// store on a driver-role node, which persists with no ConfigDb. The ConfigDb-backed EF fallback
// was retired (Task 9); when no store was wired there is nowhere to persist condition state, so
// we skip the alarm host outright rather than run the engine against a store it can never save to
// (mirrors the _opcUaPublishActor-null skip above).
if (_alarmStateStore is null)
{
_log.Debug(
"DriverHost {Node}: skipping ScriptedAlarm host spawn (no condition-state store)",
_localNode);
return;
}
var store = _alarmStateStore;
var upstream = new DependencyMuxTagUpstreamSource();
var engine = new ScriptedAlarmEngine(
upstream, store, new ScriptLoggerFactory(_scriptRootLogger.Logger), _scriptRootLogger.Logger);
_scriptedAlarmHost = Context.ActorOf(
ScriptedAlarmHostActor.Props(
_opcUaPublishActor, _dependencyMux, upstream, engine, _localNode, telemetryHub: _telemetryHub),
"scripted-alarm-host");
}
private void Bootstrap()
{
// Per-cluster mesh Phase 3: a FetchAndCache node NEVER reads central SQL for config, at boot
// or otherwise. It restores its served state from the LocalDb pointer instead.
if (_fetchAndCacheMode)
{
BootstrapFromCache();
return;
}
// Belt-and-suspenders (Phase 4): every ConfigDb read below is on the Direct path only — a
// FetchAndCache node already returned via BootstrapFromCache above, and only a FetchAndCache node
// runs DB-less. If a null factory ever reaches a Direct path it is a misconfiguration; enter
// Steady with no revision rather than NRE, and let the first dispatch drive the apply.
if (_dbFactory is null)
{
_log.Info("DriverHost {Node}: no ConfigDb on the Direct bootstrap path; entering Steady (no revision)", _localNode);
BecomeSteady();
return;
}
// Read the most-recent NodeDeploymentState for this node; if it's Applied, jump
// to Steady with that revision. If Applying (orphan from a crash), discard and replay.
// If the DB is unreachable, fall back to Stale and start the reconnect loop.
try
{
using var db = _dbFactory.CreateDbContext();
var latest = db.NodeDeploymentStates
.Where(s => s.NodeId == _localNode.Value)
.OrderByDescending(s => s.StartedAtUtc)
.Select(s => new { s.DeploymentId, s.Status, s.StartedAtUtc })
.FirstOrDefault();
if (latest is null)
{
_log.Info("DriverHost {Node}: no prior deployments; entering Steady (no revision)", _localNode);
BecomeSteady();
return;
}
var deployment = db.Deployments
.AsNoTracking()
.FirstOrDefault(d => d.DeploymentId == latest.DeploymentId);
var revision = deployment is null
? (RevisionHash?)null
: RevisionHash.Parse(deployment.RevisionHash);
switch (latest.Status)
{
case NodeDeploymentStatus.Applied:
_currentRevision = revision;
_log.Info("DriverHost {Node}: recovered Applied state at rev {Rev}", _localNode, revision);
BecomeSteady();
// The revision is recovered but the in-memory driver children + OPC UA address
// space were lost on restart. Re-spawn + re-materialise + re-subscribe from the
// applied deployment so a restarted/rebuilt node restores its served state instead
// of waiting for a config change (whose identical-config revision would no-op).
RestoreApplied(new DeploymentId(latest.DeploymentId), revision);
break;
case NodeDeploymentStatus.Applying:
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying",
_localNode, latest.DeploymentId);
if (revision is not null) BeginDirectApply(new DeploymentId(latest.DeploymentId), revision.Value, CorrelationId.NewId());
else BecomeSteady();
break;
case NodeDeploymentStatus.Failed:
default:
_log.Info("DriverHost {Node}: prior deployment {Id} failed; entering Steady at last known rev",
_localNode, latest.DeploymentId);
BecomeSteady();
break;
}
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: ConfigDb unreachable on bootstrap; entering Stale", _localNode);
// Central is unreachable, so try the node-local cache before giving up. On a hit this
// node serves its last-known-good configuration through the outage instead of coming up
// with an empty address space. On a miss, behaviour is exactly what it always was.
if (!TryBootFromCache())
Become(Stale);
}
}
/// <summary>
/// Per-cluster mesh Phase 3 boot for <c>FetchAndCache</c> mode. Restores served state from the
/// LocalDb pointer without any central-SQL read.
/// </summary>
/// <remarks>
/// <para>
/// Distinct from <see cref="TryBootFromCache"/> (the Direct-mode SQL-unreachable fallback):
/// reading from the cache is <b>normal</b> operation here, not a degraded state, so
/// <see cref="_isRunningFromCache"/> stays <see langword="false"/> — the node can still
/// fetch new deployments. An empty or unreadable cache lands Steady-with-no-revision (the
/// first dispatch fetches), never <c>Stale</c>: Stale means "central SQL is down, retry it",
/// and FetchAndCache has no config SQL read to recover.
/// </para>
/// </remarks>
private void BootstrapFromCache()
{
try
{
var cached = _deploymentArtifactCache?.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
if (cached is null)
{
_log.Info(
"DriverHost {Node}: FetchAndCache boot — the LocalDb cache is empty; entering Steady " +
"(no revision). The first dispatch will fetch this node's configuration from central.",
_localNode);
BecomeSteady();
return;
}
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
// Restore drivers + address space + subscriptions from the cached bytes — no SQL read, no
// re-ack (the deployment is already Applied). Same in-hand-bytes apply as the Direct-mode
// cache boot, but this is the steady-state config source, not an outage fallback.
if (!ApplyCachedArtifact(deploymentId, cached.Artifact))
{
// #601 / review R1 m4: unusable cached bytes applied nothing, so they must not become the
// current revision — that would ack a re-dispatch of the same revision as already applied
// without ever applying it. Steady-with-no-revision, exactly as for an empty cache.
_log.Warning(
"DriverHost {Node}: FetchAndCache boot — the cached artifact for deployment {Id} (rev {Rev}) " +
"is unreadable; entering Steady (no revision). The next dispatch will fetch.",
_localNode, deploymentId, revision);
BecomeSteady();
return;
}
_currentRevision = revision;
BecomeSteady();
_log.Info(
"DriverHost {Node}: FetchAndCache boot — restored served state for deployment {Id} " +
"(rev {Rev}) from the LocalDb pointer.",
_localNode, deploymentId, revision);
}
catch (Exception ex)
{
_log.Warning(ex,
"DriverHost {Node}: FetchAndCache boot — failed to read the LocalDb pointer; entering " +
"Steady (no revision). The next dispatch will fetch.",
_localNode);
BecomeSteady();
}
}
/// <summary>
/// Last-resort boot path: apply the artifact cached by a previous successful deploy (or
/// replicated from this node's pair peer) when central SQL cannot be reached.
/// </summary>
/// <returns>
/// <see langword="true"/> when a cached artifact was applied and the actor is now Steady;
/// <see langword="false"/> when the caller should fall through to <c>Stale</c>.
/// </returns>
/// <remarks>
/// <para>
/// <b>The read is unkeyed.</b> The cache is keyed by ClusterId so a pair shares one
/// entry, but ClusterId is only derivable from an artifact you already hold or from the
/// central DB — and at this seam we have neither. So the newest pointer row wins, which
/// is correct in every real topology because a node belongs to one cluster and its peer
/// replicates that same cluster's row. A node re-homed between clusters is the only
/// ambiguous case, and the cache logs a warning naming both.
/// </para>
/// <para>
/// <b>This does not make the node current.</b> <c>_currentRevision</c> is set from the
/// cached artifact, so a subsequent dispatch of that same revision correctly no-ops,
/// while any NEW revision still requires central — the cache holds the past, not the
/// future. Dispatch handling is unchanged.
/// </para>
/// <para>
/// Never throws: a fault here must degrade to Stale, which is exactly where the node
/// would have been without a cache at all.
/// </para>
/// </remarks>
private bool TryBootFromCache()
{
if (_deploymentArtifactCache is null)
return false;
try
{
var cached = _deploymentArtifactCache.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
if (cached is null)
{
_log.Info(
"DriverHost {Node}: no cached deployment available; entering Stale with no configuration.",
_localNode);
return false;
}
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
var revision = RevisionHash.Parse(cached.RevisionHash);
// #601 / review R1 m4: unusable cached bytes applied nothing — the node is NOT serving a
// configuration, so it must not claim RUNNING FROM CACHE, adopt the revision (a re-dispatch of
// it would then be acked without being applied) or skip Stale's ConfigDb retry loop.
if (!ApplyCachedArtifact(deploymentId, cached.Artifact))
{
_log.Error(
"DriverHost {Node}: the cached artifact for deployment {Id} (rev {Rev}) is unreadable; " +
"not booting from it — entering Stale with no configuration.",
_localNode, deploymentId, revision);
return false;
}
// Steady, not Stale: this node is serving a real configuration. The retry-db timer that
// Stale would have started does not run here, so recovery rides on the next dispatch —
// matching how a normally-booted node behaves.
_currentRevision = revision;
BecomeSteady();
_isRunningFromCache = true;
_log.Warning(
"DriverHost {Node}: RUNNING FROM CACHE — central ConfigDb is unreachable, so this node " +
"booted deployment {Id} (rev {Rev}) cached at {CachedAtUtc:o} from its local database. " +
"Configuration changes cannot be applied until the ConfigDb is reachable again.",
_localNode, deploymentId, revision, cached.AppliedAtUtc);
return true;
}
catch (Exception ex)
{
_log.Error(ex,
"DriverHost {Node}: failed to boot from the local deployment cache; falling back to Stale.",
_localNode);
return false;
}
}
/// <summary>
/// Applies an artifact already in hand — no ConfigDb read, no ACK.
/// </summary>
/// <remarks>
/// Mirrors <see cref="RestoreApplied"/>, but sources the artifact from the cache rather than
/// re-reading it from a database that is by definition unreachable here. It also skips
/// <c>UpsertNodeDeploymentState</c> and <c>SendAck</c> for the same reason: both write to or
/// depend on central.
/// </remarks>
/// <returns><see langword="false"/> when the bytes were empty or malformed and nothing was applied — the
/// caller must not adopt the cached revision (#601, review R1 m4).</returns>
private bool ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
{
var correlation = CorrelationId.NewId();
// #601: null = the cached bytes were empty or malformed — no answer, not an empty configuration.
// Keep last-known-good: no address-space rebuild and no subscription push from unusable bytes.
if (ReconcileDriversFromBlob(deploymentId, blob) is null)
return false;
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
// definition — and the rebuild would no-op, leaving OPC UA clients browsing an empty server
// while the drivers below poll happily. (Found on the docker-dev live gate.)
_opcUaPublishActor?.Tell(
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, blob));
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
return true;
}
/// <summary>
/// Enters <see cref="Steady"/> and releases anything the <see cref="Applying"/> state stashed.
/// </summary>
/// <remarks>
/// <para>
/// A deferred dispatch must re-enter the mailbox <b>exactly once</b> and <b>in order</b> —
/// <see cref="IStash.UnstashAll"/> prepends the stash to the front of the mailbox in stash
/// order, which is both. Forgetting it on any one path silently strands that dispatch
/// forever, and the coordinator only learns about it at its apply deadline.
/// </para>
/// <para>
/// Every <c>Become(Steady)</c> goes through here — including the bootstrap and Stale-recovery
/// paths, where the stash is provably empty (only <see cref="Applying"/> ever stashes, and
/// those paths run before or outside it). Unstashing an empty stash is a no-op, and routing
/// all of them through one helper is what makes "did we cover every transition?" answerable
/// by grep. The constructor is the one exception: <see cref="Stash"/> is assigned by the
/// ActorCell only after construction returns.
/// </para>
/// <para>
/// It is also the single place the <see cref="Applying"/> state's two side-tables are
/// released: <see cref="_deferredDispatches"/> (which must be cleared in lockstep with the
/// unstash, or the re-delivered dispatch would be treated as an already-deferred duplicate
/// and dropped) and the <see cref="ApplyWedgeTimerKey"/> watchdog.
/// </para>
/// </remarks>
private void BecomeSteady()
{
// Cancelled BEFORE the unstash so a re-stash by the very next apply starts a fresh window.
Timers.Cancel(ApplyWedgeTimerKey);
_deferredDispatches.Clear();
Become(Steady);
Stash.UnstashAll();
}
private void Steady()
{
Receive<DispatchDeployment>(HandleDispatchFromSteady);
// #601 — the held Applied ack is released by the address-space outcome (or its timeout); both can
// land in any behaviour, so all three register them.
Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
Receive<ArtifactCached>(HandleArtifactCached);
Receive<ArtifactCacheFailed>(HandleArtifactCacheFailed);
// A wedge tick already in the mailbox when the apply finished — dropped at Debug, see the handler.
Receive<ApplyWedged>(HandleApplyWedged);
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
}
private void Applying()
{
Receive<DispatchDeployment>(msg =>
{
if (_applyingDeploymentId is not null && msg.DeploymentId == _applyingDeploymentId.Value)
{
_log.Debug("DriverHost {Node}: duplicate DispatchDeployment for in-flight {Id}; ignoring",
_localNode, msg.DeploymentId);
return;
}
// At most ONE stashed copy per distinct deployment (see _deferredDispatches): the redundant
// copies would all short-circuit as duplicates on the way out anyway, so a re-dispatching
// coordinator would otherwise grow the stash with retries rather than with work.
if (!_deferredDispatches.Add(msg.DeploymentId))
{
_log.Debug("DriverHost {Node}: dispatch for {Id} is already deferred behind {Cur}; dropping the duplicate",
_localNode, msg.DeploymentId, _applyingDeploymentId);
return;
}
// Stashed, NOT re-forwarded to Self: a forward re-enqueues on this actor's own mailbox and
// is dequeued again immediately, spinning for the whole apply. Every transition back to
// Steady unstashes (see BecomeSteady), so this is logged once per deferred dispatch rather
// than once per spin iteration.
_log.Debug("DriverHost {Node}: dispatch for {Id} received while still applying {Cur}; deferring",
_localNode, msg.DeploymentId, _applyingDeploymentId);
Stash.Stash();
});
// Both apply paths pipe their artifact bytes back here — the FetchAndCache gRPC fetch and the
// Direct ConfigDb read (BeginFetchAndCacheApply / BeginDirectApply ran in Steady, then
// Become(Applying)); completing the apply transitions back to Steady.
Receive<ArtifactForApply>(HandleArtifactForApply);
Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
// The cache write is piped back too, and it outlives the apply that started it by design — so
// its completion can land in EITHER state and both must handle it or it dead-letters.
Receive<ArtifactCached>(HandleArtifactCached);
Receive<ArtifactCacheFailed>(HandleArtifactCacheFailed);
Receive<ApplyWedged>(HandleApplyWedged);
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
}
/// <summary>
/// Routes one batched driver production burst — the ONLY value ingress from a driver child, since
/// <see cref="DriverInstanceActor"/> wraps even a single data change in an
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/>.
/// <para>
/// Each item's <c>FullReference</c> (the tag's RawPath) resolves through
/// <see cref="_nodeIdByDriverRef"/> — rebuilt every apply from the composition's <c>RawTags</c>
/// ∪ <c>UnsReferenceVariables</c> — to the v3 Batch 4 <b>single-source fan-out</b>: the RAW
/// NodeId (<see cref="AddressSpaceRealm.Raw"/>) AND every referencing UNS NodeId
/// (<see cref="AddressSpaceRealm.Uns"/>), each carrying IDENTICAL value/status/timestamp, so
/// there is no independent buffer to drift. An unmapped ref is Debug-logged and dropped without
/// disturbing the rest of the burst.
/// </para>
/// <para>
/// The whole burst costs ONE hop to the dependency mux (which filters per item itself, feeding
/// VirtualTag inputs keyed by RawPath) and ONE
/// <see cref="ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AttributeValueBatchUpdate"/> to
/// the publish actor — never one message per value. Both sinks are optional: without a mux,
/// VirtualTagActor evaluation can't fire (the dev/Mac path, no virtual tags registered);
/// production binds both via the RuntimeActors extension.
/// </para>
/// </summary>
private void ForwardBatchToMux(DriverInstanceActor.AttributeValueBatchPublished msg)
{
// One envelope per burst; the mux filters per item (it registered a batch receive in Task 9).
_dependencyMux?.Tell(msg);
// 03/S15 recovery audit: clear whatever this burst re-published from the driver's Bad latch. Guarded
// by ONE dictionary probe per burst — with no outage outstanding (the steady state) this is a single
// miss, not per-item work.
if (_commsLatched.ContainsKey(msg.DriverInstanceId))
{
for (var i = 0; i < msg.Items.Count; i++)
TrackCommsLatchPrune(msg.DriverInstanceId, msg.Items[i].FullReference);
}
if (_opcUaPublishActor is null) return;
// Indexed for-loop, not foreach: IReadOnlyList<AttributeValueItem> enumerates through an interface
// enumerator that would allocate once per burst on the hottest routing seam in the server.
var count = msg.Items.Count;
List<ValueWrite>? writes = null;
for (var idx = 0; idx < count; idx++)
{
var item = msg.Items[idx];
if (_nodeIdByDriverRef.TryGetValue((msg.DriverInstanceId, item.FullReference), out var nodeIds))
{
// Sized for the dominant case — every item resolving to its raw node alone, no UNS
// reference — where this is exact and the list never grows. A burst that does fan out
// pays at most a doubling or two, which beats over-allocating on every burst that doesn't.
writes ??= new List<ValueWrite>(count);
foreach (var n in nodeIds)
writes.Add(new ValueWrite(n.NodeId, item.Value, item.StatusCode, item.TimestampUtc, n.Realm));
}
else if (_log.IsDebugEnabled)
{
_log.Debug("DriverHost {Node}: no bound NodeId for ({Driver},{Ref}) — value dropped",
_localNode, msg.DriverInstanceId, item.FullReference);
}
}
// Nothing resolved ⇒ no message at all, rather than an empty batch for the sink to unpack.
if (writes is not null)
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AttributeValueBatchUpdate(writes));
}
/// <summary>
/// Routes a native alarm transition (published by a driver child as
/// <see cref="DriverInstanceActor.AttributeAlarmPublished"/>) to its materialised Part 9 condition
/// node(s). The alarm path analogue of <see cref="ForwardBatchToMux"/>: the transition's <c>ConditionId</c>
/// (the dotted alarm full-reference, which equals the authored equipment-tag FullName — NOT the bare
/// <c>SourceNodeId</c> owning object) is resolved by the <see cref="_alarmNodeIdByDriverRef"/> map —
/// built each apply from the alarm-bearing EquipmentTags — to the folder-scoped condition NodeId(s)
/// the materialiser placed the condition(s) at.
/// For each node the <see cref="_nativeAlarmProjector"/> projects the transition delta into a full
/// <c>AlarmConditionSnapshot</c>, then this Tells <see cref="ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate"/>
/// — the SAME message scripted alarms use, so it routes through <c>WriteAlarmCondition</c>. An
/// unknown ref is Debug-logged and dropped (mirrors the value drop).
///
/// <para>
/// Each transition is ALSO published as an <see cref="AlarmTransitionEvent"/> to the cluster
/// <c>alerts</c> topic — the single historization + live <c>/alerts</c> fan-out path, exactly as
/// scripted alarms do (<c>ScriptedAlarmHostActor.OnEngineEmission</c>). The OPC UA condition
/// write above stays UNGATED (a Secondary keeps its address space warm for failover), but the
/// cluster-wide alerts publish is Primary-gated on the same <see cref="_localRole"/> redundancy
/// signal the inbound-write gate uses — only the Primary publishes the single fleet-wide copy.
/// </para>
/// </summary>
/// <summary>
/// #477 — a child driver's connectivity transition. Annotates the source-data Quality of EVERY native
/// alarm condition the driver owns: comms lost → <see cref="OpcUaStatus.BadNoCommunication"/>, restored →
/// <see cref="OpcUaStatus.Good"/>. This is the ONLY signal for a comms-lost native source, because a
/// disconnected driver emits no alarm transitions — the alarm feed goes silent, so without this a
/// comms-lost condition would keep reporting the accidentally-Good default forever.
/// <para>
/// UNGATED by redundancy role (like the condition write in <see cref="ForwardNativeAlarm"/>): a
/// Secondary keeps its address space — including condition quality — warm for failover. Quality is
/// a pure annotation: <c>WriteAlarmQuality</c> touches ONLY the condition's Quality, never its
/// Active/Acked/Retain, and fires a Part 9 event only on a real quality-bucket change. No cluster
/// <c>alerts</c> row is published here — driver comms health has its own status surface
/// (<see cref="IDriverHealthPublisher"/>); a row per condition would be alarm-fatigue.
/// </para>
/// <para>
/// <b>#481 (Layer 4) — the same transition also reaches SCRIPTED alarms, via the dependency
/// mux.</b> #478 gave a scripted condition the worst quality of its input tags, but only for
/// inputs whose driver actually <i>publishes</i> a Bad data change. A poll driver whose device
/// goes unreachable publishes nothing at all — it emits this signal and goes silent — so the
/// engine kept its last Good value, <c>WorstInputStatus</c> stayed Good, and worse,
/// <c>AreInputsReady</c> stayed true, so the predicate went on evaluating a stale value after
/// the source had died. <see cref="FanCommsLossToDependencyMux"/> closes that by pushing a
/// synthetic Bad for every value ref the driver owns.
/// </para>
/// <para>
/// <b>Archreview 03/S15 — and the EXTERNAL data plane, which the two layers above still left
/// out.</b> Conditions and internal consumers learned of the outage; the driver's raw value
/// variables and their UNS mirrors — what an OPC UA client actually subscribes to — did not, so
/// they served the last value at <c>Good</c> for the whole outage.
/// <see cref="FanCommsLossToValueNodes"/> closes that with one status-only batch that keeps each
/// node's value and changes only its status. All three fan-outs run on the disconnect edge only;
/// the reconnect edge writes nothing to values (recovery is the driver's own re-publish) and
/// re-annotates conditions Good, which is a statement about the LINK, not about any tag's value.
/// </para>
/// </summary>
private void OnDriverConnectivityChanged(DriverInstanceActor.ConnectivityChanged msg)
{
// #481: the mux fan-out is independent of the OPC UA sink — a node with scripted alarms but no
// publish actor (the mux-only test/dev path) must still see comms loss. Done before the early
// return for exactly that reason.
if (!msg.Connected)
{
FanCommsLossToDependencyMux(msg.DriverInstanceId);
// 03/S15: and the EXTERNAL data plane, which #481 stopped one seam short of.
FanCommsLossToValueNodes(msg.DriverInstanceId);
}
else
{
// 03/S15: nothing Good is published here — instead, check LATER whether the driver republished
// what this host marked Bad. See HandleCommsRecoveryAudit.
TrackCommsLatchArmAudit(msg.DriverInstanceId);
}
if (_opcUaPublishActor is null) return;
// #519: name the actual cause. This is a connectivity transition, so the Bad case is
// BadNoCommunication — the bare OpcUaQuality.Bad it used to send reached clients as 0x80000000,
// indistinguishable from any other fault.
var statusCode = msg.Connected ? OpcUaStatus.Good : OpcUaStatus.BadNoCommunication;
var ts = DateTime.UtcNow;
var annotated = 0;
// Fan out to every condition this driver owns, resolved through the per-driver index rather than by
// scanning every condition on the node: _alarmNodeIdByDriverRef is keyed by the composite
// (DriverInstanceId, RawPath), so a filtered walk of it costs the whole address space per flap. One
// driver ref can still back several condition NodeIds (identical machines) — hence the inner loop.
if (_alarmRefsByDriver.TryGetValue(msg.DriverInstanceId, out var alarmEntries))
{
foreach (var (_, nodeIds) in alarmEntries)
{
foreach (var n in nodeIds)
{
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmQualityUpdate(
n.NodeId, statusCode, ts, n.Realm));
annotated++;
}
}
}
if (annotated > 0)
_log.Debug("DriverHost {Node}: driver {Driver} {State} — annotated {Count} native condition(s) 0x{Status:X8}",
_localNode, msg.DriverInstanceId, msg.Connected ? "connected" : "disconnected", annotated, statusCode);
}
/// <summary>
/// Archreview <b>03/S15</b> — the value-node half of the comms-loss fan-out: stamp
/// <see cref="OpcUaStatus.BadNoCommunication"/> on every raw NodeId <b>and</b> every referencing UNS
/// NodeId the just-disconnected driver owns, as ONE status batch through the same publish actor the
/// value fan-out uses.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why it was missing.</b> #481 built the per-driver ref index and wired comms loss to the
/// internal consumers (the mux) and to Part 9 conditions, and stopped one seam short of the
/// external OPC UA data plane. A client subscribed to a value node therefore saw the last value
/// at <b>Good</b> quality for the whole outage, for every driver whose failure shape is "the
/// reader stops producing" rather than "publish a Bad snapshot per tag". (Modbus does the
/// latter, quickly, which is why the gap looked smaller than it was — a coalesced-read throw, a
/// dropped session on a push driver, or simply a driver whose last act was a Good publish all
/// held Good indefinitely.)
/// </para>
/// <para>
/// <b>Status-only, so the VALUE is retained.</b> This actor holds no value cache — v3's
/// single-source design deliberately removed the independent buffer — so only the node can keep
/// the last known value, which is what
/// <see cref="OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate"/> exists for. A Bad node still
/// carrying its last reading is OPC UA Part 8's convention and is what a SCADA consumer expects.
/// </para>
/// <para>
/// <b>Nothing is written on reconnect.</b> Publishing Good would assert a quality this actor has
/// not observed; recovery is the driver's own re-publish (see
/// <see cref="FanCommsLossToDependencyMux"/>'s remark on the detach that makes the poll engine
/// force-raise, and MQTT's initial push).
/// </para>
/// <para>
/// Ungated by redundancy role, like the condition annotation: a Secondary keeps its address
/// space honest for failover. Re-stamping an already-Bad node is harmless — the SDK's
/// <c>StatusValue</c> trigger dedups an unchanged status at the monitored item.
/// </para>
/// </remarks>
/// <param name="driverInstanceId">The driver child that just reported comms loss.</param>
private void FanCommsLossToValueNodes(string driverInstanceId)
{
if (_opcUaPublishActor is null) return;
if (!_valueRefsByDriver.TryGetValue(driverInstanceId, out var entries) || entries.Count == 0) return;
var ts = DateTime.UtcNow;
var writes = new List<StatusWrite>(entries.Count);
foreach (var (_, nodes) in entries)
{
foreach (var n in nodes)
{
writes.Add(new StatusWrite(n.NodeId, OpcUaStatus.BadNoCommunication, ts, n.Realm));
}
}
if (writes.Count == 0) return;
_opcUaPublishActor.Tell(new OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate(writes));
TrackCommsLatchSet(driverInstanceId, entries);
_log.Info(
"DriverHost {Node}: driver {Driver} disconnected — {Count} value node(s) marked BadNoCommunication (value retained)",
_localNode, driverInstanceId, writes.Count);
}
/// <summary>
/// Records every RawPath just latched <c>BadNoCommunication</c> for <paramref name="driverInstanceId"/>
/// (03/S15's recovery audit — see <see cref="_commsLatched"/>). Replaces any previous latch wholesale:
/// a second outage re-arms from the current ref set rather than accumulating refs a redeploy removed.
/// </summary>
/// <param name="driverInstanceId">The driver that just went comms-lost.</param>
/// <param name="entries">Its value refs, from the per-driver index.</param>
private void TrackCommsLatchSet(
string driverInstanceId,
List<(string RawPath, HashSet<NodeRealmRef> Nodes)> entries)
{
var latched = new HashSet<string>(entries.Count, StringComparer.Ordinal);
foreach (var (rawPath, _) in entries) latched.Add(rawPath);
_commsLatched[driverInstanceId] = latched;
// A previous reconnect's audit is now meaningless — this driver is down again.
Timers.Cancel(CommsAuditTimerKey(driverInstanceId));
}
/// <summary>
/// Clears <paramref name="fullReference"/> from <paramref name="driverInstanceId"/>'s latch, and drops
/// the driver's entry once it empties. Called per routed value item, so the steady-state cost with no
/// latch outstanding is one dictionary miss per burst — not per item.
/// </summary>
/// <param name="driverInstanceId">The publishing driver.</param>
/// <param name="fullReference">The RawPath that just published a value.</param>
private void TrackCommsLatchPrune(string driverInstanceId, string fullReference)
{
if (!_commsLatched.TryGetValue(driverInstanceId, out var latched)) return;
if (!latched.Remove(fullReference)) return;
if (latched.Count == 0) _commsLatched.Remove(driverInstanceId);
}
/// <summary>
/// Arms the recovery audit for a driver that has just reconnected: after
/// <see cref="_commsRecoveryAuditDelay"/>, <see cref="HandleCommsRecoveryAudit"/> reports whatever is
/// still latched. Nothing is armed when the driver has no latch — it never went Bad here.
/// </summary>
/// <param name="driverInstanceId">The driver that just reconnected.</param>
private void TrackCommsLatchArmAudit(string driverInstanceId)
{
if (!_commsLatched.ContainsKey(driverInstanceId)) return;
Timers.StartSingleTimer(
CommsAuditTimerKey(driverInstanceId),
new CommsRecoveryAudit(driverInstanceId),
_commsRecoveryAuditDelay);
}
private static string CommsAuditTimerKey(string driverInstanceId) => $"comms-audit:{driverInstanceId}";
/// <summary>
/// The 03/S15 tripwire firing: refs this host latched <c>BadNoCommunication</c> that the driver has
/// still not re-published <see cref="_commsRecoveryAuditDelay"/> after it reconnected.
/// </summary>
/// <remarks>
/// <para>
/// <b>It reports, it does not repair.</b> Writing Good here would assert a quality nothing has
/// observed — the very thing this whole path refuses to do. What the operator gets instead is a
/// named driver and a count, plus the <c>otopcua.driver.comms_latched_refs</c> counter, which is
/// the operator-visible signature of a driver that reconnects without re-publishing (the
/// pre-STAB-18 MQTT shape, or any future push driver that forgets its initial push).
/// </para>
/// <para>
/// The latch is cleared afterwards either way: it has been reported, and leaving it would make
/// the next outage's audit report refs from this one.
/// </para>
/// </remarks>
/// <param name="msg">Names the driver whose audit is due.</param>
private void HandleCommsRecoveryAudit(CommsRecoveryAudit msg)
{
if (!_commsLatched.TryGetValue(msg.DriverInstanceId, out var latched) || latched.Count == 0)
{
_commsLatched.Remove(msg.DriverInstanceId);
return;
}
var driverType = _children.TryGetValue(msg.DriverInstanceId, out var child) ? child.DriverType : "unknown";
OtOpcUaTelemetry.DriverCommsLatchedRefs.Add(
latched.Count,
new KeyValuePair<string, object?>("driver_instance", msg.DriverInstanceId),
new KeyValuePair<string, object?>("driver_type", driverType));
_log.Warning(
"DriverHost {Node}: driver {Driver} ({Type}) — {Count} ref(s) still BadNoCommunication {Delay} after reconnect; "
+ "the driver did not re-publish them (missing initial push, or a value-only dedup swallowed an unchanged value). "
+ "First: {First}",
_localNode, msg.DriverInstanceId, driverType, latched.Count, _commsRecoveryAuditDelay,
latched.OrderBy(r => r, StringComparer.Ordinal).First());
_commsLatched.Remove(msg.DriverInstanceId);
}
/// <summary>
/// #481 (Layer 4) — push a synthetic <see cref="OpcUaStatus.BadNoCommunication"/> into the dependency
/// mux for every value ref the just-disconnected driver owns, so scripted alarms and virtual tags see
/// the source die instead of holding its last Good value indefinitely.
/// </summary>
/// <remarks>
/// <para>
/// <b>The ref set already existed.</b> #481 asked for a driver → all-tag-refs map to be built;
/// <see cref="_nodeIdByDriverRef"/> is exactly that, keyed <c>(DriverInstanceId, RawPath)</c> and
/// rebuilt from the composition's <c>RawTags</c> ∪ <c>UnsReferenceVariables</c> on every apply —
/// read here through the per-driver <see cref="_valueRefsByDriver"/> index, since that composite
/// key cannot answer "this driver's refs" without walking the node's whole address space.
/// Alarm-bearing tags live in the separate <see cref="_alarmNodeIdByDriverRef"/> and are
/// deliberately absent here — a native condition is not a mux value, and its comms quality is
/// annotated by the caller through the Part 9 path instead.
/// </para>
/// <para>
/// <b>Value <c>null</c>, not the last-known value.</b> The honest statement is "there is no
/// current value", and the engine's <c>AreInputsReady</c> gate holds the predicate on any Bad
/// regardless, so no script observes the null. It does mean the Bad must survive
/// <c>WorstInputStatus</c>, which until #481 skipped every null-valued entry as cold start —
/// hence the paired change stamping never-pushed paths <c>BadWaitingForInitialData</c> so the two
/// are tellable apart. Without that half this fan-out would be silently inert.
/// </para>
/// <para>
/// <b>Nothing is pushed on reconnect, deliberately.</b> Publishing Good would assert a value
/// quality this actor has not observed. Recovery rides the driver's own republish:
/// <c>DriverInstanceActor</c> calls <c>DetachSubscription</c> on the Connected → Reconnecting
/// edge, which nulls the subscription handle, so the reconnect's <c>ResubscribeDesired</c> builds
/// a <i>fresh</i> subscription whose <c>PollGroupEngine</c> initial push runs with
/// <c>forceRaise: true</c> and re-publishes every tag with its real quality. That detach is what
/// makes recovery work: without it the poll engine would compare against its retained
/// <c>LastValues</c>, see an unchanged Good value, raise nothing, and leave these refs Bad
/// forever — the value-dedup trap that bit the v3 Batch 4 virtual-tag reassert.
/// </para>
/// <para>
/// Ungated by redundancy role, for the same reason as the native-condition annotation above: a
/// Secondary must keep its scripted-alarm evaluation coherent for failover. The mux fan-out
/// reaches only refs something is actually subscribed to, so a driver whose tags feed no script
/// and no virtual tag costs one dictionary lookup, one envelope, and no delivery.
/// </para>
/// <para>
/// <b>ONE batch, not one message per ref.</b> The mux's batch receive filters per item against the
/// same registered-interest map its per-value receive consulted, so what each subscriber is handed
/// is unchanged — but a driver owning 5k tags now costs one Tell on its disconnect edge instead of
/// 5k, on the actor that also carries the node's steady value ingress.
/// </para>
/// </remarks>
/// <param name="driverInstanceId">The driver child that just reported comms loss.</param>
private void FanCommsLossToDependencyMux(string driverInstanceId)
{
if (_dependencyMux is null) return;
// Indexed, not a filtered scan of the composite-keyed primary: this used to walk every value ref on
// the node to find one driver's.
if (!_valueRefsByDriver.TryGetValue(driverInstanceId, out var entries) || entries.Count == 0) return;
var ts = DateTime.UtcNow;
var pushed = entries.Count;
// ONE envelope for the whole driver, not one Tell per ref: a driver owning 5k tags cost 5k messages
// on a single connectivity edge. The mux's batch path filters the items per registered interest
// exactly as it does a steady production burst, so the delivery is unchanged — only the framing is.
var items = new DriverInstanceActor.AttributeValueItem[pushed];
for (var i = 0; i < pushed; i++)
{
items[i] = new DriverInstanceActor.AttributeValueItem(
entries[i].RawPath, Value: null, OpcUaStatus.BadNoCommunication, ts);
}
_dependencyMux.Tell(new DriverInstanceActor.AttributeValueBatchPublished(driverInstanceId, items));
// Unconditional: the empty case returned above, so reaching here means at least one ref was pushed.
_log.Debug(
"DriverHost {Node}: driver {Driver} disconnected — pushed BadNoCommunication for {Count} mux ref(s) in one batch",
_localNode, driverInstanceId, pushed);
}
private void ForwardNativeAlarm(DriverInstanceActor.AttributeAlarmPublished msg)
{
if (_opcUaPublishActor is null) return;
// Resolve on ConditionId, NOT SourceNodeId: for Galaxy the dotted alarm full-reference — which
// equals the authored equipment-tag FullName the map is keyed by — is carried in ConditionId
// (AlarmFullReference), while SourceNodeId is the bare owning object (SourceObjectReference).
if (!_alarmNodeIdByDriverRef.TryGetValue((msg.DriverInstanceId, msg.Args.ConditionId), out var nodeIds))
{
_log.Debug("DriverHost {Node}: no alarm condition for ({Driver},{Ref}) — transition dropped",
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
return;
}
// #626 — a QUALITY-ONLY annotation (an MTConnect condition gone Unavailable, or back) is not an alarm
// transition. It rides the SAME quality-only node path the #477 connectivity annotation uses
// (AlarmQualityUpdate → WriteAlarmQuality, which touches Quality and nothing else), never a projected
// snapshot: the projector never sees a client's acknowledgement (that is applied to the node by the
// SDK) and forgets a condition a deploy removed, so a projected "keep the state" can disagree with
// the node — silently acknowledging a retained, unacked condition or rewriting its message and
// severity. Ungated by role (a Secondary keeps quality
// warm, as for connectivity) and no alerts row.
if (msg.Args is { IsQualityOnly: true, QualityStatusCode: { } quality })
{
foreach (var n in nodeIds)
{
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmQualityUpdate(
n.NodeId, quality, msg.Args.SourceTimestampUtc, n.Realm));
}
return;
}
// PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write
// below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary
// publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver
// cluster while the role is unknown (default-deny closes the dual-primary boot window that would
// otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied.
var serviceAlertsAsPrimary = ShouldServiceAsPrimary();
if (!serviceAlertsAsPrimary)
{
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "alarm-emit"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
_log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary",
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
}
foreach (var n in nodeIds)
{
var nodeId = n.NodeId;
var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args, out IReadOnlyList<AlarmTransitionKind> historyKinds);
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate(
nodeId, snapshot, msg.Args.SourceTimestampUtc, n.Realm));
// Only the Primary publishes the cluster-wide alerts transition (see the gate above). The projector
// decides WHICH rows the transition warrants: none for a re-statement (Gitea #647 review: a Raise on an
// already-active condition from a respawned driver; #656: a flagged snapshot/first-sight Retrigger or
// Clear that changes nothing the condition shows), and up to two for an ActiveAcknowledged (#656:
// "Activated" only if it was not already active, "Acknowledged" only if it was not already acked) or an
// InactiveUnacknowledged (#657: "Activated" + "Cleared" only if it was acked, "Cleared" if it was active).
if (!serviceAlertsAsPrimary) continue;
var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null,
ReferencingEquipmentPaths: (IReadOnlyList<string>)Array.Empty<string>());
foreach (var historyKind in historyKinds)
PublishNativeAlertRow(msg.Args, nodeId, snapshot, meta, historyKind);
}
}
/// <summary>
/// Publish one native-alarm <c>alerts</c> row (DPS + the node-local telemetry hub). Primary-gated by the caller.
/// </summary>
/// <param name="args">The native transition.</param>
/// <param name="nodeId">The condition node id.</param>
/// <param name="snapshot">The projected snapshot (its severity is the row's).</param>
/// <param name="meta">The condition's authored metadata.</param>
/// <param name="historyKind">The row's transition kind (one of the projector's history kinds).</param>
private void PublishNativeAlertRow(
AlarmEventArgs args, string nodeId, AlarmConditionSnapshot snapshot,
(string EquipmentId, string Name, string AlarmType, bool? HistorizeToAveva, IReadOnlyList<string> ReferencingEquipmentPaths) meta,
AlarmTransitionKind historyKind)
{
// #656: an ActiveAcknowledged expands to "Activated" + "Acknowledged". The operator comment/user belong to
// the acknowledgement, and the activation happened at the original raise time when the source knows it.
// #657: an InactiveUnacknowledged's "Activated" row is likewise the occurrence, at its raise time when known.
var expandedActivation = (args.Kind == AlarmTransitionKind.ActiveAcknowledged
&& historyKind != AlarmTransitionKind.Acknowledge)
|| (args.Kind == AlarmTransitionKind.InactiveUnacknowledged && historyKind == AlarmTransitionKind.Raise);
var comment = expandedActivation ? null : args.OperatorComment;
var timestamp = expandedActivation ? args.OriginalRaiseTimestampUtc ?? args.SourceTimestampUtc : args.SourceTimestampUtc;
var alertEvent = new AlarmTransitionEvent(
AlarmId: nodeId,
EquipmentPath: meta.EquipmentId,
AlarmName: meta.Name,
TransitionKind: ToEventKind(historyKind),
// The projector mapped the four-bucket AlarmSeverity onto the OPC UA 1..1000 scale already;
// reuse its ushort so the condition node + the alerts row agree on severity.
Severity: snapshot.Severity,
Message: args.Message,
// Native transitions are device-driven; an operator comment only rides along on an
// acknowledge from the upstream alarm system. "device" marks a non-operator origin.
User: comment is null ? string.Empty : "device",
TimestampUtc: timestamp,
AlarmTypeName: meta.AlarmType,
Comment: comment,
// Per-condition opt-out parsed from TagConfig.alarm.historizeToAveva (bool?, null ⇒ absent ⇒
// historize). The HistorianAdapterActor gate (historizeToAveva is not false) historizes null +
// true and suppresses the durable AVEVA row only on an explicit false — the same posture as the
// scripted-alarm opt-out. null here rides through unchanged (the gate treats it as default-on).
HistorizeToAveva: meta.HistorizeToAveva,
// WP4: the Area/Line/Equipment UNS paths that reference this raw condition — the /alerts row
// renders them as the equipment list (empty for a raw condition no equipment references yet).
ReferencingEquipmentPaths: meta.ReferencingEquipmentPaths);
_mediator.Tell(new Publish(ScriptedAlarmHostActor.AlertsTopic, alertEvent));
// Phase 5: fan the same transition into the node-local live-telemetry hub (no-op until a gRPC
// client subscribes). The DPS publish above is unchanged — a strictly additive tap, riding the
// same Primary gate (only reached when serviceAlertsAsPrimary).
_telemetryHub?.Emit(new TelemetryItem.Alarm(alertEvent));
}
/// <summary>Maps a native <see cref="AlarmTransitionKind"/> onto the canonical alarm event-kind
/// vocabulary scripted alarms emit (the <c>EmissionKind</c> names) so a native row renders with the
/// correct chip on the <c>/alerts</c> page and historizes into the same <c>EventKind</c> column as
/// scripted alarms. An unmapped/unknown transition surfaces as <c>Activated</c> (visible) rather than
/// a grey unknown label.</summary>
private static string ToEventKind(AlarmTransitionKind kind) => kind switch
{
AlarmTransitionKind.Raise or AlarmTransitionKind.Retrigger => "Activated",
AlarmTransitionKind.Clear => "Cleared",
AlarmTransitionKind.Acknowledge => "Acknowledged",
// #656: never reached in practice — the projector expands ActiveAcknowledged into Raise + Acknowledge rows.
AlarmTransitionKind.ActiveAcknowledged => "Acknowledged",
// #657: likewise never reached — expanded into Raise + Clear (or Clear alone) rows.
AlarmTransitionKind.InactiveUnacknowledged => "Cleared",
_ => "Activated",
};
/// <summary>
/// Routes an inbound operator write (Asked from the OPC UA node-manager side) to the
/// owning driver child. Order matters:
/// <list type="number">
/// <item>PRIMARY gate FIRST — reuses the same <see cref="_localRole"/> redundancy signal the
/// scripted-alarm emit gate uses. Only the Primary services writes (default-allow while the
/// role is unknown, so single-node deploys + the boot window never reject). A Secondary/Detached
/// node keeps its address space + driver state warm for failover but must NOT push writes to the
/// shared field device.</item>
/// <item>Resolve the <see cref="_driverRefByNodeId"/> reverse map to the owning
/// <c>(DriverInstanceId, FullName)</c>.</item>
/// <item>Resolve the running driver child.</item>
/// <item>Ask the child a bounded <see cref="DriverInstanceActor.WriteAttribute"/> of the driver-side
/// <see cref="DriverInstanceActor.WriteAttribute.TagId">FullName</see> and pipe the translated
/// result back to the asker.</item>
/// </list>
/// Every branch replies the asker a <see cref="NodeWriteResult"/> exactly once.
/// </summary>
private void HandleRouteNodeWrite(RouteNodeWrite msg)
{
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN
// Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a
// multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is
// still serviced. The rejection rides the established optimistic-write self-correction surface (the
// node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued.
if (!ShouldServiceAsPrimary())
{
var reason = _localRole is null ? "not primary (role unknown)" : "not primary";
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "write"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
Sender.Tell(new NodeWriteResult(false, reason));
return;
}
// v3 Batch 4 (review H1): the node manager's write hook passes the FULL ns-qualified NodeId string
// (node.NodeId.ToString(), e.g. "ns=3;s=<id>") PLUS the realm it resolved from the namespace index.
// The routing map is keyed by (realm, BARE s= identifier). Normalise the id to its bare form and look
// up (realm, bareId) — the realm disambiguates a raw RawPath from a UNS path that happen to share a
// bare string, so a write always routes to its OWN driver ref (never a colliding sibling's).
var bareNodeId = BareNodeId(msg.NodeId);
if (!_driverRefByNodeId.TryGetValue((msg.Realm, bareNodeId), out var target))
{
Sender.Tell(new NodeWriteResult(false, $"no driver mapping for node {msg.NodeId} ({msg.Realm})"));
return;
}
if (!_children.TryGetValue(target.DriverInstanceId, out var entry))
{
Sender.Tell(new NodeWriteResult(false, "driver not running"));
return;
}
// Ask the child a bounded write — DriverInstanceActor.HandleWriteAsync already bounds the backend
// call to 5s, so the 8s Ask is a safety net for an actor that never replies. Capture Sender before
// the continuation: it runs off the actor thread, where raw Sender is unsafe to read.
var replyTo = Sender;
entry.Actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute(target.RawPath, msg.Value!), TimeSpan.FromSeconds(8))
.ContinueWith(
t => t.IsCompletedSuccessfully
? new NodeWriteResult(t.Result.Success, t.Result.Reason)
: new NodeWriteResult(false, "write timeout"),
CancellationToken.None,
TaskContinuationOptions.None,
TaskScheduler.Default)
.PipeTo(replyTo);
}
/// <summary>
/// Normalise an OPC UA NodeId string to its BARE <c>s=</c> identifier. The node manager's inbound-write
/// hook passes the full ns-qualified form (<c>NodeId.ToString()</c> == <c>"ns=&lt;N&gt;;s=&lt;id&gt;"</c>);
/// the routing maps are keyed by the bare id (the RawPath / UNS path). The SDK format always prefixes
/// <c>"ns=N;s="</c> for a string identifier in a non-zero namespace, so the first <c>";s="</c> is the
/// delimiter (an id may itself contain <c>";s="</c> later — <c>IndexOf</c> takes the FIRST, which is the
/// namespace delimiter). A bare id (no prefix) passes through unchanged.
/// </summary>
/// <param name="nodeId">The (possibly ns-qualified) NodeId string.</param>
/// <returns>The bare <c>s=</c> identifier.</returns>
internal static string BareNodeId(string nodeId)
{
if (string.IsNullOrEmpty(nodeId)) return nodeId;
var i = nodeId.IndexOf(";s=", StringComparison.Ordinal);
if (i >= 0) return nodeId[(i + 3)..];
if (nodeId.StartsWith("s=", StringComparison.Ordinal)) return nodeId[2..];
return nodeId;
}
/// <summary>
/// Routes an inbound native-condition acknowledge (the host Tells this from the OPC UA node-manager
/// side when a client Acknowledges a NATIVE Part 9 condition) to the owning driver child. Mirrors
/// <see cref="HandleRouteNodeWrite"/>'s order + gating, but the ack is fire-and-forget (no reply):
/// the OPC UA Part 9 acknowledge already committed the local condition state and the driver's
/// <see cref="IAlarmSource.AcknowledgeAsync"/> returns no per-condition status.
/// <list type="number">
/// <item>PRIMARY gate FIRST — reuses the same <see cref="_localRole"/> redundancy signal as the
/// inbound-write gate. Only the Primary pushes the ack to the shared upstream alarm system
/// (default-allow while the role is unknown). A Secondary/Detached node keeps its condition
/// state warm for failover but must NOT push the command. Drop (debug-log) on non-primary.</item>
/// <item>Resolve the <see cref="_driverRefByAlarmNodeId"/> inverse map to the owning
/// <c>(DriverInstanceId, FullName = ConditionId)</c>; an unmapped node is debug-logged + dropped
/// (no throw) — mirrors <see cref="ForwardNativeAlarm"/>'s unknown-ref drop.</item>
/// <item>Resolve the running driver child and Tell it a
/// <see cref="DriverInstanceActor.RouteAlarmAck"/> carrying the wire-ref FullName + principal.</item>
/// </list>
/// </summary>
private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg)
{
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as
// the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role
// on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream
// alarm system is a swallowed failure worth surfacing).
if (!ShouldServiceAsPrimary())
{
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "ack"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
if (_localRole is null)
{
_log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)",
_localNode, msg.ConditionNodeId);
}
else
{
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
_localNode, msg.ConditionNodeId);
}
return;
}
if (!_driverRefByAlarmNodeId.TryGetValue(msg.ConditionNodeId, out var target))
{
_log.Debug("DriverHost {Node}: no driver mapping for alarm condition node {Cond} — ack dropped",
_localNode, msg.ConditionNodeId);
return;
}
// Gitea #641 review — the node is already acknowledged (the SDK applies the client's ack); tell the
// projector too, or the next driver Clear projects (Active=false, Acked=prev=false) and un-acknowledges it.
// Most sources (MTConnect, FOCAS) send no Acknowledge transition back, and the projector now survives a
// deploy rather than resetting to its acked default.
_nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId);
if (!_children.TryGetValue(target.DriverInstanceId, out var entry))
{
_log.Debug("DriverHost {Node}: driver {Driver} not running for alarm ack of {Cond} — dropped",
_localNode, target.DriverInstanceId, msg.ConditionNodeId);
return;
}
// Fire-and-forget: the OPC UA Part 9 ack already committed the local condition state, and the
// driver's AcknowledgeAsync surfaces no per-condition status, so there is nothing to reply. The
// driver correlates on ConditionId (= the authored alarm FullName the inverse map keyed on).
entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.RawPath, msg.Comment, msg.OperatorUser));
}
/// <summary>Whether this node should service a Primary-only data-plane operation right now — the single
/// gate decision (archreview 03/S4), delegating to <see cref="PrimaryGatePolicy"/> with the live driver
/// member count.</summary>
/// <returns><c>true</c> to service as Primary; <c>false</c> to deny.</returns>
private bool ShouldServiceAsPrimary() =>
PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount());
/// <summary>Count of Up cluster members carrying the <c>driver</c> role. Uses the injected test seam when
/// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster
/// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture.</summary>
/// <returns>The number of Up driver-role members, or 0 when the cluster extension is unavailable.</returns>
private int DriverMemberCount()
{
if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
try
{
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
// 'Driver' (rolling-upgrade window) must still count toward the R2-04 primary-gate member
// count, or a genuine 2-node pair silently takes the single-node allow branch.
return Akka.Cluster.Cluster.Get(Context.System).State.Members
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up
&& RoleParser.GossipedRolesHave(m.Roles, ServiceCollectionExtensions.DriverRole));
}
catch (Exception)
{
return 0; // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture
}
}
/// <summary>Whether <paramref name="nodeId"/> holds this node's replicated alarm queue: the PeerAddress host when
/// this node dials, else the passive-half predicate (review R6 MINOR-4). This node itself never counts.</summary>
private bool SharesAlarmQueue(CommonsNodeId nodeId) =>
nodeId != _localNode
&& (_replicationPeerHost is not null
? HostOf(nodeId) == _replicationPeerHost
: _sharesAlarmQueueWithLocal?.Invoke(nodeId) == true);
/// <summary>Host part of a <c>host:port</c> node id.</summary>
private static string HostOf(NodeId nodeId)
{
var text = nodeId.ToString();
var colon = text.LastIndexOf(':');
return colon < 0 ? text : text[..colon];
}
/// <summary>The Primary-gate deny reason tag for the denial meter (<c>secondary|detached|role-unknown</c>).</summary>
private string PrimaryGateDenyReason() => _localRole switch
{
RedundancyRole.Secondary => "secondary",
RedundancyRole.Detached => "detached",
_ => "role-unknown",
};
/// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so the
/// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves
/// the cached role unchanged; when this node has a real driver peer (count &gt; 1) that omission is the
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
if (local is not null)
{
_localRole = local.Role;
}
else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
{
// S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
// Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
// identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
_warnedSnapshotMissingLocalNode = true;
_log.Warning(
"DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
_localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
}
// Publish on EVERY snapshot, including ones that leave the cached role untouched, so a role
// that changes without this node being named still reaches the drain within one heartbeat.
//
// NOTE the different policy: the drain gate keys on the role ALONE and opens when it is
// unknown, where the data-plane gate above resolves an unknown role by member count and
// denies. Publishing ShouldServiceAsPrimary() here suspended the drain on every node of a
// multi-driver cluster that had no redundancy configured — see ShouldDrainAlarmHistory.
// Defer ONLY to the node that holds a copy of our rows. The redundancy election is
// cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop
// draining — that Primary is usually in another pair entirely, and cannot deliver our events.
//
// The dialling half knows its peer by PeerAddress host. The passive half (review R6 MINOR-4) knows it
// only as its mesh partner — the predicate — and must be gated too, or both halves drain the same rows.
var peerIsPrimary = msg.Nodes.Any(n => n.Role == RedundancyRole.Primary && SharesAlarmQueue(n.NodeId));
_redundancyRoleView?.Publish(
PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary));
// Gitea #537: the continuous-historization CAPTURE verdict. Same role rule as the drain gate
// above, but a different peer question — historization is not pair-local, it is fleet-wide by
// historian tag name, so the only node worth standing down for is one recording the SAME refs:
// a mesh partner serving this node's address space. With no way to establish that (a legacy
// node carrying no cluster-scoped role, or a test harness), the predicate is null and this
// node keeps capturing. Duplicate samples are recoverable; a hole in the trend is not.
var historizationPeerIsPrimary =
_sharesHistorizationScopeWithLocal is { } sharesScope
&& msg.Nodes.Any(n => n.Role == RedundancyRole.Primary
&& n.NodeId != _localNode
&& sharesScope(n.NodeId));
_historizationRoleView?.PublishHistorizationRole(
PrimaryGatePolicy.ShouldRecordHistorizedValues(
_localRole, addressSpaceSharingPeerIsPrimary: historizationPeerIsPrimary));
// The DATA-PLANE verdict, deliberately — the same one HandleRouteNodeWrite gates on, not the
// drain verdict published immediately above. A driver reading this claims an exclusive identity
// on a shared external system (Gitea #511: the Sparkplug host id), where two claimants is the
// harm, so an unknown role with a driver peer present must deny.
PublishPrimaryVerdict();
}
private void Stale()
{
Receive<DispatchDeployment>(_ =>
{
_log.Warning("DriverHost {Node}: ignoring DispatchDeployment while Stale (DB unreachable)", _localNode);
});
// A cache store started before this node fell Stale still pipes back here (see
// CacheAppliedArtifact: Bootstrap's orphan-replay BeginDirectApply and RestoreApplied both start
// stores, and either can be followed by the ConfigDb failure that lands us in Stale). Without
// these the pipe-back dead-letters and a failed cache write goes unreported in exactly the
// situation — central unreachable — where the local cache is the thing keeping the node alive.
Receive<ArtifactCached>(HandleArtifactCached);
Receive<ArtifactCacheFailed>(HandleArtifactCacheFailed);
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
// We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
// #601 — an apply that completed just before the host went Stale still owes its ack; the
// coordinator is not behind the ConfigDb, so release it exactly as in Steady.
Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
// An inbound operator write can't be serviced while the config DB is unreachable — fast-fail so the
// node-manager's bounded Ask gets an immediate clear status instead of dead-lettering into a timeout.
Receive<RouteNodeWrite>(_ =>
Sender.Tell(new NodeWriteResult(false, "driver host stale (config DB unreachable)")));
// An inbound native-condition ack can't be serviced while Stale either (the alarm inverse map is
// empty until an apply runs). The ack is fire-and-forget (no reply), so just drop it with a log —
// the local OPC UA condition state already committed on the Part 9 ack.
Receive<RouteNativeAlarmAck>(msg =>
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Node2} while Stale (config DB unreachable)",
_localNode, msg.ConditionNodeId));
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
Receive<CommsRecoveryAudit>(_ => { });
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
Timers.StartPeriodicTimer("retry-db", RetryConfigDbConnection.Instance, ReconnectInterval);
}
private void HandleGetDiagnostics(GetDiagnostics msg)
{
var drivers = _children
.Select(kv => new DriverInstanceDiagnostics(
DriverInstanceId: Guid.Empty,
Name: kv.Key,
State: kv.Value.Stubbed ? "Stubbed" : "Spawned",
ConnectedDevices: 0,
FaultedDevices: 0,
LastChangeUtc: DateTime.UtcNow))
.ToArray();
var snapshot = new NodeDiagnosticsSnapshot(
NodeId: _localNode,
CurrentRevision: _currentRevision,
Drivers: drivers,
AsOfUtc: DateTime.UtcNow,
RunningFromCache: _isRunningFromCache);
Sender.Tell(snapshot);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
{
// #601: the short-circuit claims a CLEAN Applied, so it is only honest while the current revision's
// address space is known-good — not after a degraded apply, and not while its outcome is still
// pending. Either way the dispatch re-applies instead, which is also what repairs a degraded apply.
var currentRevisionUnconfirmed = _currentRevisionDegraded
|| _pendingAcks.Values.Any(p => p.Revision == msg.RevisionHash);
if (_currentRevision is { } cur && cur == msg.RevisionHash && !currentRevisionUnconfirmed)
{
// Idempotent — already at this rev. Ack and stay Steady.
_log.Debug("DriverHost {Node}: dispatch {Id} matches current rev {Rev}; immediate ACK",
_localNode, msg.DeploymentId, msg.RevisionHash);
SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.CorrelationId);
return;
}
if (_currentRevision is { } same && same == msg.RevisionHash)
_log.Info(
"DriverHost {Node}: dispatch {Id} matches current rev {Rev}, but its address-space apply was degraded or is unconfirmed — re-applying rather than acking clean",
_localNode, msg.DeploymentId, msg.RevisionHash);
if (_fetchAndCacheMode)
{
BeginFetchAndCacheApply(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
return;
}
BeginDirectApply(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
}
/// <summary>
/// Per-cluster mesh Phase 3 (<c>FetchAndCache</c>). Kicks off a gRPC artifact fetch and pipes
/// the result back to <see cref="Self"/> as an <see cref="ArtifactForApply"/> — the fetch is I/O
/// and MUST NOT block the mailbox for the fetch deadline, so it never runs as an awaited call
/// inside a receive. The apply itself completes synchronously on the actor thread in
/// <see cref="HandleArtifactForApply"/>, the continuation both modes share.
/// </summary>
/// <remarks>
/// Both begin-paths arm the <see cref="ApplyWedgeTimerKey"/> watchdog, because both leave the
/// host alive and dequeuing while it waits for bytes it does not have yet — which is exactly
/// what makes a silent wedge possible. (Before the Direct read moved off the mailbox this was
/// the only such path: an inline read could not be "stuck in Applying" in any sense a watchdog
/// could report, since a mailbox message cannot be dequeued by an actor thread that is hung.)
/// </remarks>
private void BeginFetchAndCacheApply(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
{
_applyingDeploymentId = deploymentId;
Become(Applying);
Timers.StartSingleTimer(ApplyWedgeTimerKey, new ApplyWedged(deploymentId), _applyWedgeTimeout);
// Persist Applying row (idempotent on PK) — same crash-boundary marker as the Direct apply.
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applying, failureReason: null);
if (_artifactFetcher is null)
{
// Misconfiguration: FetchAndCache with no fetcher wired. Fail the apply rather than fetch
// nothing forever; the node keeps serving last-known-good.
const string reason = "FetchAndCache mode is set but no artifact fetcher is configured on this node";
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Failed, reason);
SendAck(deploymentId, ApplyAckOutcome.Failed, reason, correlation);
_log.Error("DriverHost {Node}: cannot apply {Id} — {Reason}", _localNode, deploymentId, reason);
_applyingDeploymentId = null;
BecomeSteady();
return;
}
_log.Debug("DriverHost {Node}: fetching artifact for {Id} (rev {Rev}) from central over gRPC",
_localNode, deploymentId, revision);
// A faulted fetch task maps to null bytes — the #485 apply-failure path, exactly like an
// all-endpoints-down fetch — so a fault can never escape as an unhandled actor message.
_artifactFetcher.FetchAsync(deploymentId.ToString(), revision.Value, CancellationToken.None)
.PipeTo(
Self,
success: bytes => new ArtifactForApply(
deploymentId, revision, correlation, bytes, ArtifactSource.CentralFetch),
failure: _ => new ArtifactForApply(
deploymentId, revision, correlation, Bytes: null, ArtifactSource.CentralFetch));
}
/// <summary>
/// Perf roadmap Task 26 (<c>Direct</c>). Kicks off the central-SQL artifact read on a pool
/// thread and pipes the bytes back to <see cref="Self"/> as an <see cref="ArtifactForApply"/>,
/// the same shape <see cref="BeginFetchAndCacheApply"/> has always had for its gRPC fetch. The
/// apply itself completes on the actor thread in <see cref="HandleArtifactForApply"/>.
/// </summary>
/// <remarks>
/// <para>
/// The read used to run INLINE in the receive. <c>Deployment.ArtifactBlob</c> is a
/// potentially multi-MB gzip blob in central SQL, so for the whole of that round trip the
/// host dequeued nothing: no driver value batch, no diagnostics request, no driver command,
/// no next dispatch. Only the byte SOURCE differs between the two modes, and only the
/// source ever should — everything downstream of "bytes in hand" is one continuation.
/// </para>
/// <para>
/// The bytes come back as a MESSAGE, never as an escaping fault (#491): the task closes over
/// <see cref="_dbFactory"/>, <see cref="_localNode"/> and <see cref="_log"/> and touches
/// neither <see cref="Self"/> nor <c>Context</c>, <see cref="LoadArtifactBlob"/> already
/// swallows its own DB failures into a null return, and the failure mapper turns anything it
/// cannot (an OOM on an oversized blob, a factory that throws) into the same null-bytes
/// apply failure. Supervision never sees a throw from here.
/// </para>
/// </remarks>
private void BeginDirectApply(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
{
_applyingDeploymentId = deploymentId;
Become(Applying);
Timers.StartSingleTimer(ApplyWedgeTimerKey, new ApplyWedged(deploymentId), _applyWedgeTimeout);
// Persist Applying row (idempotent on PK) — the crash-boundary marker, and deliberately still
// on the actor thread: one bounded row write, and the apply must be recorded as in flight
// before the read that could outlive this process starts. See UpsertNodeDeploymentState.
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applying, failureReason: null);
_log.Debug("DriverHost {Node}: reading artifact for {Id} (rev {Rev}) from the ConfigDb, off the mailbox",
_localNode, deploymentId, revision);
Task.Run(() => LoadArtifactBlob(deploymentId))
.PipeTo(
Self,
success: bytes => new ArtifactForApply(
deploymentId, revision, correlation, bytes, ArtifactSource.ConfigDb),
failure: _ => new ArtifactForApply(
deploymentId, revision, correlation, Bytes: null, ArtifactSource.ConfigDb));
}
/// <summary>
/// Completes an apply once the artifact bytes are in hand — the ONE continuation both modes
/// share, whether the bytes were read from the ConfigDb or fetched from central. Null bytes are
/// an apply FAILURE (keep last-known-good, do not advance the revision — #485/#486); non-null
/// bytes are applied from hand and cached.
/// </summary>
private void HandleArtifactForApply(ArtifactForApply msg)
{
// Ignore a result for anything but the in-flight apply (a stale/duplicate pipe-back).
if (_applyingDeploymentId is null || msg.DeploymentId != _applyingDeploymentId.Value)
{
_log.Debug("DriverHost {Node}: dropping stale artifact result for {Id} (in-flight={Cur})",
_localNode, msg.DeploymentId, _applyingDeploymentId);
return;
}
using var span = OtOpcUaTelemetry.StartDeployApplySpan(msg.DeploymentId.ToString());
span?.SetTag("otopcua.node_id", _localNode.ToString());
span?.SetTag("otopcua.revision", msg.Revision.ToString());
span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
var sw = Stopwatch.StartNew();
// Owned by this handler, disposed in the finally: ONE JsonDocument for the whole apply. Declared
// out here (rather than a `using` inside the try) so the catch/finally still see it.
ParsedDeploymentArtifact? parsed = null;
try
{
// #485/#486: null bytes = "no answer" — the ConfigDb was unreachable or the row is
// missing/empty (Direct), or every endpoint was unreachable / NotFound / SHA mismatch
// (FetchAndCache). Either way NOTHING was applied, so fail the apply and stay on the
// current revision: HandleDispatchFromSteady short-circuits on a revision match, so
// claiming Applied here would make the node unhealable by re-dispatching that revision.
// The last-known-good address space, drivers and subscriptions keep serving throughout.
var (appliedBlob, parsedArtifact) = ReconcileDriversFromParsed(msg.DeploymentId, msg.Bytes);
parsed = parsedArtifact;
if (appliedBlob is null || parsed is null)
{
var reason = msg.Source == ArtifactSource.ConfigDb
? "the deployment artifact could not be read (ConfigDb unreachable, or the row is missing/empty); nothing was applied"
: "the deployment artifact could not be fetched from central (all endpoints unreachable, NotFound, or SHA-256 mismatch); nothing was applied";
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, reason);
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, reason, msg.Correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, reason);
_log.Error(
msg.Source == ArtifactSource.ConfigDb
? "DriverHost {Node}: apply of {Id} FAILED — {Reason}. Staying on revision {Rev} and continuing to serve it; re-dispatch once the ConfigDb is readable"
: "DriverHost {Node}: FetchAndCache apply of {Id} FAILED — {Reason}. Staying on revision {Rev} and continuing to serve it; re-dispatch once central is reachable",
_localNode, msg.DeploymentId, reason, _currentRevision);
return;
}
_currentRevision = msg.Revision;
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
// Gitea #601: the Applied ack is NOT sent here any more. It used to go out right now, before the
// address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults,
// _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held
// until OpcUaPublishActor replies RebuildCompleted (HandleRebuildCompleted) or the outcome
// timeout fires (sent degraded, "outcome unknown"). With no publish actor there is no address
// space to wait for, so the ack goes out clean immediately — exactly the old behaviour.
AwaitAddressSpaceOutcomeThenAck(msg.DeploymentId, msg.Revision, msg.Correlation);
// Rebuild the address space + push subscriptions FROM the bytes this apply already has, so
// nothing downstream re-reads the artifact: under FetchAndCache OpcUaPublishActor has no
// central SQL to read at all, and under Direct a bytes-less RebuildAddressSpace would make
// it re-read the same immutable row while the SubscribeBulk pass read it a THIRD time.
// Consequence: OpcUaPublishActor.LoadArtifact / LoadLatestArtifact are on no apply path;
// they remain only for a legacy/dev caller that Tells RebuildAddressSpace without bytes.
//
// What rides along is the raw BYTES, deliberately — not `parsed`. The publish actor is a
// different actor with its own mailbox, and ParsedDeploymentArtifact is a disposable,
// non-thread-safe object owned by this handler (disposed in the finally below, likely before
// the other actor even dequeues). It opens its own single parse on the far side.
// ReportOutcome: the reply (to Self — this Tell runs inside a receive, so the implicit sender is
// this actor) releases the held ack. Plain Tell + reply message, no Ask/await, so nothing here
// resumes off the actor thread.
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(
msg.Correlation, msg.DeploymentId, appliedBlob, ReportOutcome: true));
// SubscribeBulk pass: hand each driver its desired tag references so live values flow into
// the just-rebuilt address space instead of staying BadWaitingForInitialData.
PushDesiredSubscriptionsFromParsed(msg.DeploymentId, parsed);
// Cache the applied bytes as this node's last-known-good (idempotent store; skips empty).
CacheAppliedArtifact(msg.DeploymentId, msg.Revision, appliedBlob, parsed);
// Reaching here means the artifact came from central, so the node is no longer serving a
// cache-sourced configuration. Note this only clears on a real apply: a dispatch of the
// revision already booted from cache short-circuits in HandleDispatchFromSteady without
// touching the ConfigDb, and the flag correctly stays set.
_isRunningFromCache = false;
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "ack"));
_log.Info(
msg.Source == ArtifactSource.ConfigDb
? "DriverHost {Node}: applied deployment {Id} (rev {Rev}, children={Count})"
: "DriverHost {Node}: applied fetched deployment {Id} (rev {Rev}, children={Count})",
_localNode, msg.DeploymentId, msg.Revision, _children.Count);
}
catch (Exception ex)
{
// #601: a throw after the Applied ack was put on hold — the Failed ack below is this apply's one
// answer, so the held Applied must never follow it when the address-space outcome lands.
DropPendingAck(msg.Correlation);
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, ex.Message);
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, ex.Message, msg.Correlation);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "reject"));
span?.SetStatus(ActivityStatusCode.Error, ex.Message);
_log.Error(
ex,
msg.Source == ArtifactSource.ConfigDb
? "DriverHost {Node}: apply of {Id} failed"
: "DriverHost {Node}: FetchAndCache apply of {Id} failed",
_localNode, msg.DeploymentId);
}
finally
{
parsed?.Dispose();
OtOpcUaTelemetry.DeploymentApplyDurationSec.Record(sw.Elapsed.TotalSeconds);
_applyingDeploymentId = null;
BecomeSteady();
}
}
/// <summary>Gitea #601 — an Applied ack held until the address-space apply reports its outcome.</summary>
private sealed record PendingAck(DeploymentId DeploymentId, RevisionHash Revision, CorrelationId Correlation);
/// <summary>Gitea #601 — self-timer: the address-space outcome for this correlation did not arrive in time.</summary>
private sealed record ApplyOutcomeTimedOut(CorrelationId Correlation);
private static string ApplyOutcomeTimerKey(CorrelationId correlation) => $"apply-outcome-{correlation}";
/// <summary>
/// Gitea #601 — holds the Applied ack for an apply whose revision has just advanced until the
/// address-space apply outcome is known, arming <see cref="_applyOutcomeTimeout"/> as the backstop.
/// With no publish actor wired (tests, a node with no OPC UA data plane) there is no outcome to wait
/// for, so the ack is sent clean at once — byte-for-byte the pre-#601 behaviour.
/// </summary>
private void AwaitAddressSpaceOutcomeThenAck(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
{
if (_opcUaPublishActor is null)
{
_currentRevisionDegraded = false;
SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
return;
}
if (_pendingAcks.Remove(correlation, out var superseded))
_log.Warning(
"DriverHost {Node}: a second apply reused correlation {Correlation} while deployment {Old}'s ack was still waiting for its address-space outcome — that ack is superseded",
_localNode, correlation, superseded.DeploymentId);
_pendingAcks[correlation] = new PendingAck(deploymentId, revision, correlation);
Timers.StartSingleTimer(ApplyOutcomeTimerKey(correlation), new ApplyOutcomeTimedOut(correlation), _applyOutcomeTimeout);
}
/// <summary>Gitea #601 — forgets a held ack (its apply failed after all, and acks Failed instead).</summary>
private void DropPendingAck(CorrelationId correlation)
{
if (_pendingAcks.Remove(correlation))
Timers.Cancel(ApplyOutcomeTimerKey(correlation));
}
/// <summary>
/// Gitea #601 — the address-space apply reported its outcome: release the held ack, clean or
/// degraded. Registered in every behaviour (Steady, Applying, Stale) because the reply can land in
/// any of them — the host returns to Steady before the publish actor has even dequeued the rebuild.
/// </summary>
private void HandleRebuildCompleted(OpcUa.OpcUaPublishActor.RebuildCompleted msg)
{
if (!_pendingAcks.Remove(msg.Correlation, out var pending))
{
_log.Debug("DriverHost {Node}: address-space outcome for {Correlation} has no held ack (already timed out or superseded); dropping",
_localNode, msg.Correlation);
return;
}
Timers.Cancel(ApplyOutcomeTimerKey(msg.Correlation));
ReleasePendingAck(pending, msg.Degraded, msg.DegradedReason);
}
/// <summary>
/// Gitea #601 — the address-space outcome did not arrive within <see cref="_applyOutcomeTimeout"/>.
/// The ack is sent anyway so the coordinator does not time the deployment out on a node that DID
/// advance its revision — but degraded, never clean: nobody has confirmed the address space.
/// </summary>
private void HandleApplyOutcomeTimedOut(ApplyOutcomeTimedOut msg)
{
if (!_pendingAcks.Remove(msg.Correlation, out var pending)) return;
ReleasePendingAck(pending, degraded: true,
$"{OutcomeNotReportedReasonPrefix} ({_applyOutcomeTimeout.TotalSeconds:0}s) — no sink failure was observed; the apply may still be running (slow) or its outcome was lost");
}
private void ReleasePendingAck(PendingAck pending, bool degraded, string? reason)
{
// Only the CURRENT revision's flag is ours to set — an older apply's late outcome must not mark a
// newer revision degraded (or clean).
if (_currentRevision is { } cur && cur == pending.Revision)
_currentRevisionDegraded = degraded;
if (degraded)
{
// #601 review — the apply wrote this node's own row Applied/null before the outcome was known. Central
// adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
// is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
_log.Warning(
"DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
_localNode, pending.DeploymentId, pending.Revision, reason);
}
SendAck(pending.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, pending.Correlation,
degraded, degraded ? reason ?? "address-space apply degraded" : null);
}
/// <summary>
/// Self-timer telling this host it has been in <see cref="Applying"/> for
/// <see cref="_applyWedgeTimeout"/> without finishing.
/// </summary>
private sealed record ApplyWedged(DeploymentId DeploymentId);
/// <summary>
/// Reports a host that is still in <see cref="Applying"/> long after it should have finished.
/// </summary>
/// <remarks>
/// Purely a signal — nothing is cancelled and no ACK is sent, because the apply may yet land and
/// the coordinator's apply deadline already owns "this deployment is dead". Registered in
/// <see cref="Applying"/> AND <see cref="Steady"/>: <see cref="BecomeSteady"/> cancels the timer,
/// but a fire already in the mailbox when the apply completes still has to land somewhere, and
/// dead-lettering it would be a second, more confusing signal. Such a late fire finds
/// <see cref="_applyingDeploymentId"/> null (or naming a different deployment) and drops at Debug.
/// </remarks>
private void HandleApplyWedged(ApplyWedged msg)
{
if (_applyingDeploymentId is null || msg.DeploymentId != _applyingDeploymentId.Value)
{
_log.Debug("DriverHost {Node}: late apply-wedge tick for {Id} (in-flight={Cur}); dropping",
_localNode, msg.DeploymentId, _applyingDeploymentId);
return;
}
_log.Warning(
"DriverHost {Node}: deployment {Id} appears WEDGED — still applying after {Timeout} " +
"(the artifact read/fetch has neither answered nor failed); {Deferred} further dispatch(es) " +
"are deferred behind it. The node keeps serving revision {Rev}.",
_localNode, msg.DeploymentId, _applyWedgeTimeout, _deferredDispatches.Count, _currentRevision);
}
/// <summary>Where an apply's artifact bytes came from — the ONLY difference between the two apply
/// paths, and therefore the only thing <see cref="HandleArtifactForApply"/> branches on.</summary>
private enum ArtifactSource
{
/// <summary><c>Direct</c> mode: read from <c>Deployment.ArtifactBlob</c> in central SQL.</summary>
ConfigDb,
/// <summary><c>FetchAndCache</c> mode: fetched from central over gRPC.</summary>
CentralFetch,
}
/// <summary>Pipe-back carrier for an apply's artifact bytes (null bytes = no answer).</summary>
private sealed record ArtifactForApply(
DeploymentId DeploymentId,
RevisionHash Revision,
CorrelationId Correlation,
byte[]? Bytes,
ArtifactSource Source);
/// <summary>
/// Read the deployment artifact + reconcile the set of running <see cref="DriverInstanceActor"/>
/// children. Spawn missing, respawn on config change, stop removed/disabled drivers.
/// When the artifact blob is empty (legacy ControlPlane tests, smoke fixtures) the reconcile is
/// SKIPPED outright — see the issue #485 guard below — and when the configured
/// <see cref="IDriverFactory"/> can't materialise any of the requested types this is a no-op.
/// </summary>
/// <returns>
/// The artifact blob that was reconciled, or <see langword="null"/> when it could not be
/// loaded at all.
/// </returns>
/// <remarks>
/// <para>
/// The <b>bootstrap-restore</b> entry point (<see cref="RestoreApplied"/>). The apply paths use
/// <see cref="ReconcileDriversFromParsed"/>, which keeps its parse for the rest of the apply.
/// </para>
/// <para>
/// Returning the blob rather than swallowing it is load-bearing for the artifact cache. This
/// method catches its own DB failures, logs a warning and returns WITHOUT rethrowing — so the
/// caller proceeds to its success path having spawned zero drivers. A cache write that trusted
/// "the apply succeeded" would then persist an empty artifact as this node's last-known-good
/// configuration, and the node would later boot from it into an empty address space. The caller
/// distinguishes the two cases by this return value.
/// </para>
/// </remarks>
private byte[]? ReconcileDrivers(DeploymentId deploymentId)
{
var blob = LoadArtifactBlob(deploymentId);
return blob is null ? null : ReconcileDriversFromBlob(deploymentId, blob);
}
/// <summary>
/// The ConfigDb read half of <see cref="ReconcileDrivers"/>, shared with the Direct apply path
/// (<see cref="BeginDirectApply"/>) so both read the artifact exactly once and log the
/// same way when they cannot.
/// </summary>
/// <remarks>
/// Called on the actor thread by the PreStart restore path and on a POOL thread by
/// <see cref="BeginDirectApply"/>, so it must stay free of <see cref="Self"/> / <c>Context</c>
/// (#491). It touches only <see cref="_dbFactory"/>, <see cref="_localNode"/> and
/// <see cref="_log"/>, all of which are safe off the actor thread — the logger publishes to the
/// EventStream and captures nothing from the cell.
/// </remarks>
/// <returns>The artifact bytes (possibly empty, when the row is missing), or <see langword="null"/>
/// when there is no ConfigDb to read or the read failed.</returns>
private byte[]? LoadArtifactBlob(DeploymentId deploymentId)
{
// Phase 4 belt-and-suspenders: this ConfigDb read is Direct-only (the FetchAndCache apply uses
// ReconcileDriversFromParsed with bytes in hand). A null factory can't read, so return null — the
// caller treats that as "artifact could not be read" (#486) and keeps last-known-good.
if (_dbFactory is null)
{
_log.Warning("DriverHost {Node}: no ConfigDb to load artifact for {Id}; skipping reconcile", _localNode, deploymentId);
return null;
}
try
{
using var db = _dbFactory.CreateDbContext();
// Bound this read EXPLICITLY. It runs off the mailbox on a pool thread with no
// CancellationToken (BeginDirectApply's Task.Run), so the actor itself cannot be wedged by a
// hung central-SQL read — but the read would pin a pool thread for as long as SQL takes to
// answer. Until now the only bound was SqlClient's IMPLICIT 30 s command-timeout default:
// nothing on the ConfigDb path sets one (AddPooledDbContextFactory uses a plain
// UseSqlServer(connectionString), and no connection string carries "Command Timeout="), so a
// future connection-string edit or provider-default change could silently unbound it. Stating
// it here pins the bound to this call site.
//
// Deliberately LOCAL to this lease, not on the pooled factory's options: re-timing the global
// factory would re-time every ConfigDb consumer — migrations and long admin queries included.
// Verified against EF Core 10.0.7 (pool size 1, same context instance re-leased):
// Database.SetCommandTimeout does NOT survive the return to the pool — RelationalConnection
// resets it as an IResettableService — so the next lease sees the default again and there is
// no pool pollution to undo.
//
// Deliberately NOT a CancellationToken: threading one would mean making the FirstOrDefault
// below async, and the only caller-side token is actor lifetime — the timeout gives the same
// bound without touching the actor seam. A timeout surfaces as an exception through this
// method's existing catch-to-null, so the apply FAILS and the revision does not advance,
// keeping last-known-good served (the #485-family "no answer, never tear down" semantics —
// already correct, unchanged here).
//
// Guarded on IsRelational: the timeout is a relational-only concept and the extension throws
// on a non-relational provider (the in-memory provider the actor tests run against).
// ⚠️ Consequently the relational branch itself is UNTESTED in-suite — every actor test runs
// InMemory, so the suite only ever exercises the guard's false side. The 30 s bound is pinned
// by the EF 10.0.7 verification recorded above, not by a test.
if (db.Database.IsRelational())
db.Database.SetCommandTimeout(TimeSpan.FromSeconds(30));
return db.Deployments.AsNoTracking()
.Where(d => d.DeploymentId == deploymentId.Value)
.Select(d => d.ArtifactBlob)
.FirstOrDefault() ?? Array.Empty<byte>();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to load artifact for {Id}; skipping reconcile",
_localNode, deploymentId);
return null;
}
}
/// <summary>
/// An apply's artifact: the bytes that were reconciled plus the ONE
/// <see cref="ParsedDeploymentArtifact"/> every later consumer of that apply reads. Both are
/// <see langword="null"/> when nothing was applied (#485/#486).
/// </summary>
/// <param name="Blob">The reconciled artifact bytes, or null when there was no usable artifact.</param>
/// <param name="Parsed">The parsed artifact, owned by the apply handler — dispose before it returns.</param>
private readonly record struct ReconciledArtifact(byte[]? Blob, ParsedDeploymentArtifact? Parsed);
/// <summary>
/// The parse-once reconcile every apply runs, with the artifact bytes already in hand: parse
/// ONCE and reconcile the driver children from that parse. The caller
/// (<see cref="HandleArtifactForApply"/>) owns and disposes the returned
/// <see cref="ParsedDeploymentArtifact"/> and threads it through the rest of the apply, so an
/// apply builds exactly one <c>JsonDocument</c> in this actor.
/// </summary>
/// <returns>The reconciled blob + its parsed artifact, or a default (both null) when the blob was
/// empty or absent (#485).</returns>
/// <exception cref="System.Text.Json.JsonException">The artifact bytes are not valid JSON. Unlike the
/// lenient blob wrappers (malformed ⇒ no specs ⇒ every running child planned for StopChild) this
/// surfaces, and <see cref="HandleArtifactForApply"/>'s catch turns it into a Failed ACK with the
/// children left running — the honest reading of bytes that cannot be understood.</exception>
private ReconciledArtifact ReconcileDriversFromParsed(DeploymentId deploymentId, byte[]? blob)
{
blob ??= Array.Empty<byte>();
if (blob.Length == 0)
{
WarnEmptyArtifactSkippingReconcile(deploymentId);
return default;
}
// Opened here, NOT inside the try below: DeploymentArtifact.Open throws on malformed bytes and
// that throw must reach the apply handler's catch (Failed ACK), not be mistaken for a reconcile
// failure. Ownership transfers to the caller on success; a throw from the reconcile itself
// disposes it here, since the caller never saw it.
var parsed = DeploymentArtifact.Open(blob);
try
{
ApplyDriverSpecs(parsed.ParseDriverInstances(_localNode.Value));
}
catch
{
parsed.Dispose();
throw;
}
return new ReconciledArtifact(blob, parsed);
}
/// <summary>
/// The blob-processing half of <see cref="ReconcileDrivers"/>, split out so the boot-from-cache
/// and bootstrap-restore paths can reconcile from bytes with the lenient
/// "malformed ⇒ no specs" parse those paths have always had. The apply paths use
/// <see cref="ReconcileDriversFromParsed"/> instead, so they parse once and keep the parse.
/// </summary>
/// <returns>The reconciled blob, or <see langword="null"/> when it was empty (#485).</returns>
private byte[]? ReconcileDriversFromBlob(DeploymentId deploymentId, byte[]? blob)
{
blob ??= Array.Empty<byte>();
if (blob.Length == 0)
{
WarnEmptyArtifactSkippingReconcile(deploymentId);
return null;
}
// #601 (S-10): non-empty but malformed bytes are "no answer" too. The lenient static
// ParseDriverInstances swallows the JsonException into an empty list, and an empty list plans every
// running child for StopChild. Open throws instead, so a corrupt cache row / restored blob leaves
// the running drivers alone, exactly like the zero-length case above.
IReadOnlyList<DriverInstanceSpec> specs;
try
{
using var parsed = DeploymentArtifact.Open(blob);
specs = parsed.ParseDriverInstances(_localNode.Value);
}
catch (JsonException ex)
{
_log.Error(ex,
"DriverHost {Node}: artifact for {Id} is malformed ({Bytes} bytes); skipping reconcile and keeping the {Count} running driver(s)",
_localNode, deploymentId, blob.Length, _children.Count);
return null;
}
ApplyDriverSpecs(specs);
return blob;
}
/// <summary>
/// Issue #485 — no bytes is no answer, not "a configuration with no drivers". Parsing zero bytes
/// yields zero specs, and <c>DriverSpawnPlanner</c> then plans EVERY running child for StopChild: a
/// missing deployment row (or a half-written one) silently takes this node's entire field I/O down
/// while the apply still ACKs Applied. Nothing legitimate produces a zero-length blob — deleting the
/// last driver still deploys a JSON document with empty arrays — so treat it exactly like a load
/// failure and leave the running children alone.
/// </summary>
private void WarnEmptyArtifactSkippingReconcile(DeploymentId deploymentId) =>
_log.Warning(
"DriverHost {Node}: artifact for {Id} read back empty; skipping reconcile and keeping the {Count} running driver(s)",
_localNode, deploymentId, _children.Count);
/// <summary>Spawn / delta / stop the driver children to match <paramref name="specs"/>.</summary>
/// <param name="specs">The driver-instance specs this node should be running.</param>
private void ApplyDriverSpecs(IReadOnlyList<DriverInstanceSpec> specs)
{
var snapshots = _children.ToDictionary(
kv => kv.Key,
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
StringComparer.Ordinal);
var plan = DriverSpawnPlanner.Compute(snapshots, specs);
foreach (var id in plan.ToStop) StopChild(id);
foreach (var spec in plan.ToSpawn) SpawnChild(spec);
// v3 (WP7): a dependency-consumer driver's declared dependency refs change when its tags/scripts are
// edited. That edit changes its DriverConfig, which respawns it (#516), and the respawned driver's
// adapter registers the NEW set in PreStart — no separate re-register is needed (#665).
}
/// <summary>
/// Store a successfully applied artifact in the node-local cache, so this node can boot from
/// it while central SQL is unreachable.
/// </summary>
/// <remarks>
/// <para>
/// <b>Never throws.</b> By the time this runs the deployment is already recorded Applied
/// in central SQL and an Applied ACK has been sent to the coordinator. An exception
/// escaping here would unwind into <see cref="HandleArtifactForApply"/>'s catch and send a second,
/// contradictory Failed ACK for a deployment the fleet already believes is live.
/// </para>
/// <para>
/// <b>An empty or unloadable blob is skipped, not cached.</b> See
/// <see cref="ReconcileDrivers"/>: a DB failure there degrades silently, so "the apply
/// succeeded" does not imply "a real configuration was applied". Caching an empty
/// artifact would make it this node's last-known-good and boot it into an empty address
/// space during the next outage — strictly worse than having no cache at all.
/// </para>
/// <para>
/// <b>The store runs OFF the actor thread.</b> It is hundreds of chunk inserts into SQLite;
/// blocking on it (as this did) parked the host for the duration, so it could not dequeue a
/// driver value burst, a diagnostics request or the next dispatch. The task is piped back as
/// <see cref="ArtifactCached"/>/<see cref="ArtifactCacheFailed"/> so completion is observed
/// rather than fire-and-forgotten — the earlier objection to this shape was that the
/// pipe-back needs a handler in every state it can land in, which is why
/// <see cref="Steady"/>, <see cref="Applying"/> AND <see cref="Stale"/> all register one.
/// </para>
/// <para>
/// <b>Stale is included on purpose.</b> This used to claim Stale could not be reached after a
/// store had started, because it is only ever entered from <see cref="Bootstrap"/>. That is
/// not a property of the Stale entry — <see cref="Bootstrap"/> starts stores of its own, via
/// the orphan-replay <see cref="BeginDirectApply"/> and via <see cref="RestoreApplied"/>, and both
/// run inside the very try block whose catch Becomes Stale. Today no statement between those
/// calls and the catch happens to throw, so the old claim held by accident of ordering rather
/// than by design; a restart can orphan an in-flight pipe-back the same way. Registering the
/// handlers costs two lines and removes the need to re-derive that argument.
/// </para>
/// <para>
/// <b>The apply does not depend on the outcome</b> — it never did, and the async shape keeps
/// it that way. By the time this runs the deployment is recorded Applied and an Applied ACK
/// has gone to the coordinator; a failure logs a Warning and changes nothing else.
/// </para>
/// <para>
/// <b>Idempotence is unchanged.</b> The skip-if-already-cached check lives inside
/// <c>LocalDbDeploymentArtifactCache.StoreAsync</c>, so it moves off-thread with the rest of
/// the store rather than being lost.
/// </para>
/// <para>
/// <b>No boot-path race.</b> The readers — <see cref="BootstrapFromCache"/> and
/// <see cref="TryBootFromCache"/> — run in <see cref="PreStart"/>, before this actor has
/// processed a single message and therefore before any apply could have started a store.
/// The only concurrent writer a read can meet is the pair peer's replication, which the
/// cache already handles.
/// </para>
/// </remarks>
/// <param name="deploymentId">The deployment whose artifact is being cached.</param>
/// <param name="revision">The revision hash to store alongside the bytes.</param>
/// <param name="blob">The applied artifact BYTES — what actually gets stored.</param>
/// <param name="parsed">The apply's already-parsed artifact, when the caller has one, so the cluster
/// scope is read off that parse instead of re-parsing the same bytes. Null on the boot/restore paths,
/// which fall back to the one-shot static resolve.</param>
private void CacheAppliedArtifact(
DeploymentId deploymentId, RevisionHash revision, byte[]? blob, ParsedDeploymentArtifact? parsed = null)
{
if (_deploymentArtifactCache is null)
return;
if (blob is null || blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: not caching deployment {Id} — its artifact was empty or could not " +
"be loaded, so it is not a usable last-known-good configuration.",
_localNode, deploymentId);
return;
}
try
{
// The node's ClusterId is carried inside the artifact (ClusterNode rows), not in config
// or on IClusterRoleInfo. A single-cluster or unscoped artifact has none, so it shares
// one sentinel key — the pair still converges on a single pointer row either way.
//
// Resolved HERE, synchronously, and not one step later: `parsed` is owned by the calling
// apply and DISPOSED in its finally, which runs long before a slow store completes. The
// async continuation below must therefore capture only these scalars — never `parsed`.
var scope = parsed is not null
? parsed.ResolveClusterScope(_localNode.Value)
: DeploymentArtifact.ResolveClusterScope(blob, _localNode.Value);
var clusterId = scope.Mode == ClusterFilterMode.ScopeTo && !string.IsNullOrWhiteSpace(scope.ClusterId)
? scope.ClusterId
: SingleClusterCacheKey;
_log.Debug("DriverHost {Node}: caching deployment {Id} (rev {Rev}, cluster {ClusterId}, {Bytes} bytes)",
_localNode, deploymentId, revision, clusterId, blob.Length);
_deploymentArtifactCache
.StoreAsync(clusterId, deploymentId.ToString(), revision.Value, blob)
.PipeTo(
Self,
success: () => new ArtifactCached(deploymentId),
failure: ex => new ArtifactCacheFailed(deploymentId, ex));
}
catch (Exception ex)
{
// A cache that throws SYNCHRONOUSLY (before returning a task) never produces a pipe-back,
// so route it to the same handler by hand rather than letting it unwind into the apply's
// catch — which would send a second, contradictory Failed ACK for a live deployment.
Self.Tell(new ArtifactCacheFailed(deploymentId, ex));
}
}
/// <summary>The node-local artifact cache accepted this deployment's bytes.</summary>
/// <remarks>
/// Piped back from <see cref="CacheAppliedArtifact"/>. Carries the id only — deliberately no
/// <c>ParsedDeploymentArtifact</c> and no other apply-owned state, since the apply that started
/// the store has long since disposed its parse by the time this lands.
/// </remarks>
private sealed record ArtifactCached(DeploymentId DeploymentId);
/// <summary>The node-local artifact cache could not store this deployment's bytes.</summary>
/// <remarks>
/// Carries the <b>exception</b>, not a pre-flattened message string. A LocalDb/SQLite failure is
/// routinely a wrapped inner exception, and <c>ex.Message</c> alone ("An error occurred while
/// …") names neither the SQLite result code nor the frame that produced it — the detail an
/// operator needs to tell a full disk from a locked file from a schema mismatch. Logging it as
/// the log event's <c>Cause</c> also lets a test assert on the exception TYPE rather than on
/// prose.
/// </remarks>
private sealed record ArtifactCacheFailed(DeploymentId DeploymentId, Exception Cause);
private void HandleArtifactCached(ArtifactCached msg) =>
_log.Info("DriverHost {Node}: cached applied deployment {Id} as this node's local fallback",
_localNode, msg.DeploymentId);
private void HandleArtifactCacheFailed(ArtifactCacheFailed msg) =>
_log.Warning(msg.Cause,
"DriverHost {Node}: deployment {Id} could not be cached locally ({Reason}). The apply itself " +
"succeeded and is unaffected; this node simply has no local fallback for that revision.",
_localNode, msg.DeploymentId, msg.Cause.Message);
/// <summary>
/// Restore the served state for an already-applied deployment after a process restart.
/// <see cref="Bootstrap"/> recovers <see cref="_currentRevision"/> from NodeDeploymentState,
/// but the driver children and OPC UA address space are in-memory and gone after a restart —
/// so without this a restarted node serves an empty address space until the next config
/// change (and an identical-config redeploy no-ops on the unchanged revision). Re-spawns
/// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk.
/// No re-ack: the deployment is already Applied.
/// </summary>
/// <remarks>
/// Its two ConfigDb reads (<see cref="ReconcileDrivers"/> and
/// <see cref="PushDesiredSubscriptions"/>) stay SYNCHRONOUS deliberately: this runs in
/// <see cref="PreStart"/>, before the host has processed a single message, so there is no live
/// traffic behind them to block — unlike the apply path, which moved its read off the mailbox.
/// </remarks>
private void RestoreApplied(DeploymentId deploymentId, RevisionHash? revision)
{
var correlation = CorrelationId.NewId();
try
{
var appliedBlob = ReconcileDrivers(deploymentId);
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, appliedBlob));
PushDesiredSubscriptions(deploymentId);
// Populate the node-local cache from the artifact this restored node is now serving. The
// cache invariant is "holds what the node currently serves"; without this only a FRESH
// apply (BeginDirectApply) ever wrote it, so a node whose cache was lost — a wiped/fresh volume,
// a disk failure — would recover its served state here yet stay cache-less, unable to
// boot-from-cache on the NEXT central outage until some future new deploy happened to land.
// Replication does not heal that gap either: a peer's already-acked rows are pruned from its
// oplog, so a fully-wiped node is never back-filled. Re-caching on restore is what closes
// it. (Found on the docker-dev live gate.)
if (revision is { } rev)
CacheAppliedArtifact(deploymentId, rev, appliedBlob);
_log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to restore served state for {Id} on bootstrap", _localNode, deploymentId);
}
}
/// <summary>
/// SubscribeBulk pass. After an apply, read the deployment's Equipment-namespace tags,
/// group their driver-side FullName references by driver instance, and hand each running driver
/// child its desired subscription set via <see cref="DriverInstanceActor.SetDesiredSubscriptions"/>.
/// The child retains the set and (re)subscribes on every Connected entry, so values stream into
/// the OPC UA sink and resume after reconnects. Drivers with no configured tags get an empty set
/// (which clears any stale subscription from a previous deployment).
/// </summary>
/// <remarks>
/// The <b>bootstrap-restore</b> entry point only. Neither apply path reaches it any more:
/// <see cref="HandleArtifactForApply"/> already holds the apply's single parse and calls
/// <see cref="PushDesiredSubscriptionsFromParsed"/> directly, so this ConfigDb read exists for
/// <see cref="RestoreApplied"/>, which restores from a deployment id recovered out of
/// NodeDeploymentState and may hold no blob of its own.
/// </remarks>
private void PushDesiredSubscriptions(DeploymentId deploymentId)
{
// Phase 4 belt-and-suspenders: a ConfigDb read, so it can only be the restore path on a node that
// has one (every apply path calls PushDesiredSubscriptionsFromParsed with its parse in hand).
// No ConfigDb ⇒ nothing to load.
if (_dbFactory is null)
{
_log.Warning("DriverHost {Node}: no ConfigDb to load artifact for SubscribeBulk ({Id}); skipping", _localNode, deploymentId);
return;
}
byte[] blob;
try
{
// Synchronous by design: the only caller is the PreStart restore path above, where nothing
// is queued behind this read (the apply path reads off the mailbox instead).
using var db = _dbFactory.CreateDbContext();
blob = db.Deployments.AsNoTracking()
.Where(d => d.DeploymentId == deploymentId.Value)
.Select(d => d.ArtifactBlob)
.FirstOrDefault() ?? Array.Empty<byte>();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to load artifact for SubscribeBulk ({Id})", _localNode, deploymentId);
return;
}
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
}
/// <summary>
/// The artifact-processing half of <see cref="PushDesiredSubscriptions"/>, split out so the
/// boot-from-cache path can drive it with bytes already in hand.
/// </summary>
/// <remarks>
/// Separated because the cache path runs precisely when the ConfigDb read above cannot
/// succeed — re-reading the artifact there would fail by definition.
/// </remarks>
private void PushDesiredSubscriptionsFromArtifact(DeploymentId deploymentId, byte[] blob)
{
// Issue #485 — the same "no bytes is no answer" rule as ReconcileDrivers. An empty artifact parses
// to a composition with no raw tags, which hands every child an EMPTY desired set — and an empty set
// drops the live subscription handle rather than re-subscribing. Reachable independently of the
// reconcile guard: this method does its OWN ConfigDb read, so the row can go missing between the two.
if (blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: artifact for {Id} read back empty; skipping SubscribeBulk and keeping the live subscriptions",
_localNode, deploymentId);
return;
}
// #601 (S-10): Open, not the lenient static ParseComposition — that wrapper swallows malformed JSON
// into an EMPTY composition, which would hand every child an empty desired set and drop its live
// subscription. A malformed artifact is no answer: leave the subscriptions as they are.
AddressSpaceComposition composition;
try
{
using var parsed = DeploymentArtifact.Open(blob);
composition = parsed.ParseComposition(_localNode.Value);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to parse composition for SubscribeBulk ({Id})", _localNode, deploymentId);
return;
}
ApplyDesiredSubscriptions(composition);
}
/// <summary>
/// The apply paths' SubscribeBulk entry point: reads the composition off the apply's SINGLE parse
/// instead of re-parsing the same bytes a second time.
/// </summary>
/// <remarks>
/// No empty-artifact guard here, unlike <see cref="PushDesiredSubscriptionsFromArtifact"/>: a
/// <see cref="ParsedDeploymentArtifact"/> only exists for a non-empty blob (the apply's reconcile
/// already applied that guard and never opened one otherwise).
/// </remarks>
/// <param name="deploymentId">The deployment being applied — diagnostics only.</param>
/// <param name="parsed">The apply's parsed artifact.</param>
private void PushDesiredSubscriptionsFromParsed(DeploymentId deploymentId, ParsedDeploymentArtifact parsed)
{
AddressSpaceComposition composition;
try
{
composition = parsed.ParseComposition(_localNode.Value);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to parse composition for SubscribeBulk ({Id})", _localNode, deploymentId);
return;
}
ApplyDesiredSubscriptions(composition);
}
/// <summary>Gets (creating on first use) the per-driver bucket of a secondary index.</summary>
/// <param name="index">Either <see cref="_valueRefsByDriver"/> or <see cref="_alarmRefsByDriver"/>.</param>
/// <param name="driverInstanceId">The driver whose bucket is wanted.</param>
/// <returns>The mutable bucket for that driver.</returns>
private static List<(string RawPath, HashSet<NodeRealmRef> Nodes)> IndexFor(
Dictionary<string, List<(string RawPath, HashSet<NodeRealmRef> Nodes)>> index, string driverInstanceId)
{
if (!index.TryGetValue(driverInstanceId, out var bucket))
index[driverInstanceId] = bucket = new List<(string, HashSet<NodeRealmRef>)>();
return bucket;
}
/// <summary>
/// Hand each running driver child its desired value + alarm subscription set for
/// <paramref name="composition"/>, rebuild the live-value/write routing maps, and forward the
/// Equipment VirtualTag + ScriptedAlarm plans to their hosts.
/// </summary>
/// <param name="composition">The composition this node should serve.</param>
private void ApplyDesiredSubscriptions(AddressSpaceComposition composition)
{
// v3 Batch 4: the driver subscribes + publishes by RawPath (the wire-ref == RawPath == RawTagPlan.NodeId
// per the v3 identity contract). Value-subscription set = the non-alarm raw tags' RawPaths; alarm-bearing
// raw tags are Part 9 conditions, fed via the native alarm event stream (ForwardNativeAlarm), so they are
// excluded from the value set.
var refsByDriver = composition.RawTags
.Where(t => t.Alarm is null)
.GroupBy(t => t.DriverInstanceId, StringComparer.Ordinal)
.ToDictionary(
g => g.Key,
g => (IReadOnlyList<string>)g.Select(t => t.NodeId)
.Distinct(StringComparer.Ordinal)
.ToArray(),
StringComparer.Ordinal);
// Native-alarm subscription set: the alarm-bearing raw tags' RawPaths (= the driver's ConditionId).
// An IAlarmSource driver suppresses OnAlarmEvent until at least one alarm subscription exists, so the
// instance actor must SubscribeAlarmsAsync these refs to un-gate the feed. Routing stays by ConditionId
// in ForwardNativeAlarm; this set just opens (and scopes) the subscription.
var alarmRefsByDriver = composition.RawTags
.Where(t => t.Alarm is not null)
.GroupBy(t => t.DriverInstanceId, StringComparer.Ordinal)
.ToDictionary(
g => g.Key,
g => (IReadOnlyList<string>)g.Select(t => t.NodeId)
.Distinct(StringComparer.Ordinal)
.ToArray(),
StringComparer.Ordinal);
// Referencing UNS nodes by backing RawPath — each raw value tag fans out to its own raw NodeId PLUS
// every UNS reference that projects it (dual-NodeId registration against the SAME driver ref).
var unsRefsByRawPath = new Dictionary<string, List<UnsReferenceVariable>>(StringComparer.Ordinal);
foreach (var v in composition.UnsReferenceVariables)
{
if (!unsRefsByRawPath.TryGetValue(v.BackingRawPath, out var list))
unsRefsByRawPath[v.BackingRawPath] = list = new List<UnsReferenceVariable>();
list.Add(v);
}
// Rebuild the driver live-value + write routing maps from the RawTags ∪ UnsReferenceVariables pass.
// Clear-and-repopulate every apply so renames/removals/re-points are reflected.
_nodeIdByDriverRef.Clear();
_driverRefByNodeId.Clear();
_alarmNodeIdByDriverRef.Clear();
_driverRefByAlarmNodeId.Clear();
_alarmMetaByNodeId.Clear();
// NOT _nativeAlarmProjector.Clear() (Gitea #641): it is pruned to the still-deployed conditions after the
// maps are rebuilt below, so a redeploy that keeps an alarm keeps its projected prior state.
// The per-driver secondary indexes are rebuilt WHOLE alongside their primaries below — same pass,
// same clear — so no incremental maintenance exists to get out of step with them.
_valueRefsByDriver.Clear();
_alarmRefsByDriver.Clear();
foreach (var t in composition.RawTags)
{
var key = (t.DriverInstanceId, t.NodeId); // (DriverInstanceId, RawPath) — RawPath IS the wire-ref
if (t.Alarm is not null)
{
// Native alarm → a single Part 9 condition at the RawPath (Raw realm). WP4 adds the referencing
// equipment notifier NodeIds; WP3 wires the single raw condition.
if (!_alarmNodeIdByDriverRef.TryGetValue(key, out var aset))
{
_alarmNodeIdByDriverRef[key] = aset = new HashSet<NodeRealmRef>();
// Indexed on CREATION only, so a repeated key contributes exactly one entry — and the set
// is captured by reference, so later additions to it are seen through the index.
IndexFor(_alarmRefsByDriver, t.DriverInstanceId).Add((t.NodeId, aset));
}
aset.Add(new NodeRealmRef(t.NodeId, AddressSpaceRealm.Raw));
// Inverse 1:1 map for the inbound native-condition ack path (keyed by BARE condition NodeId).
_driverRefByAlarmNodeId[t.NodeId] = key;
// Per-condition metadata for the alerts fan-out. WP4 refines /alerts identity to the referencing
// equipment paths; WP3 keys the display off the RawPath. The referencing-equipment paths (the
// Area/Line/Equipment UNS folder paths carried on the RawTagPlan) ride onto every transition so
// the /alerts row lists which equipment reference this raw condition.
_alarmMetaByNodeId[t.NodeId] = (t.NodeId, t.Name, t.Alarm.AlarmType, t.Alarm.HistorizeToAveva, t.ReferencingEquipmentPaths);
continue;
}
// Value tag: register the RAW NodeId (Raw realm) AND every referencing UNS NodeId (Uns realm)
// against the SAME driver ref — the single value source fans to all of them.
if (!_nodeIdByDriverRef.TryGetValue(key, out var set))
{
_nodeIdByDriverRef[key] = set = new HashSet<NodeRealmRef>();
IndexFor(_valueRefsByDriver, t.DriverInstanceId).Add((t.NodeId, set));
}
set.Add(new NodeRealmRef(t.NodeId, AddressSpaceRealm.Raw));
_driverRefByNodeId[(AddressSpaceRealm.Raw, t.NodeId)] = key;
if (unsRefsByRawPath.TryGetValue(t.NodeId, out var refs))
{
foreach (var v in refs)
{
set.Add(new NodeRealmRef(v.NodeId, AddressSpaceRealm.Uns));
// A UNS write resolves to the same driver ref — keyed under the Uns realm so it can never
// collide with a raw NodeId that shares its bare string.
_driverRefByNodeId[(AddressSpaceRealm.Uns, v.NodeId)] = key;
}
}
}
// Gitea #641 — forget the projected prior state of conditions this composition no longer carries, and keep
// the rest. Clearing it whole on every apply made the next driver Acknowledge of a still-active alarm
// project Active=false (the default prior), and the next Clear of an unacknowledged one project
// Acked=true — on any deploy, rebuild or not. The node manager carries the NODE's state across a rebuild,
// so the two stay in step. Keyed by the bare condition NodeId, exactly as ForwardNativeAlarm projects.
_nativeAlarmProjector.RetainOnly(_driverRefByAlarmNodeId.Keys.ToHashSet(StringComparer.Ordinal));
// Cache the per-driver payload as THE authority, in the same clear-and-repopulate pass as the routing
// maps above so a rename/removal can never leave a stale entry behind. Keyed over the UNION of the two
// ref maps AND the running children: a child with no authored tags must get an explicit EMPTY set (which
// it routes to Unsubscribe), not be absent from the authority — absence and empty must not differ here.
_desiredByDriver.Clear();
foreach (var driverId in refsByDriver.Keys
.Concat(alarmRefsByDriver.Keys)
.Concat(_children.Keys)
.Distinct(StringComparer.Ordinal))
{
_desiredByDriver[driverId] = new DesiredSubscriptionSet(
refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty<string>(),
alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty<string>());
}
var total = 0;
foreach (var (driverId, entry) in _children)
{
total += SendDesiredSubscriptions(entry.Actor, driverId);
}
if (total > 0)
{
_log.Info("DriverHost {Node}: SubscribeBulk pushed {Refs} references across {Drivers} driver(s)",
_localNode, total, refsByDriver.Count);
}
// Hand the Equipment-namespace VirtualTags to the host so it spawns/reconciles a
// VirtualTagActor per plan and streams their evaluated values back onto the just-rebuilt
// address space. Runs on BOTH the fresh-apply path (HandleArtifactForApply) and the bootstrap-restore
// path (RestoreApplied) because both call this method, so one send covers both.
// NOTE: the Stale-recovery path (TryRecoverFromStale) does NOT call PushDesiredSubscriptions,
// so — like drivers — VirtualTags remain empty after a Stale recovery until the next
// deployment dispatch. This is intentional and consistent with driver recovery: the Stale
// path only restores the revision marker + NodeDeploymentState; a subsequent dispatch
// (or a redeploy from AdminUI) triggers the full apply + subscribe pass.
_virtualTagHost?.Tell(new VirtualTagHostActor.ApplyVirtualTags(composition.EquipmentVirtualTags));
if (composition.EquipmentVirtualTags.Count > 0)
{
_log.Info("DriverHost {Node}: applied {Count} Equipment VirtualTag(s) to the VirtualTag host",
_localNode, composition.EquipmentVirtualTags.Count);
}
// Same pass for Equipment-namespace ScriptedAlarms: hand the plans to the host so it
// (re)loads its engine + re-registers mux interest for the union of dependency refs. Covers
// both the fresh-apply and bootstrap-restore paths (both call this method); the Stale-recovery
// path deliberately does not, matching driver + VirtualTag recovery.
_scriptedAlarmHost?.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(composition.EquipmentScriptedAlarms));
if (composition.EquipmentScriptedAlarms.Count > 0)
{
_log.Info("DriverHost {Node}: applied {Count} Equipment ScriptedAlarm(s) to the ScriptedAlarm host",
_localNode, composition.EquipmentScriptedAlarms.Count);
}
}
/// <summary>
/// Send <paramref name="driverId"/>'s desired subscription payload to <paramref name="actor"/>.
/// </summary>
/// <remarks>
/// The ONE place a <see cref="DriverInstanceActor.SetDesiredSubscriptions"/> is built, so the apply's
/// bulk push, a respawn (<see cref="SpawnChild"/>) and a supervision reprovision
/// (<c>HandleReprovisionRequested</c>) cannot drift. That is structural rather than
/// comment-maintained on purpose: a second sender that assembled its own payload is exactly how
/// 03/S16 happened — the restart path got a child without the set that makes a child useful.
/// <para>
/// An EMPTY set is valid and MEANINGFUL — the child routes it to <c>Unsubscribe</c> (dropping a
/// stale handle) rather than to a spurious subscribe — so a driver missing from
/// <see cref="_desiredByDriver"/> is sent an empty set, never skipped.
/// </para>
/// </remarks>
/// <param name="actor">The driver child to hand the set to.</param>
/// <param name="driverId">The driver instance whose set to send.</param>
/// <returns>The value-ref count, for the bulk loop's log total.</returns>
private int SendDesiredSubscriptions(IActorRef actor, string driverId)
{
var set = _desiredByDriver.TryGetValue(driverId, out var s)
? s
: new DesiredSubscriptionSet(Array.Empty<string>(), Array.Empty<string>());
actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(
set.ValueRefs, SubscriptionPublishingInterval, set.AlarmRefs));
return set.ValueRefs.Count;
}
private void SpawnChild(DriverInstanceSpec spec)
{
// #609: an instance id removed by an earlier deployment (StopChild ⇒ Evict, which also holds back late
// items) may come back; lift the hold BEFORE the child exists, since it publishes health as it starts.
// The resilience tracker's equivalent is lifted by _invokerFactory.Create below.
_telemetryHub?.Readmit(spec.DriverInstanceId);
var stub = DriverInstanceActor.ShouldStub(spec.DriverType, _localRoles);
IDriver? driver = null;
if (!stub)
{
try { driver = _driverFactory.TryCreate(spec.DriverType, spec.DriverInstanceId, spec.DriverConfig); }
catch (Exception ex)
{
// A factory throw is a CONFIG error, not a connectivity one — TryCreate is pure parsing;
// device I/O happens later in InitializeAsync. Logged at Error because the node silently
// degrades to a stub that answers nothing, and since #516 routed DriverConfig changes
// through a respawn this path is now reachable by an ordinary operator edit rather than
// only by a brand-new driver. It still does NOT fail the deployment — making it do so is
// a deliberate follow-up, since it would let one malformed driver block a fleet deploy.
_log.Error(ex,
"DriverHost {Node}: factory for {Type} REJECTED the config for {Id} — the driver is " +
"stubbed and will serve nothing until the config is fixed and redeployed",
_localNode, spec.DriverType, spec.DriverInstanceId);
}
if (driver is null)
{
_log.Warning(
"DriverHost {Node}: no factory for driver type {Type} (instance {Id}); falling back to stub",
_localNode, spec.DriverType, spec.DriverInstanceId);
stub = true;
}
}
IActorRef child;
// Prefer the real ClusterId from the deployment artifact; fall back to the local node
// identity for pre-PR artifacts that don't carry it yet (older deploys persisted before
// ClusterId was added to DriverInstanceSpec).
var clusterId = !string.IsNullOrEmpty(spec.ClusterId) ? spec.ClusterId : _localNode.Value;
// A fresh generation-suffixed name on EVERY spawn so a respawn can never collide with a child
// still tearing down: Akka frees a stopped actor's name only after it FULLY terminates (async),
// but HandleRestartDriver stops + respawns within one (synchronous) message handler — the old
// child is still registered, so reusing the base name throws InvalidActorNameException
// ("actor name is not unique"). Children are tracked by the _children dict (by IActorRef), never
// by path, so the suffix is invisible to every caller.
var actorName = ActorNameFor(spec.DriverInstanceId, _childSpawnGeneration++);
if (stub)
{
child = Context.ActorOf(
DriverInstanceActor.Props(
new StubbedDriver(spec.DriverInstanceId, spec.DriverType),
reconnectInterval: null,
startStubbed: true,
healthPublisher: _healthPublisher,
clusterId: clusterId),
actorName);
}
else
{
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
// route through the retry/breaker/telemetry pipeline. The tier defaults are layered
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
// no-op invoker on nodes without the real resilience factory bound.
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
child = Context.ActorOf(
DriverInstanceActor.Props(
driver!,
healthPublisher: _healthPublisher,
clusterId: clusterId,
invoker: invoker),
actorName);
child.Tell(new DriverInstanceActor.InitializeRequested(spec.DriverConfig));
}
_children[spec.DriverInstanceId] = new ChildEntry(child, spec, stub);
// EVERY spawn hands the child its desired set — deploy reconcile, operator Restart (03/S16), and the
// supervision reprovision. Sending it HERE rather than at each call site is the point: a child that
// exists without its subscription set is a dead driver, so the two facts are established together.
// Before this, the only sender was the apply's bulk push, so an operator's Restart produced a child
// that connected and then read nothing until the next deployment — the #488 periodic reconcile cannot
// help, because it early-returns on an empty desired set.
//
// ORDERING, and its one accepted cost: ApplyDriverSpecs (which spawns) runs BEFORE
// ApplyDesiredSubscriptions (which rebuilds _desiredByDriver), so on a deploy that RESPAWNS a child
// (#516 — the merged DriverConfig folds in the raw tags, so a tag-set edit changes the spec) the new
// child is handed the PREVIOUS apply's set, and nothing at all on the first deploy. That transient is
// superseded a moment later by the bulk push, on the same sender→receiver pair, so Akka's per-pair
// ordering guarantees the child CONVERGES on the new set — but it is NOT guaranteed to be inert on
// the way there: the child runs concurrently with the rest of this handler, so a driver whose
// InitializeAsync completes quickly can reach Connected and subscribe the stale set before the bulk
// push arrives, then re-subscribe. The cost is one extra subscribe/unsubscribe round-trip on a
// respawning deploy (HandleSubscribeAsync drops the prior subscription before establishing the new
// one, so it converges); a real device driver, whose initialise opens a socket, rarely wins that race.
// Deliberately not "fixed" by withholding the send: a child that never gets a set is the 03/S16
// defect, and the operator-Restart and supervision paths have no bulk push coming at all.
// Sent after InitializeRequested above, which keeps init-before-subscribe ordering.
SendDesiredSubscriptions(child, spec.DriverInstanceId);
// v3 (WP7): if the (real, non-stub) driver consumes other tags' live values, spawn a mux-adapter
// child that registers its dependency refs on the per-node dependency mux and forwards changes
// into the driver. Requires a wired mux (dev/None deployments have none — then calc inputs simply
// can't flow, matching the VirtualTag dev path).
if (!stub && driver is IDependencyConsumer consumer && _dependencyMux is not null)
{
var adapter = Context.ActorOf(
ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags.DependencyConsumerMuxAdapter.Props(consumer, _dependencyMux),
"depmux-" + actorName);
_depConsumerAdapters[spec.DriverInstanceId] = adapter;
_log.Info("DriverHost {Node}: wired dependency-consumer mux adapter for {Id} ({Count} ref(s))",
_localNode, spec.DriverInstanceId, consumer.DependencyRefs.Count);
}
_log.Info("DriverHost {Node}: spawned {Type} driver {Id} (stub={Stub})",
_localNode, spec.DriverType, spec.DriverInstanceId, stub);
}
/// <summary>Stops + forgets the dependency-consumer mux adapter for a driver (if any). Called wherever a
/// driver child is stopped/respawned so a stale adapter never keeps a dead driver's refs registered.</summary>
private void StopDependencyAdapter(string driverInstanceId)
{
if (_depConsumerAdapters.Remove(driverInstanceId, out var adapter))
Context.Stop(adapter);
}
private void StopChild(string driverInstanceId)
{
if (!_children.TryGetValue(driverInstanceId, out var entry)) return;
StopDependencyAdapter(driverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(driverInstanceId);
// #609: the instance left the plan (StopChild is only reached via plan.ToStop), so its cached
// Health/Resilience snapshots must not replay to new telemetry subscribers. Release the shared
// resilience state FIRST: the resilience publisher re-emits the tracker's snapshots every tick, so
// evicting the hub alone was undone within one interval. Context.Stop is asynchronous, so the child can
// still finish an in-flight capability call (observed: its SubscribeBulk) and publish health AFTER these
// two calls; both therefore HOLD the id — late writes are dropped — until a later SpawnChild of the same
// id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
_invokerFactory.Release(driverInstanceId);
_telemetryHub?.Evict(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}
private static string ActorNameFor(string driverInstanceId, long generation)
{
// Akka actor names cannot contain '/', ':', or whitespace. Mangle defensively. The monotonic
// generation suffix guarantees a never-before-used name on every spawn (see SpawnChild) so a
// restart's respawn can never collide with the still-terminating predecessor.
var chars = driverInstanceId.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' or '.' ? c : '_').ToArray();
return "drv-" + new string(chars) + "-g" + generation;
}
/// <summary>
/// Minimal placeholder driver used when no factory is registered for a driver type or when
/// <see cref="DriverInstanceActor.ShouldStub"/> returns true. <see cref="DriverInstanceActor"/>
/// is started with <c>startStubbed:true</c> so the driver methods on this object never run.
/// </summary>
private sealed class StubbedDriver : IDriver
{
/// <inheritdoc />
public string DriverInstanceId { get; }
/// <inheritdoc />
public string DriverType { get; }
/// <summary>Initializes a new stubbed driver with the specified ID and type.</summary>
/// <param name="id">The driver instance identifier.</param>
/// <param name="type">The driver type name.</param>
public StubbedDriver(string id, string type) { DriverInstanceId = id; DriverType = type; }
/// <inheritdoc />
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);
}
private void HandleRestartDriver(RestartDriver msg)
{
// DPS broadcast — only act if this node hosts the requested instance.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry))
return;
_log.Info("DriverHost {Node}: restarting driver {Id} by request of {User}",
_localNode, msg.DriverInstanceId, msg.ActorByUserName);
// Stop the existing child actor — DriverInstanceActor.PostStop calls ShutdownAsync.
StopDependencyAdapter(msg.DriverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(msg.DriverInstanceId);
// Respawn from the same spec the last reconcile used — preserves RowId, Name, ClusterId.
SpawnChild(entry.Spec);
}
/// <summary>
/// Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6,
/// 03/S16) — the third and least visible of the three incarnation routes.
/// </summary>
/// <remarks>
/// A restart recreates the actor object around the SAME driver instance (the Props closure captured
/// it) without going through <see cref="SpawnChild"/>, so neither the <c>InitializeRequested</c> nor
/// the desired-subscription send that a spawn performs ever happens — and the child's
/// <see cref="IActorRef"/> is unchanged, so <c>_children</c> needs no maintenance and the host has no
/// other notification that a new incarnation exists. Its <c>PreRestart</c> ran <c>PostStop</c>, which
/// called <c>ShutdownAsync</c> on the driver, so re-initialising is not optional: without it the
/// child sits in <c>Connecting</c> forever around a shut-down driver.
/// <para>
/// The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the
/// instance actor restarting, so it is deliberately not touched here.
/// </para>
/// </remarks>
/// <param name="msg">The restarted child's self-announcement.</param>
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) =>
ReprovisionChild(msg, Sender);
/// <summary>A reprovision held back by <see cref="_reprovisionBackoff"/>, carrying the incarnation it
/// was requested by so the identity check still applies when it is re-delivered by the timer.</summary>
/// <param name="Request">The original request.</param>
/// <param name="Child">The incarnation that asked; re-validated against <c>_children</c> on delivery,
/// so a driver restarted or removed in the meantime discards it.</param>
private sealed record ReprovisionDeferred(
DriverInstanceActor.ReprovisionRequested Request, IActorRef Child);
private static string ReprovisionTimerKey(string driverInstanceId) => "reprovision-" + driverInstanceId;
private void ReprovisionChild(DriverInstanceActor.ReprovisionRequested msg, IActorRef child)
{
// Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
// Route by INCARNATION, not just by id. This message is delivered asynchronously, so one can still
// be in flight from an OLD incarnation after HandleRestartDriver stopped that child and spawned a
// replacement under the same DriverInstanceId. Delivered to the new child — which is in Connecting,
// where InitializeRequested IS handled — it would start a second, concurrent InitializeAsync,
// racing the dispose-before-overwrite that #527's teardown exists to serialise
// (ModbusDriver._transport, MqttDriver._connection each leak a transport on that race).
if (!Equals(child, entry.Actor))
{
_log.Debug(
"DriverHost {Node}: dropping a reprovision for {Id} from a superseded incarnation ({Stale})",
_localNode, msg.DriverInstanceId, child);
return;
}
// Rate-limit per driver (review F3). DEFERRED, never dropped: the last request must still be
// honoured, or a child whose final crash fell inside the window is left un-provisioned forever —
// exactly the 03/S16 failure this hook exists to close. One timer key per driver, so a burst
// collapses to a single pending reprovision rather than a queue of them.
var now = DateTime.UtcNow;
if (_lastReprovisionUtc.TryGetValue(msg.DriverInstanceId, out var last) &&
now - last < _reprovisionBackoff)
{
var delay = _reprovisionBackoff - (now - last);
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision again after {Reason} " +
"within {Backoff} — deferring the re-provision by {Delay} (crash loop?)",
_localNode, msg.DriverInstanceId, msg.Reason, _reprovisionBackoff, delay);
Timers.StartSingleTimer(
ReprovisionTimerKey(msg.DriverInstanceId), new ReprovisionDeferred(msg, child), delay);
return;
}
_lastReprovisionUtc[msg.DriverInstanceId] = now;
// A crash that Akka silently restarted is otherwise invisible to operators; name it.
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",
_localNode, msg.DriverInstanceId, msg.Reason);
// A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested
// anyway), but it still stores a desired set, so only the initialise is conditional.
if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig));
SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId);
}
private void HandleReconnectDriver(ReconnectDriver msg)
{
// DPS broadcast — only act if this node hosts the requested instance.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry))
return;
_log.Info("DriverHost {Node}: reconnecting driver {Id} by request of {User}",
_localNode, msg.DriverInstanceId, msg.ActorByUserName);
// Tell the child to drop its transport and re-enter the Reconnecting state.
entry.Actor.Tell(new DriverInstanceActor.ForceReconnect());
}
private void TryRecoverFromStale()
{
// Defensive: FetchAndCache never enters Stale (BootstrapFromCache always Becomes Steady, so
// the retry-db timer that drives this is never started), and its recovery must NOT read
// central SQL. A DB-less node (null factory, Phase 4) likewise has nothing to recover from. If
// we somehow land here, leave Steady rather than re-reading Deployments.
if (_fetchAndCacheMode || _dbFactory is null)
{
Timers.Cancel("retry-db");
BecomeSteady();
return;
}
try
{
// Synchronous, like the Applying/Applied row writes below: one bounded row read on a timer
// tick, not a multi-MB blob. Recorded follow-up — if the Stale-recovery tick is ever shown
// to stall a busy host, it moves off the mailbox the same way BeginDirectApply's read did.
using var db = _dbFactory.CreateDbContext();
var latestSealed = db.Deployments
.AsNoTracking()
.Where(d => d.Status == DeploymentStatus.Sealed)
.OrderByDescending(d => d.SealedAtUtc)
.Select(d => new { d.DeploymentId, d.RevisionHash })
.FirstOrDefault();
_log.Info("DriverHost {Node}: ConfigDb back; recovering from Stale", _localNode);
Timers.Cancel("retry-db");
if (latestSealed is not null)
{
_currentRevision = RevisionHash.Parse(latestSealed.RevisionHash);
UpsertNodeDeploymentState(new DeploymentId(latestSealed.DeploymentId),
NodeDeploymentStatus.Applied, failureReason: null);
}
BecomeSteady();
}
catch (Exception ex)
{
_log.Debug(ex, "DriverHost {Node}: still Stale; will retry in {Interval}", _localNode, ReconnectInterval);
}
}
private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason)
{
// Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of
// record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every
// ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is
// redundant and simply no-ops here with no loss of information. Logged once to keep the reason
// discoverable without spamming every apply transition.
if (_dbFactory is null)
{
if (!_loggedNoConfigDbAck)
{
_loggedNoConfigDbAck = true;
_log.Debug(
"DriverHost {Node}: no ConfigDb — NodeDeploymentState written by central from the ApplyAck",
_localNode);
}
return;
}
try
{
// Synchronous by design, and a recorded follow-up rather than an oversight: this is a
// bounded per-apply write of ONE row (at most three per deployment), not the multi-MB
// artifact read that Task 26 moved off the mailbox. It is also the apply's crash-boundary
// marker, which is easiest to reason about while it is ordered on the actor thread.
using var db = _dbFactory.CreateDbContext();
var existing = db.NodeDeploymentStates.FirstOrDefault(
x => x.NodeId == _localNode.Value && x.DeploymentId == deploymentId.Value);
if (existing is null)
{
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = _localNode.Value,
DeploymentId = deploymentId.Value,
Status = status,
FailureReason = failureReason,
AppliedAtUtc = status == NodeDeploymentStatus.Applied ? DateTime.UtcNow : null,
});
}
else
{
existing.Status = status;
existing.FailureReason = failureReason;
if (status == NodeDeploymentStatus.Applied) existing.AppliedAtUtc = DateTime.UtcNow;
}
db.SaveChanges();
}
catch (Exception ex)
{
_log.Warning(ex, "DriverHost {Node}: failed to upsert NodeDeploymentState for {Id}", _localNode, deploymentId);
}
}
private void SendAck(
DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation,
bool degraded = false, string? degradedReason = null)
{
var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation, degraded, degradedReason);
if (_ackRouter is not null)
{
// Either a direct coordinator handle (tests) or, under MeshTransport:Mode = Grpc, the
// NodeCommunicationActor that relays this across the mesh boundary (it owns the transport,
// not this actor). Renamed from _coordinatorOverride in per-cluster mesh Phase 2: under
// Grpc this ref is emphatically NOT a coordinator, and the old name would send the next
// reader looking for one.
_ackRouter.Tell(ack);
}
else
{
// No router — publish on the dedicated ACK topic. The coordinator singleton subscribes
// there in PreStart so the ACK reaches whichever admin node hosts it without an
// actor-path lookup.
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentAcksTopic, ack));
}
}
}
Reference in New Issue View Git Blame Copy Permalink