feat(localdb): DriverHostActor stores applied artifacts in the pair-local cache

New Props/ctor parameter appended LAST at all three positions - the forwarding
list inside Props.Create is positional and compiles into an expression tree, so a
mis-ordered argument is usually type-compatible and binds the wrong dependency at
runtime. All 26 existing test construction sites use named arguments and needed no
change.

ReconcileDrivers now returns the blob it loaded. It catches its own DB failures and
returns without rethrowing, so ApplyAndAck can reach its success path having spawned
zero drivers; caching on 'the apply succeeded' would persist an empty artifact as
last-known-good and boot the node into an empty address space during the next
outage. Verified: weakening the guard to null-only turns EmptyArtifact_IsNotCached
red.

The store runs in its own try/catch because an Applied ACK has already been sent by
then - an escaping exception would land in ApplyAndAck's catch and emit a second,
contradictory Failed ACK for a deployment the fleet believes is live.
This commit is contained in:
Joseph Doherty
2026-07-20 11:06:57 -04:00
parent a38a52b831
commit 1becf59168
4 changed files with 402 additions and 7 deletions
@@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
@@ -90,6 +91,11 @@ public static class LocalDbRegistration
services.AddZbLocalDb(configuration, LocalDbSetup.OnReady);
services.AddZbLocalDbReplication(configuration);
// The consumer-facing seam. WithOtOpcUaRuntimeActors resolves this optionally and threads it
// into DriverHostActor; registering the store without this leaves the cache tables present,
// replicating, and never written to.
services.AddSingleton<IDeploymentArtifactCache, LocalDbDeploymentArtifactCache>();
return services;
}
}
@@ -23,6 +23,7 @@ using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
@@ -57,6 +58,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <summary>Publishing interval handed to each driver's SubscribeBulk pass after an apply.</summary>
private static readonly TimeSpan SubscriptionPublishingInterval = TimeSpan.FromSeconds(1);
/// <summary>
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
/// someone reads the table during an incident.
/// </summary>
private const string SingleClusterCacheKey = "__single";
/// <summary>
/// Node-local cache of applied deployment artifacts, or null when this node has none
/// (admin-only graphs, and tests that do not exercise it).
/// </summary>
private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly CommonsNodeId _localNode;
private readonly IActorRef? _coordinatorOverride;
@@ -339,11 +353,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null) =>
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
// the wrong dependency at runtime. New parameters go LAST in all three places — this
// signature, the constructor's, and this call — and nothing else moves.
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider));
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -372,6 +393,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <param name="driverMemberCountProvider">Test seam (archreview 03/S4): overrides the count of Up
/// <c>driver</c>-role cluster members the Primary gate reads while the role is unknown. When null the
/// default reads <c>Cluster.Get(Context.System).State.Members</c> (0 on a non-cluster ActorRefProvider).</param>
/// <param name="deploymentArtifactCache">Optional node-local cache of applied deployment artifacts.
/// When supplied, each successful apply stores its artifact so the node can boot from its
/// last-known-good configuration while central SQL is unreachable. Null on admin-only nodes and in
/// tests that do not exercise the cache — caching is then simply skipped.</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
CommonsNodeId localNode,
@@ -388,8 +413,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null)
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null)
{
_deploymentArtifactCache = deploymentArtifactCache;
_dbFactory = dbFactory;
_localNode = localNode;
_coordinatorOverride = coordinator;
@@ -1426,7 +1453,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
try
{
ReconcileDrivers(deploymentId);
var appliedBlob = ReconcileDrivers(deploymentId);
_currentRevision = revision;
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null);
SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
@@ -1437,6 +1464,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// SubscribeBulk pass: hand each driver its desired tag references so live values flow into
// the just-rebuilt address space instead of staying BadWaitingForInitialData.
PushDesiredSubscriptions(deploymentId);
CacheAppliedArtifact(deploymentId, revision, appliedBlob);
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "ack"));
_log.Info("DriverHost {Node}: applied deployment {Id} (rev {Rev}, children={Count})",
_localNode, deploymentId, revision, _children.Count);
@@ -1464,7 +1492,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// configured <see cref="IDriverFactory"/> can't materialise any of the requested
/// types, this is effectively a no-op.
/// </summary>
private void ReconcileDrivers(DeploymentId deploymentId)
/// <returns>
/// The artifact blob that was reconciled, or <see langword="null"/> when it could not be
/// loaded at all.
/// </returns>
/// <remarks>
/// Returning the blob rather than swallowing it is load-bearing for the artifact cache. This
/// method catches its own DB failures, logs a warning and returns WITHOUT rethrowing — so
/// <see cref="ApplyAndAck"/> proceeds to its success path, ACKs Applied and logs success
/// having spawned zero drivers. A cache write that trusted "the apply succeeded" would then
/// persist an empty artifact as this node's last-known-good configuration, and the node would
/// later boot from it into an empty address space. The caller distinguishes the two cases by
/// this return value.
/// </remarks>
private byte[]? ReconcileDrivers(DeploymentId deploymentId)
{
byte[] blob;
try
@@ -1479,7 +1520,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
{
_log.Warning(ex, "DriverHost {Node}: failed to load artifact for {Id}; skipping reconcile",
_localNode, deploymentId);
return;
return null;
}
var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
@@ -1500,6 +1541,74 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// sequenced AFTER ReinitializeAsync rebuilds DependencyRefs — a synchronous re-register HERE would
// re-read the STALE ref set (ApplyDelta runs asynchronously on the child), so it is deliberately NOT
// done inline.
return blob;
}
/// <summary>
/// Store a successfully applied artifact in the node-local cache, so this node can boot from
/// it while central SQL is unreachable.
/// </summary>
/// <remarks>
/// <para>
/// <b>Never throws.</b> By the time this runs the deployment is already recorded Applied
/// in central SQL and an Applied ACK has been sent to the coordinator. An exception
/// escaping here would unwind into <see cref="ApplyAndAck"/>'s catch and send a second,
/// contradictory Failed ACK for a deployment the fleet already believes is live.
/// </para>
/// <para>
/// <b>An empty or unloadable blob is skipped, not cached.</b> See
/// <see cref="ReconcileDrivers"/>: a DB failure there degrades silently, so "the apply
/// succeeded" does not imply "a real configuration was applied". Caching an empty
/// artifact would make it this node's last-known-good and boot it into an empty address
/// space during the next outage — strictly worse than having no cache at all.
/// </para>
/// <para>
/// Runs synchronously on the actor thread. That matches every other DB call on this path
/// (all of which already block), and the alternative — piping the result back as a
/// self-message — would need a handler registered in Steady, Applying AND Stale or it
/// dead-letters after the <c>Become(Steady)</c> in the enclosing finally.
/// </para>
/// </remarks>
private void CacheAppliedArtifact(DeploymentId deploymentId, RevisionHash revision, byte[]? blob)
{
if (_deploymentArtifactCache is null)
return;
if (blob is null || blob.Length == 0)
{
_log.Warning(
"DriverHost {Node}: not caching deployment {Id} — its artifact was empty or could not " +
"be loaded, so it is not a usable last-known-good configuration.",
_localNode, deploymentId);
return;
}
try
{
// The node's ClusterId is carried inside the artifact (ClusterNode rows), not in config
// or on IClusterRoleInfo. A single-cluster or unscoped artifact has none, so it shares
// one sentinel key — the pair still converges on a single pointer row either way.
var scope = DeploymentArtifact.ResolveClusterScope(blob, _localNode.Value);
var clusterId = scope.Mode == ClusterFilterMode.ScopeTo && !string.IsNullOrWhiteSpace(scope.ClusterId)
? scope.ClusterId
: SingleClusterCacheKey;
_deploymentArtifactCache
.StoreAsync(clusterId, deploymentId.ToString(), revision.Value, blob)
.GetAwaiter()
.GetResult();
_log.Debug("DriverHost {Node}: cached deployment {Id} (rev {Rev}, cluster {ClusterId}, {Bytes} bytes)",
_localNode, deploymentId, revision, clusterId, blob.Length);
}
catch (Exception ex)
{
_log.Error(ex,
"DriverHost {Node}: failed to cache applied deployment {Id}. The apply itself succeeded " +
"and is unaffected; this node simply has no local fallback for that revision.",
_localNode, deploymentId);
}
}
/// <summary>
@@ -15,6 +15,7 @@ using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
@@ -221,6 +222,11 @@ public static class ServiceCollectionExtensions
var serviceLevel = resolver.GetService<IServiceLevelPublisher>() ?? NullServiceLevelPublisher.Instance;
var loggerFactory = resolver.GetService<ILoggerFactory>() ?? NullLoggerFactory.Instance;
var healthPublisher = resolver.GetService<IDriverHealthPublisher>() ?? NullDriverHealthPublisher.Instance;
// Node-local deployment-artifact cache. Registered by the Host's AddOtOpcUaLocalDb on
// driver-role nodes only; deliberately left null elsewhere (admin-only graphs, test
// harnesses) rather than given a null-object, so DriverHostActor skips caching outright
// instead of pretending to cache into a sink that drops everything.
var deploymentArtifactCache = resolver.GetService<IDeploymentArtifactCache>();
// Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in
// Host DI inside the hasDriver block; may be absent in some role configs / test harnesses,
// in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host.
@@ -338,7 +344,8 @@ public static class ServiceCollectionExtensions
historyWriter: historyWriter,
loggerFactory: loggerFactory,
scriptRootLogger: scriptRootLogger,
invokerFactory: invokerFactory),
invokerFactory: invokerFactory,
deploymentArtifactCache: deploymentArtifactCache),
DriverHostActorName);
registry.Register<DriverHostActorKey>(driverHost);