fix(localdb): materialise the address space from the cached blob on boot-from-cache

Boot-from-cache told OpcUaPublishActor to RebuildAddressSpace(deploymentId), which
re-loads the artifact from the ConfigDb by id — the very database that is
unreachable during the outage this path exists to survive. The rebuild no-op'd, so
a cache-booted node ran its drivers but served OPC UA clients an EMPTY address
space. RebuildAddressSpace now carries an optional in-hand Artifact blob;
ApplyCachedArtifact passes the cached bytes so materialisation happens from them
directly. Found on the docker-dev live gate: healthy peer served 16 nodes, the
cache-booted node served 1.
This commit is contained in:
Joseph Doherty
2026-07-20 22:38:54 -04:00
parent ce9fa07ff8
commit 9137cb41eb
3 changed files with 50 additions and 9 deletions
@@ -705,8 +705,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
foreach (var spec in plan.ToApplyDelta) ApplyChildDelta(spec);
foreach (var spec in plan.ToSpawn) SpawnChild(spec);
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
// definition — and the rebuild would no-op, leaving OPC UA clients browsing an empty server
// while the drivers below poll happily. (Found on the docker-dev live gate.)
_opcUaPublishActor?.Tell(
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, blob));
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
}
@@ -73,7 +73,17 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// applied config + the SubscribeBulk pass. It is null only for legacy/dev callers, which
/// fall back to the latest sealed deployment (lags a not-yet-sealed apply by one revision).
/// </summary>
public sealed record RebuildAddressSpace(CorrelationId Correlation, DeploymentId? DeploymentId = null);
/// <param name="Correlation">Correlation id for tracing this rebuild.</param>
/// <param name="DeploymentId">The applied deployment whose artifact to materialise, or null for the latest-sealed fallback.</param>
/// <param name="Artifact">
/// The artifact bytes already in hand, used INSTEAD of loading them from the ConfigDb. This is
/// what makes boot-from-cache actually serve its address space: on a central-SQL outage the
/// host has the cached blob but <see cref="DeploymentId"/> alone would drive a ConfigDb read
/// that cannot succeed, leaving the rebuild a no-op and clients browsing an empty server. Null
/// on the normal path, where loading from the ConfigDb by id is correct.
/// </param>
public sealed record RebuildAddressSpace(
CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null);
/// <summary>Inject driver-discovered nodes (FixedTree) under an equipment at runtime (post-connect).</summary>
/// <param name="EquipmentRootNodeId">The OPC UA NodeId of the equipment root folder to inject the
@@ -345,13 +355,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
try
{
// Prefer the artifact of the deployment the host just applied — at apply time it is not
// yet Sealed, so LoadLatestArtifact would return the PREVIOUS revision and materialise a
// stale composition (variables that don't match the SubscribeBulk refs). Fall back to
// latest-sealed only for legacy callers that don't carry a DeploymentId.
var artifact = msg.DeploymentId is { } depId
? LoadArtifact(depId)
: LoadLatestArtifact();
// An in-hand artifact (boot-from-cache) wins: the host already has the cached bytes, and
// the ConfigDb read the DeploymentId path would do is exactly what is unreachable during
// the outage this exists to survive. Otherwise prefer the artifact of the deployment the
// host just applied — at apply time it is not yet Sealed, so LoadLatestArtifact would
// return the PREVIOUS revision and materialise a stale composition (variables that don't
// match the SubscribeBulk refs). Fall back to latest-sealed only for legacy callers that
// carry neither.
var artifact = msg.Artifact
?? (msg.DeploymentId is { } depId ? LoadArtifact(depId) : LoadLatestArtifact());
var composition = _localNode is { } ln
? DeploymentArtifact.ParseComposition(artifact, ln.Value,
inconsistency => _log.Warning("OpcUaPublish {Node}: cross-cluster binding — {Message}", ln, inconsistency))
@@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
@@ -55,6 +56,30 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
cache.UnkeyedReads.ShouldBe(1);
}
[Fact]
public void BootFromCache_HandsTheCachedArtifactToTheAddressSpaceRebuild()
{
// The client-facing address space rebuild must materialise from the CACHED bytes, not from a
// ConfigDb read keyed by deployment id — that read is exactly what is unreachable during the
// outage boot-from-cache exists to survive. Without the blob, the node logs "running from
// cache", spins up its drivers, and still serves OPC UA clients an empty server. (Regression
// for the defect the docker-dev live gate caught: 16 nodes on the healthy peer, 1 here.)
var blob = EmptyArtifact();
var cached = new CachedDeploymentArtifact(
DeploymentId.NewId().ToString(), CachedRev.Value, blob, DateTimeOffset.UtcNow.AddHours(-1));
var publishProbe = CreateTestProbe();
Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, coordinator: null,
localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publishProbe.Ref,
deploymentArtifactCache: new StubArtifactCache(cached)));
var rebuild = publishProbe.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5));
rebuild.Artifact.ShouldNotBeNull();
rebuild.Artifact.ShouldBe(blob);
}
[Fact]
public void CentralUnreachableAndCacheEmpty_StaysStaleWithNoConfiguration()
{