dohertj2/lmxopcua
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(opcua): hold last-known-good when a deploy's artifact cannot be read (#485)

Browse Source 
A transient ConfigDb error while loading a deployment's artifact emptied the
served address space. LoadArtifact caught the exception, logged "rebuild
becomes no-op" and returned zero bytes; those parsed to an EMPTY composition,
which the planner diffed against the live one as a PureRemove and the applier
faithfully executed — 16 nodes gone, while the log claimed nothing happened.

An artifact we could not obtain is not an empty configuration. Nothing
legitimate produces a zero-length blob (an operator who really deletes
everything still deploys a JSON document with empty arrays), so "no bytes" can
only mean "no answer". HandleRebuild now abandons the rebuild on one, leaving
both the materialised nodes and _lastApplied intact so the retry diffs against
what is actually being served. The loader's own log line is now true.

The two cases are logged differently: warn when there is a live address space
being protected, debug when nothing has been deployed to this node yet.

Covered by a RED-first actor test (verified failing: the load error tore down
eq-2's subtree), plus a positive control proving a READABLE artifact that
genuinely drops an equipment still removes it — the guard suppresses teardown
only when the read failed.

Found on the LocalDb Phase 1 follow-up live gate, which induced it by flapping
SQL; that gate's log is the production evidence for the seam.

Closes #485

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-21 02:29:39 -04:00
parent 5fdb5a5a5e
commit 4b0be1335e
3 changed files with 122 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs
+32 -4
View File
@@ -112,6 +112,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
private int _writes;
private byte _lastServiceLevel;
private bool _publishedAtLeastOnce;
/// <summary>True once a real composition has been applied, i.e. there is a served address space worth
/// protecting. Distinguishes "the artifact went missing under a running server" (issue #485 — warn, and
/// hold what is materialised) from "nothing has been deployed here yet" (a quiet no-op at boot).</summary>
private bool _hasAppliedComposition;
private DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
private RedundancyStateChanged? _lastSnapshot;
private (bool Ok, DateTime At)? _probeAboutMe;
@@ -364,6 +368,28 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
// carry neither.
var artifact = msg.Artifact
?? (msg.DeploymentId is { } depId ? LoadArtifact(depId) : LoadLatestArtifact());
// Issue #485 — an artifact we could not obtain is NOT an empty configuration. Parsing zero
// bytes yields an empty composition, which the planner diffs against the live one as a
// PureRemove and the applier then faithfully tears down: a transient ConfigDb blip empties the
// served address space. Nothing legitimate produces a zero-length blob (an operator who really
// deletes everything still deploys a JSON document with empty arrays), so the only honest
// reading of "no bytes" is "no answer" — hold the last-known-good address space and let the
// next deploy (or the next boot) supply a real one. Returning here also leaves _lastApplied
// intact, so the retry diffs against what is actually materialised.
if (artifact is null or { Length: 0 })
{
if (_hasAppliedComposition)
_log.Warning(
"OpcUaPublish: no usable artifact for deployment {Id} (correlation={Correlation}) — KEEPING the currently served address space rather than tearing it down",
msg.DeploymentId, msg.Correlation);
else
_log.Debug(
"OpcUaPublish: no artifact to materialise yet (deployment={Id}, correlation={Correlation})",
msg.DeploymentId, msg.Correlation);
return;
}
var composition = _localNode is { } ln
? DeploymentArtifact.ParseComposition(artifact, ln.Value,
inconsistency => _log.Warning("OpcUaPublish {Node}: cross-cluster binding — {Message}", ln, inconsistency))
@@ -379,6 +405,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
var outcome = _applier.Apply(plan);
_lastApplied = composition;
_hasAppliedComposition = true;
// Sum swallowed per-node materialise failures across every pass together with Apply's own
// removal-pass tally (archreview 01/S-1). A degraded apply used to vanish into per-node
@@ -443,7 +470,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
/// <summary>Read a specific deployment's artifact blob from ConfigDb (the one just applied,
/// which may not be Sealed yet). Empty array on any failure — parser treats it as "no composition".</summary>
/// which may not be Sealed yet). Empty array on any failure — <see cref="HandleRebuild"/> treats that as
/// "no answer" and abandons the rebuild, leaving the served address space untouched (issue #485).</summary>
private byte[] LoadArtifact(DeploymentId deploymentId)
{
try
@@ -456,13 +484,13 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
catch (Exception ex)
{
_log.Warning(ex, "OpcUaPublish: failed to load artifact for deployment {Id}; rebuild becomes no-op", deploymentId);
_log.Warning(ex, "OpcUaPublish: failed to load artifact for deployment {Id}; the rebuild is abandoned", deploymentId);
return Array.Empty<byte>();
}
}
/// <summary>Read the most recent <c>Sealed</c> deployment's artifact blob from ConfigDb.
/// Empty array on any failure — the parser treats empty blob as "no composition".</summary>
/// Empty array on any failure — see <see cref="LoadArtifact"/> for how that is handled.</summary>
private byte[] LoadLatestArtifact()
{
try
@@ -476,7 +504,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
catch (Exception ex)
{
_log.Warning(ex, "OpcUaPublish: failed to load latest deployment artifact; rebuild becomes no-op");
_log.Warning(ex, "OpcUaPublish: failed to load latest deployment artifact; the rebuild is abandoned");
return Array.Empty<byte>();
}
}