feat(localdb): boot from the pair-local artifact cache when central SQL is unreachable
Hooks the Bootstrap catch that previously went straight to Stale. On a cache hit the node serves its last-known-good configuration through the outage instead of coming up with an empty address space; on a miss, behaviour is byte-for-byte what it was. The read is unkeyed because ClusterId is only derivable from an artifact you already hold or from the unreachable central DB (recon D-1). Newest pointer wins, which is correct in every real topology - a node belongs to one cluster and its peer replicates that same row. Splits PushDesiredSubscriptionsFromArtifact out of PushDesiredSubscriptions: the cache path runs precisely when the ConfigDb read cannot succeed, so re-reading the artifact there would fail by definition. RunningFromCache is surfaced on NodeDiagnosticsSnapshot (optional param, existing call sites unaffected). A node running from cache looks entirely healthy from outside - full address space, live values - but its config is frozen and no deploy can reach it. It clears only on a real apply from central.
This commit is contained in:
@@ -71,6 +71,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
|
||||
|
||||
/// <summary>
|
||||
/// True once this node has booted a cached artifact because central SQL was unreachable.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Surfaced on <see cref="NodeDiagnosticsSnapshot"/> because a node running from cache looks
|
||||
/// completely healthy from the outside — it serves a full address space with live values.
|
||||
/// The difference is that its configuration is frozen at whatever the cache held, and no
|
||||
/// deployment can reach it. Without an explicit signal that is invisible until someone
|
||||
/// wonders why a deploy "succeeded" everywhere but did not take effect here.
|
||||
/// </remarks>
|
||||
private bool _isRunningFromCache;
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly CommonsNodeId _localNode;
|
||||
private readonly IActorRef? _coordinatorOverride;
|
||||
@@ -587,10 +599,118 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "DriverHost {Node}: ConfigDb unreachable on bootstrap; entering Stale", _localNode);
|
||||
Become(Stale);
|
||||
|
||||
// Central is unreachable, so try the node-local cache before giving up. On a hit this
|
||||
// node serves its last-known-good configuration through the outage instead of coming up
|
||||
// with an empty address space. On a miss, behaviour is exactly what it always was.
|
||||
if (!TryBootFromCache())
|
||||
Become(Stale);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Last-resort boot path: apply the artifact cached by a previous successful deploy (or
|
||||
/// replicated from this node's pair peer) when central SQL cannot be reached.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// <see langword="true"/> when a cached artifact was applied and the actor is now Steady;
|
||||
/// <see langword="false"/> when the caller should fall through to <c>Stale</c>.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The read is unkeyed.</b> The cache is keyed by ClusterId so a pair shares one
|
||||
/// entry, but ClusterId is only derivable from an artifact you already hold or from the
|
||||
/// central DB — and at this seam we have neither. So the newest pointer row wins, which
|
||||
/// is correct in every real topology because a node belongs to one cluster and its peer
|
||||
/// replicates that same cluster's row. A node re-homed between clusters is the only
|
||||
/// ambiguous case, and the cache logs a warning naming both.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>This does not make the node current.</b> <c>_currentRevision</c> is set from the
|
||||
/// cached artifact, so a subsequent dispatch of that same revision correctly no-ops,
|
||||
/// while any NEW revision still requires central — the cache holds the past, not the
|
||||
/// future. Dispatch handling is unchanged.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Never throws: a fault here must degrade to Stale, which is exactly where the node
|
||||
/// would have been without a cache at all.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private bool TryBootFromCache()
|
||||
{
|
||||
if (_deploymentArtifactCache is null)
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var cached = _deploymentArtifactCache.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
|
||||
if (cached is null)
|
||||
{
|
||||
_log.Info(
|
||||
"DriverHost {Node}: no cached deployment available; entering Stale with no configuration.",
|
||||
_localNode);
|
||||
return false;
|
||||
}
|
||||
|
||||
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
|
||||
var revision = RevisionHash.Parse(cached.RevisionHash);
|
||||
|
||||
// Steady, not Stale: this node is serving a real configuration. The retry-db timer that
|
||||
// Stale would have started does not run here, so recovery rides on the next dispatch —
|
||||
// matching how a normally-booted node behaves.
|
||||
_currentRevision = revision;
|
||||
Become(Steady);
|
||||
|
||||
ApplyCachedArtifact(deploymentId, cached.Artifact);
|
||||
|
||||
_isRunningFromCache = true;
|
||||
_log.Warning(
|
||||
"DriverHost {Node}: RUNNING FROM CACHE — central ConfigDb is unreachable, so this node " +
|
||||
"booted deployment {Id} (rev {Rev}) cached at {CachedAtUtc:o} from its local database. " +
|
||||
"Configuration changes cannot be applied until the ConfigDb is reachable again.",
|
||||
_localNode, deploymentId, revision, cached.AppliedAtUtc);
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Error(ex,
|
||||
"DriverHost {Node}: failed to boot from the local deployment cache; falling back to Stale.",
|
||||
_localNode);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies an artifact already in hand — no ConfigDb read, no ACK.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Mirrors <see cref="RestoreApplied"/>, but sources the artifact from the cache rather than
|
||||
/// re-reading it from a database that is by definition unreachable here. It also skips
|
||||
/// <c>UpsertNodeDeploymentState</c> and <c>SendAck</c> for the same reason: both write to or
|
||||
/// depend on central.
|
||||
/// </remarks>
|
||||
private void ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
|
||||
{
|
||||
var correlation = CorrelationId.NewId();
|
||||
|
||||
var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
|
||||
var snapshots = _children.ToDictionary(
|
||||
kv => kv.Key,
|
||||
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
|
||||
StringComparer.Ordinal);
|
||||
var plan = DriverSpawnPlanner.Compute(snapshots, specs);
|
||||
|
||||
foreach (var id in plan.ToStop) StopChild(id);
|
||||
foreach (var spec in plan.ToApplyDelta) ApplyChildDelta(spec);
|
||||
foreach (var spec in plan.ToSpawn) SpawnChild(spec);
|
||||
|
||||
_opcUaPublishActor?.Tell(
|
||||
new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
|
||||
|
||||
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
|
||||
}
|
||||
|
||||
private void Steady()
|
||||
{
|
||||
Receive<DispatchDeployment>(HandleDispatchFromSteady);
|
||||
@@ -1420,7 +1540,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
NodeId: _localNode,
|
||||
CurrentRevision: _currentRevision,
|
||||
Drivers: drivers,
|
||||
AsOfUtc: DateTime.UtcNow);
|
||||
AsOfUtc: DateTime.UtcNow,
|
||||
RunningFromCache: _isRunningFromCache);
|
||||
Sender.Tell(snapshot);
|
||||
}
|
||||
|
||||
@@ -1465,6 +1586,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
// the just-rebuilt address space instead of staying BadWaitingForInitialData.
|
||||
PushDesiredSubscriptions(deploymentId);
|
||||
CacheAppliedArtifact(deploymentId, revision, appliedBlob);
|
||||
// Reaching here means the artifact came from central, so the node is no longer serving a
|
||||
// cache-sourced configuration. Note this only clears on a real apply: a dispatch of the
|
||||
// revision already booted from cache short-circuits in HandleDispatchFromSteady without
|
||||
// touching the ConfigDb, and the flag correctly stays set.
|
||||
_isRunningFromCache = false;
|
||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "ack"));
|
||||
_log.Info("DriverHost {Node}: applied deployment {Id} (rev {Rev}, children={Count})",
|
||||
_localNode, deploymentId, revision, _children.Count);
|
||||
@@ -1661,6 +1787,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The artifact-processing half of <see cref="PushDesiredSubscriptions"/>, split out so the
|
||||
/// boot-from-cache path can drive it with bytes already in hand.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Separated because the cache path runs precisely when the ConfigDb read above cannot
|
||||
/// succeed — re-reading the artifact there would fail by definition.
|
||||
/// </remarks>
|
||||
private void PushDesiredSubscriptionsFromArtifact(DeploymentId deploymentId, byte[] blob)
|
||||
{
|
||||
AddressSpaceComposition composition;
|
||||
try
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user