fix(localdb): re-cache the artifact on RestoreApplied so a wiped node self-heals

CacheAppliedArtifact ran only on a fresh apply (ApplyAndAck), never on the
bootstrap RestoreApplied path. So a node whose cache was lost — fresh/wiped
volume, disk failure — recovered its served state from central yet stayed
cache-less, unable to boot-from-cache on the NEXT central outage until some
future new deploy happened to land. Replication does not heal this: a peer's
already-acked cache rows are pruned from its oplog, so a fully-wiped, converged
node is never back-filled. RestoreApplied now writes the served artifact back to
the cache (invariant: the cache holds what the node serves). Found on the
docker-dev live gate wiping a node's LocalDb volume.
This commit is contained in:
Joseph Doherty
2026-07-20 22:56:23 -04:00
parent 9137cb41eb
commit c6a9f93a0c
2 changed files with 58 additions and 5 deletions
@@ -580,7 +580,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// space were lost on restart. Re-spawn + re-materialise + re-subscribe from the // space were lost on restart. Re-spawn + re-materialise + re-subscribe from the
// applied deployment so a restarted/rebuilt node restores its served state instead // applied deployment so a restarted/rebuilt node restores its served state instead
// of waiting for a config change (whose identical-config revision would no-op). // of waiting for a config change (whose identical-config revision would no-op).
RestoreApplied(new DeploymentId(latest.DeploymentId)); RestoreApplied(new DeploymentId(latest.DeploymentId), revision);
break; break;
case NodeDeploymentStatus.Applying: case NodeDeploymentStatus.Applying:
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying", _log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying",
@@ -1750,14 +1750,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk. /// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk.
/// No re-ack: the deployment is already Applied. /// No re-ack: the deployment is already Applied.
/// </summary> /// </summary>
private void RestoreApplied(DeploymentId deploymentId) private void RestoreApplied(DeploymentId deploymentId, RevisionHash? revision)
{ {
var correlation = CorrelationId.NewId(); var correlation = CorrelationId.NewId();
try try
{ {
ReconcileDrivers(deploymentId); var appliedBlob = ReconcileDrivers(deploymentId);
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId)); _opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, appliedBlob));
PushDesiredSubscriptions(deploymentId); PushDesiredSubscriptions(deploymentId);
// Populate the node-local cache from the artifact this restored node is now serving. The
// cache invariant is "holds what the node currently serves"; without this only a FRESH
// apply (ApplyAndAck) ever wrote it, so a node whose cache was lost — a wiped/fresh volume,
// a disk failure — would recover its served state here yet stay cache-less, unable to
// boot-from-cache on the NEXT central outage until some future new deploy happened to land.
// Replication does not heal that gap either: a peer's already-acked rows are pruned from its
// oplog, so a fully-wiped node is never back-filled. Re-caching on restore is what closes
// it. (Found on the docker-dev live gate.)
if (revision is { } rev)
CacheAppliedArtifact(deploymentId, rev, appliedBlob);
_log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId); _log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId);
} }
catch (Exception ex) catch (Exception ex)
@@ -166,6 +166,38 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
snapshot.CurrentRevision.ShouldBeNull(); snapshot.CurrentRevision.ShouldBeNull();
} }
[Fact]
public void RestoringAnAppliedDeploymentOnBootstrap_RepopulatesTheCache()
{
// The cache invariant is "holds what the node serves". A node recovering an already-applied
// deployment (the RestoreApplied path) must (re)write its cache — otherwise a node whose cache
// was lost (fresh volume, disk failure) recovers its served state from central yet stays
// cache-less, unable to boot-from-cache on the NEXT outage. Replication does not heal it: a
// peer's already-acked rows are pruned from its oplog, so a wiped node is never back-filled.
// (Regression for the docker-dev live-gate finding.)
var db = NewInMemoryDbFactory();
var rev = RevisionHash.Parse(new string('e', 64));
var deploymentId = SeedAppliedDeployment(db, rev);
// A cache that starts empty — exactly the wiped-node state.
var cache = new StubArtifactCache(current: null);
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator: null,
localRoles: new HashSet<string> { "driver" },
deploymentArtifactCache: cache));
var snapshot = AskDiagnostics(actor);
snapshot.CurrentRevision.ShouldBe(rev); // recovered the applied deployment
snapshot.RunningFromCache.ShouldBeFalse(); // via central, not the cache
// The point: the restore path wrote the served artifact back into the cache.
AwaitAssert(
() => cache.Stores.ShouldBe(1),
duration: TimeSpan.FromSeconds(5));
cache.LastStoredDeploymentId.ShouldBe(deploymentId.ToString());
}
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor) private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
{ {
var probe = CreateTestProbe(); var probe = CreateTestProbe();
@@ -222,13 +254,24 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
{ {
private int _unkeyedReads; private int _unkeyedReads;
private int _stores;
/// <summary>How many times the boot path consulted this cache.</summary> /// <summary>How many times the boot path consulted this cache.</summary>
public int UnkeyedReads => Volatile.Read(ref _unkeyedReads); public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);
/// <summary>How many times the node wrote an applied artifact to this cache.</summary>
public int Stores => Volatile.Read(ref _stores);
/// <summary>The deployment id of the most recent store.</summary>
public string? LastStoredDeploymentId { get; private set; }
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash, public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
byte[] artifact, CancellationToken ct = default) byte[] artifact, CancellationToken ct = default)
=> Task.CompletedTask; {
Interlocked.Increment(ref _stores);
LastStoredDeploymentId = deploymentId;
return Task.CompletedTask;
}
public Task<CachedDeploymentArtifact?> GetCurrentAsync( public Task<CachedDeploymentArtifact?> GetCurrentAsync(
string clusterId, CancellationToken ct = default) string clusterId, CancellationToken ct = default)