fix(localdb): re-cache the artifact on RestoreApplied so a wiped node self-heals

CacheAppliedArtifact ran only on a fresh apply (ApplyAndAck), never on the
bootstrap RestoreApplied path. So a node whose cache was lost — fresh/wiped
volume, disk failure — recovered its served state from central yet stayed
cache-less, unable to boot-from-cache on the NEXT central outage until some
future new deploy happened to land. Replication does not heal this: a peer's
already-acked cache rows are pruned from its oplog, so a fully-wiped, converged
node is never back-filled. RestoreApplied now writes the served artifact back to
the cache (invariant: the cache holds what the node serves). Found on the
docker-dev live gate wiping a node's LocalDb volume.
This commit is contained in:
Joseph Doherty
2026-07-20 22:56:23 -04:00
parent 9137cb41eb
commit c6a9f93a0c
2 changed files with 58 additions and 5 deletions
@@ -580,7 +580,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// space were lost on restart. Re-spawn + re-materialise + re-subscribe from the
// applied deployment so a restarted/rebuilt node restores its served state instead
// of waiting for a config change (whose identical-config revision would no-op).
RestoreApplied(new DeploymentId(latest.DeploymentId));
RestoreApplied(new DeploymentId(latest.DeploymentId), revision);
break;
case NodeDeploymentStatus.Applying:
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying",
@@ -1750,14 +1750,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk.
/// No re-ack: the deployment is already Applied.
/// </summary>
private void RestoreApplied(DeploymentId deploymentId)
private void RestoreApplied(DeploymentId deploymentId, RevisionHash? revision)
{
var correlation = CorrelationId.NewId();
try
{
ReconcileDrivers(deploymentId);
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
var appliedBlob = ReconcileDrivers(deploymentId);
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, appliedBlob));
PushDesiredSubscriptions(deploymentId);
// Populate the node-local cache from the artifact this restored node is now serving. The
// cache invariant is "holds what the node currently serves"; without this only a FRESH
// apply (ApplyAndAck) ever wrote it, so a node whose cache was lost — a wiped/fresh volume,
// a disk failure — would recover its served state here yet stay cache-less, unable to
// boot-from-cache on the NEXT central outage until some future new deploy happened to land.
// Replication does not heal that gap either: a peer's already-acked rows are pruned from its
// oplog, so a fully-wiped node is never back-filled. Re-caching on restore is what closes
// it. (Found on the docker-dev live gate.)
if (revision is { } rev)
CacheAppliedArtifact(deploymentId, rev, appliedBlob);
_log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId);
}
catch (Exception ex)
@@ -166,6 +166,38 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
snapshot.CurrentRevision.ShouldBeNull();
}
[Fact]
public void RestoringAnAppliedDeploymentOnBootstrap_RepopulatesTheCache()
{
// The cache invariant is "holds what the node serves". A node recovering an already-applied
// deployment (the RestoreApplied path) must (re)write its cache — otherwise a node whose cache
// was lost (fresh volume, disk failure) recovers its served state from central yet stays
// cache-less, unable to boot-from-cache on the NEXT outage. Replication does not heal it: a
// peer's already-acked rows are pruned from its oplog, so a wiped node is never back-filled.
// (Regression for the docker-dev live-gate finding.)
var db = NewInMemoryDbFactory();
var rev = RevisionHash.Parse(new string('e', 64));
var deploymentId = SeedAppliedDeployment(db, rev);
// A cache that starts empty — exactly the wiped-node state.
var cache = new StubArtifactCache(current: null);
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator: null,
localRoles: new HashSet<string> { "driver" },
deploymentArtifactCache: cache));
var snapshot = AskDiagnostics(actor);
snapshot.CurrentRevision.ShouldBe(rev); // recovered the applied deployment
snapshot.RunningFromCache.ShouldBeFalse(); // via central, not the cache
// The point: the restore path wrote the served artifact back into the cache.
AwaitAssert(
() => cache.Stores.ShouldBe(1),
duration: TimeSpan.FromSeconds(5));
cache.LastStoredDeploymentId.ShouldBe(deploymentId.ToString());
}
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
{
var probe = CreateTestProbe();
@@ -222,13 +254,24 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
{
private int _unkeyedReads;
private int _stores;
/// <summary>How many times the boot path consulted this cache.</summary>
public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);
/// <summary>How many times the node wrote an applied artifact to this cache.</summary>
public int Stores => Volatile.Read(ref _stores);
/// <summary>The deployment id of the most recent store.</summary>
public string? LastStoredDeploymentId { get; private set; }
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
byte[] artifact, CancellationToken ct = default)
=> Task.CompletedTask;
{
Interlocked.Increment(ref _stores);
LastStoredDeploymentId = deploymentId;
return Task.CompletedTask;
}
public Task<CachedDeploymentArtifact?> GetCurrentAsync(
string clusterId, CancellationToken ct = default)