fix(localdb): re-cache the artifact on RestoreApplied so a wiped node self-heals
CacheAppliedArtifact ran only on a fresh apply (ApplyAndAck), never on the bootstrap RestoreApplied path. So a node whose cache was lost — fresh/wiped volume, disk failure — recovered its served state from central yet stayed cache-less, unable to boot-from-cache on the NEXT central outage until some future new deploy happened to land. Replication does not heal this: a peer's already-acked cache rows are pruned from its oplog, so a fully-wiped, converged node is never back-filled. RestoreApplied now writes the served artifact back to the cache (invariant: the cache holds what the node serves). Found on the docker-dev live gate wiping a node's LocalDb volume.
This commit is contained in:
@@ -580,7 +580,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
// space were lost on restart. Re-spawn + re-materialise + re-subscribe from the
|
||||
// applied deployment so a restarted/rebuilt node restores its served state instead
|
||||
// of waiting for a config change (whose identical-config revision would no-op).
|
||||
RestoreApplied(new DeploymentId(latest.DeploymentId));
|
||||
RestoreApplied(new DeploymentId(latest.DeploymentId), revision);
|
||||
break;
|
||||
case NodeDeploymentStatus.Applying:
|
||||
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying",
|
||||
@@ -1750,14 +1750,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
/// drivers, rebuilds the address space from the applied artifact, and re-pushes SubscribeBulk.
|
||||
/// No re-ack: the deployment is already Applied.
|
||||
/// </summary>
|
||||
private void RestoreApplied(DeploymentId deploymentId)
|
||||
private void RestoreApplied(DeploymentId deploymentId, RevisionHash? revision)
|
||||
{
|
||||
var correlation = CorrelationId.NewId();
|
||||
try
|
||||
{
|
||||
ReconcileDrivers(deploymentId);
|
||||
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
|
||||
var appliedBlob = ReconcileDrivers(deploymentId);
|
||||
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId, appliedBlob));
|
||||
PushDesiredSubscriptions(deploymentId);
|
||||
// Populate the node-local cache from the artifact this restored node is now serving. The
|
||||
// cache invariant is "holds what the node currently serves"; without this only a FRESH
|
||||
// apply (ApplyAndAck) ever wrote it, so a node whose cache was lost — a wiped/fresh volume,
|
||||
// a disk failure — would recover its served state here yet stay cache-less, unable to
|
||||
// boot-from-cache on the NEXT central outage until some future new deploy happened to land.
|
||||
// Replication does not heal that gap either: a peer's already-acked rows are pruned from its
|
||||
// oplog, so a fully-wiped node is never back-filled. Re-caching on restore is what closes
|
||||
// it. (Found on the docker-dev live gate.)
|
||||
if (revision is { } rev)
|
||||
CacheAppliedArtifact(deploymentId, rev, appliedBlob);
|
||||
_log.Info("DriverHost {Node}: restored served state for applied deployment {Id} on bootstrap", _localNode, deploymentId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
+44
-1
@@ -166,6 +166,38 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
snapshot.CurrentRevision.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RestoringAnAppliedDeploymentOnBootstrap_RepopulatesTheCache()
|
||||
{
|
||||
// The cache invariant is "holds what the node serves". A node recovering an already-applied
|
||||
// deployment (the RestoreApplied path) must (re)write its cache — otherwise a node whose cache
|
||||
// was lost (fresh volume, disk failure) recovers its served state from central yet stays
|
||||
// cache-less, unable to boot-from-cache on the NEXT outage. Replication does not heal it: a
|
||||
// peer's already-acked rows are pruned from its oplog, so a wiped node is never back-filled.
|
||||
// (Regression for the docker-dev live-gate finding.)
|
||||
var db = NewInMemoryDbFactory();
|
||||
var rev = RevisionHash.Parse(new string('e', 64));
|
||||
var deploymentId = SeedAppliedDeployment(db, rev);
|
||||
|
||||
// A cache that starts empty — exactly the wiped-node state.
|
||||
var cache = new StubArtifactCache(current: null);
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
var snapshot = AskDiagnostics(actor);
|
||||
snapshot.CurrentRevision.ShouldBe(rev); // recovered the applied deployment
|
||||
snapshot.RunningFromCache.ShouldBeFalse(); // via central, not the cache
|
||||
|
||||
// The point: the restore path wrote the served artifact back into the cache.
|
||||
AwaitAssert(
|
||||
() => cache.Stores.ShouldBe(1),
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
cache.LastStoredDeploymentId.ShouldBe(deploymentId.ToString());
|
||||
}
|
||||
|
||||
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
@@ -222,13 +254,24 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
|
||||
{
|
||||
private int _unkeyedReads;
|
||||
private int _stores;
|
||||
|
||||
/// <summary>How many times the boot path consulted this cache.</summary>
|
||||
public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);
|
||||
|
||||
/// <summary>How many times the node wrote an applied artifact to this cache.</summary>
|
||||
public int Stores => Volatile.Read(ref _stores);
|
||||
|
||||
/// <summary>The deployment id of the most recent store.</summary>
|
||||
public string? LastStoredDeploymentId { get; private set; }
|
||||
|
||||
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
=> Task.CompletedTask;
|
||||
{
|
||||
Interlocked.Increment(ref _stores);
|
||||
LastStoredDeploymentId = deploymentId;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(
|
||||
string clusterId, CancellationToken ct = default)
|
||||
|
||||
Reference in New Issue
Block a user