fix(localdb): re-cache the artifact on RestoreApplied so a wiped node self-heals

CacheAppliedArtifact ran only on a fresh apply (ApplyAndAck), never on the bootstrap RestoreApplied path. So a node whose cache was lost — fresh/wiped volume, disk failure — recovered its served state from central yet stayed cache-less, unable to boot-from-cache on the NEXT central outage until some future new deploy happened to land. Replication does not heal this: a peer's already-acked cache rows are pruned from its oplog, so a fully-wiped, converged node is never back-filled. RestoreApplied now writes the served artifact back to the cache (invariant: the cache holds what the node serves). Found on the docker-dev live gate wiping a node's LocalDb volume.
2026-07-20 22:56:23 -04:00
parent 9137cb41eb
commit c6a9f93a0c
2 changed files with 58 additions and 5 deletions
@@ -166,6 +166,38 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
        snapshot.CurrentRevision.ShouldBeNull();
    }

+    [Fact]
+    public void RestoringAnAppliedDeploymentOnBootstrap_RepopulatesTheCache()
+    {
+        // The cache invariant is "holds what the node serves". A node recovering an already-applied
+        // deployment (the RestoreApplied path) must (re)write its cache — otherwise a node whose cache
+        // was lost (fresh volume, disk failure) recovers its served state from central yet stays
+        // cache-less, unable to boot-from-cache on the NEXT outage. Replication does not heal it: a
+        // peer's already-acked rows are pruned from its oplog, so a wiped node is never back-filled.
+        // (Regression for the docker-dev live-gate finding.)
+        var db = NewInMemoryDbFactory();
+        var rev = RevisionHash.Parse(new string('e', 64));
+        var deploymentId = SeedAppliedDeployment(db, rev);
+
+        // A cache that starts empty — exactly the wiped-node state.
+        var cache = new StubArtifactCache(current: null);
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            db, TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: cache));
+
+        var snapshot = AskDiagnostics(actor);
+        snapshot.CurrentRevision.ShouldBe(rev);       // recovered the applied deployment
+        snapshot.RunningFromCache.ShouldBeFalse();     // via central, not the cache
+
+        // The point: the restore path wrote the served artifact back into the cache.
+        AwaitAssert(
+            () => cache.Stores.ShouldBe(1),
+            duration: TimeSpan.FromSeconds(5));
+        cache.LastStoredDeploymentId.ShouldBe(deploymentId.ToString());
+    }
+
    private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
    {
        var probe = CreateTestProbe();
@@ -222,13 +254,24 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
    private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
    {
        private int _unkeyedReads;
+        private int _stores;

        /// <summary>How many times the boot path consulted this cache.</summary>
        public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);

+        /// <summary>How many times the node wrote an applied artifact to this cache.</summary>
+        public int Stores => Volatile.Read(ref _stores);
+
+        /// <summary>The deployment id of the most recent store.</summary>
+        public string? LastStoredDeploymentId { get; private set; }
+
        public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
                               byte[] artifact, CancellationToken ct = default)
-            => Task.CompletedTask;
+        {
+            Interlocked.Increment(ref _stores);
+            LastStoredDeploymentId = deploymentId;
+            return Task.CompletedTask;
+        }

        public Task<CachedDeploymentArtifact?> GetCurrentAsync(
            string clusterId, CancellationToken ct = default)