feat(localdb): boot from the pair-local artifact cache when central SQL is unreachable

Hooks the Bootstrap catch that previously went straight to Stale. On a cache hit the
node serves its last-known-good configuration through the outage instead of coming
up with an empty address space; on a miss, behaviour is byte-for-byte what it was.

The read is unkeyed because ClusterId is only derivable from an artifact you already
hold or from the unreachable central DB (recon D-1). Newest pointer wins, which is
correct in every real topology - a node belongs to one cluster and its peer
replicates that same row.

Splits PushDesiredSubscriptionsFromArtifact out of PushDesiredSubscriptions: the
cache path runs precisely when the ConfigDb read cannot succeed, so re-reading the
artifact there would fail by definition.

RunningFromCache is surfaced on NodeDiagnosticsSnapshot (optional param, existing
call sites unaffected). A node running from cache looks entirely healthy from
outside - full address space, live values - but its config is frozen and no deploy
can reach it. It clears only on a real apply from central.

src/Core/ZB.MOM.WW.OtOpcUa.Commons/Interfaces/NodeDiagnosticsSnapshot.cs

@@ -14,8 +14,16 @@ public sealed record DriverInstanceDiagnostics(
 /// Per-node diagnostics returned by <c>IFleetDiagnosticsClient</c>. Populated by the node's
 /// local <c>DriverHostActor</c> via a request/response over Akka.
 /// </summary>
+/// <param name="RunningFromCache">
+///     True when the node booted its configuration from the node-local artifact cache because the
+///     central ConfigDb was unreachable. Such a node looks entirely healthy — full address space,
+///     live values — but its configuration is frozen and no deployment can reach it, so the state
+///     needs to be explicitly visible rather than inferred. Defaults to false, which is also what
+///     every node that booted normally reports.
+/// </param>
 public sealed record NodeDiagnosticsSnapshot(
     NodeId NodeId,
     RevisionHash? CurrentRevision,
     IReadOnlyList<DriverInstanceDiagnostics> Drivers,
-    DateTime AsOfUtc);
+    DateTime AsOfUtc,
+    bool RunningFromCache = false);

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -71,6 +71,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// </summary>
     private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
 
+    /// <summary>
+    ///     True once this node has booted a cached artifact because central SQL was unreachable.
+    /// </summary>
+    /// <remarks>
+    ///     Surfaced on <see cref="NodeDiagnosticsSnapshot"/> because a node running from cache looks
+    ///     completely healthy from the outside — it serves a full address space with live values.
+    ///     The difference is that its configuration is frozen at whatever the cache held, and no
+    ///     deployment can reach it. Without an explicit signal that is invisible until someone
+    ///     wonders why a deploy "succeeded" everywhere but did not take effect here.
+    /// </remarks>
+    private bool _isRunningFromCache;
+
     private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
     private readonly CommonsNodeId _localNode;
     private readonly IActorRef? _coordinatorOverride;
@@ -587,10 +599,118 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         catch (Exception ex)
         {
             _log.Warning(ex, "DriverHost {Node}: ConfigDb unreachable on bootstrap; entering Stale", _localNode);
-            Become(Stale);
+
+            // Central is unreachable, so try the node-local cache before giving up. On a hit this
+            // node serves its last-known-good configuration through the outage instead of coming up
+            // with an empty address space. On a miss, behaviour is exactly what it always was.
+            if (!TryBootFromCache())
+                Become(Stale);
         }
     }
 
+    /// <summary>
+    ///     Last-resort boot path: apply the artifact cached by a previous successful deploy (or
+    ///     replicated from this node's pair peer) when central SQL cannot be reached.
+    /// </summary>
+    /// <returns>
+    ///     <see langword="true"/> when a cached artifact was applied and the actor is now Steady;
+    ///     <see langword="false"/> when the caller should fall through to <c>Stale</c>.
+    /// </returns>
+    /// <remarks>
+    ///     <para>
+    ///         <b>The read is unkeyed.</b> The cache is keyed by ClusterId so a pair shares one
+    ///         entry, but ClusterId is only derivable from an artifact you already hold or from the
+    ///         central DB — and at this seam we have neither. So the newest pointer row wins, which
+    ///         is correct in every real topology because a node belongs to one cluster and its peer
+    ///         replicates that same cluster's row. A node re-homed between clusters is the only
+    ///         ambiguous case, and the cache logs a warning naming both.
+    ///     </para>
+    ///     <para>
+    ///         <b>This does not make the node current.</b> <c>_currentRevision</c> is set from the
+    ///         cached artifact, so a subsequent dispatch of that same revision correctly no-ops,
+    ///         while any NEW revision still requires central — the cache holds the past, not the
+    ///         future. Dispatch handling is unchanged.
+    ///     </para>
+    ///     <para>
+    ///         Never throws: a fault here must degrade to Stale, which is exactly where the node
+    ///         would have been without a cache at all.
+    ///     </para>
+    /// </remarks>
+    private bool TryBootFromCache()
+    {
+        if (_deploymentArtifactCache is null)
+            return false;
+
+        try
+        {
+            var cached = _deploymentArtifactCache.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
+            if (cached is null)
+            {
+                _log.Info(
+                    "DriverHost {Node}: no cached deployment available; entering Stale with no configuration.",
+                    _localNode);
+                return false;
+            }
+
+            var deploymentId = DeploymentId.Parse(cached.DeploymentId);
+            var revision = RevisionHash.Parse(cached.RevisionHash);
+
+            // Steady, not Stale: this node is serving a real configuration. The retry-db timer that
+            // Stale would have started does not run here, so recovery rides on the next dispatch —
+            // matching how a normally-booted node behaves.
+            _currentRevision = revision;
+            Become(Steady);
+
+            ApplyCachedArtifact(deploymentId, cached.Artifact);
+
+            _isRunningFromCache = true;
+            _log.Warning(
+                "DriverHost {Node}: RUNNING FROM CACHE — central ConfigDb is unreachable, so this node " +
+                "booted deployment {Id} (rev {Rev}) cached at {CachedAtUtc:o} from its local database. " +
+                "Configuration changes cannot be applied until the ConfigDb is reachable again.",
+                _localNode, deploymentId, revision, cached.AppliedAtUtc);
+
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _log.Error(ex,
+                "DriverHost {Node}: failed to boot from the local deployment cache; falling back to Stale.",
+                _localNode);
+            return false;
+        }
+    }
+
+    /// <summary>
+    ///     Applies an artifact already in hand — no ConfigDb read, no ACK.
+    /// </summary>
+    /// <remarks>
+    ///     Mirrors <see cref="RestoreApplied"/>, but sources the artifact from the cache rather than
+    ///     re-reading it from a database that is by definition unreachable here. It also skips
+    ///     <c>UpsertNodeDeploymentState</c> and <c>SendAck</c> for the same reason: both write to or
+    ///     depend on central.
+    /// </remarks>
+    private void ApplyCachedArtifact(DeploymentId deploymentId, byte[] blob)
+    {
+        var correlation = CorrelationId.NewId();
+
+        var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
+        var snapshots = _children.ToDictionary(
+            kv => kv.Key,
+            kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
+            StringComparer.Ordinal);
+        var plan = DriverSpawnPlanner.Compute(snapshots, specs);
+
+        foreach (var id in plan.ToStop) StopChild(id);
+        foreach (var spec in plan.ToApplyDelta) ApplyChildDelta(spec);
+        foreach (var spec in plan.ToSpawn) SpawnChild(spec);
+
+        _opcUaPublishActor?.Tell(
+            new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(correlation, deploymentId));
+
+        PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
+    }
+
     private void Steady()
     {
         Receive<DispatchDeployment>(HandleDispatchFromSteady);
@@ -1420,7 +1540,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
             NodeId: _localNode,
             CurrentRevision: _currentRevision,
             Drivers: drivers,
-            AsOfUtc: DateTime.UtcNow);
+            AsOfUtc: DateTime.UtcNow,
+            RunningFromCache: _isRunningFromCache);
         Sender.Tell(snapshot);
     }
 
@@ -1465,6 +1586,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
             // the just-rebuilt address space instead of staying BadWaitingForInitialData.
             PushDesiredSubscriptions(deploymentId);
             CacheAppliedArtifact(deploymentId, revision, appliedBlob);
+            // Reaching here means the artifact came from central, so the node is no longer serving a
+            // cache-sourced configuration. Note this only clears on a real apply: a dispatch of the
+            // revision already booted from cache short-circuits in HandleDispatchFromSteady without
+            // touching the ConfigDb, and the flag correctly stays set.
+            _isRunningFromCache = false;
             OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "ack"));
             _log.Info("DriverHost {Node}: applied deployment {Id} (rev {Rev}, children={Count})",
                 _localNode, deploymentId, revision, _children.Count);
@@ -1661,6 +1787,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
             return;
         }
 
+        PushDesiredSubscriptionsFromArtifact(deploymentId, blob);
+    }
+
+    /// <summary>
+    ///     The artifact-processing half of <see cref="PushDesiredSubscriptions"/>, split out so the
+    ///     boot-from-cache path can drive it with bytes already in hand.
+    /// </summary>
+    /// <remarks>
+    ///     Separated because the cache path runs precisely when the ConfigDb read above cannot
+    ///     succeed — re-reading the artifact there would fail by definition.
+    /// </remarks>
+    private void PushDesiredSubscriptionsFromArtifact(DeploymentId deploymentId, byte[] blob)
+    {
         AddressSpaceComposition composition;
         try
         {

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs

@@ -0,0 +1,232 @@
+using System.Text.Json;
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
+
+/// <summary>
+///     LocalDb Phase 1 (Task 8) — booting from the node-local artifact cache when central SQL is
+///     unreachable.
+/// </summary>
+/// <remarks>
+///     The behaviour that matters is asymmetric: the cache must rescue a node whose ConfigDb read
+///     failed, and must be completely invisible otherwise. A cache consulted on the happy path would
+///     be a route for stale configuration to override fresh configuration.
+/// </remarks>
+public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
+{
+    private static readonly NodeId TestNode = NodeId.Parse("driver-test");
+    private static readonly RevisionHash CachedRev = RevisionHash.Parse(new string('c', 64));
+
+    [Fact]
+    public void CentralUnreachableAndCacheHasArtifact_BootsFromCache()
+    {
+        var cached = new CachedDeploymentArtifact(
+            DeploymentId.NewId().ToString(),
+            CachedRev.Value,
+            EmptyArtifact(),
+            DateTimeOffset.UtcNow.AddHours(-1));
+        var cache = new StubArtifactCache(cached);
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: cache));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeTrue();
+        snapshot.CurrentRevision.ShouldBe(CachedRev);
+
+        // Positive control for the UnkeyedReads counter the "never consults the cache" tests rely
+        // on. Without this, those ShouldBe(0) assertions would pass just as happily if the counter
+        // were never incremented at all.
+        cache.UnkeyedReads.ShouldBe(1);
+    }
+
+    [Fact]
+    public void CentralUnreachableAndCacheEmpty_StaysStaleWithNoConfiguration()
+    {
+        // The pre-cache behaviour, unchanged. A cache miss must not invent a configuration.
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: new StubArtifactCache(current: null)));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeFalse();
+        snapshot.CurrentRevision.ShouldBeNull();
+    }
+
+    [Fact]
+    public void CentralUnreachableAndNoCacheConfigured_StaysStale()
+    {
+        // Admin-only graphs and older harnesses pass null; behaviour is identical to a cache miss.
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" }));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeFalse();
+        snapshot.CurrentRevision.ShouldBeNull();
+    }
+
+    [Fact]
+    public void CentralReachable_NeverConsultsTheCache()
+    {
+        // THE guard against stale config winning. If the cache were read on the happy path, a node
+        // whose cache held an older revision could quietly serve it over the deployed one.
+        var cache = new StubArtifactCache(new CachedDeploymentArtifact(
+            DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            NewInMemoryDbFactory(), TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: cache));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeFalse();
+        cache.UnkeyedReads.ShouldBe(0);
+    }
+
+    [Fact]
+    public void CentralReachableWithAPriorAppliedDeployment_NeverConsultsTheCache()
+    {
+        // The RestoreApplied path — the other way a boot can succeed against central.
+        var db = NewInMemoryDbFactory();
+        var rev = RevisionHash.Parse(new string('d', 64));
+        var deploymentId = SeedAppliedDeployment(db, rev);
+
+        var cache = new StubArtifactCache(new CachedDeploymentArtifact(
+            deploymentId.ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            db, TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: cache));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeFalse();
+        snapshot.CurrentRevision.ShouldBe(rev);
+        cache.UnkeyedReads.ShouldBe(0);
+    }
+
+    [Fact]
+    public void AThrowingCache_DegradesToStale_RatherThanFailingTheActor()
+    {
+        // A cache fault must leave the node exactly where it would have been without a cache.
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, coordinator: null,
+            localRoles: new HashSet<string> { "driver" },
+            deploymentArtifactCache: new ThrowingReadCache()));
+
+        var snapshot = AskDiagnostics(actor);
+
+        snapshot.RunningFromCache.ShouldBeFalse();
+        snapshot.CurrentRevision.ShouldBeNull();
+    }
+
+    private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
+    {
+        var probe = CreateTestProbe();
+        // GetDiagnostics is serviced in Steady, Applying AND Stale, so this observes the node
+        // whichever way the boot resolved.
+        AwaitAssert(
+            () =>
+            {
+                actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
+                probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(2));
+            },
+            duration: TimeSpan.FromSeconds(5));
+
+        actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
+        return probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(5));
+    }
+
+    /// <summary>A syntactically valid artifact with no driver instances.</summary>
+    private static byte[] EmptyArtifact()
+        => JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
+
+    private static DeploymentId SeedAppliedDeployment(
+        IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev)
+    {
+        var id = DeploymentId.NewId();
+        using var ctx = db.CreateDbContext();
+        ctx.Deployments.Add(new Deployment
+        {
+            DeploymentId = id.Value,
+            RevisionHash = rev.Value,
+            Status = DeploymentStatus.Sealed,
+            CreatedBy = "test",
+            SealedAtUtc = DateTime.UtcNow,
+            ArtifactBlob = EmptyArtifact(),
+        });
+        ctx.NodeDeploymentStates.Add(new NodeDeploymentState
+        {
+            NodeId = TestNode.Value,
+            DeploymentId = id.Value,
+            Status = NodeDeploymentStatus.Applied,
+            StartedAtUtc = DateTime.UtcNow,
+        });
+        ctx.SaveChanges();
+        return id;
+    }
+
+    /// <summary>Stands in for an unreachable central SQL Server.</summary>
+    private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext()
+            => throw new InvalidOperationException("ConfigDb unreachable");
+    }
+
+    private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
+    {
+        private int _unkeyedReads;
+
+        /// <summary>How many times the boot path consulted this cache.</summary>
+        public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);
+
+        public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
+                               byte[] artifact, CancellationToken ct = default)
+            => Task.CompletedTask;
+
+        public Task<CachedDeploymentArtifact?> GetCurrentAsync(
+            string clusterId, CancellationToken ct = default)
+            => Task.FromResult(current);
+
+        public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
+        {
+            Interlocked.Increment(ref _unkeyedReads);
+            return Task.FromResult(current);
+        }
+    }
+
+    private sealed class ThrowingReadCache : IDeploymentArtifactCache
+    {
+        public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
+                               byte[] artifact, CancellationToken ct = default)
+            => Task.CompletedTask;
+
+        public Task<CachedDeploymentArtifact?> GetCurrentAsync(
+            string clusterId, CancellationToken ct = default)
+            => throw new InvalidOperationException("cache unreadable");
+
+        public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
+            => throw new InvalidOperationException("cache unreadable");
+    }
+}