fix(site-runtime): retry the startup deployed-config load on SQLite failure (S5) — no more silent zero-instance node

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -61,6 +61,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// nodes/tests that never receive a refresh; the active site path supplies it via DI.
     /// </summary>
     private readonly IDeploymentConfigFetcher? _configFetcher;
+    /// <summary>
+    /// Delay before re-attempting the startup deployed-config load after a
+    /// transient failure (S5). Fixed-interval, unlimited retries — consistent with
+    /// the system's fixed-interval retry philosophy. Overridable via the ctor for tests.
+    /// </summary>
+    private readonly TimeSpan _startupLoadRetryInterval;
+    /// <summary>
+    /// Loads all deployed configs at startup. Defaults to
+    /// <see cref="SiteStorageService.GetAllDeployedConfigsAsync"/>; overridable via the
+    /// ctor so a test can drive a transient load failure (S5).
+    /// </summary>
+    private readonly Func<Task<List<DeployedInstance>>> _configLoader;
+    /// <summary>
+    /// Set once the startup deployed-config load has succeeded. Guards against a
+    /// stale <see cref="RetryStartupLoad"/> re-running the load after success.
+    /// </summary>
+    private bool _startupLoadCompleted;
     private readonly Dictionary<string, IActorRef> _instanceActors = new();
     /// <summary>
     /// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
@@ -125,7 +142,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         ISiteHealthCollector? healthCollector = null,
         IServiceProvider? serviceProvider = null,
         ILoggerFactory? loggerFactory = null,
-        IDeploymentConfigFetcher? configFetcher = null)
+        IDeploymentConfigFetcher? configFetcher = null,
+        TimeSpan? startupLoadRetryInterval = null,
+        Func<Task<List<DeployedInstance>>>? configLoader = null)
     {
         _storage = storage;
         _compilationService = compilationService;
@@ -138,6 +157,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _healthCollector = healthCollector;
         _serviceProvider = serviceProvider;
         _configFetcher = configFetcher;
+        _startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5);
+        _configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
         _logger = logger;
         // Reuse a single logger factory for all Instance Actors.
         // Prefer an explicitly injected factory, fall back to one resolved from
@@ -227,6 +248,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
         // Internal startup messages
         Receive<StartupConfigsLoaded>(HandleStartupConfigsLoaded);
+        // S5: re-attempt the startup deployed-config load after a transient
+        // failure. Ignored once the load has already completed so a stale timer
+        // (e.g. a failure retry that raced a subsequent success) cannot re-run it.
+        Receive<RetryStartupLoad>(_ =>
+        {
+            if (_startupLoadCompleted)
+                return;
+            LoadDeployedConfigs();
+        });
         Receive<SharedScriptsLoaded>(HandleSharedScriptsLoaded);
         Receive<StartNextBatch>(HandleStartNextBatch);
 
@@ -248,8 +278,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _healthCollector?.SetActiveNode(true);
         _logger.LogInformation("DeploymentManagerActor starting — loading deployed configs from SQLite...");
 
-        // Load all configs asynchronously and pipe to self
+        LoadDeployedConfigs();
-        _storage.GetAllDeployedConfigsAsync().ContinueWith(t =>
+    }
+
+    /// <summary>
+    /// Loads all deployed configs off the actor thread and pipes the result back as a
+    /// <see cref="StartupConfigsLoaded"/>. Called from <see cref="PreStart"/> and re-invoked
+    /// by a <see cref="RetryStartupLoad"/> timer when a load fails (S5) — a transient SQLite
+    /// failure must NOT leave the site as a silent zero-instance node, so the load is retried
+    /// at a fixed interval until it succeeds.
+    /// </summary>
+    private void LoadDeployedConfigs()
+    {
+        _configLoader().ContinueWith(t =>
         {
             if (t.IsCompletedSuccessfully)
                 return new StartupConfigsLoaded(t.Result, null);
@@ -297,10 +338,20 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     {
         if (msg.Error != null)
         {
-            _logger.LogError("Failed to load deployed configs: {Error}", msg.Error);
+            // S5: a transient SQLite failure must not strand the site as a silent
+            // zero-instance node. Log and re-arm the load at a fixed interval; the
+            // retry is unlimited and self-cancels once a load succeeds.
+            _logger.LogError(
+                "Failed to load deployed configs: {Error} — retrying in {Interval}",
+                msg.Error, _startupLoadRetryInterval);
+            Timers.StartSingleTimer(
+                "startup-load-retry", RetryStartupLoad.Instance, _startupLoadRetryInterval);
             return;
         }
 
+        // Mark the startup load complete so a stale retry timer is ignored.
+        _startupLoadCompleted = true;
+
         var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList();
         _deployedInstanceNames.Clear();
         foreach (var c in msg.Configs)
@@ -1713,6 +1764,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
     internal record StartupConfigsLoaded(List<DeployedInstance> Configs, string? Error);
 
+    /// <summary>
+    /// Internal singleton signal that re-attempts the startup deployed-config load
+    /// after a transient SQLite failure (S5). Delivered by a single-shot Akka timer.
+    /// </summary>
+    internal sealed record RetryStartupLoad
+    {
+        /// <summary>The shared singleton instance.</summary>
+        public static readonly RetryStartupLoad Instance = new();
+        private RetryStartupLoad() { }
+    }
+
     /// <summary>
     /// Internal message piped back once shared scripts have been compiled off-thread
     /// Carries the enabled configs so staggered Instance Actor

tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs

@@ -15,6 +15,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
 using System.Text.Json;
+using System.Threading;
 
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
 
@@ -266,6 +267,46 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
         Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "bad-instance");
     }
 
+    // ── S5: startup deployed-config load is retried on transient failure ──
+
+    [Fact]
+    public void StartupLoadFailure_IsRetried_UntilSuccess()
+    {
+        // The first startup load throws (transient SQLite failure); the second
+        // succeeds and returns one enabled config. The DM must retry rather than
+        // stall as a silent zero-instance node — so the Instance Actor eventually
+        // exists.
+        var calls = 0;
+        Func<Task<List<DeployedInstance>>> loader = () =>
+            Interlocked.Increment(ref calls) == 1
+                ? Task.FromException<List<DeployedInstance>>(new IOException("db locked"))
+                : Task.FromResult(new List<DeployedInstance>
+                {
+                    new()
+                    {
+                        InstanceUniqueName = "inst-1",
+                        ConfigJson = MakeConfigJson("inst-1"),
+                        DeploymentId = "d1",
+                        RevisionHash = "h1",
+                        IsEnabled = true
+                    }
+                });
+
+        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
+            _storage, _compilationService, _sharedScriptLibrary, null,
+            new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance,
+            null, null, null, null, null, null,
+            TimeSpan.FromMilliseconds(200), loader)));
+
+        AwaitAssert(() =>
+        {
+            Assert.True(Volatile.Read(ref calls) >= 2,
+                $"expected the load to be retried; calls={Volatile.Read(ref calls)}");
+            Sys.ActorSelection(dm.Path / "inst-1").Tell(new Identify(1));
+            Assert.NotNull(ExpectMsg<ActorIdentity>(TimeSpan.FromSeconds(1)).Subject);
+        }, TimeSpan.FromSeconds(10));
+    }
+
     // ── M1.6: site event log `deployment` category ─────────────────────────
 
     [Fact]