fix(site-runtime): retry the startup deployed-config load on SQLite failure (S5) — no more silent zero-instance node

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -61,6 +61,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// nodes/tests that never receive a refresh; the active site path supplies it via DI.
     /// </summary>
     private readonly IDeploymentConfigFetcher? _configFetcher;
+    /// <summary>
+    /// Delay before re-attempting the startup deployed-config load after a
+    /// transient failure (S5). Fixed-interval, unlimited retries — consistent with
+    /// the system's fixed-interval retry philosophy. Overridable via the ctor for tests.
+    /// </summary>
+    private readonly TimeSpan _startupLoadRetryInterval;
+    /// <summary>
+    /// Loads all deployed configs at startup. Defaults to
+    /// <see cref="SiteStorageService.GetAllDeployedConfigsAsync"/>; overridable via the
+    /// ctor so a test can drive a transient load failure (S5).
+    /// </summary>
+    private readonly Func<Task<List<DeployedInstance>>> _configLoader;
+    /// <summary>
+    /// Set once the startup deployed-config load has succeeded. Guards against a
+    /// stale <see cref="RetryStartupLoad"/> re-running the load after success.
+    /// </summary>
+    private bool _startupLoadCompleted;
     private readonly Dictionary<string, IActorRef> _instanceActors = new();
     /// <summary>
     /// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
@@ -125,7 +142,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         ISiteHealthCollector? healthCollector = null,
         IServiceProvider? serviceProvider = null,
         ILoggerFactory? loggerFactory = null,
-        IDeploymentConfigFetcher? configFetcher = null)
+        IDeploymentConfigFetcher? configFetcher = null,
+        TimeSpan? startupLoadRetryInterval = null,
+        Func<Task<List<DeployedInstance>>>? configLoader = null)
     {
         _storage = storage;
         _compilationService = compilationService;
@@ -138,6 +157,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _healthCollector = healthCollector;
         _serviceProvider = serviceProvider;
         _configFetcher = configFetcher;
+        _startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5);
+        _configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
         _logger = logger;
         // Reuse a single logger factory for all Instance Actors.
         // Prefer an explicitly injected factory, fall back to one resolved from
@@ -227,6 +248,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
         // Internal startup messages
         Receive<StartupConfigsLoaded>(HandleStartupConfigsLoaded);
+        // S5: re-attempt the startup deployed-config load after a transient
+        // failure. Ignored once the load has already completed so a stale timer
+        // (e.g. a failure retry that raced a subsequent success) cannot re-run it.
+        Receive<RetryStartupLoad>(_ =>
+        {
+            if (_startupLoadCompleted)
+                return;
+            LoadDeployedConfigs();
+        });
         Receive<SharedScriptsLoaded>(HandleSharedScriptsLoaded);
         Receive<StartNextBatch>(HandleStartNextBatch);
 
@@ -248,8 +278,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _healthCollector?.SetActiveNode(true);
         _logger.LogInformation("DeploymentManagerActor starting — loading deployed configs from SQLite...");
 
-        // Load all configs asynchronously and pipe to self
-        _storage.GetAllDeployedConfigsAsync().ContinueWith(t =>
+        LoadDeployedConfigs();
+    }
+
+    /// <summary>
+    /// Loads all deployed configs off the actor thread and pipes the result back as a
+    /// <see cref="StartupConfigsLoaded"/>. Called from <see cref="PreStart"/> and re-invoked
+    /// by a <see cref="RetryStartupLoad"/> timer when a load fails (S5) — a transient SQLite
+    /// failure must NOT leave the site as a silent zero-instance node, so the load is retried
+    /// at a fixed interval until it succeeds.
+    /// </summary>
+    private void LoadDeployedConfigs()
+    {
+        _configLoader().ContinueWith(t =>
         {
             if (t.IsCompletedSuccessfully)
                 return new StartupConfigsLoaded(t.Result, null);
@@ -297,10 +338,20 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     {
         if (msg.Error != null)
         {
-            _logger.LogError("Failed to load deployed configs: {Error}", msg.Error);
+            // S5: a transient SQLite failure must not strand the site as a silent
+            // zero-instance node. Log and re-arm the load at a fixed interval; the
+            // retry is unlimited and self-cancels once a load succeeds.
+            _logger.LogError(
+                "Failed to load deployed configs: {Error} — retrying in {Interval}",
+                msg.Error, _startupLoadRetryInterval);
+            Timers.StartSingleTimer(
+                "startup-load-retry", RetryStartupLoad.Instance, _startupLoadRetryInterval);
             return;
         }
 
+        // Mark the startup load complete so a stale retry timer is ignored.
+        _startupLoadCompleted = true;
+
         var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList();
         _deployedInstanceNames.Clear();
         foreach (var c in msg.Configs)
@@ -1713,6 +1764,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
     internal record StartupConfigsLoaded(List<DeployedInstance> Configs, string? Error);
 
+    /// <summary>
+    /// Internal singleton signal that re-attempts the startup deployed-config load
+    /// after a transient SQLite failure (S5). Delivered by a single-shot Akka timer.
+    /// </summary>
+    internal sealed record RetryStartupLoad
+    {
+        /// <summary>The shared singleton instance.</summary>
+        public static readonly RetryStartupLoad Instance = new();
+        private RetryStartupLoad() { }
+    }
+
     /// <summary>
     /// Internal message piped back once shared scripts have been compiled off-thread
     /// Carries the enabled configs so staggered Instance Actor