diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
index b2c8d54c..5c33bbf8 100644
--- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
@@ -61,6 +61,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// nodes/tests that never receive a refresh; the active site path supplies it via DI.
///
private readonly IDeploymentConfigFetcher? _configFetcher;
+ ///
+ /// Delay before re-attempting the startup deployed-config load after a
+ /// transient failure (S5). Fixed-interval, unlimited retries — consistent with
+ /// the system's fixed-interval retry philosophy. Overridable via the ctor for tests.
+ ///
+ private readonly TimeSpan _startupLoadRetryInterval;
+ ///
+ /// Loads all deployed configs at startup. Defaults to
+ /// ; overridable via the
+ /// ctor so a test can drive a transient load failure (S5).
+ ///
+ private readonly Func>> _configLoader;
+ ///
+ /// Set once the startup deployed-config load has succeeded. Guards against a
+ /// stale re-running the load after success.
+ ///
+ private bool _startupLoadCompleted;
private readonly Dictionary _instanceActors = new();
///
/// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
@@ -125,7 +142,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
ISiteHealthCollector? healthCollector = null,
IServiceProvider? serviceProvider = null,
ILoggerFactory? loggerFactory = null,
- IDeploymentConfigFetcher? configFetcher = null)
+ IDeploymentConfigFetcher? configFetcher = null,
+ TimeSpan? startupLoadRetryInterval = null,
+ Func>>? configLoader = null)
{
_storage = storage;
_compilationService = compilationService;
@@ -138,6 +157,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_configFetcher = configFetcher;
+ _startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5);
+ _configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
_logger = logger;
// Reuse a single logger factory for all Instance Actors.
// Prefer an explicitly injected factory, fall back to one resolved from
@@ -227,6 +248,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// Internal startup messages
Receive(HandleStartupConfigsLoaded);
+ // S5: re-attempt the startup deployed-config load after a transient
+ // failure. Ignored once the load has already completed so a stale timer
+ // (e.g. a failure retry that raced a subsequent success) cannot re-run it.
+ Receive(_ =>
+ {
+ if (_startupLoadCompleted)
+ return;
+ LoadDeployedConfigs();
+ });
Receive(HandleSharedScriptsLoaded);
Receive(HandleStartNextBatch);
@@ -248,8 +278,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_healthCollector?.SetActiveNode(true);
_logger.LogInformation("DeploymentManagerActor starting — loading deployed configs from SQLite...");
- // Load all configs asynchronously and pipe to self
- _storage.GetAllDeployedConfigsAsync().ContinueWith(t =>
+ LoadDeployedConfigs();
+ }
+
+ ///
+ /// Loads all deployed configs off the actor thread and pipes the result back as a
+ /// . Called from and re-invoked
+ /// by a timer when a load fails (S5) — a transient SQLite
+ /// failure must NOT leave the site as a silent zero-instance node, so the load is retried
+ /// at a fixed interval until it succeeds.
+ ///
+ private void LoadDeployedConfigs()
+ {
+ _configLoader().ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
return new StartupConfigsLoaded(t.Result, null);
@@ -297,10 +338,20 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
if (msg.Error != null)
{
- _logger.LogError("Failed to load deployed configs: {Error}", msg.Error);
+ // S5: a transient SQLite failure must not strand the site as a silent
+ // zero-instance node. Log and re-arm the load at a fixed interval; the
+ // retry is unlimited and self-cancels once a load succeeds.
+ _logger.LogError(
+ "Failed to load deployed configs: {Error} — retrying in {Interval}",
+ msg.Error, _startupLoadRetryInterval);
+ Timers.StartSingleTimer(
+ "startup-load-retry", RetryStartupLoad.Instance, _startupLoadRetryInterval);
return;
}
+ // Mark the startup load complete so a stale retry timer is ignored.
+ _startupLoadCompleted = true;
+
var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList();
_deployedInstanceNames.Clear();
foreach (var c in msg.Configs)
@@ -1713,6 +1764,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
internal record StartupConfigsLoaded(List Configs, string? Error);
+ ///
+ /// Internal singleton signal that re-attempts the startup deployed-config load
+ /// after a transient SQLite failure (S5). Delivered by a single-shot Akka timer.
+ ///
+ internal sealed record RetryStartupLoad
+ {
+ /// The shared singleton instance.
+ public static readonly RetryStartupLoad Instance = new();
+ private RetryStartupLoad() { }
+ }
+
///
/// Internal message piped back once shared scripts have been compiled off-thread
/// Carries the enabled configs so staggered Instance Actor
diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs
index 1f82cf81..99a186b6 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs
@@ -15,6 +15,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using System.Text.Json;
+using System.Threading;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
@@ -266,6 +267,46 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "bad-instance");
}
+ // ── S5: startup deployed-config load is retried on transient failure ──
+
+ [Fact]
+ public void StartupLoadFailure_IsRetried_UntilSuccess()
+ {
+ // The first startup load throws (transient SQLite failure); the second
+ // succeeds and returns one enabled config. The DM must retry rather than
+ // stall as a silent zero-instance node — so the Instance Actor eventually
+ // exists.
+ var calls = 0;
+ Func>> loader = () =>
+ Interlocked.Increment(ref calls) == 1
+ ? Task.FromException>(new IOException("db locked"))
+ : Task.FromResult(new List
+ {
+ new()
+ {
+ InstanceUniqueName = "inst-1",
+ ConfigJson = MakeConfigJson("inst-1"),
+ DeploymentId = "d1",
+ RevisionHash = "h1",
+ IsEnabled = true
+ }
+ });
+
+ var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
+ _storage, _compilationService, _sharedScriptLibrary, null,
+ new SiteRuntimeOptions(), NullLogger.Instance,
+ null, null, null, null, null, null,
+ TimeSpan.FromMilliseconds(200), loader)));
+
+ AwaitAssert(() =>
+ {
+ Assert.True(Volatile.Read(ref calls) >= 2,
+ $"expected the load to be retried; calls={Volatile.Read(ref calls)}");
+ Sys.ActorSelection(dm.Path / "inst-1").Tell(new Identify(1));
+ Assert.NotNull(ExpectMsg(TimeSpan.FromSeconds(1)).Subject);
+ }, TimeSpan.FromSeconds(10));
+ }
+
// ── M1.6: site event log `deployment` category ─────────────────────────
[Fact]