diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs index b2c8d54c..5c33bbf8 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs @@ -61,6 +61,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers /// nodes/tests that never receive a refresh; the active site path supplies it via DI. /// private readonly IDeploymentConfigFetcher? _configFetcher; + /// + /// Delay before re-attempting the startup deployed-config load after a + /// transient failure (S5). Fixed-interval, unlimited retries — consistent with + /// the system's fixed-interval retry philosophy. Overridable via the ctor for tests. + /// + private readonly TimeSpan _startupLoadRetryInterval; + /// + /// Loads all deployed configs at startup. Defaults to + /// ; overridable via the + /// ctor so a test can drive a transient load failure (S5). + /// + private readonly Func>> _configLoader; + /// + /// Set once the startup deployed-config load has succeeded. Guards against a + /// stale re-running the load after success. + /// + private bool _startupLoadCompleted; private readonly Dictionary _instanceActors = new(); /// /// Tracks Instance Actors that are terminating as part of a redeployment, keyed by @@ -125,7 +142,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers ISiteHealthCollector? healthCollector = null, IServiceProvider? serviceProvider = null, ILoggerFactory? loggerFactory = null, - IDeploymentConfigFetcher? configFetcher = null) + IDeploymentConfigFetcher? configFetcher = null, + TimeSpan? startupLoadRetryInterval = null, + Func>>? configLoader = null) { _storage = storage; _compilationService = compilationService; @@ -138,6 +157,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers _healthCollector = healthCollector; _serviceProvider = serviceProvider; _configFetcher = configFetcher; + _startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5); + _configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync()); _logger = logger; // Reuse a single logger factory for all Instance Actors. // Prefer an explicitly injected factory, fall back to one resolved from @@ -227,6 +248,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers // Internal startup messages Receive(HandleStartupConfigsLoaded); + // S5: re-attempt the startup deployed-config load after a transient + // failure. Ignored once the load has already completed so a stale timer + // (e.g. a failure retry that raced a subsequent success) cannot re-run it. + Receive(_ => + { + if (_startupLoadCompleted) + return; + LoadDeployedConfigs(); + }); Receive(HandleSharedScriptsLoaded); Receive(HandleStartNextBatch); @@ -248,8 +278,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers _healthCollector?.SetActiveNode(true); _logger.LogInformation("DeploymentManagerActor starting — loading deployed configs from SQLite..."); - // Load all configs asynchronously and pipe to self - _storage.GetAllDeployedConfigsAsync().ContinueWith(t => + LoadDeployedConfigs(); + } + + /// + /// Loads all deployed configs off the actor thread and pipes the result back as a + /// . Called from and re-invoked + /// by a timer when a load fails (S5) — a transient SQLite + /// failure must NOT leave the site as a silent zero-instance node, so the load is retried + /// at a fixed interval until it succeeds. + /// + private void LoadDeployedConfigs() + { + _configLoader().ContinueWith(t => { if (t.IsCompletedSuccessfully) return new StartupConfigsLoaded(t.Result, null); @@ -297,10 +338,20 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers { if (msg.Error != null) { - _logger.LogError("Failed to load deployed configs: {Error}", msg.Error); + // S5: a transient SQLite failure must not strand the site as a silent + // zero-instance node. Log and re-arm the load at a fixed interval; the + // retry is unlimited and self-cancels once a load succeeds. + _logger.LogError( + "Failed to load deployed configs: {Error} — retrying in {Interval}", + msg.Error, _startupLoadRetryInterval); + Timers.StartSingleTimer( + "startup-load-retry", RetryStartupLoad.Instance, _startupLoadRetryInterval); return; } + // Mark the startup load complete so a stale retry timer is ignored. + _startupLoadCompleted = true; + var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList(); _deployedInstanceNames.Clear(); foreach (var c in msg.Configs) @@ -1713,6 +1764,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers internal record StartupConfigsLoaded(List Configs, string? Error); + /// + /// Internal singleton signal that re-attempts the startup deployed-config load + /// after a transient SQLite failure (S5). Delivered by a single-shot Akka timer. + /// + internal sealed record RetryStartupLoad + { + /// The shared singleton instance. + public static readonly RetryStartupLoad Instance = new(); + private RetryStartupLoad() { } + } + /// /// Internal message piped back once shared scripts have been compiled off-thread /// Carries the enabled configs so staggered Instance Actor diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs index 1f82cf81..99a186b6 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs @@ -15,6 +15,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; using System.Text.Json; +using System.Threading; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; @@ -266,6 +267,46 @@ public class DeploymentManagerActorTests : TestKit, IDisposable Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "bad-instance"); } + // ── S5: startup deployed-config load is retried on transient failure ── + + [Fact] + public void StartupLoadFailure_IsRetried_UntilSuccess() + { + // The first startup load throws (transient SQLite failure); the second + // succeeds and returns one enabled config. The DM must retry rather than + // stall as a silent zero-instance node — so the Instance Actor eventually + // exists. + var calls = 0; + Func>> loader = () => + Interlocked.Increment(ref calls) == 1 + ? Task.FromException>(new IOException("db locked")) + : Task.FromResult(new List + { + new() + { + InstanceUniqueName = "inst-1", + ConfigJson = MakeConfigJson("inst-1"), + DeploymentId = "d1", + RevisionHash = "h1", + IsEnabled = true + } + }); + + var dm = ActorOf(Props.Create(() => new DeploymentManagerActor( + _storage, _compilationService, _sharedScriptLibrary, null, + new SiteRuntimeOptions(), NullLogger.Instance, + null, null, null, null, null, null, + TimeSpan.FromMilliseconds(200), loader))); + + AwaitAssert(() => + { + Assert.True(Volatile.Read(ref calls) >= 2, + $"expected the load to be retried; calls={Volatile.Read(ref calls)}"); + Sys.ActorSelection(dm.Path / "inst-1").Tell(new Identify(1)); + Assert.NotNull(ExpectMsg(TimeSpan.FromSeconds(1)).Subject); + }, TimeSpan.FromSeconds(10)); + } + // ── M1.6: site event log `deployment` category ───────────────────────── [Fact]