merge: startup reconcile starts the healed InstanceActor (register row 67) — EnsureInstanceStartedCommand to the DM singleton, idempotent, disable-wins, Warning site event on self-heal

src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Deployment/EnsureInstanceStartedCommand.cs

@@ -0,0 +1,29 @@
+namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
+
+/// <summary>
+/// Site-internal request to make sure an already-persisted instance is actually RUNNING.
+/// Routes via the deployment-manager singleton proxy so the Instance Actor is only ever
+/// created under the singleton, whichever node raised the request.
+/// <para>
+/// Emitted by <c>SiteReconciliationActor</c> after its node-STARTUP pass fetches and stores
+/// a config the node was missing. The reconcile heal path writes the FILE; without this the
+/// runtime was left untouched, so an instance whose config was absent at boot (lost volume,
+/// first boot after a restore, rehomed data dir) stayed dead until the NEXT restart, whose
+/// SQLite load finally started it.
+/// </para>
+/// <para>
+/// <b>Fire-and-forget and idempotent.</b> There is no reply: the handler no-ops when the
+/// instance is already running, when an in-flight deploy will create it anyway, or when the
+/// durable row says the instance is disabled. The stored row — not this message — is the
+/// authority on both the config body and enabled/disabled state, so a disable that landed
+/// after the reconcile fetch still wins.
+/// </para>
+/// </summary>
+/// <param name="InstanceUniqueName">Unique name of the instance that should be running.</param>
+/// <param name="Reason">
+/// Short human-readable provenance for the request (e.g. <c>"startup reconcile"</c>), carried
+/// verbatim into the log line and the operational site event so the heal is not silent.
+/// </param>
+public record EnsureInstanceStartedCommand(
+    string InstanceUniqueName,
+    string Reason);

src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs

@@ -1024,6 +1024,11 @@ akka {{
         // guarded-writes it (orphans are logged, never deleted). Requires the HTTP
         // config fetcher; if it is somehow absent the self-heal is skipped (best-effort —
         // replication remains the primary path and the next restart retries).
+        // The dmProxy is passed so the pass heals the RUNTIME too (register row 67): each
+        // enabled instance it restores gets an EnsureInstanceStartedCommand routed to the
+        // singleton, which starts the Instance Actor if the boot-time load missed it. Going
+        // through the PROXY (not a local ActorOf) keeps Instance Actors under the singleton
+        // regardless of which node's reconcile pass found the gap.
         if (deploymentConfigFetcher != null)
         {
             var reconcileLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
@@ -1037,7 +1042,8 @@ akka {{
                     _nodeOptions.NodeName,
                     reconcileLogger,
                     null,
-                    null)),
+                    null,
+                    dmProxy)),
                 "site-reconciliation");
             _logger.LogInformation(
                 "SiteReconciliationActor created (per-node startup self-heal) for site {SiteId} node {Node}",

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -255,6 +255,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         Receive<RefreshFetched>(HandleRefreshFetched);
         Receive<RefreshFetchFailed>(HandleRefreshFetchFailed);
 
+        // Reconcile self-heal: a config the node was missing at boot has just been
+        // fetched + stored by SiteReconciliationActor, which heals the FILE only. Start the
+        // Instance Actor too, so a missing-config boot no longer needs a SECOND restart to
+        // run. Idempotent and enabled-state-aware; the durable row is the authority.
+        Receive<EnsureInstanceStartedCommand>(HandleEnsureInstanceStarted);
+        Receive<EnsureInstanceStartedResult>(HandleEnsureInstanceStartedResult);
+
         // Query-the-site-before-redeploy idempotency.
         // Central asks for the instance's currently-applied deployment identity
         // before re-sending a deployment whose prior record is stuck InProgress
@@ -1283,6 +1290,129 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _logger.LogInformation("Instance {Instance} enabled", instanceName);
     }
 
+    /// <summary>
+    /// Reconcile self-heal (deferred register row 67): makes sure an instance whose config was
+    /// (re)stored out-of-band is actually RUNNING.
+    ///
+    /// <para>
+    /// <b>Why this exists.</b> Instance Actors were created from exactly two paths — the
+    /// boot-time SQLite load (<see cref="HandleStartupConfigsLoaded"/>) and an explicit
+    /// deploy/enable command. <c>SiteReconciliationActor</c>'s node-STARTUP fetch runs AFTER
+    /// that boot load has already read storage, so a config it healed into the file started no
+    /// actor: the instance stayed dead until the next restart's load finally picked it up.
+    /// This handler closes that gap without touching the deploy pipeline.
+    /// </para>
+    ///
+    /// <para>
+    /// <b>Guards.</b> The command is advisory and fire-and-forget, so every reason to do
+    /// nothing is a silent return: the actor is already running (idempotent — both nodes'
+    /// reconcile actors legitimately raise this for the same instance), or an in-flight
+    /// deploy/redeploy owns the instance and will create the actor itself. Deferring to the
+    /// deploy paths is what keeps the deploy gate's ordering guarantees and the
+    /// all-or-nothing-per-instance rule intact — this never applies a config, it only starts
+    /// one that is already fully persisted.
+    /// </para>
+    /// </summary>
+    private void HandleEnsureInstanceStarted(EnsureInstanceStartedCommand command)
+    {
+        var instanceName = command.InstanceUniqueName;
+
+        if (!ShouldAttemptEnsureStart(instanceName))
+            return;
+
+        // Re-read the durable row off the actor thread. The stored row — NOT the requesting
+        // message — is the authority on the config body AND on enabled/disabled state, so a
+        // disable that landed after the reconcile fetch still wins and the instance stays
+        // stopped.
+        Task.Run(async () =>
+        {
+            var configs = await _storage.GetAllDeployedConfigsAsync();
+            var config = configs.FirstOrDefault(c => c.InstanceUniqueName == instanceName);
+            return new EnsureInstanceStartedResult(command, config, null);
+        }).ContinueWith(t => t.IsCompletedSuccessfully
+                ? t.Result
+                : new EnsureInstanceStartedResult(command, null, t.Exception?.GetBaseException().Message))
+            .PipeTo(Self);
+    }
+
+    /// <summary>
+    /// Shared guard for the two halves of the ensure-started round trip. Re-evaluated after
+    /// the storage read because the instance's state can change while that read is in flight
+    /// (a deploy or delete may have arrived on the mailbox in between).
+    /// </summary>
+    private bool ShouldAttemptEnsureStart(string instanceName) =>
+        !_instanceActors.ContainsKey(instanceName) &&
+        !_deployWarms.ContainsKey(instanceName) &&
+        !_terminatingActorsByName.ContainsKey(instanceName) &&
+        !_inFlightDeployments.ContainsKey(instanceName);
+
+    /// <summary>
+    /// Actor-thread half of the reconcile self-heal: creates the Instance Actor for a config
+    /// that is present and enabled in storage but has no running actor, and SURFACES the heal
+    /// (log + operational site event) so "config stored, actor not running" is no longer
+    /// silent.
+    /// </summary>
+    private void HandleEnsureInstanceStartedResult(EnsureInstanceStartedResult result)
+    {
+        var instanceName = result.Command.InstanceUniqueName;
+        var reason = result.Command.Reason;
+
+        if (result.Error != null)
+        {
+            _logger.LogWarning(
+                "Ensure-started ({Reason}) for {Instance} could not read the deployed config: {Error}",
+                reason, instanceName, result.Error);
+            return;
+        }
+
+        // No stored row: nothing to start. Normal when the config was superseded/removed
+        // between the reconcile fetch and this read, and on an UNREPLICATED site whose peer
+        // node healed its own file — this singleton's storage legitimately has no such row.
+        if (result.Config == null)
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: no deployed config on this node — nothing to start",
+                reason, instanceName);
+            return;
+        }
+
+        // Disabled instances stay stopped. The heal restores the config row either way; a
+        // disabled instance having a row but no actor is the correct steady state.
+        if (!result.Config.IsEnabled)
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: instance is disabled — leaving it stopped",
+                reason, instanceName);
+            return;
+        }
+
+        // Re-check the guards: a deploy/delete for this instance may have arrived while the
+        // storage read was in flight, and it is authoritative over this advisory start.
+        if (!ShouldAttemptEnsureStart(instanceName))
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: already running or superseded by an in-flight deploy",
+                reason, instanceName);
+            return;
+        }
+
+        CreateInstanceActor(instanceName, result.Config.ConfigJson);
+        _deployedInstanceNames.Add(instanceName);
+        UpdateInstanceCounts();
+
+        // Surface the heal. Warning (not Info) on purpose: reaching here means the node
+        // booted WITHOUT a config for an enabled, successfully-deployed instance — the
+        // condition is repaired, but it is abnormal and worth an operator's attention.
+        _logger.LogWarning(
+            "Instance {Instance} started by {Reason} self-heal — its config was missing at boot "
+            + "and has been restored; the instance was NOT running until now",
+            instanceName, reason);
+        LogDeploymentEvent("Warning", instanceName,
+            $"Instance {instanceName} started by {reason} self-heal",
+            "Config was missing at node boot and was restored by reconciliation; "
+            + "the Instance Actor was not running and has now been started.");
+    }
+
     /// <summary>
     /// Deletes an instance: stops the actor and removes config from SQLite.
     /// Note: store-and-forward messages are NOT cleared per design decision.
@@ -2419,6 +2549,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
     internal record EnableResult(
         EnableInstanceCommand Command, DeployedInstance? Config, string? Error, IActorRef OriginalSender);
+
+    /// <summary>
+    /// Carries the durable row read for an <see cref="EnsureInstanceStartedCommand"/> back onto
+    /// the actor thread. No original sender: the ensure-started path is fire-and-forget, so
+    /// there is nobody to answer.
+    /// </summary>
+    internal record EnsureInstanceStartedResult(
+        EnsureInstanceStartedCommand Command, DeployedInstance? Config, string? Error);
     internal record DeployPersistenceResult(
         string DeploymentId, string InstanceName, bool Success, string? Error,
         IActorRef OriginalSender, bool IsRedeploy);

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReconciliationActor.cs

@@ -14,7 +14,9 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 ///   <item>read the node's local deployed inventory from SQLite,</item>
 ///   <item>report it to central via the <c>SiteCommunicationActor</c> Ask
 ///         (<see cref="ReconcileSiteRequest"/> → <see cref="ReconcileSiteResponse"/>),</item>
-///   <item>fetch each gap item's config over HTTP and guarded-write it, and</item>
+///   <item>fetch each gap item's config over HTTP and guarded-write it,</item>
+///   <item>ask the Deployment Manager singleton to START each enabled instance it just
+///         healed, and</item>
 ///   <item>LOG (never delete) any orphan the node still has but central no longer deploys.</item>
 /// </list>
 /// </summary>
@@ -36,6 +38,17 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 /// The site does NOT carry the central fetch base URL in its own config — it uses
 /// <see cref="ReconcileSiteResponse.CentralFetchBaseUrl"/> from central's reply.
 /// </para>
+/// <para>
+/// <b>The pass heals the runtime, not just the file</b> (deferred register row 67). Storing a
+/// config starts nothing by itself: the Deployment Manager creates Instance Actors from the
+/// boot-time SQLite load, which has already run by the time this pass fires. So for every
+/// ENABLED instance whose config this pass stored, an <see cref="EnsureInstanceStartedCommand"/>
+/// is sent to the Deployment Manager singleton, which starts the actor if it is not already
+/// running. Without it, an instance whose config was missing at boot (lost volume, first boot
+/// after a restore, rehomed data dir) needed a SECOND restart before it ran again.
+/// The command is advisory, idempotent, and deliberately fire-and-forget — a site with no
+/// Deployment Manager ref configured simply keeps the old file-only behavior.
+/// </para>
 /// </remarks>
 public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
 {
@@ -44,6 +57,7 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
     private readonly SiteStorageService _storage;
     private readonly IDeploymentConfigFetcher _configFetcher;
     private readonly IActorRef _siteCommunicationActor;
+    private readonly IActorRef? _deploymentManager;
     private readonly string _siteIdentifier;
     private readonly string _nodeId;
     private readonly ILogger<SiteReconciliationActor> _logger;
@@ -71,6 +85,14 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
     /// Defaults to 5 seconds.
     /// </param>
     /// <param name="askTimeout">Round-trip timeout for the reconcile Ask to central. Defaults to 30 seconds.</param>
+    /// <param name="deploymentManager">
+    /// The Deployment Manager singleton PROXY. Every enabled instance this pass heals is
+    /// followed by a fire-and-forget <see cref="EnsureInstanceStartedCommand"/> so the runtime
+    /// heals with the file (register row 67). Routing through the proxy — rather than starting
+    /// anything locally — is what keeps Instance Actors under the singleton no matter which
+    /// node's reconcile pass found the gap. Optional: when null the pass stores configs and
+    /// starts nothing, exactly as it did before.
+    /// </param>
     public SiteReconciliationActor(
         SiteStorageService storage,
         IDeploymentConfigFetcher configFetcher,
@@ -79,11 +101,13 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
         string nodeId,
         ILogger<SiteReconciliationActor> logger,
         TimeSpan? initialDelay = null,
-        TimeSpan? askTimeout = null)
+        TimeSpan? askTimeout = null,
+        IActorRef? deploymentManager = null)
     {
         _storage = storage;
         _configFetcher = configFetcher;
         _siteCommunicationActor = siteCommunicationActor;
+        _deploymentManager = deploymentManager;
         _siteIdentifier = siteIdentifier;
         _nodeId = nodeId;
         _logger = logger;
@@ -153,6 +177,9 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
 
         var fetched = 0;
         var failed = 0;
+        // Enabled instances healed by this pass, to be started via the Deployment Manager once
+        // the pass reports back on the actor thread (row 67).
+        var healedEnabled = new List<string>();
 
         // Fetch + guarded-write each gap item. Per-item failure must not abort the rest.
         foreach (var item in response.Gap)
@@ -168,6 +195,12 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
                     .ConfigureAwait(false);
 
                 fetched++;
+                // Only ENABLED instances are candidates to start; a disabled one correctly
+                // keeps a config row with no running actor. The Deployment Manager re-checks
+                // this against the durable row anyway — this is just the cheap filter.
+                if (item.IsEnabled)
+                    healedEnabled.Add(item.InstanceUniqueName);
+
                 _logger.LogInformation(
                     "Reconcile: fetched + stored config for {Instance} (deployment {DeploymentId}, rev {Revision})",
                     item.InstanceUniqueName, item.DeploymentId, item.RevisionHash);
@@ -198,7 +231,7 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
                 name);
         }
 
-        return ReconcilePassResult.Completed(fetched, failed, response.OrphanNames.Count);
+        return ReconcilePassResult.Completed(fetched, failed, response.OrphanNames.Count, healedEnabled);
     }
 
     private void HandleReconcilePassResult(ReconcilePassResult result)
@@ -216,6 +249,23 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
         _logger.LogInformation(
             "Reconcile pass for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
             _siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
+
+        // Heal the RUNTIME, not just the file (row 67). Storing a config starts nothing: the
+        // Deployment Manager's boot-time SQLite load has already run by now, so without this
+        // an instance missing at boot would stay dead until the next restart. Fire-and-forget
+        // through the singleton proxy — the handler is idempotent, re-checks the durable row's
+        // enabled state, and defers to any in-flight deploy.
+        if (_deploymentManager == null || result.HealedEnabled.Count == 0)
+            return;
+
+        foreach (var instanceName in result.HealedEnabled)
+        {
+            _logger.LogInformation(
+                "Reconcile: requesting start of healed instance {Instance} on site {Site}",
+                instanceName, _siteIdentifier);
+            _deploymentManager.Tell(
+                new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
+        }
     }
 
     // ── Internal messages ──
@@ -227,21 +277,31 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
         private RunReconcile() { }
     }
 
-    /// <summary>Summary of one reconcile pass, piped to <c>Self</c> for logging.</summary>
-    private sealed record ReconcilePassResult(int Fetched, int Failed, int Orphans, Exception? Error)
+    /// <summary>
+    /// Summary of one reconcile pass, piped to <c>Self</c> for logging and for the
+    /// ensure-started dispatch. <see cref="HealedEnabled"/> is built off the actor thread but
+    /// only ever read on it, so no synchronization is needed.
+    /// </summary>
+    private sealed record ReconcilePassResult(
+        int Fetched, int Failed, int Orphans, IReadOnlyList<string> HealedEnabled, Exception? Error)
     {
         /// <summary>Creates a result for a reconcile pass that ran to completion.</summary>
         /// <param name="fetched">Number of instances successfully fetched/reconciled.</param>
         /// <param name="failed">Number of instances that failed to reconcile.</param>
         /// <param name="orphans">Number of local instances no longer deployed at central.</param>
+        /// <param name="healedEnabled">
+        /// Names of the ENABLED instances whose config this pass stored — each gets an
+        /// <see cref="EnsureInstanceStartedCommand"/> so the runtime heals with the file.
+        /// </param>
         /// <returns>A completed <see cref="ReconcilePassResult"/> with no error.</returns>
-        public static ReconcilePassResult Completed(int fetched, int failed, int orphans)
-            => new(fetched, failed, orphans, null);
+        public static ReconcilePassResult Completed(
+            int fetched, int failed, int orphans, IReadOnlyList<string> healedEnabled)
+            => new(fetched, failed, orphans, healedEnabled, null);
 
         /// <summary>Creates a result for a reconcile pass that failed before completing.</summary>
         /// <param name="error">The exception that caused the pass to fail.</param>
         /// <returns>A faulted <see cref="ReconcilePassResult"/> carrying the error.</returns>
         public static ReconcilePassResult Faulted(Exception error)
-            => new(0, 0, 0, error);
+            => new(0, 0, 0, [], error);
     }
 }