merge: startup reconcile starts the healed InstanceActor (register row 67) — EnsureInstanceStartedCommand to the DM singleton, idempotent, disable-wins, Warning site event on self-heal

2026-08-15 17:56:21 -04:00
parent e8a90f18dc abf864b4e5
commit 744ff36a7b
7 changed files with 631 additions and 9 deletions
@@ -255,6 +255,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
        Receive<RefreshFetched>(HandleRefreshFetched);
        Receive<RefreshFetchFailed>(HandleRefreshFetchFailed);

+        // Reconcile self-heal: a config the node was missing at boot has just been
+        // fetched + stored by SiteReconciliationActor, which heals the FILE only. Start the
+        // Instance Actor too, so a missing-config boot no longer needs a SECOND restart to
+        // run. Idempotent and enabled-state-aware; the durable row is the authority.
+        Receive<EnsureInstanceStartedCommand>(HandleEnsureInstanceStarted);
+        Receive<EnsureInstanceStartedResult>(HandleEnsureInstanceStartedResult);
+
        // Query-the-site-before-redeploy idempotency.
        // Central asks for the instance's currently-applied deployment identity
        // before re-sending a deployment whose prior record is stuck InProgress
@@ -1283,6 +1290,129 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
        _logger.LogInformation("Instance {Instance} enabled", instanceName);
    }

+    /// <summary>
+    /// Reconcile self-heal (deferred register row 67): makes sure an instance whose config was
+    /// (re)stored out-of-band is actually RUNNING.
+    ///
+    /// <para>
+    /// <b>Why this exists.</b> Instance Actors were created from exactly two paths — the
+    /// boot-time SQLite load (<see cref="HandleStartupConfigsLoaded"/>) and an explicit
+    /// deploy/enable command. <c>SiteReconciliationActor</c>'s node-STARTUP fetch runs AFTER
+    /// that boot load has already read storage, so a config it healed into the file started no
+    /// actor: the instance stayed dead until the next restart's load finally picked it up.
+    /// This handler closes that gap without touching the deploy pipeline.
+    /// </para>
+    ///
+    /// <para>
+    /// <b>Guards.</b> The command is advisory and fire-and-forget, so every reason to do
+    /// nothing is a silent return: the actor is already running (idempotent — both nodes'
+    /// reconcile actors legitimately raise this for the same instance), or an in-flight
+    /// deploy/redeploy owns the instance and will create the actor itself. Deferring to the
+    /// deploy paths is what keeps the deploy gate's ordering guarantees and the
+    /// all-or-nothing-per-instance rule intact — this never applies a config, it only starts
+    /// one that is already fully persisted.
+    /// </para>
+    /// </summary>
+    private void HandleEnsureInstanceStarted(EnsureInstanceStartedCommand command)
+    {
+        var instanceName = command.InstanceUniqueName;
+
+        if (!ShouldAttemptEnsureStart(instanceName))
+            return;
+
+        // Re-read the durable row off the actor thread. The stored row — NOT the requesting
+        // message — is the authority on the config body AND on enabled/disabled state, so a
+        // disable that landed after the reconcile fetch still wins and the instance stays
+        // stopped.
+        Task.Run(async () =>
+        {
+            var configs = await _storage.GetAllDeployedConfigsAsync();
+            var config = configs.FirstOrDefault(c => c.InstanceUniqueName == instanceName);
+            return new EnsureInstanceStartedResult(command, config, null);
+        }).ContinueWith(t => t.IsCompletedSuccessfully
+                ? t.Result
+                : new EnsureInstanceStartedResult(command, null, t.Exception?.GetBaseException().Message))
+            .PipeTo(Self);
+    }
+
+    /// <summary>
+    /// Shared guard for the two halves of the ensure-started round trip. Re-evaluated after
+    /// the storage read because the instance's state can change while that read is in flight
+    /// (a deploy or delete may have arrived on the mailbox in between).
+    /// </summary>
+    private bool ShouldAttemptEnsureStart(string instanceName) =>
+        !_instanceActors.ContainsKey(instanceName) &&
+        !_deployWarms.ContainsKey(instanceName) &&
+        !_terminatingActorsByName.ContainsKey(instanceName) &&
+        !_inFlightDeployments.ContainsKey(instanceName);
+
+    /// <summary>
+    /// Actor-thread half of the reconcile self-heal: creates the Instance Actor for a config
+    /// that is present and enabled in storage but has no running actor, and SURFACES the heal
+    /// (log + operational site event) so "config stored, actor not running" is no longer
+    /// silent.
+    /// </summary>
+    private void HandleEnsureInstanceStartedResult(EnsureInstanceStartedResult result)
+    {
+        var instanceName = result.Command.InstanceUniqueName;
+        var reason = result.Command.Reason;
+
+        if (result.Error != null)
+        {
+            _logger.LogWarning(
+                "Ensure-started ({Reason}) for {Instance} could not read the deployed config: {Error}",
+                reason, instanceName, result.Error);
+            return;
+        }
+
+        // No stored row: nothing to start. Normal when the config was superseded/removed
+        // between the reconcile fetch and this read, and on an UNREPLICATED site whose peer
+        // node healed its own file — this singleton's storage legitimately has no such row.
+        if (result.Config == null)
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: no deployed config on this node — nothing to start",
+                reason, instanceName);
+            return;
+        }
+
+        // Disabled instances stay stopped. The heal restores the config row either way; a
+        // disabled instance having a row but no actor is the correct steady state.
+        if (!result.Config.IsEnabled)
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: instance is disabled — leaving it stopped",
+                reason, instanceName);
+            return;
+        }
+
+        // Re-check the guards: a deploy/delete for this instance may have arrived while the
+        // storage read was in flight, and it is authoritative over this advisory start.
+        if (!ShouldAttemptEnsureStart(instanceName))
+        {
+            _logger.LogDebug(
+                "Ensure-started ({Reason}) for {Instance}: already running or superseded by an in-flight deploy",
+                reason, instanceName);
+            return;
+        }
+
+        CreateInstanceActor(instanceName, result.Config.ConfigJson);
+        _deployedInstanceNames.Add(instanceName);
+        UpdateInstanceCounts();
+
+        // Surface the heal. Warning (not Info) on purpose: reaching here means the node
+        // booted WITHOUT a config for an enabled, successfully-deployed instance — the
+        // condition is repaired, but it is abnormal and worth an operator's attention.
+        _logger.LogWarning(
+            "Instance {Instance} started by {Reason} self-heal — its config was missing at boot "
+            + "and has been restored; the instance was NOT running until now",
+            instanceName, reason);
+        LogDeploymentEvent("Warning", instanceName,
+            $"Instance {instanceName} started by {reason} self-heal",
+            "Config was missing at node boot and was restored by reconciliation; "
+            + "the Instance Actor was not running and has now been started.");
+    }
+
    /// <summary>
    /// Deletes an instance: stops the actor and removes config from SQLite.
    /// Note: store-and-forward messages are NOT cleared per design decision.
@@ -2419,6 +2549,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
    internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
    internal record EnableResult(
        EnableInstanceCommand Command, DeployedInstance? Config, string? Error, IActorRef OriginalSender);
+
+    /// <summary>
+    /// Carries the durable row read for an <see cref="EnsureInstanceStartedCommand"/> back onto
+    /// the actor thread. No original sender: the ensure-started path is fire-and-forget, so
+    /// there is nobody to answer.
+    /// </summary>
+    internal record EnsureInstanceStartedResult(
+        EnsureInstanceStartedCommand Command, DeployedInstance? Config, string? Error);
    internal record DeployPersistenceResult(
        string DeploymentId, string InstanceName, bool Success, string? Error,
        IActorRef OriginalSender, bool IsRedeploy);