fix(siteruntime): startup reconcile starts the healed InstanceActor (register row 67)
SiteReconciliationActor's node-STARTUP fetch healed the config FILE but never the RUNTIME. Instance Actors are created from exactly two paths -- the Deployment Manager's boot-time SQLite load and an explicit deploy/enable command -- and the reconcile pass runs AFTER that boot load has already read storage. So a config reconcile restored started no actor: an enabled, successfully-deployed instance whose config was missing at boot (lost volume, first boot after a restore, rehomed data dir) stayed dead until the NEXT restart, whose load finally picked it up. Proven live on site-a 2026-08-15: "Reconcile: fetched + stored config" with no "InstanceActor started" after it, and a debug snapshot answering instanceNotFound. The pass now raises a fire-and-forget EnsureInstanceStartedCommand (new Commons Deployment message) for every ENABLED instance it healed, routed through the deployment-manager singleton PROXY so Instance Actors stay under the singleton no matter which node's reconcile found the gap. The handler is deliberately advisory: - idempotent -- no-op when the actor is already running, which both nodes' reconcile actors legitimately trigger for the same instance; - enabled-state-aware -- it re-reads the durable row, which is the authority on both the config body and enabled/disabled, so a disable landing after the fetch still wins and the instance stays stopped; - it defers to any in-flight deploy/redeploy/warm for the instance, which keeps the deploy gate's ordering guarantees and the all-or-nothing-per-instance rule intact -- this never applies a config, it only starts one already persisted; - a missing row is a quiet no-op, the normal case on an unreplicated site whose peer node healed its own file. The heal is no longer silent: a successful reconcile start emits a Warning log line and an operational `deployment` site event, because reaching it means the node booted without a config for an instance that should have been running.
This commit is contained in:
Joseph Doherty
@@ -255,6 +255,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
Receive<RefreshFetched>(HandleRefreshFetched);
|
||||
Receive<RefreshFetchFailed>(HandleRefreshFetchFailed);
|
||||
|
||||
// Reconcile self-heal: a config the node was missing at boot has just been
|
||||
// fetched + stored by SiteReconciliationActor, which heals the FILE only. Start the
|
||||
// Instance Actor too, so a missing-config boot no longer needs a SECOND restart to
|
||||
// run. Idempotent and enabled-state-aware; the durable row is the authority.
|
||||
Receive<EnsureInstanceStartedCommand>(HandleEnsureInstanceStarted);
|
||||
Receive<EnsureInstanceStartedResult>(HandleEnsureInstanceStartedResult);
|
||||
|
||||
// Query-the-site-before-redeploy idempotency.
|
||||
// Central asks for the instance's currently-applied deployment identity
|
||||
// before re-sending a deployment whose prior record is stuck InProgress
|
||||
@@ -1283,6 +1290,129 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
_logger.LogInformation("Instance {Instance} enabled", instanceName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconcile self-heal (deferred register row 67): makes sure an instance whose config was
|
||||
/// (re)stored out-of-band is actually RUNNING.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Why this exists.</b> Instance Actors were created from exactly two paths — the
|
||||
/// boot-time SQLite load (<see cref="HandleStartupConfigsLoaded"/>) and an explicit
|
||||
/// deploy/enable command. <c>SiteReconciliationActor</c>'s node-STARTUP fetch runs AFTER
|
||||
/// that boot load has already read storage, so a config it healed into the file started no
|
||||
/// actor: the instance stayed dead until the next restart's load finally picked it up.
|
||||
/// This handler closes that gap without touching the deploy pipeline.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Guards.</b> The command is advisory and fire-and-forget, so every reason to do
|
||||
/// nothing is a silent return: the actor is already running (idempotent — both nodes'
|
||||
/// reconcile actors legitimately raise this for the same instance), or an in-flight
|
||||
/// deploy/redeploy owns the instance and will create the actor itself. Deferring to the
|
||||
/// deploy paths is what keeps the deploy gate's ordering guarantees and the
|
||||
/// all-or-nothing-per-instance rule intact — this never applies a config, it only starts
|
||||
/// one that is already fully persisted.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void HandleEnsureInstanceStarted(EnsureInstanceStartedCommand command)
|
||||
{
|
||||
var instanceName = command.InstanceUniqueName;
|
||||
|
||||
if (!ShouldAttemptEnsureStart(instanceName))
|
||||
return;
|
||||
|
||||
// Re-read the durable row off the actor thread. The stored row — NOT the requesting
|
||||
// message — is the authority on the config body AND on enabled/disabled state, so a
|
||||
// disable that landed after the reconcile fetch still wins and the instance stays
|
||||
// stopped.
|
||||
Task.Run(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var config = configs.FirstOrDefault(c => c.InstanceUniqueName == instanceName);
|
||||
return new EnsureInstanceStartedResult(command, config, null);
|
||||
}).ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? t.Result
|
||||
: new EnsureInstanceStartedResult(command, null, t.Exception?.GetBaseException().Message))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shared guard for the two halves of the ensure-started round trip. Re-evaluated after
|
||||
/// the storage read because the instance's state can change while that read is in flight
|
||||
/// (a deploy or delete may have arrived on the mailbox in between).
|
||||
/// </summary>
|
||||
private bool ShouldAttemptEnsureStart(string instanceName) =>
|
||||
!_instanceActors.ContainsKey(instanceName) &&
|
||||
!_deployWarms.ContainsKey(instanceName) &&
|
||||
!_terminatingActorsByName.ContainsKey(instanceName) &&
|
||||
!_inFlightDeployments.ContainsKey(instanceName);
|
||||
|
||||
/// <summary>
|
||||
/// Actor-thread half of the reconcile self-heal: creates the Instance Actor for a config
|
||||
/// that is present and enabled in storage but has no running actor, and SURFACES the heal
|
||||
/// (log + operational site event) so "config stored, actor not running" is no longer
|
||||
/// silent.
|
||||
/// </summary>
|
||||
private void HandleEnsureInstanceStartedResult(EnsureInstanceStartedResult result)
|
||||
{
|
||||
var instanceName = result.Command.InstanceUniqueName;
|
||||
var reason = result.Command.Reason;
|
||||
|
||||
if (result.Error != null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Ensure-started ({Reason}) for {Instance} could not read the deployed config: {Error}",
|
||||
reason, instanceName, result.Error);
|
||||
return;
|
||||
}
|
||||
|
||||
// No stored row: nothing to start. Normal when the config was superseded/removed
|
||||
// between the reconcile fetch and this read, and on an UNREPLICATED site whose peer
|
||||
// node healed its own file — this singleton's storage legitimately has no such row.
|
||||
if (result.Config == null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Ensure-started ({Reason}) for {Instance}: no deployed config on this node — nothing to start",
|
||||
reason, instanceName);
|
||||
return;
|
||||
}
|
||||
|
||||
// Disabled instances stay stopped. The heal restores the config row either way; a
|
||||
// disabled instance having a row but no actor is the correct steady state.
|
||||
if (!result.Config.IsEnabled)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Ensure-started ({Reason}) for {Instance}: instance is disabled — leaving it stopped",
|
||||
reason, instanceName);
|
||||
return;
|
||||
}
|
||||
|
||||
// Re-check the guards: a deploy/delete for this instance may have arrived while the
|
||||
// storage read was in flight, and it is authoritative over this advisory start.
|
||||
if (!ShouldAttemptEnsureStart(instanceName))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Ensure-started ({Reason}) for {Instance}: already running or superseded by an in-flight deploy",
|
||||
reason, instanceName);
|
||||
return;
|
||||
}
|
||||
|
||||
CreateInstanceActor(instanceName, result.Config.ConfigJson);
|
||||
_deployedInstanceNames.Add(instanceName);
|
||||
UpdateInstanceCounts();
|
||||
|
||||
// Surface the heal. Warning (not Info) on purpose: reaching here means the node
|
||||
// booted WITHOUT a config for an enabled, successfully-deployed instance — the
|
||||
// condition is repaired, but it is abnormal and worth an operator's attention.
|
||||
_logger.LogWarning(
|
||||
"Instance {Instance} started by {Reason} self-heal — its config was missing at boot "
|
||||
+ "and has been restored; the instance was NOT running until now",
|
||||
instanceName, reason);
|
||||
LogDeploymentEvent("Warning", instanceName,
|
||||
$"Instance {instanceName} started by {reason} self-heal",
|
||||
"Config was missing at node boot and was restored by reconciliation; "
|
||||
+ "the Instance Actor was not running and has now been started.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes an instance: stops the actor and removes config from SQLite.
|
||||
/// Note: store-and-forward messages are NOT cleared per design decision.
|
||||
@@ -2418,6 +2548,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
|
||||
internal record EnableResult(
|
||||
EnableInstanceCommand Command, DeployedInstance? Config, string? Error, IActorRef OriginalSender);
|
||||
|
||||
/// <summary>
|
||||
/// Carries the durable row read for an <see cref="EnsureInstanceStartedCommand"/> back onto
|
||||
/// the actor thread. No original sender: the ensure-started path is fire-and-forget, so
|
||||
/// there is nobody to answer.
|
||||
/// </summary>
|
||||
internal record EnsureInstanceStartedResult(
|
||||
EnsureInstanceStartedCommand Command, DeployedInstance? Config, string? Error);
|
||||
internal record DeployPersistenceResult(
|
||||
string DeploymentId, string InstanceName, bool Success, string? Error,
|
||||
IActorRef OriginalSender, bool IsRedeploy);
|
||||
|
||||
Reference in New Issue
Block a user