merge(agent-a): rows 85+83 — EnsureDclConnections moved into the CreateInstanceActor funnel (self-healed/enabled instances get their DCL connections); ReconcileStartsInstanceTests gated on first instance-count report

This commit is contained in:
Joseph Doherty
2026-08-17 03:52:35 -04:00
10 changed files with 680 additions and 60 deletions
@@ -646,7 +646,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
for (var i = startIdx; i < endIdx; i++)
{
var config = state.Configs[i];
EnsureDclConnections(config.ConfigJson);
// CreateInstanceActor ensures this config's DCL connections (register row 85).
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson, config.RevisionHash);
}
@@ -1076,10 +1076,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
var instanceName = command.InstanceUniqueName;
// Ensure DCL connections exist for any data-sourced attributes
EnsureDclConnections(command.FlattenedConfigurationJson);
// Create the Instance Actor immediately
// Create the Instance Actor immediately. CreateInstanceActor first ensures the DCL
// connections this config's data-sourced attributes subscribe through (register row 85).
CreateInstanceActor(instanceName, command.FlattenedConfigurationJson, command.RevisionHash);
if (!isRedeploy)
_deployedInstanceNames.Add(instanceName);
@@ -2725,6 +2723,31 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </param>
internal void CreateInstanceActor(string instanceName, string configJson, string revisionHash)
{
// Deferred register row 85 — an Instance Actor is useless without the DCL connection
// actors its data-sourced attributes subscribe through, so ensuring them belongs in the
// single creation funnel rather than at each caller.
//
// This used to be the caller's job, and only TWO of the four callers did it: the
// boot-time batch and ApplyDeployment. The two self-heal starts did not —
// StartStoredInstance (the shared row-67 ensure-started / row-78 sweep path) and
// HandleEnableResult. Both therefore created an instance whose every tag subscribe was
// answered "Unknown connection: X" by the DCL manager, forever, because nothing else on
// the node would ever create that connection actor:
//
// - Row 85's live symptom (2026-08-16): a replication-seeded deployed_configurations row
// is started by the row-78 sweep, but the connection actor is only ever created as a
// side effect of a deploy or the boot load, so tag subscribes failed for 6+ minutes
// until a second pair restart let the boot load create it.
// - The same defect is reachable with no replication at all: the boot load reads only
// ENABLED configs, so a node that boots with an instance DISABLED never creates that
// instance's connections, and enabling it afterwards hit the identical dead end.
//
// Placed BEFORE the already-exists guard so the two paths that already called it keep
// their exact previous semantics (they called it unconditionally). It is cheap and
// silent when there is nothing to do: EnsureDclConnection is hash-guarded on
// _createdConnections, so an unchanged connection is a no-op with no message and no log.
EnsureDclConnections(configJson);
if (_instanceActors.ContainsKey(instanceName))
{
_logger.LogWarning("Instance Actor {Instance} already exists, skipping creation", instanceName);