fix(siteruntime): self-healed and newly-enabled instances now get their DCL connections (register row 85)
An Instance Actor started by a self-heal came up with NO DCL connection actors, so every tag it subscribed was answered "Unknown connection: <name>" by the DCL manager indefinitely. Proven live 2026-08-16 by the row-78 close-out probe: the sweep correctly started the replication-seeded instances, then every subscribe failed for 6+ min until a second pair restart. Root cause is narrower than the register assumed. A DCL connection actor is never created from data_connection_definitions -- that table has no production reader at all (GetAllDataConnectionDefinitionsAsync is called only by a test). It exists only as a side effect of starting an instance, from the Connections map inside that instance's own flattened config, via EnsureDclConnections. That ensure was the CALLER's job and only two of the four callers made it: CreateInstanceActorBatch (boot) ensured OK ApplyDeployment (deploy) ensured OK StartStoredInstance MISSING <- row-67 ensure-started + row-78 sweep HandleEnableResult MISSING <- enabling a disabled instance The two misses are the two paths added after the original pair -- the signature of a convention living at call sites instead of in the funnel. It was never replication-specific: the boot load reads only ENABLED configs, so a node booting with an instance disabled never creates its connections either, and enabling it afterwards hit the identical dead end on an ordinary unreplicated site. Fix: EnsureDclConnections moves into CreateInstanceActor, already documented as the single Instance Actor creation funnel (it is where the running revision is stamped for the same reason), and the two caller-side calls are removed. Placed before the already-exists guard so the two paths that already called it keep their exact previous semantics. No new message, option or DCL change -- it reuses EnsureDclConnection's existing per-name hash guard, so steady state is silent. Structural rather than another guard: a fifth start path cannot forget. Deliberately NOT done, with rationale in the memo: a sweep pass over data_connection_definitions. It would not have fixed row 85 (that table is empty on a site that has only ever received instance deploys), and re-hashing from it would fight the deploy path over the same per-name hash slot, recreating connection actors and dropping live subscriptions on a loop. Two residual sibling gaps (connection-only artifact deploy replicating to a peer that never adopts it, and the mirror-image removal case) are recorded in the memo and the component docs as open, not silently skipped. Tests: SelfHealCreatesDclConnectionTests, 7 tests asserting on the CreateConnectionCommand messages sent to a recording DCL stand-in. Verified fail-before against a behavioral revert (call removed from the funnel, restored at the two original sites): 4 fail / 3 pass, the 3 being steady-state silence plus two refactor guards that must pass on both sides. Docs: sweep design memo gains section 7; Component-SiteRuntime, Component-DataConnectionLayer, docs/components/SiteRuntime and docs/components/DataConnectionLayer updated (the deploy-flow step ordering in docs/components/SiteRuntime was stale as written).
This commit is contained in:
@@ -6,7 +6,6 @@ using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
@@ -89,7 +88,7 @@ public class InstanceSweepSelfHealTests : TestKit, IDisposable
|
||||
FakeSiteEventLogger? siteLog = null,
|
||||
int sweepSeconds = SweepSeconds)
|
||||
{
|
||||
var health = new StartupProbe();
|
||||
var health = new DeploymentManagerStartupProbe();
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage,
|
||||
_compilationService,
|
||||
@@ -511,39 +510,4 @@ public class InstanceSweepSelfHealTests : TestKit, IDisposable
|
||||
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal <see cref="ISiteHealthCollector"/> whose only job is to expose the observable edge
|
||||
/// "the Deployment Manager's startup config load has completed". The manager calls
|
||||
/// <see cref="SetInstanceCounts"/> unconditionally the moment that load finishes — including
|
||||
/// for an empty site — so a non-zero <see cref="ReportCount"/> replaces a wall-clock sleep.
|
||||
/// </summary>
|
||||
private sealed class StartupProbe : ISiteHealthCollector
|
||||
{
|
||||
private int _reports;
|
||||
|
||||
public int ReportCount => Volatile.Read(ref _reports);
|
||||
|
||||
public void SetInstanceCounts(int deployed, int enabled, int disabled) =>
|
||||
Interlocked.Increment(ref _reports);
|
||||
|
||||
public void IncrementScriptError() { }
|
||||
public void IncrementAlarmError() { }
|
||||
public void IncrementDeadLetter() { }
|
||||
public void IncrementSiteAuditWriteFailures() { }
|
||||
public void IncrementAuditRedactionFailure() { }
|
||||
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
|
||||
public void RemoveConnection(string connectionName) { }
|
||||
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
|
||||
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
|
||||
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
|
||||
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
|
||||
public void SetParkedMessageCount(int count) { }
|
||||
public void SetNodeHostname(string hostname) { }
|
||||
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
|
||||
public void SetActiveNode(bool isActive) { }
|
||||
public bool IsActiveNode => true;
|
||||
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId) =>
|
||||
throw new NotSupportedException("Startup probe collects no report.");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user