merge(agent-a): rows 85+83 — EnsureDclConnections moved into the CreateInstanceActor funnel (self-healed/enabled instances get their DCL connections); ReconcileStartsInstanceTests gated on first instance-count report
This commit is contained in:
@@ -6,7 +6,6 @@ using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
@@ -89,7 +88,7 @@ public class InstanceSweepSelfHealTests : TestKit, IDisposable
|
||||
FakeSiteEventLogger? siteLog = null,
|
||||
int sweepSeconds = SweepSeconds)
|
||||
{
|
||||
var health = new StartupProbe();
|
||||
var health = new DeploymentManagerStartupProbe();
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage,
|
||||
_compilationService,
|
||||
@@ -511,39 +510,4 @@ public class InstanceSweepSelfHealTests : TestKit, IDisposable
|
||||
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal <see cref="ISiteHealthCollector"/> whose only job is to expose the observable edge
|
||||
/// "the Deployment Manager's startup config load has completed". The manager calls
|
||||
/// <see cref="SetInstanceCounts"/> unconditionally the moment that load finishes — including
|
||||
/// for an empty site — so a non-zero <see cref="ReportCount"/> replaces a wall-clock sleep.
|
||||
/// </summary>
|
||||
private sealed class StartupProbe : ISiteHealthCollector
|
||||
{
|
||||
private int _reports;
|
||||
|
||||
public int ReportCount => Volatile.Read(ref _reports);
|
||||
|
||||
public void SetInstanceCounts(int deployed, int enabled, int disabled) =>
|
||||
Interlocked.Increment(ref _reports);
|
||||
|
||||
public void IncrementScriptError() { }
|
||||
public void IncrementAlarmError() { }
|
||||
public void IncrementDeadLetter() { }
|
||||
public void IncrementSiteAuditWriteFailures() { }
|
||||
public void IncrementAuditRedactionFailure() { }
|
||||
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
|
||||
public void RemoveConnection(string connectionName) { }
|
||||
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
|
||||
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
|
||||
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
|
||||
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
|
||||
public void SetParkedMessageCount(int count) { }
|
||||
public void SetNodeHostname(string hostname) { }
|
||||
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
|
||||
public void SetActiveNode(bool isActive) { }
|
||||
public bool IsActiveNode => true;
|
||||
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId) =>
|
||||
throw new NotSupportedException("Startup probe collects no report.");
|
||||
}
|
||||
}
|
||||
|
||||
+39
-11
@@ -68,20 +68,49 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
private IActorRef CreateDeploymentManager(IServiceProvider? serviceProvider = null) =>
|
||||
ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
/// <summary>
|
||||
/// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read
|
||||
/// storage — its first <c>ISiteHealthCollector.SetInstanceCounts</c> report.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Deferred register row 83.</b> This used to be a bare <c>Task.Delay(500)</c>. Every test
|
||||
/// here writes its <c>deployed_configurations</c> row AFTER the manager is constructed, and a
|
||||
/// row that lands before the boot load reads storage is picked up by that load — so the test
|
||||
/// would still pass, while proving the BOOT LOAD works rather than the startup-reconcile path
|
||||
/// it claims to cover. A sleep only makes that race unlikely; on a loaded machine the load can
|
||||
/// still be in flight when it expires, and the test silently changes what it proves instead of
|
||||
/// failing. The row-78 fix session hit exactly this trap and closed it in
|
||||
/// <c>InstanceSweepSelfHealTests</c>; this is the same gate, applied to row 67's suite.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The sweep is disabled (<c>InstanceSweepIntervalSeconds = 0</c>) so these tests keep proving
|
||||
/// the row-67 <see cref="EnsureInstanceStartedCommand"/> path specifically — with the sweep
|
||||
/// armed, its periodic self-heal would independently start the same instances and the suite
|
||||
/// would no longer distinguish the two mechanisms.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private IActorRef CreateDeploymentManager(IServiceProvider? serviceProvider = null)
|
||||
{
|
||||
var health = new DeploymentManagerStartupProbe();
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage,
|
||||
_compilationService,
|
||||
_sharedScriptLibrary,
|
||||
null,
|
||||
new SiteRuntimeOptions(),
|
||||
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
dclManager: null,
|
||||
healthCollector: null,
|
||||
healthCollector: health,
|
||||
serviceProvider: serviceProvider,
|
||||
loggerFactory: null,
|
||||
configFetcher: null)));
|
||||
|
||||
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
|
||||
TimeSpan.FromMilliseconds(25));
|
||||
return dm;
|
||||
}
|
||||
|
||||
private IActorRef CreateReconciliationActor(
|
||||
IActorRef siteCommunicationActor,
|
||||
IDeploymentConfigFetcher fetcher,
|
||||
@@ -143,7 +172,6 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
// Boot the Deployment Manager against EMPTY storage — this is the missing-config boot
|
||||
// (lost volume / first boot after a restore / rehomed data dir).
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
// Baseline: the instance is genuinely not running.
|
||||
Assert.False(await InstanceIsRunningAsync(dm, "HealedPump"));
|
||||
@@ -188,7 +216,6 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var dm = CreateDeploymentManager(new SingleServiceProvider(siteLog));
|
||||
await Task.Delay(500);
|
||||
|
||||
const string instanceName = "SurfacedPump";
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
@@ -286,18 +313,21 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
public async Task EnsureInstanceStarted_IsIdempotent_WhenAlreadyRunning()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
const string instanceName = "IdempotentPump";
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-idem", instanceName, "sha256:abc",
|
||||
MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
|
||||
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
|
||||
await Task.Delay(1000);
|
||||
|
||||
// Wait on the instance actually being routable, not on the clock.
|
||||
await AwaitAssertAsync(async () =>
|
||||
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10));
|
||||
|
||||
var before = await ResolveInstanceChildAsync(dm, instanceName);
|
||||
|
||||
// Two redundant ensure-started requests, as the two nodes would send.
|
||||
// Two redundant ensure-started requests, as the two nodes would send. This one IS a
|
||||
// settle wait: the assertion below is that nothing happened, which cannot be awaited on.
|
||||
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
||||
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
||||
await Task.Delay(1000);
|
||||
@@ -317,7 +347,6 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
public async Task EnsureInstanceStarted_DisabledInstance_StaysStopped()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
const string instanceName = "DisabledPump";
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
@@ -338,7 +367,6 @@ public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
||||
public async Task EnsureInstanceStarted_UnknownInstance_IsAQuietNoOp()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
dm.Tell(new EnsureInstanceStartedCommand("NeverHeardOf", "startup reconcile"));
|
||||
await Task.Delay(1500);
|
||||
|
||||
+374
@@ -0,0 +1,374 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Regression tests for deferred register row <b>85</b> — <b>an Instance Actor started by a
|
||||
/// self-heal got no DCL connection actors</b>, so every tag it subscribed was answered
|
||||
/// <c>Unknown connection: X</c> forever.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>The defect.</b> A DCL connection actor is never created from the
|
||||
/// <c>data_connection_definitions</c> table — that table has no production reader at all. It is
|
||||
/// created only as a side effect of starting an instance, from the <c>Connections</c> map inside
|
||||
/// that instance's own flattened config, via <c>EnsureDclConnections</c>. That call used to be
|
||||
/// each CALLER's job, and only two of the four callers made it: the boot-time batch and
|
||||
/// <c>ApplyDeployment</c>. The two self-heal starts did not:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item><c>StartStoredInstance</c> — the path shared by the row-67 startup-reconcile
|
||||
/// ensure-started command and the row-78 sweep.</item>
|
||||
/// <item><c>HandleEnableResult</c> — enabling an instance that was disabled at boot.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Row 85's live sequence (2026-08-16).</b> A fresh node-a was seeded by LocalDb CDC
|
||||
/// replication; the row-78 sweep correctly started the enabled instances, and then every tag
|
||||
/// subscribe failed for 6+ minutes with <c>Unknown connection: OPC PLC Simulator</c> /
|
||||
/// <c>No connection actor for OPC PLC Simulator</c>, because nothing on that node had created the
|
||||
/// connection actor. A second pair restart healed it — the boot load creates connections, so it
|
||||
/// papered over the gap.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>The enable path proves this is not a replication-only bug.</b> The boot-time load reads
|
||||
/// only ENABLED configs, so a node that boots with an instance disabled never creates that
|
||||
/// instance's connections either; enabling it afterwards hit the identical dead end with no
|
||||
/// replication anywhere in the picture.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>The fix</b> moves <c>EnsureDclConnections</c> into <c>CreateInstanceActor</c> — already
|
||||
/// documented as the single Instance Actor creation funnel — so all four paths get it and a
|
||||
/// fifth path cannot forget. Design note:
|
||||
/// <c>docs/plans/2026-08-15-instance-sweep-self-heal-design.md</c> §7.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SelfHealCreatesDclConnectionTests : TestKit, IDisposable
|
||||
{
|
||||
private const string ConnectionName = "OPC PLC Simulator";
|
||||
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
/// <summary>
|
||||
/// Every <see cref="CreateConnectionCommand"/> the Deployment Manager sent to the DCL, in
|
||||
/// order. This is the exact message whose absence WAS register row 85.
|
||||
/// </summary>
|
||||
private readonly ConcurrentQueue<CreateConnectionCommand> _createdConnections = new();
|
||||
|
||||
public SelfHealCreatesDclConnectionTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("dcl-selfheal-test");
|
||||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedScriptLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
|
||||
Shutdown();
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
/// <summary>Shortest sweep the (whole-second) option allows.</summary>
|
||||
private const int SweepSeconds = 1;
|
||||
|
||||
/// <summary>Long enough for at least three sweep ticks to have come and gone.</summary>
|
||||
private static readonly TimeSpan SeveralSweeps = TimeSpan.FromSeconds(4);
|
||||
|
||||
/// <summary>
|
||||
/// Starts a Deployment Manager wired to a recording stand-in for the DCL manager, and blocks
|
||||
/// until its boot-time config load has PROVABLY read storage (its first instance-count
|
||||
/// report — see <see cref="DeploymentManagerStartupProbe"/>).
|
||||
///
|
||||
/// <para>
|
||||
/// That gate is load-bearing for the post-boot tests here for the same reason it is in
|
||||
/// <c>InstanceSweepSelfHealTests</c>: a config row written before the boot load reads storage
|
||||
/// is picked up by the boot load, which DOES create connections — so an ungated test would
|
||||
/// pass while proving the boot load works rather than the self-heal path it names.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private IActorRef CreateBootedDeploymentManager(int sweepSeconds = SweepSeconds)
|
||||
{
|
||||
var dclSpy = ActorOf(Props.Create(() => new DclConnectionSpy(_createdConnections)));
|
||||
var health = new DeploymentManagerStartupProbe();
|
||||
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage,
|
||||
_compilationService,
|
||||
_sharedScriptLibrary,
|
||||
null,
|
||||
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = sweepSeconds },
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
dclManager: dclSpy,
|
||||
healthCollector: health,
|
||||
serviceProvider: null,
|
||||
loggerFactory: null,
|
||||
configFetcher: null)));
|
||||
|
||||
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
|
||||
TimeSpan.FromMilliseconds(25));
|
||||
return dm;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A flattened config carrying one data connection — the <c>Connections</c> map is what
|
||||
/// <c>EnsureDclConnections</c> reads, and it travels inside every instance's own config.
|
||||
/// </summary>
|
||||
private static string MakeConfigJson(string instanceName, string endpoint = "opc.tcp://plc:4840") =>
|
||||
JsonSerializer.Serialize(new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
Attributes =
|
||||
[
|
||||
new ResolvedAttribute
|
||||
{
|
||||
CanonicalName = "TestAttr", Value = "42", DataType = "Int32"
|
||||
}
|
||||
],
|
||||
Connections = new Dictionary<string, ConnectionConfig>
|
||||
{
|
||||
[ConnectionName] = new()
|
||||
{
|
||||
Protocol = "OpcUa",
|
||||
ConfigurationJson = $"{{\"endpointUrl\":\"{endpoint}\"}}",
|
||||
FailoverRetryCount = 3
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
private IReadOnlyList<CreateConnectionCommand> CreatesFor(string connectionName) =>
|
||||
_createdConnections.Where(c => c.ConnectionName == connectionName).ToArray();
|
||||
|
||||
private async Task AwaitConnectionCreatedAsync(string connectionName, string because) =>
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.True(CreatesFor(connectionName).Count > 0, because);
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(20));
|
||||
|
||||
// ── The row-85 reproduction: a self-heal start must bring its connections with it ──
|
||||
|
||||
/// <summary>
|
||||
/// THE row-85 reproduction. The node boots with an empty store, then a config for an ENABLED
|
||||
/// instance appears in storage on its own — exactly what LocalDb's CDC sync applier does when
|
||||
/// it seeds a fresh node from its peer. The row-78 sweep starts the Instance Actor; the DCL
|
||||
/// must be told to create the connection that instance's tags subscribe through.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Fails before the fix</b> at the connection assert: the instance starts (row 78 works)
|
||||
/// but NO <see cref="CreateConnectionCommand"/> is ever sent, which is the live symptom —
|
||||
/// the instance is up and every tag subscribe is answered <c>Unknown connection</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SweepStartedInstance_AlsoCreatesItsDclConnection()
|
||||
{
|
||||
const string instanceName = "ReplicatedPump";
|
||||
var dm = CreateBootedDeploymentManager();
|
||||
|
||||
// Baseline: the node genuinely booted with nothing.
|
||||
Assert.Empty(CreatesFor(ConnectionName));
|
||||
|
||||
// Replication seeds the row — a bare storage write, the CDC applier's only side effect.
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
instanceName, MakeConfigJson(instanceName), "dep-replicated", "rev-replicated", true);
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName,
|
||||
"The sweep started a replication-seeded instance but never asked the DCL to create "
|
||||
+ "its connection — every tag subscribe would be answered 'Unknown connection' "
|
||||
+ "until the node was restarted again (register row 85)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The row-67 startup-reconcile heal shares <c>StartStoredInstance</c> with the row-78 sweep,
|
||||
/// so it had the identical gap: the reconcile fetched and stored a config, the Deployment
|
||||
/// Manager started the instance, and its connections were never created.
|
||||
///
|
||||
/// <para><b>Fails before the fix.</b></para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReconcileEnsureStartedInstance_AlsoCreatesItsDclConnection()
|
||||
{
|
||||
const string instanceName = "ReconciledPump";
|
||||
var dm = CreateBootedDeploymentManager(sweepSeconds: 0); // isolate the reconcile path
|
||||
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
instanceName, MakeConfigJson(instanceName), "dep-r", "rev-r", true);
|
||||
|
||||
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName,
|
||||
"The startup-reconcile ensure-started heal started the instance without creating "
|
||||
+ "its DCL connection (register row 85, row-67 path)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Row 85 without any replication at all.</b> The boot-time load reads only ENABLED
|
||||
/// configs, so a node that boots with this instance DISABLED never created its connection.
|
||||
/// Enabling it afterwards then started the Instance Actor through
|
||||
/// <c>HandleEnableResult</c> — the second caller that never ensured connections — leaving a
|
||||
/// running instance with no connection actor on a perfectly ordinary single site.
|
||||
///
|
||||
/// <para><b>Fails before the fix.</b></para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task EnabledInstance_DisabledAtBoot_AlsoCreatesItsDclConnection()
|
||||
{
|
||||
const string instanceName = "EnabledLaterPump";
|
||||
|
||||
// Stored DISABLED before boot, so the boot load skips it entirely.
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
instanceName, MakeConfigJson(instanceName), "dep-e", "rev-e", false);
|
||||
|
||||
var dm = CreateBootedDeploymentManager(sweepSeconds: 0); // isolate the enable path
|
||||
Assert.Empty(CreatesFor(ConnectionName));
|
||||
|
||||
dm.Tell(new EnableInstanceCommand("cmd-enable", instanceName, DateTimeOffset.UtcNow));
|
||||
ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName,
|
||||
"Enabling an instance that was disabled at boot started it without creating its DCL "
|
||||
+ "connection — register row 85 reached with no replication involved");
|
||||
}
|
||||
|
||||
// ── Guards: the repair must be one-shot and silent in steady state ──
|
||||
|
||||
/// <summary>
|
||||
/// Idempotence: many sweeps, ONE create command. The sweep runs forever on a live node, and
|
||||
/// re-sending the create every tick would be an unbounded message stream — and, since
|
||||
/// <c>HandleCreateConnection</c> warns and returns on a name it already holds, an unbounded
|
||||
/// stream of Warning log lines too.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Fails before the fix</b> (zero commands, not one), and would fail again if the
|
||||
/// <c>_createdConnections</c> hash guard inside <c>EnsureDclConnection</c> were ever lost.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SweepStartedInstance_CreatesTheConnection_ExactlyOnce_AcrossManySweeps()
|
||||
{
|
||||
const string instanceName = "IdempotentConnectionPump";
|
||||
var dm = CreateBootedDeploymentManager();
|
||||
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
instanceName, MakeConfigJson(instanceName), "dep-i", "rev-i", true);
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName, "the sweep never created the connection");
|
||||
|
||||
// Let several more sweeps run over the now-healthy state.
|
||||
await Task.Delay(SeveralSweeps);
|
||||
|
||||
Assert.Single(CreatesFor(ConnectionName));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sweep over a healthy, deployed-the-ordinary-way instance must send the DCL nothing at
|
||||
/// all. The hash guard is what makes the connection-ensure safe to sit in the creation
|
||||
/// funnel, and this pins it against a config that never changes.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Sweep_SendsNoConnectionCommands_WhenThereIsNothingToHeal()
|
||||
{
|
||||
const string instanceName = "HealthyConnectionPump";
|
||||
var dm = CreateBootedDeploymentManager();
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-1", instanceName, "rev-1", MakeConfigJson(instanceName),
|
||||
"admin", DateTimeOffset.UtcNow));
|
||||
Assert.Equal(
|
||||
DeploymentStatus.Success,
|
||||
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName, "the deploy never created the connection");
|
||||
var afterDeploy = _createdConnections.Count;
|
||||
|
||||
await Task.Delay(SeveralSweeps);
|
||||
|
||||
Assert.Equal(afterDeploy, _createdConnections.Count);
|
||||
}
|
||||
|
||||
// ── Regression guards for the two paths that already worked ──
|
||||
|
||||
/// <summary>
|
||||
/// The fix moved <c>EnsureDclConnections</c> out of <c>ApplyDeployment</c> and into
|
||||
/// <c>CreateInstanceActor</c>. This pins that an ordinary central deploy still creates its
|
||||
/// connections — it passes both before and after the fix, which is the point of a
|
||||
/// refactor guard.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OrdinaryDeploy_StillCreatesItsDclConnection()
|
||||
{
|
||||
const string instanceName = "DeployedPump";
|
||||
var dm = CreateBootedDeploymentManager(sweepSeconds: 0);
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-1", instanceName, "rev-1", MakeConfigJson(instanceName),
|
||||
"admin", DateTimeOffset.UtcNow));
|
||||
Assert.Equal(
|
||||
DeploymentStatus.Success,
|
||||
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName,
|
||||
"an ordinary deploy stopped creating its DCL connection");
|
||||
|
||||
var created = CreatesFor(ConnectionName).Single();
|
||||
Assert.Equal("OpcUa", created.ProtocolType);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The other path the fix moved the call out of: the boot-time staggered batch. The config is
|
||||
/// stored BEFORE the manager starts, so the boot load — not the sweep — is what creates the
|
||||
/// instance, and it must still bring its connections with it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task BootLoadedInstance_StillCreatesItsDclConnection()
|
||||
{
|
||||
const string instanceName = "BootLoadedPump";
|
||||
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
instanceName, MakeConfigJson(instanceName), "dep-b", "rev-b", true);
|
||||
|
||||
CreateBootedDeploymentManager(sweepSeconds: 0);
|
||||
|
||||
await AwaitConnectionCreatedAsync(ConnectionName,
|
||||
"the boot-time load stopped creating its DCL connection");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stand-in for the DCL manager actor: records every
|
||||
/// <see cref="CreateConnectionCommand"/> and swallows everything else (Instance Actors send
|
||||
/// their tag subscribes here too, and those are not what these tests assert on).
|
||||
/// </summary>
|
||||
private sealed class DclConnectionSpy : ReceiveActor
|
||||
{
|
||||
public DclConnectionSpy(ConcurrentQueue<CreateConnectionCommand> creates)
|
||||
{
|
||||
Receive<CreateConnectionCommand>(creates.Enqueue);
|
||||
ReceiveAny(_ => { });
|
||||
}
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
/// <summary>
|
||||
/// Minimal <see cref="ISiteHealthCollector"/> whose only job is to expose the observable edge
|
||||
/// "the Deployment Manager's boot-time config load has PROVABLY read storage". The manager calls
|
||||
/// <see cref="SetInstanceCounts"/> unconditionally the moment that load finishes — including for
|
||||
/// an empty site — so a non-zero <see cref="ReportCount"/> replaces a wall-clock sleep.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Why every post-boot start-path test must gate on this (deferred register rows 83 + 78).</b>
|
||||
/// A test that writes a <c>deployed_configurations</c> row BEFORE that read lands has its row
|
||||
/// picked up by the boot load itself — so the test passes while proving the boot load works,
|
||||
/// not the post-boot path (startup reconcile, sweep, enable) it claims to cover. A bare
|
||||
/// <c>Task.Delay</c> only makes that race unlikely, not impossible: on a loaded machine the load
|
||||
/// can still be in flight when the delay expires, and the test silently changes what it proves
|
||||
/// instead of failing. Gating on the first instance-count report removes the race entirely.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class DeploymentManagerStartupProbe : ISiteHealthCollector
|
||||
{
|
||||
private int _reports;
|
||||
|
||||
/// <summary>
|
||||
/// Number of <see cref="SetInstanceCounts"/> calls seen. Greater than zero means the
|
||||
/// boot-time config load has completed and read storage.
|
||||
/// </summary>
|
||||
public int ReportCount => Volatile.Read(ref _reports);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetInstanceCounts(int deployed, int enabled, int disabled) =>
|
||||
Interlocked.Increment(ref _reports);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IncrementScriptError() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IncrementAlarmError() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IncrementDeadLetter() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IncrementSiteAuditWriteFailures() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IncrementAuditRedactionFailure() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void RemoveConnection(string connectionName) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetParkedMessageCount(int count) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetNodeHostname(string hostname) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetActiveNode(bool isActive) { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool IsActiveNode => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId) =>
|
||||
throw new NotSupportedException("Startup probe collects no report.");
|
||||
}
|
||||
Reference in New Issue
Block a user