dohertj2/lmxopcua
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): add ClusterNode.MaintenanceMode — the hatch the live gate proved missing

Browse Source 
Phase 1 gate step 4 failed: setting Enabled = 0 to take a node out of service
returns 422 ClusterEnabledNodeCountMismatch, because
DraftValidator.ValidateClusterTopology requires the enabled-node count to equal
ServerCluster.NodeCount. On a Warm/Hot pair — every cluster on the rig, and
every cluster in the target topology — Enabled can therefore never be the
maintenance hatch. The plan called step 4 "the check that makes step 3
acceptable to ship", so Task 3's behaviour change was not shippable as it stood:
a node down for maintenance would block every deployment to its cluster.

The two rules were each reasonable and contradictory together — the validator
reads Enabled as "part of the declared topology", Phase 1 additionally read it
as "expect an ack". Split the meanings rather than weaken either rule:

  Enabled          part of the declared topology  (validator, untouched)
  MaintenanceMode  expected to participate now    (coordinator + reconciler)

Rejected alternatives: counting configured rather than enabled nodes (drops the
guard against booting a pair into InvalidTopology); downgrading the rule to a
warning (weakens a deploy gate for everyone); shipping with no hatch.

Sabotage: dropping !n.MaintenanceMode from the coordinator query turns the new
test red. It also asserts both nodes remain Enabled, so the fix cannot quietly
regress to disabling the row after all.

Configuration.Tests 95/95, ControlPlane.Tests 101/101, solution builds clean.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-22 09:07:29 -04:00
parent 57e1d01766
commit ee69caa270
12 changed files with 2039 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbSourcedAcksTests.cs
+53 -9
View File
@@ -48,7 +48,7 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDispatchingDeployment(dbFactory);
SeedNodes(dbFactory, ("site-a-1:4053", true));
SeedNodes(dbFactory, ("site-a-1:4053", true, false));
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
@@ -73,16 +73,18 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
}
/// <summary>
/// The <c>Enabled = 0</c> escape hatch: a disabled row is NOT expected, so a deployment seals
/// without it. This is what makes the "a stopped node now fails the deploy" behaviour change
/// acceptable to ship — without it, a node down for maintenance blocks every deployment.
/// A disabled row is NOT expected, so a deployment seals without it — for a node genuinely
/// decommissioned rather than temporarily down. <b>This is not the maintenance hatch</b>: the
/// topology gate rejects a deployment whose enabled-node count differs from the cluster's
/// <c>NodeCount</c>, so on any 2-node pair clearing <c>Enabled</c> never reaches this code.
/// See <see cref="MaintenanceMode_rows_are_not_expected_to_ack_while_staying_enabled"/>.
/// </summary>
[Fact]
public void Disabled_ClusterNode_rows_are_not_expected_to_ack()
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDispatchingDeployment(dbFactory);
SeedNodes(dbFactory, ("site-a-1:4053", true), ("site-a-2:4053", false));
SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", false, false));
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
@@ -116,7 +118,7 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDispatchingDeployment(dbFactory);
SeedNodes(dbFactory, ("site-a-1:4053", true));
SeedNodes(dbFactory, ("site-a-1:4053", true, false));
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
@@ -158,7 +160,7 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDispatchingDeployment(dbFactory);
SeedNodes(dbFactory, ("site-a-1:4053", false), ("site-a-2:4053", false));
SeedNodes(dbFactory, ("site-a-1:4053", false, false), ("site-a-2:4053", false, false));
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
@@ -171,9 +173,50 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
}, duration: TimeSpan.FromSeconds(3));
}
/// <summary>
/// The maintenance hatch, and the reason it is a separate flag. The Phase 1 live gate set
/// <c>Enabled = 0</c> on one node of a 2-node pair and the deployment was rejected outright by
/// <c>DraftValidator.ValidateClusterTopology</c> (enabled-node count must equal
/// <c>NodeCount</c>) — so the intended escape hatch could not be used on any pair, which is
/// every cluster in the target topology. <c>MaintenanceMode</c> excludes a node from the
/// expected-ack set while leaving it enabled, so the topology gate still passes.
/// </summary>
[Fact]
public void MaintenanceMode_rows_are_not_expected_to_ack_while_staying_enabled()
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDispatchingDeployment(dbFactory);
SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", true, true));
// The topology gate counts Enabled, so both nodes still count — this is what Enabled = 0
// broke. Assert it here so the fix cannot be "quietly disable it after all".
using (var check = dbFactory.CreateDbContext())
{
check.ClusterNodes.Count(n => n.Enabled).ShouldBe(2);
}
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
}, duration: TimeSpan.FromSeconds(3));
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: TimeSpan.FromSeconds(3));
}
private static void SeedNodes(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
params (string NodeId, bool Enabled)[] nodes)
params (string NodeId, bool Enabled, bool Maintenance)[] nodes)
{
using var db = dbFactory.CreateDbContext();
db.ServerClusters.Add(new ServerCluster
@@ -186,7 +229,7 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
RedundancyMode = RedundancyMode.Warm,
CreatedBy = "test",
});
foreach (var (nodeId, enabled) in nodes)
foreach (var (nodeId, enabled, maintenance) in nodes)
{
db.ClusterNodes.Add(new ClusterNode
{
@@ -195,6 +238,7 @@ public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneAct
Host = nodeId.Split(':')[0],
ApplicationUri = $"urn:OtOpcUa:{nodeId}",
Enabled = enabled,
MaintenanceMode = maintenance,
CreatedBy = "test",
});
}