dohertj2/lmxopcua
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): add ClusterNode.MaintenanceMode — the hatch the live gate proved missing

Browse Source 
Phase 1 gate step 4 failed: setting Enabled = 0 to take a node out of service
returns 422 ClusterEnabledNodeCountMismatch, because
DraftValidator.ValidateClusterTopology requires the enabled-node count to equal
ServerCluster.NodeCount. On a Warm/Hot pair — every cluster on the rig, and
every cluster in the target topology — Enabled can therefore never be the
maintenance hatch. The plan called step 4 "the check that makes step 3
acceptable to ship", so Task 3's behaviour change was not shippable as it stood:
a node down for maintenance would block every deployment to its cluster.

The two rules were each reasonable and contradictory together — the validator
reads Enabled as "part of the declared topology", Phase 1 additionally read it
as "expect an ack". Split the meanings rather than weaken either rule:

  Enabled          part of the declared topology  (validator, untouched)
  MaintenanceMode  expected to participate now    (coordinator + reconciler)

Rejected alternatives: counting configured rather than enabled nodes (drops the
guard against booting a pair into InvalidTopology); downgrading the rule to a
warning (weakens a deploy gate for everyone); shipping with no hatch.

Sabotage: dropping !n.MaintenanceMode from the coordinator query turns the new
test red. It also asserts both nodes remain Enabled, so the fix cannot quietly
regress to disabling the row after all.

Configuration.Tests 95/95, ControlPlane.Tests 101/101, solution builds clean.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-22 09:07:29 -04:00
parent 57e1d01766
commit ee69caa270
12 changed files with 2039 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs
+13 -5
View File
@@ -27,8 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
/// <b>Behaviour change:</b> a configured node that is switched off is now <i>expected</i>, so a
/// deployment dispatched while it is down fails at the apply deadline instead of sealing green
/// without it. That is deliberate — under the membership rule the operator was told the fleet was
/// deployed when it was not. <c>ClusterNode.Enabled = 0</c> is the escape hatch for a node taken
/// down for maintenance.
/// deployed when it was not. <c>ClusterNode.MaintenanceMode = 1</c> is the escape hatch for a node
/// taken down for maintenance.
/// </para>
/// <para>
/// <b>Not <c>Enabled = 0</c>.</b> That was the intended hatch until the Phase 1 live gate found it
/// unusable: <c>DraftValidator.ValidateClusterTopology</c> requires the enabled-node count to equal
/// <c>ServerCluster.NodeCount</c>, so clearing <c>Enabled</c> on either node of a Warm/Hot pair
/// rejects the deployment outright — and every cluster in the target topology is a pair. The two
/// rules were independently reasonable and contradictory together, so the meanings were split:
/// <c>Enabled</c> = part of the declared topology, <c>MaintenanceMode</c> = do not expect it now.
/// </para>
/// <para>
/// <b>Every <c>ClusterNode</c> row is assumed to be a driver node.</b> The membership rule filtered
@@ -271,8 +279,8 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
var missing = _expectedAcks.Where(n => !_acks.ContainsKey(n)).Select(n => n.Value).Order().ToList();
_log.Warning(
"Deployment {Id} timed out after {Deadline} ({Acked}/{Total} acks landed). " +
"No ack from: {MissingNodes}. Each is an enabled ClusterNode row — start the node, or " +
"set ClusterNode.Enabled = 0 if it is down for maintenance, then redeploy",
"No ack from: {MissingNodes}. Each is an enabled, non-maintenance ClusterNode row — start " +
"the node, or set ClusterNode.MaintenanceMode = 1 if it is down for maintenance, then redeploy",
_current.Value, _applyDeadline, _acks.Count, _expectedAcks.Count, string.Join(", ", missing));
ResetForNext();
}
@@ -316,7 +324,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
private static HashSet<NodeId> DiscoverDriverNodes(OtOpcUaConfigDbContext db) =>
db.ClusterNodes
.AsNoTracking()
.Where(n => n.Enabled)
.Where(n => n.Enabled && !n.MaintenanceMode)
.Select(n => n.NodeId)
.ToList()
.Select(NodeId.Parse)
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconciler.cs
+10 -9
View File
@@ -75,21 +75,22 @@ public static class ClusterNodeAddressReconciler
/// them would report a mismatch for correct configuration.
/// </param>
/// <param name="rows">Every <c>ClusterNode</c> row, enabled or not.</param>
/// <param name="enabledNodeIds">
/// The subset of <paramref name="rows"/> that is enabled. Disabled rows are deliberately
/// still matched against membership (a disabled row for a running node is worth knowing) but
/// never reported as <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> — being absent
/// is the whole point of disabling one.
/// <param name="expectedPresentNodeIds">
/// The subset of <paramref name="rows"/> that should currently be in the cluster: enabled and
/// not in maintenance. Excluded rows are deliberately still matched against membership (a
/// disabled row for a running node is worth knowing) but never reported as
/// <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> — being absent is the whole point
/// of disabling one or putting it in maintenance.
/// </param>
/// <returns>Every disagreement found, ordered by node id for stable logging.</returns>
public static IReadOnlyList<ClusterNodeAddressMismatch> Reconcile(
IReadOnlyCollection<(string Host, int Port)> observedDriverMembers,
IReadOnlyCollection<ClusterNodeAddress> rows,
IReadOnlyCollection<string> enabledNodeIds)
IReadOnlyCollection<string> expectedPresentNodeIds)
{
var byNodeId = new Dictionary<string, ClusterNodeAddress>(StringComparer.OrdinalIgnoreCase);
foreach (var row in rows) byNodeId[row.NodeId] = row;
var enabled = new HashSet<string>(enabledNodeIds, StringComparer.OrdinalIgnoreCase);
var expectedPresent = new HashSet<string>(expectedPresentNodeIds, StringComparer.OrdinalIgnoreCase);
var findings = new List<ClusterNodeAddressMismatch>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -121,12 +122,12 @@ public static class ClusterNodeAddressReconciler
foreach (var row in rows)
{
if (seen.Contains(row.NodeId) || !enabled.Contains(row.NodeId)) continue;
if (seen.Contains(row.NodeId) || !expectedPresent.Contains(row.NodeId)) continue;
findings.Add(new ClusterNodeAddressMismatch(
AddressMismatchKind.EnabledRowNotInCluster, row.NodeId,
$"enabled ClusterNode row {row.NodeId} has no matching driver member; the next " +
"deployment will fail at the apply deadline waiting for it. Start the node, or set " +
"ClusterNode.Enabled = 0 while it is down for maintenance"));
"ClusterNode.MaintenanceMode = 1 while it is down for maintenance"));
}
return findings
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs
+4 -1
View File
@@ -89,8 +89,11 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer
rows = db.ClusterNodes.AsNoTracking()
.Select(n => new ClusterNodeAddress(n.NodeId, n.Host, n.AkkaPort))
.ToList();
// Same predicate the coordinator's expected-ack set uses — a node the deploy path will
// not wait for must not be reported as missing either, or the maintenance hatch trades a
// failed deployment for a permanent warning.
enabled = db.ClusterNodes.AsNoTracking()
.Where(n => n.Enabled).Select(n => n.NodeId).ToList();
.Where(n => n.Enabled && !n.MaintenanceMode).Select(n => n.NodeId).ToList();
}
catch (Exception ex)
{