feat(fleet): reconcile ClusterNode dial targets against live membership
ClusterNode.AkkaPort and the node's own Cluster:Port are the same fact in two places and nothing made them agree. Phase 2 dials the row instead of gossiping, so a node binding 4054 while its row says 4053 becomes unreachable from central — and the symptom is a silent absence of acks rather than an error. That is the shape of the Modbus/ModbusTcp and TwinCat/Focas drifts already in this repo. Since Phase 1 also made the rows the deploy path's expected-ack set, drift already costs a failed deployment today. Implemented as the plan's preferred option: an admin-role singleton comparing rows against the membership an admin node can already see, rather than each driver node asserting its own row — Phase 4 removes the driver nodes' ConfigDb connection, so a self-assertion written there would have to be deleted again. Three shapes, split by severity: a row whose dial target disagrees with its own NodeId and a running node with no row are Errors; an enabled row with no matching member is a Warning, because a node down for maintenance is a legitimate state. Findings are logged only when the set changes — a check that reprints the same warning every sweep trains operators to filter it out. Documented limitation: Phase 2 must revisit this. Once the fleet splits into one mesh per cluster an admin node cannot see site members, and every site row would report EnabledRowNotInCluster forever. Sabotage: removing the change-detection guard turns the repeat test red. ControlPlane.Tests 100/100. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,137 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||
|
||||
/// <summary>What kind of disagreement was found between a node's real address and its row.</summary>
|
||||
public enum AddressMismatchKind
|
||||
{
|
||||
/// <summary>
|
||||
/// The row was matched to a live member by <c>NodeId</c>, but its <c>Host</c> / <c>AkkaPort</c>
|
||||
/// columns name a different address. Phase 2 dials those columns, so central would dial
|
||||
/// somewhere the node is not — and the symptom there is a silent absence of acks.
|
||||
/// </summary>
|
||||
RowDialTargetDisagrees,
|
||||
|
||||
/// <summary>
|
||||
/// A driver-role member is in the cluster with no enabled <c>ClusterNode</c> row at its
|
||||
/// address. Its deployment acks are discarded (see <c>ConfigPublishCoordinator</c>), so it
|
||||
/// silently receives deployments it is never credited for.
|
||||
/// </summary>
|
||||
RunningNodeHasNoRow,
|
||||
|
||||
/// <summary>
|
||||
/// An enabled row names a node that is not currently a driver-role member. Either it is down
|
||||
/// — in which case the next deployment fails at the apply deadline — or it binds an address
|
||||
/// other than the one its row claims.
|
||||
/// </summary>
|
||||
EnabledRowNotInCluster,
|
||||
}
|
||||
|
||||
/// <summary>One disagreement, with enough detail to act on without opening the DB.</summary>
|
||||
/// <param name="Kind">Which disagreement this is.</param>
|
||||
/// <param name="NodeId">The <c>host:port</c> identity the finding is about.</param>
|
||||
/// <param name="Detail">Human-readable specifics, including what to do about it.</param>
|
||||
public sealed record ClusterNodeAddressMismatch(AddressMismatchKind Kind, string NodeId, string Detail);
|
||||
|
||||
/// <summary>A <c>ClusterNode</c> row reduced to the address facts this check cares about.</summary>
|
||||
/// <param name="NodeId">The row's primary key, in the <c>host:port</c> space.</param>
|
||||
/// <param name="Host">The host central will dial in Phase 2.</param>
|
||||
/// <param name="AkkaPort">The remoting port central will dial in Phase 2.</param>
|
||||
public sealed record ClusterNodeAddress(string NodeId, string Host, int AkkaPort);
|
||||
|
||||
/// <summary>
|
||||
/// Compares the addresses driver nodes actually occupy against the <c>ClusterNode</c> rows that
|
||||
/// claim to describe them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why this exists.</b> <c>ClusterNode.AkkaPort</c> and the node's own <c>Cluster:Port</c>
|
||||
/// are the same fact stored in two places, and nothing makes them agree. A node that binds
|
||||
/// 4054 while its row says 4053 is unreachable from central once Phase 2 dials the row
|
||||
/// instead of gossiping — and an unenforced duplicated address is the exact shape of the
|
||||
/// <c>Modbus</c>/<c>ModbusTcp</c> and <c>TwinCat</c>/<c>Focas</c> drifts already recorded in
|
||||
/// this repo. Since Phase 1 also made the rows the deploy path's expected-ack set, a drifted
|
||||
/// row already costs a failed deployment today.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>It runs on an admin node</b>, comparing rows against the membership an admin node can
|
||||
/// already see, rather than having each driver node assert its own row — Phase 4 removes the
|
||||
/// driver nodes' ConfigDb connection entirely, so a self-assertion written there would have
|
||||
/// to be deleted again.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Phase 2 must revisit this.</b> Once the fleet splits into one mesh per cluster, an
|
||||
/// admin node no longer sees site members at all, and every site row would report
|
||||
/// <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> forever. The check then has to
|
||||
/// move to whatever transport replaces gossip — or be scoped to the central mesh.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class ClusterNodeAddressReconciler
|
||||
{
|
||||
/// <summary>
|
||||
/// Reconciles observed driver-role member addresses against the configured rows.
|
||||
/// </summary>
|
||||
/// <param name="observedDriverMembers">
|
||||
/// <c>host</c>/<c>port</c> of every Up member carrying the <c>driver</c> role. Admin-only
|
||||
/// members must be excluded — they legitimately have no <c>ClusterNode</c> row, and including
|
||||
/// them would report a mismatch for correct configuration.
|
||||
/// </param>
|
||||
/// <param name="rows">Every <c>ClusterNode</c> row, enabled or not.</param>
|
||||
/// <param name="enabledNodeIds">
|
||||
/// The subset of <paramref name="rows"/> that is enabled. Disabled rows are deliberately
|
||||
/// still matched against membership (a disabled row for a running node is worth knowing) but
|
||||
/// never reported as <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> — being absent
|
||||
/// is the whole point of disabling one.
|
||||
/// </param>
|
||||
/// <returns>Every disagreement found, ordered by node id for stable logging.</returns>
|
||||
public static IReadOnlyList<ClusterNodeAddressMismatch> Reconcile(
|
||||
IReadOnlyCollection<(string Host, int Port)> observedDriverMembers,
|
||||
IReadOnlyCollection<ClusterNodeAddress> rows,
|
||||
IReadOnlyCollection<string> enabledNodeIds)
|
||||
{
|
||||
var byNodeId = new Dictionary<string, ClusterNodeAddress>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var row in rows) byNodeId[row.NodeId] = row;
|
||||
var enabled = new HashSet<string>(enabledNodeIds, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var findings = new List<ClusterNodeAddressMismatch>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var (host, port) in observedDriverMembers)
|
||||
{
|
||||
var nodeId = $"{host}:{port}";
|
||||
seen.Add(nodeId);
|
||||
|
||||
if (!byNodeId.TryGetValue(nodeId, out var row))
|
||||
{
|
||||
findings.Add(new ClusterNodeAddressMismatch(
|
||||
AddressMismatchKind.RunningNodeHasNoRow, nodeId,
|
||||
$"driver node is in the cluster at {nodeId} but no ClusterNode row has that NodeId; " +
|
||||
"its deployment acks are discarded. Either register the row or correct the node's " +
|
||||
"Cluster:PublicHostname / Cluster:Port to match an existing one"));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.Equals(row.Host, host, StringComparison.OrdinalIgnoreCase) || row.AkkaPort != port)
|
||||
{
|
||||
findings.Add(new ClusterNodeAddressMismatch(
|
||||
AddressMismatchKind.RowDialTargetDisagrees, nodeId,
|
||||
$"node is at {nodeId} but its row's dial target is {row.Host}:{row.AkkaPort}; " +
|
||||
"central will dial the wrong address once Phase 2 stops using gossip. " +
|
||||
"Correct ClusterNode.Host / ClusterNode.AkkaPort"));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (seen.Contains(row.NodeId) || !enabled.Contains(row.NodeId)) continue;
|
||||
findings.Add(new ClusterNodeAddressMismatch(
|
||||
AddressMismatchKind.EnabledRowNotInCluster, row.NodeId,
|
||||
$"enabled ClusterNode row {row.NodeId} has no matching driver member; the next " +
|
||||
"deployment will fail at the apply deadline waiting for it. Start the node, or set " +
|
||||
"ClusterNode.Enabled = 0 while it is down for maintenance"));
|
||||
}
|
||||
|
||||
return findings
|
||||
.OrderBy(f => f.NodeId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(f => f.Kind)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user