diff --git a/docs/plans/2026-07-21-per-cluster-mesh-phase1.md.tasks.json b/docs/plans/2026-07-21-per-cluster-mesh-phase1.md.tasks.json index 21816038..1c388c3e 100644 --- a/docs/plans/2026-07-21-per-cluster-mesh-phase1.md.tasks.json +++ b/docs/plans/2026-07-21-per-cluster-mesh-phase1.md.tasks.json @@ -8,9 +8,9 @@ "adminui-edit": "Deferred to Phase 2 (nothing reads the columns until then)" }, "tasks": [ - { "id": 1, "subject": "Address columns on ClusterNode", "status": "pending", "blockedBy": [] }, - { "id": 2, "subject": "EF migration AddClusterNodeTransportPorts", "status": "pending", "blockedBy": [1] }, - { "id": 3, "subject": "Coordinator sources expected-ack set from ClusterNode", "status": "pending", "blockedBy": [2] }, + { "id": 1, "subject": "Address columns on ClusterNode", "status": "completed", "commit": "2bbb0271", "blockedBy": [] }, + { "id": 2, "subject": "EF migration AddClusterNodeTransportPorts", "status": "completed", "commit": "da74ebd6", "blockedBy": [1] }, + { "id": 3, "subject": "Coordinator sources expected-ack set from ClusterNode", "status": "completed", "commit": "d88e2455", "blockedBy": [2] }, { "id": 4, "subject": "Reconcile duplicated AkkaPort with the node's own Cluster:Port", "status": "pending", "blockedBy": [2] }, { "id": 5, "subject": "Rig seed + docs", "status": "pending", "blockedBy": [3, 4] }, { "id": 6, "subject": "Live gate on docker-dev", "status": "pending", "blockedBy": [5] } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconciler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconciler.cs new file mode 100644 index 00000000..d35204d6 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconciler.cs @@ -0,0 +1,137 @@ +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; + +/// What kind of disagreement was found between a node's real address and its row. +public enum AddressMismatchKind +{ + /// + /// The row was matched to a live member by NodeId, but its Host / AkkaPort + /// columns name a different address. Phase 2 dials those columns, so central would dial + /// somewhere the node is not — and the symptom there is a silent absence of acks. + /// + RowDialTargetDisagrees, + + /// + /// A driver-role member is in the cluster with no enabled ClusterNode row at its + /// address. Its deployment acks are discarded (see ConfigPublishCoordinator), so it + /// silently receives deployments it is never credited for. + /// + RunningNodeHasNoRow, + + /// + /// An enabled row names a node that is not currently a driver-role member. Either it is down + /// — in which case the next deployment fails at the apply deadline — or it binds an address + /// other than the one its row claims. + /// + EnabledRowNotInCluster, +} + +/// One disagreement, with enough detail to act on without opening the DB. +/// Which disagreement this is. +/// The host:port identity the finding is about. +/// Human-readable specifics, including what to do about it. +public sealed record ClusterNodeAddressMismatch(AddressMismatchKind Kind, string NodeId, string Detail); + +/// A ClusterNode row reduced to the address facts this check cares about. +/// The row's primary key, in the host:port space. +/// The host central will dial in Phase 2. +/// The remoting port central will dial in Phase 2. +public sealed record ClusterNodeAddress(string NodeId, string Host, int AkkaPort); + +/// +/// Compares the addresses driver nodes actually occupy against the ClusterNode rows that +/// claim to describe them. +/// +/// +/// +/// Why this exists. ClusterNode.AkkaPort and the node's own Cluster:Port +/// are the same fact stored in two places, and nothing makes them agree. A node that binds +/// 4054 while its row says 4053 is unreachable from central once Phase 2 dials the row +/// instead of gossiping — and an unenforced duplicated address is the exact shape of the +/// Modbus/ModbusTcp and TwinCat/Focas drifts already recorded in +/// this repo. Since Phase 1 also made the rows the deploy path's expected-ack set, a drifted +/// row already costs a failed deployment today. +/// +/// +/// It runs on an admin node, comparing rows against the membership an admin node can +/// already see, rather than having each driver node assert its own row — Phase 4 removes the +/// driver nodes' ConfigDb connection entirely, so a self-assertion written there would have +/// to be deleted again. +/// +/// +/// Phase 2 must revisit this. Once the fleet splits into one mesh per cluster, an +/// admin node no longer sees site members at all, and every site row would report +/// forever. The check then has to +/// move to whatever transport replaces gossip — or be scoped to the central mesh. +/// +/// +public static class ClusterNodeAddressReconciler +{ + /// + /// Reconciles observed driver-role member addresses against the configured rows. + /// + /// + /// host/port of every Up member carrying the driver role. Admin-only + /// members must be excluded — they legitimately have no ClusterNode row, and including + /// them would report a mismatch for correct configuration. + /// + /// Every ClusterNode row, enabled or not. + /// + /// The subset of that is enabled. Disabled rows are deliberately + /// still matched against membership (a disabled row for a running node is worth knowing) but + /// never reported as — being absent + /// is the whole point of disabling one. + /// + /// Every disagreement found, ordered by node id for stable logging. + public static IReadOnlyList Reconcile( + IReadOnlyCollection<(string Host, int Port)> observedDriverMembers, + IReadOnlyCollection rows, + IReadOnlyCollection enabledNodeIds) + { + var byNodeId = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var row in rows) byNodeId[row.NodeId] = row; + var enabled = new HashSet(enabledNodeIds, StringComparer.OrdinalIgnoreCase); + + var findings = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var (host, port) in observedDriverMembers) + { + var nodeId = $"{host}:{port}"; + seen.Add(nodeId); + + if (!byNodeId.TryGetValue(nodeId, out var row)) + { + findings.Add(new ClusterNodeAddressMismatch( + AddressMismatchKind.RunningNodeHasNoRow, nodeId, + $"driver node is in the cluster at {nodeId} but no ClusterNode row has that NodeId; " + + "its deployment acks are discarded. Either register the row or correct the node's " + + "Cluster:PublicHostname / Cluster:Port to match an existing one")); + continue; + } + + if (!string.Equals(row.Host, host, StringComparison.OrdinalIgnoreCase) || row.AkkaPort != port) + { + findings.Add(new ClusterNodeAddressMismatch( + AddressMismatchKind.RowDialTargetDisagrees, nodeId, + $"node is at {nodeId} but its row's dial target is {row.Host}:{row.AkkaPort}; " + + "central will dial the wrong address once Phase 2 stops using gossip. " + + "Correct ClusterNode.Host / ClusterNode.AkkaPort")); + } + } + + foreach (var row in rows) + { + if (seen.Contains(row.NodeId) || !enabled.Contains(row.NodeId)) continue; + findings.Add(new ClusterNodeAddressMismatch( + AddressMismatchKind.EnabledRowNotInCluster, row.NodeId, + $"enabled ClusterNode row {row.NodeId} has no matching driver member; the next " + + "deployment will fail at the apply deadline waiting for it. Start the node, or set " + + "ClusterNode.Enabled = 0 while it is down for maintenance")); + } + + return findings + .OrderBy(f => f.NodeId, StringComparer.OrdinalIgnoreCase) + .ThenBy(f => f.Kind) + .ToList(); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs new file mode 100644 index 00000000..fefba4c0 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs @@ -0,0 +1,133 @@ +using Akka.Actor; +using Akka.Cluster; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; + +/// +/// Admin-role cluster singleton that runs against live +/// membership and logs what it finds. A singleton rather than a per-admin-node actor so a +/// two-admin fleet reports each disagreement once. +/// +/// +/// Recomputes on membership change (debounced) and on a slow periodic sweep, and logs only +/// when the finding set changes. A node legitimately down would otherwise reprint the same +/// warning every sweep until someone silences it by ignoring the log, which is the failure mode +/// this check exists to avoid. +/// +public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimers +{ + /// Driver role name — only members carrying it are expected to have a ClusterNode row. + public const string DriverRole = "driver"; + + /// Collapses a burst of membership events (a rolling restart) into one reconcile. + public static readonly TimeSpan DebounceWindow = TimeSpan.FromSeconds(2); + + /// Periodic sweep, so a row edited in the AdminUI is checked without a topology change. + public static readonly TimeSpan DefaultSweepInterval = TimeSpan.FromMinutes(5); + + private readonly IDbContextFactory _dbFactory; + private readonly Akka.Cluster.Cluster _cluster; + private readonly TimeSpan _sweepInterval; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private IReadOnlyList? _lastReported; + + /// Gets the timer scheduler for this actor. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates Props for the reconciler. + /// Factory for the config database context. + /// Periodic sweep interval; defaults to . + /// Props for actor creation. + public static Props Props( + IDbContextFactory dbFactory, TimeSpan? sweepInterval = null) => + Akka.Actor.Props.Create(() => new ClusterNodeAddressReconcilerActor(dbFactory, sweepInterval)); + + /// Initializes a new instance of the class. + /// Factory for the config database context. + /// Periodic sweep interval; defaults to . + public ClusterNodeAddressReconcilerActor( + IDbContextFactory dbFactory, TimeSpan? sweepInterval = null) + { + _dbFactory = dbFactory; + _cluster = Akka.Cluster.Cluster.Get(Context.System); + _sweepInterval = sweepInterval ?? DefaultSweepInterval; + + Receive(_ => ScheduleReconcile()); + Receive(_ => ScheduleReconcile()); + Receive(_ => Reconcile()); + } + + /// + protected override void PreStart() + { + _cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents, typeof(ClusterEvent.IMemberEvent)); + Timers.StartPeriodicTimer("sweep", ReconcileNow.Instance, _sweepInterval, _sweepInterval); + } + + /// + protected override void PostStop() => _cluster.Unsubscribe(Self); + + private void ScheduleReconcile() => + Timers.StartSingleTimer("debounce", ReconcileNow.Instance, DebounceWindow); + + private void Reconcile() + { + var observed = _cluster.State.Members + .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole)) + .Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0)) + .Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0) + .ToList(); + + List rows; + List enabled; + try + { + using var db = _dbFactory.CreateDbContext(); + rows = db.ClusterNodes.AsNoTracking() + .Select(n => new ClusterNodeAddress(n.NodeId, n.Host, n.AkkaPort)) + .ToList(); + enabled = db.ClusterNodes.AsNoTracking() + .Where(n => n.Enabled).Select(n => n.NodeId).ToList(); + } + catch (Exception ex) + { + // A consistency check must never take the admin node down with it. Central SQL being + // briefly unreachable is an operational fact, not a reason to restart this singleton. + _log.Warning(ex, "ClusterNode address reconcile skipped — config database unreachable"); + return; + } + + var findings = ClusterNodeAddressReconciler.Reconcile(observed, rows, enabled); + + if (_lastReported is not null && findings.SequenceEqual(_lastReported)) return; + _lastReported = findings; + + if (findings.Count == 0) + { + _log.Info("ClusterNode addresses reconcile with cluster membership ({Count} driver members)", + observed.Count); + return; + } + + foreach (var f in findings) + { + // Error for the two shapes that are always a misconfiguration; Warning for a row whose + // node is merely absent, which is a legitimate state during maintenance. + if (f.Kind == AddressMismatchKind.EnabledRowNotInCluster) + _log.Warning("ClusterNode address check [{Kind}] {NodeId}: {Detail}", f.Kind, f.NodeId, f.Detail); + else + _log.Error("ClusterNode address check [{Kind}] {NodeId}: {Detail}", f.Kind, f.NodeId, f.Detail); + } + } + + /// Self-message: run a reconcile pass now. + public sealed class ReconcileNow + { + /// The singleton instance. + public static readonly ReconcileNow Instance = new(); + private ReconcileNow() { } + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 20753d13..b5caebb5 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -23,6 +23,7 @@ public static class ServiceCollectionExtensions public const string AuditWriterSingletonName = "audit-writer"; public const string FleetStatusSingletonName = "fleet-status"; public const string RedundancyStateSingletonName = "redundancy-state"; + public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; /// /// Registers all five admin-role cluster singletons + their proxies on the AkkaConfigurationBuilder. @@ -90,6 +91,18 @@ public static class ServiceCollectionExtensions (system, registry, resolver) => RedundancyStateActor.Props(), singletonOptions); + // Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and + // nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes + // the driver nodes' ConfigDb connection entirely. + builder.WithSingleton( + ClusterNodeAddressReconcilerSingletonName, + (system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + return ClusterNodeAddressReconcilerActor.Props(dbFactory); + }, + singletonOptions); + return builder; } } @@ -100,3 +113,4 @@ public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class FleetStatusBroadcasterKey { } public sealed class RedundancyStateActorKey { } +public sealed class ClusterNodeAddressReconcilerKey { } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerActorTests.cs new file mode 100644 index 00000000..a25cf368 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerActorTests.cs @@ -0,0 +1,98 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Proves the reconciler is actually wired to something — subscribes to membership, reads the DB, +/// and emits. covers the comparison logic, which a +/// dormant actor would leave perfectly correct and completely inert. +/// +public sealed class ClusterNodeAddressReconcilerActorTests : ControlPlaneActorTestBase +{ + /// + /// An enabled row with no matching driver member is warned about. The harness ActorSystem + /// joins as admin with no driver members, so the seeded row is unmatched by + /// construction. + /// + [Fact] + public void Enabled_row_with_no_matching_member_is_logged() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNode(dbFactory, "site-a-1:4053", enabled: true); + + EventFilter.Warning(contains: "site-a-1:4053").ExpectOne(TimeSpan.FromSeconds(10), () => + Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props(dbFactory))); + } + + /// + /// The finding set is logged once, not once per sweep. A check that reprints the same warning + /// every five minutes trains operators to filter it out, which costs more than it catches — + /// so this runs a deliberately fast sweep and asserts the count stays at one. + /// + [Fact] + public void Unchanged_findings_are_not_repeated_on_every_sweep() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNode(dbFactory, "site-a-1:4053", enabled: true); + + // ~6 sweeps inside the window; without the change-detection guard this logs 6 times. + EventFilter.Warning(contains: "site-a-1:4053").Expect(1, TimeSpan.FromSeconds(3), () => + { + Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props( + dbFactory, sweepInterval: TimeSpan.FromMilliseconds(400))); + // Hold the filter open past several sweeps rather than returning immediately. + ExpectNoMsg(TimeSpan.FromSeconds(2.5)); + }); + } + + /// + /// A disabled row is silence — the maintenance hatch must not trade a failed deployment for a + /// permanent warning. Positive control for the test above: same setup, one flag flipped. + /// + [Fact] + public void Disabled_row_produces_no_warning() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNode(dbFactory, "site-a-1:4053", enabled: false); + + EventFilter.Warning(contains: "site-a-1:4053").Expect(0, TimeSpan.FromSeconds(2), () => + { + Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props( + dbFactory, sweepInterval: TimeSpan.FromMilliseconds(400))); + ExpectNoMsg(TimeSpan.FromSeconds(1.5)); + }); + } + + private static void SeedNode( + IDbContextFactory dbFactory, string nodeId, bool enabled) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", + Name = "Site A", + Enterprise = "zb", + Site = "site-a", + NodeCount = 2, + RedundancyMode = RedundancyMode.Warm, + CreatedBy = "test", + }); + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, + ClusterId = "SITE-A", + Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", + Enabled = enabled, + CreatedBy = "test", + }); + db.SaveChanges(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerTests.cs new file mode 100644 index 00000000..3d5416f0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ClusterNodeAddressReconcilerTests.cs @@ -0,0 +1,126 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Per-cluster mesh Phase 1 Task 4: ClusterNode.AkkaPort and the node's own +/// Cluster:Port are the same fact in two places, and nothing makes them agree. These pin +/// the shapes that drift takes. +/// +public sealed class ClusterNodeAddressReconcilerTests +{ + private static ClusterNodeAddress Row(string host, int akkaPort, string? nodeId = null) => + new(nodeId ?? $"{host}:{akkaPort}", host, akkaPort); + + /// A fleet whose rows match membership reports nothing. + [Fact] + public void Matching_rows_and_membership_produce_no_findings() + { + var findings = ClusterNodeAddressReconciler.Reconcile( + [("site-a-1", 4053), ("site-a-2", 4053)], + [Row("site-a-1", 4053), Row("site-a-2", 4053)], + ["site-a-1:4053", "site-a-2:4053"]); + + findings.ShouldBeEmpty(); + } + + /// + /// The drift the plan names: a node binds 4054 while its row says 4053. Membership shows it + /// at an address no row claims, and the row it should have matched looks absent — both halves + /// are reported, because either alone reads as a different problem. + /// + [Fact] + public void Node_bound_to_the_wrong_port_is_reported_from_both_sides() + { + var findings = ClusterNodeAddressReconciler.Reconcile( + [("site-a-1", 4054)], + [Row("site-a-1", 4053)], + ["site-a-1:4053"]); + + findings.Count.ShouldBe(2); + findings.ShouldContain(f => + f.Kind == AddressMismatchKind.RunningNodeHasNoRow && f.NodeId == "site-a-1:4054"); + findings.ShouldContain(f => + f.Kind == AddressMismatchKind.EnabledRowNotInCluster && f.NodeId == "site-a-1:4053"); + } + + /// + /// The Phase 2 shape: the row's NodeId still matches membership, but its Host / + /// AkkaPort columns — the values central will actually dial — have been edited away + /// from it. Gossip still works today, so nothing else would notice. + /// + [Fact] + public void Row_whose_dial_target_disagrees_with_its_own_NodeId_is_reported() + { + var findings = ClusterNodeAddressReconciler.Reconcile( + [("site-a-1", 4053)], + [Row("site-a-1", 4054, nodeId: "site-a-1:4053")], + ["site-a-1:4053"]); + + var only = findings.ShouldHaveSingleItem(); + only.Kind.ShouldBe(AddressMismatchKind.RowDialTargetDisagrees); + only.NodeId.ShouldBe("site-a-1:4053"); + only.Detail.ShouldContain("site-a-1:4054"); + } + + /// A running driver node nobody registered — the coordinator discards its acks. + [Fact] + public void Running_node_with_no_row_is_reported() + { + var findings = ClusterNodeAddressReconciler.Reconcile( + [("ghost-9", 4053)], [], []); + + findings.ShouldHaveSingleItem().Kind.ShouldBe(AddressMismatchKind.RunningNodeHasNoRow); + } + + /// + /// A disabled row for an absent node is silence, not a finding — being absent is the entire + /// point of disabling it. Without this the maintenance hatch would trade a failed deploy for + /// a permanent warning. + /// + [Fact] + public void Disabled_row_for_an_absent_node_is_not_reported() + { + var findings = ClusterNodeAddressReconciler.Reconcile( + [("site-a-1", 4053)], + [Row("site-a-1", 4053), Row("site-a-2", 4053)], + ["site-a-1:4053"]); + + findings.ShouldBeEmpty(); + } + + /// + /// An admin-only member has no ClusterNode row by design, so the caller filters + /// membership to driver-role members. Passing an unfiltered list would report correct + /// configuration as broken — pinned here because the filter lives at the call site. + /// + [Fact] + public void Admin_only_members_are_the_callers_responsibility_to_exclude() + { + // What the actor passes: driver members only. + ClusterNodeAddressReconciler.Reconcile( + [("central-1", 4053)], [Row("central-1", 4053)], ["central-1:4053"]) + .ShouldBeEmpty(); + + // What an unfiltered call would produce — the false positive this guards against. + ClusterNodeAddressReconciler.Reconcile( + [("central-1", 4053), ("admin-only-1", 4053)], + [Row("central-1", 4053)], + ["central-1:4053"]) + .ShouldHaveSingleItem().Kind.ShouldBe(AddressMismatchKind.RunningNodeHasNoRow); + } + + /// + /// Host matching is case-insensitive, matching the coordinator's NodeIdComparer — DNS + /// and SQL collation are both case-insensitive, so the same node surfaces with either casing. + /// + [Fact] + public void Host_matching_is_case_insensitive() + { + ClusterNodeAddressReconciler.Reconcile( + [("SITE-A-1", 4053)], [Row("site-a-1", 4053)], ["site-a-1:4053"]) + .ShouldBeEmpty(); + } +}