feat(fleet): reconcile ClusterNode dial targets against live membership
ClusterNode.AkkaPort and the node's own Cluster:Port are the same fact in two places and nothing made them agree. Phase 2 dials the row instead of gossiping, so a node binding 4054 while its row says 4053 becomes unreachable from central — and the symptom is a silent absence of acks rather than an error. That is the shape of the Modbus/ModbusTcp and TwinCat/Focas drifts already in this repo. Since Phase 1 also made the rows the deploy path's expected-ack set, drift already costs a failed deployment today. Implemented as the plan's preferred option: an admin-role singleton comparing rows against the membership an admin node can already see, rather than each driver node asserting its own row — Phase 4 removes the driver nodes' ConfigDb connection, so a self-assertion written there would have to be deleted again. Three shapes, split by severity: a row whose dial target disagrees with its own NodeId and a running node with no row are Errors; an enabled row with no matching member is a Warning, because a node down for maintenance is a legitimate state. Findings are logged only when the set changes — a check that reprints the same warning every sweep trains operators to filter it out. Documented limitation: Phase 2 must revisit this. Once the fleet splits into one mesh per cluster an admin node cannot see site members, and every site row would report EnabledRowNotInCluster forever. Sabotage: removing the change-detection guard turns the repeat test red. ControlPlane.Tests 100/100. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
@@ -8,9 +8,9 @@
|
|||||||
"adminui-edit": "Deferred to Phase 2 (nothing reads the columns until then)"
|
"adminui-edit": "Deferred to Phase 2 (nothing reads the columns until then)"
|
||||||
},
|
},
|
||||||
"tasks": [
|
"tasks": [
|
||||||
{ "id": 1, "subject": "Address columns on ClusterNode", "status": "pending", "blockedBy": [] },
|
{ "id": 1, "subject": "Address columns on ClusterNode", "status": "completed", "commit": "2bbb0271", "blockedBy": [] },
|
||||||
{ "id": 2, "subject": "EF migration AddClusterNodeTransportPorts", "status": "pending", "blockedBy": [1] },
|
{ "id": 2, "subject": "EF migration AddClusterNodeTransportPorts", "status": "completed", "commit": "da74ebd6", "blockedBy": [1] },
|
||||||
{ "id": 3, "subject": "Coordinator sources expected-ack set from ClusterNode", "status": "pending", "blockedBy": [2] },
|
{ "id": 3, "subject": "Coordinator sources expected-ack set from ClusterNode", "status": "completed", "commit": "d88e2455", "blockedBy": [2] },
|
||||||
{ "id": 4, "subject": "Reconcile duplicated AkkaPort with the node's own Cluster:Port", "status": "pending", "blockedBy": [2] },
|
{ "id": 4, "subject": "Reconcile duplicated AkkaPort with the node's own Cluster:Port", "status": "pending", "blockedBy": [2] },
|
||||||
{ "id": 5, "subject": "Rig seed + docs", "status": "pending", "blockedBy": [3, 4] },
|
{ "id": 5, "subject": "Rig seed + docs", "status": "pending", "blockedBy": [3, 4] },
|
||||||
{ "id": 6, "subject": "Live gate on docker-dev", "status": "pending", "blockedBy": [5] }
|
{ "id": 6, "subject": "Live gate on docker-dev", "status": "pending", "blockedBy": [5] }
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||||
|
|
||||||
|
/// <summary>What kind of disagreement was found between a node's real address and its row.</summary>
|
||||||
|
public enum AddressMismatchKind
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The row was matched to a live member by <c>NodeId</c>, but its <c>Host</c> / <c>AkkaPort</c>
|
||||||
|
/// columns name a different address. Phase 2 dials those columns, so central would dial
|
||||||
|
/// somewhere the node is not — and the symptom there is a silent absence of acks.
|
||||||
|
/// </summary>
|
||||||
|
RowDialTargetDisagrees,
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A driver-role member is in the cluster with no enabled <c>ClusterNode</c> row at its
|
||||||
|
/// address. Its deployment acks are discarded (see <c>ConfigPublishCoordinator</c>), so it
|
||||||
|
/// silently receives deployments it is never credited for.
|
||||||
|
/// </summary>
|
||||||
|
RunningNodeHasNoRow,
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// An enabled row names a node that is not currently a driver-role member. Either it is down
|
||||||
|
/// — in which case the next deployment fails at the apply deadline — or it binds an address
|
||||||
|
/// other than the one its row claims.
|
||||||
|
/// </summary>
|
||||||
|
EnabledRowNotInCluster,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>One disagreement, with enough detail to act on without opening the DB.</summary>
|
||||||
|
/// <param name="Kind">Which disagreement this is.</param>
|
||||||
|
/// <param name="NodeId">The <c>host:port</c> identity the finding is about.</param>
|
||||||
|
/// <param name="Detail">Human-readable specifics, including what to do about it.</param>
|
||||||
|
public sealed record ClusterNodeAddressMismatch(AddressMismatchKind Kind, string NodeId, string Detail);
|
||||||
|
|
||||||
|
/// <summary>A <c>ClusterNode</c> row reduced to the address facts this check cares about.</summary>
|
||||||
|
/// <param name="NodeId">The row's primary key, in the <c>host:port</c> space.</param>
|
||||||
|
/// <param name="Host">The host central will dial in Phase 2.</param>
|
||||||
|
/// <param name="AkkaPort">The remoting port central will dial in Phase 2.</param>
|
||||||
|
public sealed record ClusterNodeAddress(string NodeId, string Host, int AkkaPort);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compares the addresses driver nodes actually occupy against the <c>ClusterNode</c> rows that
|
||||||
|
/// claim to describe them.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Why this exists.</b> <c>ClusterNode.AkkaPort</c> and the node's own <c>Cluster:Port</c>
|
||||||
|
/// are the same fact stored in two places, and nothing makes them agree. A node that binds
|
||||||
|
/// 4054 while its row says 4053 is unreachable from central once Phase 2 dials the row
|
||||||
|
/// instead of gossiping — and an unenforced duplicated address is the exact shape of the
|
||||||
|
/// <c>Modbus</c>/<c>ModbusTcp</c> and <c>TwinCat</c>/<c>Focas</c> drifts already recorded in
|
||||||
|
/// this repo. Since Phase 1 also made the rows the deploy path's expected-ack set, a drifted
|
||||||
|
/// row already costs a failed deployment today.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>It runs on an admin node</b>, comparing rows against the membership an admin node can
|
||||||
|
/// already see, rather than having each driver node assert its own row — Phase 4 removes the
|
||||||
|
/// driver nodes' ConfigDb connection entirely, so a self-assertion written there would have
|
||||||
|
/// to be deleted again.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Phase 2 must revisit this.</b> Once the fleet splits into one mesh per cluster, an
|
||||||
|
/// admin node no longer sees site members at all, and every site row would report
|
||||||
|
/// <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> forever. The check then has to
|
||||||
|
/// move to whatever transport replaces gossip — or be scoped to the central mesh.
|
||||||
|
/// </para>
|
||||||
|
/// </remarks>
|
||||||
|
public static class ClusterNodeAddressReconciler
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Reconciles observed driver-role member addresses against the configured rows.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="observedDriverMembers">
|
||||||
|
/// <c>host</c>/<c>port</c> of every Up member carrying the <c>driver</c> role. Admin-only
|
||||||
|
/// members must be excluded — they legitimately have no <c>ClusterNode</c> row, and including
|
||||||
|
/// them would report a mismatch for correct configuration.
|
||||||
|
/// </param>
|
||||||
|
/// <param name="rows">Every <c>ClusterNode</c> row, enabled or not.</param>
|
||||||
|
/// <param name="enabledNodeIds">
|
||||||
|
/// The subset of <paramref name="rows"/> that is enabled. Disabled rows are deliberately
|
||||||
|
/// still matched against membership (a disabled row for a running node is worth knowing) but
|
||||||
|
/// never reported as <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> — being absent
|
||||||
|
/// is the whole point of disabling one.
|
||||||
|
/// </param>
|
||||||
|
/// <returns>Every disagreement found, ordered by node id for stable logging.</returns>
|
||||||
|
public static IReadOnlyList<ClusterNodeAddressMismatch> Reconcile(
|
||||||
|
IReadOnlyCollection<(string Host, int Port)> observedDriverMembers,
|
||||||
|
IReadOnlyCollection<ClusterNodeAddress> rows,
|
||||||
|
IReadOnlyCollection<string> enabledNodeIds)
|
||||||
|
{
|
||||||
|
var byNodeId = new Dictionary<string, ClusterNodeAddress>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
foreach (var row in rows) byNodeId[row.NodeId] = row;
|
||||||
|
var enabled = new HashSet<string>(enabledNodeIds, StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
var findings = new List<ClusterNodeAddressMismatch>();
|
||||||
|
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
foreach (var (host, port) in observedDriverMembers)
|
||||||
|
{
|
||||||
|
var nodeId = $"{host}:{port}";
|
||||||
|
seen.Add(nodeId);
|
||||||
|
|
||||||
|
if (!byNodeId.TryGetValue(nodeId, out var row))
|
||||||
|
{
|
||||||
|
findings.Add(new ClusterNodeAddressMismatch(
|
||||||
|
AddressMismatchKind.RunningNodeHasNoRow, nodeId,
|
||||||
|
$"driver node is in the cluster at {nodeId} but no ClusterNode row has that NodeId; " +
|
||||||
|
"its deployment acks are discarded. Either register the row or correct the node's " +
|
||||||
|
"Cluster:PublicHostname / Cluster:Port to match an existing one"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.Equals(row.Host, host, StringComparison.OrdinalIgnoreCase) || row.AkkaPort != port)
|
||||||
|
{
|
||||||
|
findings.Add(new ClusterNodeAddressMismatch(
|
||||||
|
AddressMismatchKind.RowDialTargetDisagrees, nodeId,
|
||||||
|
$"node is at {nodeId} but its row's dial target is {row.Host}:{row.AkkaPort}; " +
|
||||||
|
"central will dial the wrong address once Phase 2 stops using gossip. " +
|
||||||
|
"Correct ClusterNode.Host / ClusterNode.AkkaPort"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var row in rows)
|
||||||
|
{
|
||||||
|
if (seen.Contains(row.NodeId) || !enabled.Contains(row.NodeId)) continue;
|
||||||
|
findings.Add(new ClusterNodeAddressMismatch(
|
||||||
|
AddressMismatchKind.EnabledRowNotInCluster, row.NodeId,
|
||||||
|
$"enabled ClusterNode row {row.NodeId} has no matching driver member; the next " +
|
||||||
|
"deployment will fail at the apply deadline waiting for it. Start the node, or set " +
|
||||||
|
"ClusterNode.Enabled = 0 while it is down for maintenance"));
|
||||||
|
}
|
||||||
|
|
||||||
|
return findings
|
||||||
|
.OrderBy(f => f.NodeId, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ThenBy(f => f.Kind)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
using Akka.Actor;
|
||||||
|
using Akka.Cluster;
|
||||||
|
using Akka.Event;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Admin-role cluster singleton that runs <see cref="ClusterNodeAddressReconciler"/> against live
|
||||||
|
/// membership and logs what it finds. A singleton rather than a per-admin-node actor so a
|
||||||
|
/// two-admin fleet reports each disagreement once.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Recomputes on membership change (debounced) and on a slow periodic sweep, and <b>logs only
|
||||||
|
/// when the finding set changes</b>. A node legitimately down would otherwise reprint the same
|
||||||
|
/// warning every sweep until someone silences it by ignoring the log, which is the failure mode
|
||||||
|
/// this check exists to avoid.
|
||||||
|
/// </remarks>
|
||||||
|
public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimers
|
||||||
|
{
|
||||||
|
/// <summary>Driver role name — only members carrying it are expected to have a ClusterNode row.</summary>
|
||||||
|
public const string DriverRole = "driver";
|
||||||
|
|
||||||
|
/// <summary>Collapses a burst of membership events (a rolling restart) into one reconcile.</summary>
|
||||||
|
public static readonly TimeSpan DebounceWindow = TimeSpan.FromSeconds(2);
|
||||||
|
|
||||||
|
/// <summary>Periodic sweep, so a row edited in the AdminUI is checked without a topology change.</summary>
|
||||||
|
public static readonly TimeSpan DefaultSweepInterval = TimeSpan.FromMinutes(5);
|
||||||
|
|
||||||
|
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||||
|
private readonly Akka.Cluster.Cluster _cluster;
|
||||||
|
private readonly TimeSpan _sweepInterval;
|
||||||
|
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||||
|
private IReadOnlyList<ClusterNodeAddressMismatch>? _lastReported;
|
||||||
|
|
||||||
|
/// <summary>Gets the timer scheduler for this actor.</summary>
|
||||||
|
public ITimerScheduler Timers { get; set; } = null!;
|
||||||
|
|
||||||
|
/// <summary>Creates Props for the reconciler.</summary>
|
||||||
|
/// <param name="dbFactory">Factory for the config database context.</param>
|
||||||
|
/// <param name="sweepInterval">Periodic sweep interval; defaults to <see cref="DefaultSweepInterval"/>.</param>
|
||||||
|
/// <returns>Props for actor creation.</returns>
|
||||||
|
public static Props Props(
|
||||||
|
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, TimeSpan? sweepInterval = null) =>
|
||||||
|
Akka.Actor.Props.Create(() => new ClusterNodeAddressReconcilerActor(dbFactory, sweepInterval));
|
||||||
|
|
||||||
|
/// <summary>Initializes a new instance of the <see cref="ClusterNodeAddressReconcilerActor"/> class.</summary>
|
||||||
|
/// <param name="dbFactory">Factory for the config database context.</param>
|
||||||
|
/// <param name="sweepInterval">Periodic sweep interval; defaults to <see cref="DefaultSweepInterval"/>.</param>
|
||||||
|
public ClusterNodeAddressReconcilerActor(
|
||||||
|
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, TimeSpan? sweepInterval = null)
|
||||||
|
{
|
||||||
|
_dbFactory = dbFactory;
|
||||||
|
_cluster = Akka.Cluster.Cluster.Get(Context.System);
|
||||||
|
_sweepInterval = sweepInterval ?? DefaultSweepInterval;
|
||||||
|
|
||||||
|
Receive<ClusterEvent.IMemberEvent>(_ => ScheduleReconcile());
|
||||||
|
Receive<ClusterEvent.CurrentClusterState>(_ => ScheduleReconcile());
|
||||||
|
Receive<ReconcileNow>(_ => Reconcile());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void PreStart()
|
||||||
|
{
|
||||||
|
_cluster.Subscribe(Self, ClusterEvent.InitialStateAsEvents, typeof(ClusterEvent.IMemberEvent));
|
||||||
|
Timers.StartPeriodicTimer("sweep", ReconcileNow.Instance, _sweepInterval, _sweepInterval);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void PostStop() => _cluster.Unsubscribe(Self);
|
||||||
|
|
||||||
|
private void ScheduleReconcile() =>
|
||||||
|
Timers.StartSingleTimer("debounce", ReconcileNow.Instance, DebounceWindow);
|
||||||
|
|
||||||
|
private void Reconcile()
|
||||||
|
{
|
||||||
|
var observed = _cluster.State.Members
|
||||||
|
.Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole))
|
||||||
|
.Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0))
|
||||||
|
.Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
List<ClusterNodeAddress> rows;
|
||||||
|
List<string> enabled;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var db = _dbFactory.CreateDbContext();
|
||||||
|
rows = db.ClusterNodes.AsNoTracking()
|
||||||
|
.Select(n => new ClusterNodeAddress(n.NodeId, n.Host, n.AkkaPort))
|
||||||
|
.ToList();
|
||||||
|
enabled = db.ClusterNodes.AsNoTracking()
|
||||||
|
.Where(n => n.Enabled).Select(n => n.NodeId).ToList();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// A consistency check must never take the admin node down with it. Central SQL being
|
||||||
|
// briefly unreachable is an operational fact, not a reason to restart this singleton.
|
||||||
|
_log.Warning(ex, "ClusterNode address reconcile skipped — config database unreachable");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(observed, rows, enabled);
|
||||||
|
|
||||||
|
if (_lastReported is not null && findings.SequenceEqual(_lastReported)) return;
|
||||||
|
_lastReported = findings;
|
||||||
|
|
||||||
|
if (findings.Count == 0)
|
||||||
|
{
|
||||||
|
_log.Info("ClusterNode addresses reconcile with cluster membership ({Count} driver members)",
|
||||||
|
observed.Count);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var f in findings)
|
||||||
|
{
|
||||||
|
// Error for the two shapes that are always a misconfiguration; Warning for a row whose
|
||||||
|
// node is merely absent, which is a legitimate state during maintenance.
|
||||||
|
if (f.Kind == AddressMismatchKind.EnabledRowNotInCluster)
|
||||||
|
_log.Warning("ClusterNode address check [{Kind}] {NodeId}: {Detail}", f.Kind, f.NodeId, f.Detail);
|
||||||
|
else
|
||||||
|
_log.Error("ClusterNode address check [{Kind}] {NodeId}: {Detail}", f.Kind, f.NodeId, f.Detail);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Self-message: run a reconcile pass now.</summary>
|
||||||
|
public sealed class ReconcileNow
|
||||||
|
{
|
||||||
|
/// <summary>The singleton instance.</summary>
|
||||||
|
public static readonly ReconcileNow Instance = new();
|
||||||
|
private ReconcileNow() { }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,6 +23,7 @@ public static class ServiceCollectionExtensions
|
|||||||
public const string AuditWriterSingletonName = "audit-writer";
|
public const string AuditWriterSingletonName = "audit-writer";
|
||||||
public const string FleetStatusSingletonName = "fleet-status";
|
public const string FleetStatusSingletonName = "fleet-status";
|
||||||
public const string RedundancyStateSingletonName = "redundancy-state";
|
public const string RedundancyStateSingletonName = "redundancy-state";
|
||||||
|
public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler";
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Registers all five admin-role cluster singletons + their proxies on the AkkaConfigurationBuilder.
|
/// Registers all five admin-role cluster singletons + their proxies on the AkkaConfigurationBuilder.
|
||||||
@@ -90,6 +91,18 @@ public static class ServiceCollectionExtensions
|
|||||||
(system, registry, resolver) => RedundancyStateActor.Props(),
|
(system, registry, resolver) => RedundancyStateActor.Props(),
|
||||||
singletonOptions);
|
singletonOptions);
|
||||||
|
|
||||||
|
// Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and
|
||||||
|
// nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes
|
||||||
|
// the driver nodes' ConfigDb connection entirely.
|
||||||
|
builder.WithSingleton<ClusterNodeAddressReconcilerKey>(
|
||||||
|
ClusterNodeAddressReconcilerSingletonName,
|
||||||
|
(system, registry, resolver) =>
|
||||||
|
{
|
||||||
|
var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
||||||
|
return ClusterNodeAddressReconcilerActor.Props(dbFactory);
|
||||||
|
},
|
||||||
|
singletonOptions);
|
||||||
|
|
||||||
return builder;
|
return builder;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -100,3 +113,4 @@ public sealed class AdminOperationsActorKey { }
|
|||||||
public sealed class AuditWriterActorKey { }
|
public sealed class AuditWriterActorKey { }
|
||||||
public sealed class FleetStatusBroadcasterKey { }
|
public sealed class FleetStatusBroadcasterKey { }
|
||||||
public sealed class RedundancyStateActorKey { }
|
public sealed class RedundancyStateActorKey { }
|
||||||
|
public sealed class ClusterNodeAddressReconcilerKey { }
|
||||||
|
|||||||
+98
@@ -0,0 +1,98 @@
|
|||||||
|
using Akka.Actor;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Xunit;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Proves the reconciler is actually wired to something — subscribes to membership, reads the DB,
|
||||||
|
/// and emits. <see cref="ClusterNodeAddressReconcilerTests"/> covers the comparison logic, which a
|
||||||
|
/// dormant actor would leave perfectly correct and completely inert.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ClusterNodeAddressReconcilerActorTests : ControlPlaneActorTestBase
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// An enabled row with no matching driver member is warned about. The harness ActorSystem
|
||||||
|
/// joins as <c>admin</c> with no driver members, so the seeded row is unmatched by
|
||||||
|
/// construction.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Enabled_row_with_no_matching_member_is_logged()
|
||||||
|
{
|
||||||
|
var dbFactory = NewInMemoryDbFactory();
|
||||||
|
SeedNode(dbFactory, "site-a-1:4053", enabled: true);
|
||||||
|
|
||||||
|
EventFilter.Warning(contains: "site-a-1:4053").ExpectOne(TimeSpan.FromSeconds(10), () =>
|
||||||
|
Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props(dbFactory)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The finding set is logged once, not once per sweep. A check that reprints the same warning
|
||||||
|
/// every five minutes trains operators to filter it out, which costs more than it catches —
|
||||||
|
/// so this runs a deliberately fast sweep and asserts the count stays at one.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Unchanged_findings_are_not_repeated_on_every_sweep()
|
||||||
|
{
|
||||||
|
var dbFactory = NewInMemoryDbFactory();
|
||||||
|
SeedNode(dbFactory, "site-a-1:4053", enabled: true);
|
||||||
|
|
||||||
|
// ~6 sweeps inside the window; without the change-detection guard this logs 6 times.
|
||||||
|
EventFilter.Warning(contains: "site-a-1:4053").Expect(1, TimeSpan.FromSeconds(3), () =>
|
||||||
|
{
|
||||||
|
Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props(
|
||||||
|
dbFactory, sweepInterval: TimeSpan.FromMilliseconds(400)));
|
||||||
|
// Hold the filter open past several sweeps rather than returning immediately.
|
||||||
|
ExpectNoMsg(TimeSpan.FromSeconds(2.5));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A disabled row is silence — the maintenance hatch must not trade a failed deployment for a
|
||||||
|
/// permanent warning. Positive control for the test above: same setup, one flag flipped.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Disabled_row_produces_no_warning()
|
||||||
|
{
|
||||||
|
var dbFactory = NewInMemoryDbFactory();
|
||||||
|
SeedNode(dbFactory, "site-a-1:4053", enabled: false);
|
||||||
|
|
||||||
|
EventFilter.Warning(contains: "site-a-1:4053").Expect(0, TimeSpan.FromSeconds(2), () =>
|
||||||
|
{
|
||||||
|
Sys.ActorOf(ClusterNodeAddressReconcilerActor.Props(
|
||||||
|
dbFactory, sweepInterval: TimeSpan.FromMilliseconds(400)));
|
||||||
|
ExpectNoMsg(TimeSpan.FromSeconds(1.5));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void SeedNode(
|
||||||
|
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, string nodeId, bool enabled)
|
||||||
|
{
|
||||||
|
using var db = dbFactory.CreateDbContext();
|
||||||
|
db.ServerClusters.Add(new ServerCluster
|
||||||
|
{
|
||||||
|
ClusterId = "SITE-A",
|
||||||
|
Name = "Site A",
|
||||||
|
Enterprise = "zb",
|
||||||
|
Site = "site-a",
|
||||||
|
NodeCount = 2,
|
||||||
|
RedundancyMode = RedundancyMode.Warm,
|
||||||
|
CreatedBy = "test",
|
||||||
|
});
|
||||||
|
db.ClusterNodes.Add(new ClusterNode
|
||||||
|
{
|
||||||
|
NodeId = nodeId,
|
||||||
|
ClusterId = "SITE-A",
|
||||||
|
Host = nodeId.Split(':')[0],
|
||||||
|
ApplicationUri = $"urn:OtOpcUa:{nodeId}",
|
||||||
|
Enabled = enabled,
|
||||||
|
CreatedBy = "test",
|
||||||
|
});
|
||||||
|
db.SaveChanges();
|
||||||
|
}
|
||||||
|
}
|
||||||
+126
@@ -0,0 +1,126 @@
|
|||||||
|
using Shouldly;
|
||||||
|
using Xunit;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Per-cluster mesh Phase 1 Task 4: <c>ClusterNode.AkkaPort</c> and the node's own
|
||||||
|
/// <c>Cluster:Port</c> are the same fact in two places, and nothing makes them agree. These pin
|
||||||
|
/// the shapes that drift takes.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ClusterNodeAddressReconcilerTests
|
||||||
|
{
|
||||||
|
private static ClusterNodeAddress Row(string host, int akkaPort, string? nodeId = null) =>
|
||||||
|
new(nodeId ?? $"{host}:{akkaPort}", host, akkaPort);
|
||||||
|
|
||||||
|
/// <summary>A fleet whose rows match membership reports nothing.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void Matching_rows_and_membership_produce_no_findings()
|
||||||
|
{
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("site-a-1", 4053), ("site-a-2", 4053)],
|
||||||
|
[Row("site-a-1", 4053), Row("site-a-2", 4053)],
|
||||||
|
["site-a-1:4053", "site-a-2:4053"]);
|
||||||
|
|
||||||
|
findings.ShouldBeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The drift the plan names: a node binds 4054 while its row says 4053. Membership shows it
|
||||||
|
/// at an address no row claims, and the row it should have matched looks absent — both halves
|
||||||
|
/// are reported, because either alone reads as a different problem.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Node_bound_to_the_wrong_port_is_reported_from_both_sides()
|
||||||
|
{
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("site-a-1", 4054)],
|
||||||
|
[Row("site-a-1", 4053)],
|
||||||
|
["site-a-1:4053"]);
|
||||||
|
|
||||||
|
findings.Count.ShouldBe(2);
|
||||||
|
findings.ShouldContain(f =>
|
||||||
|
f.Kind == AddressMismatchKind.RunningNodeHasNoRow && f.NodeId == "site-a-1:4054");
|
||||||
|
findings.ShouldContain(f =>
|
||||||
|
f.Kind == AddressMismatchKind.EnabledRowNotInCluster && f.NodeId == "site-a-1:4053");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The Phase 2 shape: the row's NodeId still matches membership, but its <c>Host</c> /
|
||||||
|
/// <c>AkkaPort</c> columns — the values central will actually dial — have been edited away
|
||||||
|
/// from it. Gossip still works today, so nothing else would notice.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Row_whose_dial_target_disagrees_with_its_own_NodeId_is_reported()
|
||||||
|
{
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("site-a-1", 4053)],
|
||||||
|
[Row("site-a-1", 4054, nodeId: "site-a-1:4053")],
|
||||||
|
["site-a-1:4053"]);
|
||||||
|
|
||||||
|
var only = findings.ShouldHaveSingleItem();
|
||||||
|
only.Kind.ShouldBe(AddressMismatchKind.RowDialTargetDisagrees);
|
||||||
|
only.NodeId.ShouldBe("site-a-1:4053");
|
||||||
|
only.Detail.ShouldContain("site-a-1:4054");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A running driver node nobody registered — the coordinator discards its acks.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void Running_node_with_no_row_is_reported()
|
||||||
|
{
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("ghost-9", 4053)], [], []);
|
||||||
|
|
||||||
|
findings.ShouldHaveSingleItem().Kind.ShouldBe(AddressMismatchKind.RunningNodeHasNoRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A disabled row for an absent node is silence, not a finding — being absent is the entire
|
||||||
|
/// point of disabling it. Without this the maintenance hatch would trade a failed deploy for
|
||||||
|
/// a permanent warning.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Disabled_row_for_an_absent_node_is_not_reported()
|
||||||
|
{
|
||||||
|
var findings = ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("site-a-1", 4053)],
|
||||||
|
[Row("site-a-1", 4053), Row("site-a-2", 4053)],
|
||||||
|
["site-a-1:4053"]);
|
||||||
|
|
||||||
|
findings.ShouldBeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// An admin-only member has no <c>ClusterNode</c> row by design, so the caller filters
|
||||||
|
/// membership to driver-role members. Passing an unfiltered list would report correct
|
||||||
|
/// configuration as broken — pinned here because the filter lives at the call site.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Admin_only_members_are_the_callers_responsibility_to_exclude()
|
||||||
|
{
|
||||||
|
// What the actor passes: driver members only.
|
||||||
|
ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("central-1", 4053)], [Row("central-1", 4053)], ["central-1:4053"])
|
||||||
|
.ShouldBeEmpty();
|
||||||
|
|
||||||
|
// What an unfiltered call would produce — the false positive this guards against.
|
||||||
|
ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("central-1", 4053), ("admin-only-1", 4053)],
|
||||||
|
[Row("central-1", 4053)],
|
||||||
|
["central-1:4053"])
|
||||||
|
.ShouldHaveSingleItem().Kind.ShouldBe(AddressMismatchKind.RunningNodeHasNoRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Host matching is case-insensitive, matching the coordinator's <c>NodeIdComparer</c> — DNS
|
||||||
|
/// and SQL collation are both case-insensitive, so the same node surfaces with either casing.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Host_matching_is_case_insensitive()
|
||||||
|
{
|
||||||
|
ClusterNodeAddressReconciler.Reconcile(
|
||||||
|
[("SITE-A-1", 4053)], [Row("site-a-1", 4053)], ["site-a-1:4053"])
|
||||||
|
.ShouldBeEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user