fix(mesh): also scope the reconciler's observed membership to the own cluster

Task 3 review: scoping only the rows/enabled DB queries by ClusterId left the observed set (from Cluster.State.Members) filtered by the driver role alone. In the mixed-mesh window — DB scoping live via a cluster role, but the physical Akka mesh not yet split so central still sees foreign members via gossip — a foreign-cluster driver member stays in observed while its row is filtered out, hitting the RunningNodeHasNoRow branch, which logs at ERROR. That traded the Warning-level false positive for a worse Error-level one. Filter observed by the node's own cluster ROLE too. Extracted the projection into a static FilterOwnClusterDriverMembers(members, ownClusterRole) so the membership filter is unit-testable without seeding a live cluster (the actor harness joins as admin only, so observed is always empty). Sourced at registration from IClusterRoleInfo.ClusterRole alongside ClusterId. Null role (legacy, no cluster role) keeps every driver member — reconcile-all, unchanged. Also clarified the ownClusterId doc wording (non-null-and-non-empty). Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-24 02:07:02 -04:00
parent a4d3ab4005
commit 1ce2042e14
4 changed files with 165 additions and 23 deletions
@@ -61,13 +61,20 @@ public sealed record ClusterNodeAddress(string NodeId, string Host, int AkkaPort
 ///         <b>Scoped to the admin node's own cluster (Phase 6).</b> Once the fleet splits into one
 ///         mesh per cluster, an admin node no longer sees site members at all, and a fleet-wide sweep
 ///         would report every other cluster's row as
-///         <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> forever. The actor therefore loads
-///         only rows whose <c>ClusterId</c> matches the admin node's own
-///         (<see cref="ZB.MOM.WW.OtOpcUa.Commons.Interfaces.IClusterRoleInfo.ClusterId"/>) before
-///         calling <see cref="Reconcile"/>, so it never false-reports a cluster it cannot observe. A
-///         legacy admin node carrying no cluster role (null <c>ClusterId</c>) still sees every member
-///         via gossip and reconciles the full fleet unchanged. The pure function below is untouched by
-///         this — scoping happens entirely in the actor's DB queries.
+///         <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> forever. The actor therefore scopes
+///         <b>both</b> inputs to this function to the admin node's own cluster: the <c>rows</c>/enabled
+///         set to rows whose <c>ClusterId</c> matches
+///         <see cref="ZB.MOM.WW.OtOpcUa.Commons.Interfaces.IClusterRoleInfo.ClusterId"/>, and the
+///         observed membership to members carrying its own
+///         <see cref="ZB.MOM.WW.OtOpcUa.Commons.Interfaces.IClusterRoleInfo.ClusterRole"/>
+///         (<c>FilterOwnClusterDriverMembers</c>). Scoping only the rows would swap the
+///         <see cref="AddressMismatchKind.EnabledRowNotInCluster"/> false positive for a worse
+///         Error-level <see cref="AddressMismatchKind.RunningNodeHasNoRow"/> during the mixed-mesh
+///         window, where the DB scope is live but central still sees foreign members via gossip. A
+///         legacy admin node carrying no cluster role (null <c>ClusterId</c>/<c>ClusterRole</c>) still
+///         sees every member via gossip and reconciles the full fleet unchanged. The pure function
+///         below is untouched — scoping happens entirely in the actor's DB queries and its
+///         membership filter.
 ///     </para>
 /// </remarks>
 public static class ClusterNodeAddressReconciler
@@ -31,6 +31,7 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer

    private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
    private readonly string? _ownClusterId;
+    private readonly string? _ownClusterRole;
    private readonly Akka.Cluster.Cluster _cluster;
    private readonly TimeSpan _sweepInterval;
    private readonly ILoggingAdapter _log = Context.GetLogger();
@@ -43,33 +44,47 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer
    /// <param name="dbFactory">Factory for the config database context.</param>
    /// <param name="ownClusterId">
    ///     The admin node's own <c>ClusterId</c> (e.g. <c>MAIN</c>), from
-    ///     <see cref="IClusterRoleInfo.ClusterId"/>. Non-null/empty scopes the reconcile to rows in that
-    ///     cluster — the only members central's own mesh can see after the Phase 6 split. Null/empty
-    ///     (a legacy admin node with no cluster role) reconciles the whole fleet, unchanged.
+    ///     <see cref="IClusterRoleInfo.ClusterId"/>. When non-null-and-non-empty this scopes the row
+    ///     side of the reconcile (<c>rows</c>/<c>enabled</c>) to that cluster — the only members
+    ///     central's own mesh can see after the Phase 6 split. A null or empty value (a legacy admin
+    ///     node with no cluster role) reconciles the whole fleet, unchanged.
+    /// </param>
+    /// <param name="ownClusterRole">
+    ///     The admin node's own cluster role (e.g. <c>cluster-MAIN</c>), from
+    ///     <see cref="IClusterRoleInfo.ClusterRole"/>. Scopes the membership (<c>observed</c>) side to
+    ///     match the row side, so a foreign-cluster driver member central still sees via gossip in the
+    ///     mixed-mesh window is not mis-reported. Null/empty keeps every driver member (legacy).
    /// </param>
    /// <param name="sweepInterval">Periodic sweep interval; defaults to <see cref="DefaultSweepInterval"/>.</param>
    /// <returns>Props for actor creation.</returns>
    public static Props Props(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
        string? ownClusterId = null,
+        string? ownClusterRole = null,
        TimeSpan? sweepInterval = null) =>
        Akka.Actor.Props.Create(() =>
-            new ClusterNodeAddressReconcilerActor(dbFactory, ownClusterId, sweepInterval));
+            new ClusterNodeAddressReconcilerActor(dbFactory, ownClusterId, ownClusterRole, sweepInterval));

    /// <summary>Initializes a new instance of the <see cref="ClusterNodeAddressReconcilerActor"/> class.</summary>
    /// <param name="dbFactory">Factory for the config database context.</param>
    /// <param name="ownClusterId">
-    ///     The admin node's own <c>ClusterId</c>; scopes the reconcile to that cluster's rows when
-    ///     non-null/empty, else reconciles the whole fleet (legacy behaviour).
+    ///     The admin node's own <c>ClusterId</c>; scopes the row side to that cluster's rows when
+    ///     non-null-and-non-empty, else reconciles the whole fleet (legacy behaviour).
+    /// </param>
+    /// <param name="ownClusterRole">
+    ///     The admin node's own cluster role; scopes the observed-membership side to that role when
+    ///     non-null-and-non-empty, else keeps every driver member (legacy behaviour).
    /// </param>
    /// <param name="sweepInterval">Periodic sweep interval; defaults to <see cref="DefaultSweepInterval"/>.</param>
    public ClusterNodeAddressReconcilerActor(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
        string? ownClusterId = null,
+        string? ownClusterRole = null,
        TimeSpan? sweepInterval = null)
    {
        _dbFactory = dbFactory;
        _ownClusterId = string.IsNullOrEmpty(ownClusterId) ? null : ownClusterId;
+        _ownClusterRole = string.IsNullOrEmpty(ownClusterRole) ? null : ownClusterRole;
        _cluster = Akka.Cluster.Cluster.Get(Context.System);
        _sweepInterval = sweepInterval ?? DefaultSweepInterval;

@@ -93,11 +108,7 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer

    private void Reconcile()
    {
-        var observed = _cluster.State.Members
-            .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole))
-            .Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0))
-            .Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0)
-            .ToList();
+        var observed = FilterOwnClusterDriverMembers(_cluster.State.Members, _ownClusterRole);

        List<ClusterNodeAddress> rows;
        List<string> enabled;
@@ -151,6 +162,35 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer
        }
    }

+    /// <summary>
+    ///     Projects the Up, driver-role members central should reconcile against, scoped to the admin
+    ///     node's OWN cluster role when it carries one. Extracted so the observed-membership filter —
+    ///     the membership half of the Phase 6 own-cluster scope, mirroring the <c>ClusterId</c> row
+    ///     scope in <see cref="Reconcile"/> — is unit-testable without seeding a live cluster.
+    /// </summary>
+    /// <param name="members">The live cluster members (from <c>Cluster.State.Members</c>).</param>
+    /// <param name="ownClusterRole">
+    ///     The admin node's own cluster role (e.g. <c>cluster-MAIN</c>). When non-null-and-non-empty,
+    ///     driver members that do not also carry it are excluded — matching the row scope so a foreign
+    ///     driver member central still sees via gossip in the mixed-mesh window (row filtered out, but
+    ///     member still visible) is NOT mis-reported as an Error-level <c>RunningNodeHasNoRow</c>. A
+    ///     null or empty value (a legacy node with no cluster role) keeps every driver member —
+    ///     reconcile-all, unchanged.
+    /// </param>
+    /// <returns>The <c>host</c>/<c>port</c> of each in-scope Up driver member.</returns>
+    public static IReadOnlyList<(string Host, int Port)> FilterOwnClusterDriverMembers(
+        IEnumerable<Member> members, string? ownClusterRole)
+    {
+        var driverMembers = members
+            .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole));
+        if (!string.IsNullOrEmpty(ownClusterRole))
+            driverMembers = driverMembers.Where(m => m.Roles.Contains(ownClusterRole));
+        return driverMembers
+            .Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0))
+            .Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0)
+            .ToList();
+    }
+
    /// <summary>Self-message: run a reconcile pass now.</summary>
    public sealed class ReconcileNow
    {
@@ -153,12 +153,16 @@ public static class ServiceCollectionExtensions
            (system, registry, resolver) =>
            {
                var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
-                // Per-cluster mesh Phase 6: scope the reconcile to the admin node's OWN cluster's rows.
-                // After the mesh split central sees only its own pair via gossip, so a fleet-wide sweep
-                // would false-report every other cluster's rows as EnabledRowNotInCluster forever. A
-                // legacy node with no cluster role has null ClusterId ⇒ full-fleet reconcile, unchanged.
+                // Per-cluster mesh Phase 6: scope BOTH halves of the reconcile to the admin node's OWN
+                // cluster — rows by ClusterId, observed membership by cluster role. After the mesh split
+                // central sees only its own pair via gossip, so an unscoped sweep would false-report
+                // every other cluster's rows (EnabledRowNotInCluster). Scoping only the rows would swap
+                // that for a WORSE Error-level RunningNodeHasNoRow in the mixed-mesh window (rows already
+                // filtered, but foreign members still visible via gossip). A legacy node with no cluster
+                // role has null ClusterId/ClusterRole ⇒ full-fleet reconcile, unchanged.
                var roleInfo = resolver.GetService<IClusterRoleInfo>();
-                return ClusterNodeAddressReconcilerActor.Props(dbFactory, roleInfo.ClusterId);
+                return ClusterNodeAddressReconcilerActor.Props(
+                    dbFactory, roleInfo.ClusterId, roleInfo.ClusterRole);
            },
            singletonOptions,
            createProxyToo: false);