From 85ca86e9557330fb7ded36d2ba8b9c7ec7825fb0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 15:45:21 -0400 Subject: [PATCH] =?UTF-8?q?refactor(mesh):=20step=205=20=E2=80=94=20delete?= =?UTF-8?q?=20MeshTransport:CentralContactPoints,=20its=20rules=20and=20it?= =?UTF-8?q?s=20compose=20entries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The property, the four validator rules that graded it (empty / non-akka.tcp / actor-path suffix / one-bad-among-good) and all twelve docker-compose entries (6 nodes x 2) are gone. The compose comment blocks that explained "/system/receptionist is appended at construction time" and the client's contact rotation go with them, and the surrounding prose now describes the Grpc command plane it actually runs. CentralGrpcEndpoints' doc no longer says it "replaces" the Akka contact points — there is nothing left to replace — and ContactRefreshSeconds' failure message names the node dispatch set rather than the ClusterClient contact points. `rg CentralContactPoints src tests docker-dev` is now empty. --- docker-dev/docker-compose.yml | 93 ++++----------- .../MeshTransportOptions.cs | 26 +---- .../MeshTransportOptionsValidator.cs | 41 +------ .../MeshTransportOptionsValidatorTests.cs | 108 +----------------- 4 files changed, 34 insertions(+), 234 deletions(-) diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index 9e1069a7..3e6cbe4f 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -21,9 +21,10 @@ # # Since central no longer shares a gossip ring with the site meshes, it reaches # them over the three explicit split-topology transports instead of gossip: -# - Command plane: MeshTransport__Mode=ClusterClient (central dials each site -# mesh's ClusterClientReceptionist; DPS is gossip-only and -# would not cross the mesh boundary at all now) +# - Command plane: MeshTransport__Mode=Grpc (central dials each site node's +# NodeCommandService on :4056; a node posts its deploy ack +# back to central's CentralCommandService on :4055. DPS is +# gossip-only and would not cross the mesh boundary at all) # - Live telemetry: Telemetry__Mode=Grpc on every driver node (all six) + # TelemetryDial__Mode=Grpc on the admin pair (central-1/2) # — the site node hosts the gRPC server, central dials in @@ -53,8 +54,7 @@ # data-plane endpoints (4842-4845) and authenticate those # users against the same shared GLAuth (via the *ldap-env # anchor); central manages + deploys to them over the -# ClusterClient/gRPC/FetchAndCache transports above, not -# gossip. +# Grpc/FetchAndCache transports above, not gossip. # # Auth is real LDAP against the shared GLAuth on the Linux Docker host # (10.100.0.35:3893, dc=zb,dc=local) — there is no LDAP container here. @@ -329,33 +329,22 @@ services: Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" # Per-cluster mesh Phase 6: cluster-MAIN marks this node as a member of the MAIN mesh # (RoleParser.ClusterRolePrefix = "cluster-"), read by SplitTopologyTransportValidator to - # require ClusterClient/Grpc transports, and by ClusterRoleInfo to scope redundancy + + # require the mesh-crossing Grpc transports, and by ClusterRoleInfo to scope redundancy + # ClusterNodeAddressReconcilerActor to ClusterId=MAIN. Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Cluster__Roles__2: "cluster-MAIN" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6). Central no # longer shares a gossip ring with the site meshes, so DPS (gossip-only) cannot reach them — - # ClusterClient is the only transport that can. SplitTopologyTransportValidator refuses to - # start any node carrying a cluster-* role that is left on Dps, so this is fixed, not an + # Grpc is the only transport that can. SplitTopologyTransportValidator refuses to start any + # node carrying a cluster-* role that is left on Dps, so this is fixed, not an # env-var-overridable default, on every node in this file. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0 of the ClusterClient→gRPC migration, docs/plans/2026-08-06- - # clusterclient-to-grpc-design.md). SHIPPED DARK: Mode above stays ClusterClient, so none of - # this is dialled or served yet — only MeshTransportOptionsValidator's Grpc-mode rules read - # it. GrpcPsk is this node's OWN per-cluster key (MAIN, since central is cluster-MAIN); - # CentralGrpcEndpoints is the gRPC counterpart of CentralContactPoints above. ClusterPsks is - # CENTRAL-SIDE ONLY (IMeshPskProvider's override map, consulted before the Secrets store) — - # one entry per cluster central will dial once the switch flips, so only the central nodes - # carry it. + # gRPC command transport (docs/plans/2026-08-06-clusterclient-to-grpc-design.md). GrpcPsk is + # this node's OWN per-cluster key (MAIN, since central is cluster-MAIN); CentralGrpcEndpoints + # is where a node posts its deploy ack. ClusterPsks is CENTRAL-SIDE ONLY (IMeshPskProvider's + # override map, consulted before the Secrets store) — one entry per cluster central dials, so + # only the central nodes carry it. MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" @@ -478,16 +467,8 @@ services: Cluster__Roles__1: "driver" Cluster__Roles__2: "cluster-MAIN" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see central-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # gRPC command transport — see central-1. # ClusterPsks is central-side only; both central nodes carry the same three-cluster map. MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -549,7 +530,7 @@ services: # ── Site A cluster (2-node driver-only) ───────────────────────────────────── # Per-cluster mesh Phase 6: site-a-1 and site-a-2 form their OWN 2-node "SITE-A" Akka mesh, # separate from MAIN and SITE-B — self-first seeded WITHIN the pair, never off central. No UI, - # no user auth; central manages + deploys to this mesh over ClusterClient/gRPC/FetchAndCache, + # no user auth; central manages + deploys to this mesh over Grpc/FetchAndCache, # not gossip (central and site-a share no seed list, so they never form one ring). site-a-1: @@ -592,7 +573,7 @@ services: Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-a-1 seeds its OWN "SITE-A" mesh (itself first, site-a-2 - # second) — it no longer seeds off central-1. Central is reached over ClusterClient/gRPC/ + # second) — it no longer seeds off central-1. Central is reached over Grpc/ # FetchAndCache, never gossip; central and site-a share no seed list. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-2:4053" @@ -605,20 +586,12 @@ services: Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see - # central-1/central-2. Central dials this mesh's ClusterClientReceptionist via the contact - # points below; there is no gossip path anymore. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + # central-1/central-2. Central dials this node's own NodeCommandService; there is no gossip + # path anymore. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # gRPC command transport — see central-1. # GrpcPsk is this node's own key for its cluster (SITE-A); CentralGrpcEndpoints is the gRPC - # counterpart of CentralContactPoints above. No ClusterPsks here — that map is central-side + # where this node posts its deploy ack. No ClusterPsks here — that map is central-side # only. MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -721,15 +694,7 @@ services: Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1/site-a-1. SHIPPED DARK. Same SITE-A key as # site-a-1 (must be byte-identical — both nodes authenticate against the same cluster's PSK). MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" @@ -814,17 +779,9 @@ services: Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK. GrpcPsk is this node's own - # key for its cluster (SITE-B); CentralGrpcEndpoints mirrors CentralContactPoints above. No + # key for its cluster (SITE-B); CentralGrpcEndpoints is where it posts its deploy ack. No # ClusterPsks here — that map is central-side only. MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -892,15 +849,7 @@ services: Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1/site-b-1. SHIPPED DARK. Same SITE-B key as # site-b-1 (must be byte-identical). MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs index a52c515d..7fa62de4 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs @@ -56,21 +56,7 @@ public sealed class MeshTransportOptions public string Mode { get; set; } = ModeDps; /// - /// Akka remoting addresses of the central nodes, e.g. - /// akka.tcp://otopcua@central-1:4053. Node addresses only — the - /// /system/receptionist path is appended at construction time. - /// - /// - /// Each entry must be a central node's remoting endpoint, never this node's own port. - /// The sister project shipped a site template whose second contact was the site's own - /// remoting port; it is a permanent failure in the initial-contact rotation and stays - /// invisible until a command goes missing. - /// rejects the malformed shapes at boot instead. - /// - public string[] CentralContactPoints { get; set; } = []; - - /// - /// How often central re-reads ClusterNode rows to rebuild its contact points. + /// How often central re-reads ClusterNode rows to rebuild its dispatch set. /// Refreshing on a timer (rather than only on admin change) means a row edited by any /// process — migration, direct SQL, a second admin node — is picked up without a restart. /// @@ -83,12 +69,10 @@ public sealed class MeshTransportOptions /// (scheme://host:port), not bare host:port — http:// is h2c. /// /// - /// Tried in order, stopping at the first success, which is what preserves the - /// receptionist's one-ack-per-node-per-dispatch semantic: either central node routes the ack - /// to the same ConfigPublishCoordinator singleton proxy, so which one answers does not - /// matter, but two answering would double-count. The gRPC counterpart of - /// , and it replaces it — a node running - /// never reads the Akka contact points. + /// Tried in order, stopping at the first success, which is what keeps the ack + /// one-per-node-per-dispatch: either central node routes it to the same + /// ConfigPublishCoordinator singleton proxy, so which one answers does not matter, + /// but two answering would double-count. /// public string[] CentralGrpcEndpoints { get; set; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs index 5641672e..0bd78471 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs @@ -57,48 +57,13 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions@:), not a bare host:port."); - } - else if (contactPoint.Contains("/user/", StringComparison.Ordinal) - || contactPoint.Contains("/system/", StringComparison.Ordinal)) - { - errors.Add( - $"Contact point '{contactPoint}' carries an actor-path suffix. Configure the " - + "node address only — '/system/receptionist' is appended at construction time, " - + "and a doubled suffix produces a path that never resolves."); - } - } + + "node dispatch set on this interval; a non-positive value would leave that set " + + "frozen at whatever it held when the node booted."); } // The first Grpc block below is node-side: the PSK a driver presents (and requires), the // central endpoints it posts its ack to, and the two call deadlines (the admin-side rule that - // follows it is a separate scope). Scoped to the driver role for the same reason the - // ClusterClient block above is scoped to its mode — an admin-only central node - // dials nodes rather than being dialled and resolves per-cluster keys at runtime through + // follows it is a separate scope). Scoped to the driver role because an admin-only central node // MeshTransportOptions.ClusterPsks / the Secrets store, so requiring a single GrpcPsk and an // endpoint list of it would demand configuration it never reads. Roles come from // configuration, not IClusterRoleInfo (see the ctor remark), and through diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs index 294897b2..d03e4b86 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs @@ -86,11 +86,7 @@ public class MeshTransportOptionsValidatorTests [Fact] public void Mode_matching_is_case_insensitive() { - Validate(new MeshTransportOptions - { - Mode = "clusterclient", - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }).Succeeded.ShouldBeTrue(); + Validate(new MeshTransportOptions { Mode = "dps" }).Succeeded.ShouldBeTrue(); } [Fact] @@ -102,92 +98,7 @@ public class MeshTransportOptionsValidatorTests result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.ContactRefreshSeconds)); } - [Fact] - public void ClusterClient_mode_requires_central_contact_points() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = [], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints)); - } - - [Fact] - public void Dps_mode_does_not_require_central_contact_points() - { - // An admin-only node has no reason to carry them, and requiring them under the default mode - // would make the section mandatory everywhere for a feature that is switched off. - Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeDps, - CentralContactPoints = [], - }).Succeeded.ShouldBeTrue(); - } - - [Fact] - public void A_contact_point_that_is_not_an_akka_address_fails() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["central-1:4053"], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("akka.tcp://"); - } - - [Fact] - public void A_contact_point_carrying_an_actor_path_suffix_fails() - { - // The sister project's shipped site template listed a full receptionist path, and separately - // listed the site's OWN remoting port as the second contact — "a permanent failure in the - // initial-contact rotation". Both are invisible until a command goes missing. - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("/system/receptionist"); - } - - [Fact] - public void One_bad_contact_point_among_good_ones_still_fails() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = - [ - "akka.tcp://otopcua@central-1:4053", - "central-2:4053", - ], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("central-2:4053"); - } - - [Fact] - public void Well_formed_ClusterClient_options_succeed() - { - Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = - [ - "akka.tcp://otopcua@central-1:4053", - "akka.tcp://otopcua@central-2:4053", - ], - }).Succeeded.ShouldBeTrue(); - } - - // ---- Grpc mode (ClusterClient→gRPC migration, Phase 0) -------------------------------------- + // ---- Grpc mode (the cross-mesh command plane) ----------------------------------------------- [Fact] public void Grpc_is_an_accepted_mode() @@ -385,14 +296,9 @@ public class MeshTransportOptionsValidatorTests [Fact] public void ClusterClient_admin_node_needs_no_config_serve_listener() { - // Contact points are required by the mode itself, independently of role — supplied so this - // test fails only if the ConfigServe rule wrongly fires outside Grpc mode. + // Fails only if the ConfigServe rule wrongly fires outside Grpc mode. Validate( - new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }, + new MeshTransportOptions { Mode = MeshTransportOptions.ModeClusterClient }, "admin", "cluster-MAIN").Succeeded.ShouldBeTrue(); } @@ -446,11 +352,7 @@ public class MeshTransportOptionsValidatorTests public void ClusterClient_mode_requires_none_of_the_new_Grpc_keys() { Validate( - new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }, + new MeshTransportOptions { Mode = MeshTransportOptions.ModeClusterClient }, "driver", "cluster-SITE-A").Succeeded.ShouldBeTrue(); }