diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index 9e1069a7..3e6cbe4f 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -21,9 +21,10 @@ # # Since central no longer shares a gossip ring with the site meshes, it reaches # them over the three explicit split-topology transports instead of gossip: -# - Command plane: MeshTransport__Mode=ClusterClient (central dials each site -# mesh's ClusterClientReceptionist; DPS is gossip-only and -# would not cross the mesh boundary at all now) +# - Command plane: MeshTransport__Mode=Grpc (central dials each site node's +# NodeCommandService on :4056; a node posts its deploy ack +# back to central's CentralCommandService on :4055. DPS is +# gossip-only and would not cross the mesh boundary at all) # - Live telemetry: Telemetry__Mode=Grpc on every driver node (all six) + # TelemetryDial__Mode=Grpc on the admin pair (central-1/2) # — the site node hosts the gRPC server, central dials in @@ -53,8 +54,7 @@ # data-plane endpoints (4842-4845) and authenticate those # users against the same shared GLAuth (via the *ldap-env # anchor); central manages + deploys to them over the -# ClusterClient/gRPC/FetchAndCache transports above, not -# gossip. +# Grpc/FetchAndCache transports above, not gossip. # # Auth is real LDAP against the shared GLAuth on the Linux Docker host # (10.100.0.35:3893, dc=zb,dc=local) — there is no LDAP container here. @@ -329,33 +329,22 @@ services: Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" # Per-cluster mesh Phase 6: cluster-MAIN marks this node as a member of the MAIN mesh # (RoleParser.ClusterRolePrefix = "cluster-"), read by SplitTopologyTransportValidator to - # require ClusterClient/Grpc transports, and by ClusterRoleInfo to scope redundancy + + # require the mesh-crossing Grpc transports, and by ClusterRoleInfo to scope redundancy + # ClusterNodeAddressReconcilerActor to ClusterId=MAIN. Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Cluster__Roles__2: "cluster-MAIN" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6). Central no # longer shares a gossip ring with the site meshes, so DPS (gossip-only) cannot reach them — - # ClusterClient is the only transport that can. SplitTopologyTransportValidator refuses to - # start any node carrying a cluster-* role that is left on Dps, so this is fixed, not an + # Grpc is the only transport that can. SplitTopologyTransportValidator refuses to start any + # node carrying a cluster-* role that is left on Dps, so this is fixed, not an # env-var-overridable default, on every node in this file. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0 of the ClusterClient→gRPC migration, docs/plans/2026-08-06- - # clusterclient-to-grpc-design.md). SHIPPED DARK: Mode above stays ClusterClient, so none of - # this is dialled or served yet — only MeshTransportOptionsValidator's Grpc-mode rules read - # it. GrpcPsk is this node's OWN per-cluster key (MAIN, since central is cluster-MAIN); - # CentralGrpcEndpoints is the gRPC counterpart of CentralContactPoints above. ClusterPsks is - # CENTRAL-SIDE ONLY (IMeshPskProvider's override map, consulted before the Secrets store) — - # one entry per cluster central will dial once the switch flips, so only the central nodes - # carry it. + # gRPC command transport (docs/plans/2026-08-06-clusterclient-to-grpc-design.md). GrpcPsk is + # this node's OWN per-cluster key (MAIN, since central is cluster-MAIN); CentralGrpcEndpoints + # is where a node posts its deploy ack. ClusterPsks is CENTRAL-SIDE ONLY (IMeshPskProvider's + # override map, consulted before the Secrets store) — one entry per cluster central dials, so + # only the central nodes carry it. MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" @@ -478,16 +467,8 @@ services: Cluster__Roles__1: "driver" Cluster__Roles__2: "cluster-MAIN" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see central-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # gRPC command transport — see central-1. # ClusterPsks is central-side only; both central nodes carry the same three-cluster map. MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -549,7 +530,7 @@ services: # ── Site A cluster (2-node driver-only) ───────────────────────────────────── # Per-cluster mesh Phase 6: site-a-1 and site-a-2 form their OWN 2-node "SITE-A" Akka mesh, # separate from MAIN and SITE-B — self-first seeded WITHIN the pair, never off central. No UI, - # no user auth; central manages + deploys to this mesh over ClusterClient/gRPC/FetchAndCache, + # no user auth; central manages + deploys to this mesh over Grpc/FetchAndCache, # not gossip (central and site-a share no seed list, so they never form one ring). site-a-1: @@ -592,7 +573,7 @@ services: Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-a-1 seeds its OWN "SITE-A" mesh (itself first, site-a-2 - # second) — it no longer seeds off central-1. Central is reached over ClusterClient/gRPC/ + # second) — it no longer seeds off central-1. Central is reached over Grpc/ # FetchAndCache, never gossip; central and site-a share no seed list. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-2:4053" @@ -605,20 +586,12 @@ services: Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see - # central-1/central-2. Central dials this mesh's ClusterClientReceptionist via the contact - # points below; there is no gossip path anymore. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + # central-1/central-2. Central dials this node's own NodeCommandService; there is no gossip + # path anymore. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" - # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # gRPC command transport — see central-1. # GrpcPsk is this node's own key for its cluster (SITE-A); CentralGrpcEndpoints is the gRPC - # counterpart of CentralContactPoints above. No ClusterPsks here — that map is central-side + # where this node posts its deploy ack. No ClusterPsks here — that map is central-side # only. MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -721,15 +694,7 @@ services: Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1/site-a-1. SHIPPED DARK. Same SITE-A key as # site-a-1 (must be byte-identical — both nodes authenticate against the same cluster's PSK). MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" @@ -814,17 +779,9 @@ services: Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK. GrpcPsk is this node's own - # key for its cluster (SITE-B); CentralGrpcEndpoints mirrors CentralContactPoints above. No + # key for its cluster (SITE-B); CentralGrpcEndpoints is where it posts its deploy ack. No # ClusterPsks here — that map is central-side only. MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" @@ -892,15 +849,7 @@ services: Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. - # - # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction - # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an - # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in - # silence. Both central nodes are listed so a driver node's client can rotate to whichever one - # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Grpc}" - MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" - MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" # gRPC command transport (Phase 0) — see central-1/site-b-1. SHIPPED DARK. Same SITE-B key as # site-b-1 (must be byte-identical). MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs index a52c515d..7fa62de4 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs @@ -56,21 +56,7 @@ public sealed class MeshTransportOptions public string Mode { get; set; } = ModeDps; /// - /// Akka remoting addresses of the central nodes, e.g. - /// akka.tcp://otopcua@central-1:4053. Node addresses only — the - /// /system/receptionist path is appended at construction time. - /// - /// - /// Each entry must be a central node's remoting endpoint, never this node's own port. - /// The sister project shipped a site template whose second contact was the site's own - /// remoting port; it is a permanent failure in the initial-contact rotation and stays - /// invisible until a command goes missing. - /// rejects the malformed shapes at boot instead. - /// - public string[] CentralContactPoints { get; set; } = []; - - /// - /// How often central re-reads ClusterNode rows to rebuild its contact points. + /// How often central re-reads ClusterNode rows to rebuild its dispatch set. /// Refreshing on a timer (rather than only on admin change) means a row edited by any /// process — migration, direct SQL, a second admin node — is picked up without a restart. /// @@ -83,12 +69,10 @@ public sealed class MeshTransportOptions /// (scheme://host:port), not bare host:port — http:// is h2c. /// /// - /// Tried in order, stopping at the first success, which is what preserves the - /// receptionist's one-ack-per-node-per-dispatch semantic: either central node routes the ack - /// to the same ConfigPublishCoordinator singleton proxy, so which one answers does not - /// matter, but two answering would double-count. The gRPC counterpart of - /// , and it replaces it — a node running - /// never reads the Akka contact points. + /// Tried in order, stopping at the first success, which is what keeps the ack + /// one-per-node-per-dispatch: either central node routes it to the same + /// ConfigPublishCoordinator singleton proxy, so which one answers does not matter, + /// but two answering would double-count. /// public string[] CentralGrpcEndpoints { get; set; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs index 5641672e..0bd78471 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs @@ -57,48 +57,13 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions@:), not a bare host:port."); - } - else if (contactPoint.Contains("/user/", StringComparison.Ordinal) - || contactPoint.Contains("/system/", StringComparison.Ordinal)) - { - errors.Add( - $"Contact point '{contactPoint}' carries an actor-path suffix. Configure the " - + "node address only — '/system/receptionist' is appended at construction time, " - + "and a doubled suffix produces a path that never resolves."); - } - } + + "node dispatch set on this interval; a non-positive value would leave that set " + + "frozen at whatever it held when the node booted."); } // The first Grpc block below is node-side: the PSK a driver presents (and requires), the // central endpoints it posts its ack to, and the two call deadlines (the admin-side rule that - // follows it is a separate scope). Scoped to the driver role for the same reason the - // ClusterClient block above is scoped to its mode — an admin-only central node - // dials nodes rather than being dialled and resolves per-cluster keys at runtime through + // follows it is a separate scope). Scoped to the driver role because an admin-only central node // MeshTransportOptions.ClusterPsks / the Secrets store, so requiring a single GrpcPsk and an // endpoint list of it would demand configuration it never reads. Roles come from // configuration, not IClusterRoleInfo (see the ctor remark), and through diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs index 294897b2..d03e4b86 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs @@ -86,11 +86,7 @@ public class MeshTransportOptionsValidatorTests [Fact] public void Mode_matching_is_case_insensitive() { - Validate(new MeshTransportOptions - { - Mode = "clusterclient", - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }).Succeeded.ShouldBeTrue(); + Validate(new MeshTransportOptions { Mode = "dps" }).Succeeded.ShouldBeTrue(); } [Fact] @@ -102,92 +98,7 @@ public class MeshTransportOptionsValidatorTests result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.ContactRefreshSeconds)); } - [Fact] - public void ClusterClient_mode_requires_central_contact_points() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = [], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints)); - } - - [Fact] - public void Dps_mode_does_not_require_central_contact_points() - { - // An admin-only node has no reason to carry them, and requiring them under the default mode - // would make the section mandatory everywhere for a feature that is switched off. - Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeDps, - CentralContactPoints = [], - }).Succeeded.ShouldBeTrue(); - } - - [Fact] - public void A_contact_point_that_is_not_an_akka_address_fails() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["central-1:4053"], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("akka.tcp://"); - } - - [Fact] - public void A_contact_point_carrying_an_actor_path_suffix_fails() - { - // The sister project's shipped site template listed a full receptionist path, and separately - // listed the site's OWN remoting port as the second contact — "a permanent failure in the - // initial-contact rotation". Both are invisible until a command goes missing. - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("/system/receptionist"); - } - - [Fact] - public void One_bad_contact_point_among_good_ones_still_fails() - { - var result = Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = - [ - "akka.tcp://otopcua@central-1:4053", - "central-2:4053", - ], - }); - - result.Failed.ShouldBeTrue(); - result.FailureMessage.ShouldContain("central-2:4053"); - } - - [Fact] - public void Well_formed_ClusterClient_options_succeed() - { - Validate(new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = - [ - "akka.tcp://otopcua@central-1:4053", - "akka.tcp://otopcua@central-2:4053", - ], - }).Succeeded.ShouldBeTrue(); - } - - // ---- Grpc mode (ClusterClient→gRPC migration, Phase 0) -------------------------------------- + // ---- Grpc mode (the cross-mesh command plane) ----------------------------------------------- [Fact] public void Grpc_is_an_accepted_mode() @@ -385,14 +296,9 @@ public class MeshTransportOptionsValidatorTests [Fact] public void ClusterClient_admin_node_needs_no_config_serve_listener() { - // Contact points are required by the mode itself, independently of role — supplied so this - // test fails only if the ConfigServe rule wrongly fires outside Grpc mode. + // Fails only if the ConfigServe rule wrongly fires outside Grpc mode. Validate( - new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }, + new MeshTransportOptions { Mode = MeshTransportOptions.ModeClusterClient }, "admin", "cluster-MAIN").Succeeded.ShouldBeTrue(); } @@ -446,11 +352,7 @@ public class MeshTransportOptionsValidatorTests public void ClusterClient_mode_requires_none_of_the_new_Grpc_keys() { Validate( - new MeshTransportOptions - { - Mode = MeshTransportOptions.ModeClusterClient, - CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], - }, + new MeshTransportOptions { Mode = MeshTransportOptions.ModeClusterClient }, "driver", "cluster-SITE-A").Succeeded.ShouldBeTrue(); }