diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..bad7715e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,29 @@ +# Build context exclusions for docker-dev/Dockerfile, whose `COPY . .` takes the whole repo. +# +# Without this file the context also carries every project's bin/obj AND .claude/worktrees — agent +# worktrees that are themselves full checkouts WITH their own build outputs. That reached ~19 GB and +# exhausted the Docker VM's disk mid-COPY, failing the rig build with "no space left on device" +# rather than anything a build log would attribute to the repo. +# +# Nothing excluded here is needed: the build restores and publishes inside the image, and the +# migrator stage runs `dotnet ef` against the same restored source. + +# Agent worktrees — full checkouts, build outputs and all. +.claude/ + +# Local build outputs; the image builds its own. +**/bin/ +**/obj/ +artifacts/ + +# VCS + editor state. +.git/ +.gitignore +.vs/ +.vscode/ +.idea/ + +# Docker's own state, and anything the image would never read. +**/.DS_Store +**/*.user +docker-dev/**/*.log diff --git a/CLAUDE.md b/CLAUDE.md index d585b07b..160fbb52 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -228,7 +228,46 @@ The server supports non-transparent warm/hot redundancy via the `Redundancy` sec Assert the **effective** cluster config off a running `ActorSystem` (as `SplitBrainResolverActivationTests` does), never the typed options or the akka.conf text — several HOCON fragments compete and the losing one still reads correctly in the file. -**Still per-Akka-cluster, not per application `Cluster`.** One Primary is elected across the whole Akka mesh, so a fleet running several application clusters in one mesh (what docker-dev models) gets one Primary among all driver nodes. The fix is the per-cluster mesh workstream: `docs/plans/2026-07-21-per-cluster-mesh-design.md` (Phases 0a/0b/1 done; 2–7 not started). +**Still per-Akka-cluster, not per application `Cluster`.** One Primary is elected across the whole Akka mesh, so a fleet running several application clusters in one mesh (what docker-dev models) gets one Primary among all driver nodes. The fix is the per-cluster mesh workstream: `docs/plans/2026-07-21-per-cluster-mesh-design.md` (Phases 0a/0b/1 done; 2 code-complete behind a dark switch; 3–7 not started). + +## Mesh command transport (`MeshTransport`) + +Per-cluster mesh **Phase 2** added a second transport for the three central→node command channels +(`deployments`, `driver-control`, `alarm-commands`) and the `deployment-acks` reply, selected by +`MeshTransport:Mode`: `Dps` (the DistributedPubSub path this repo has always used, **the default**) +or `ClusterClient`, which does **not** require central and the node to share a gossip ring — the +prerequisite for splitting the fleet into one mesh per application `Cluster`. Both `CentralCommunicationActor` +(`/user/central-communication`) and `NodeCommunicationActor` (`/user/node-communication`) are spawned +and registered with the `ClusterClientReceptionist` in **both** modes, so flipping the flag is a config +change, not a redeploy. Things worth knowing before touching it: + +- **`SendToAll`, never `Send`.** Today's DPS publish reaches every `DriverHostActor`; there is no + ClusterId or node filter on the node side at all (scoping happens later, inside the artifact via + `DeploymentArtifact.ResolveClusterScope`). `ClusterClient.Send` delivers to exactly **one** registered + actor — it would deploy to a single node while every other node silently kept its old configuration, + and the deployment could still seal green. +- **Exactly ONE fleet-wide ClusterClient while the fleet is one mesh.** A receptionist serves its whole + cluster, so `SendToAll` reaches every registered node-comm actor regardless of which node's address was + dialled. One client per application `Cluster` — the obviously-right-looking shape, and what Phase 6 + wants — would fan each command out once per cluster (N× duplicate dispatch *and* N× duplicate ack). + Marked `TODO(Phase 6)` in `CentralCommunicationActor.RebuildClient`. **Corollary:** the contact set does + not scope delivery; excluding a `MaintenanceMode` node from the contacts does not stop it receiving. +- **Inbound commands re-emit on the node's `EventStream`, not on their DPS topic** — DPS is mesh-wide and + central `SendToAll`s to every node, so a DPS re-publish would deliver N copies to every subscriber. + Node-side subscribers (`DriverHostActor`, `ScriptedAlarmHostActor`) accept both. +- **Comm actors are per-node, NOT singletons.** A client rotates across contact points and must find a live + comm actor at whichever node answers; `akka.cluster.client.receptionist.role` is empty for the same reason. +- **`buffer-size = 0` is a behaviour decision, not a tuning knob.** Akka's default buffers 1000 and replays + on reconnect — silently, on the client's own timer, at an unbounded remove from the dispatch. Do not raise + it to quiet a flaky test. **But do not over-claim what it buys:** the Phase 2 live gate observed a node + applying a deployment **44 s after it was sealed `TimedOut`** anyway, via `DriverHostActor.Bootstrap()` + replaying the orphan `Applying` row on restart. Zero buffering removes the silent replay, not every replay. + +Contact points are node **addresses only** (`akka.tcp://@:`); `/system/receptionist` is +appended at construction and `MeshTransportOptionsValidator` refuses to start a node whose contact carries an +actor-path suffix. `redundancy-state` stays on DPS in both modes (bidirectional, and built from +`Cluster.State`). See `docs/Configuration.md` §`MeshTransport`, `docs/Redundancy.md` §"Command transport", +and `docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md`. **Phase 1 landed 2026-07-22 and changed a deploy-path behaviour.** `ConfigPublishCoordinator` sources its expected-ack set from **enabled `ClusterNode` rows**, not from `Akka.Cluster.State.Members` filtered by the `driver` role — central must be able to name a deployment's nodes without sharing a gossip ring with them. Consequences worth knowing before touching deploys: diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index cdfc663c..71042130 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -190,6 +190,19 @@ services: Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" Security__Jwt__Issuer: "otopcua-dev" Security__Jwt__Audience: "otopcua-dev" @@ -263,6 +276,19 @@ services: Cluster__SeedNodes__1: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" Security__Jwt__Issuer: "otopcua-dev" Security__Jwt__Audience: "otopcua-dev" @@ -329,6 +355,19 @@ services: # is in its own seed list), so these configs are exempt rather than broken. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: *secrets-env # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/ # mem_reservation are inherited from the *otopcua-host anchor. @@ -395,6 +434,19 @@ services: Cluster__PublicHostname: "site-a-2" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" @@ -453,6 +505,19 @@ services: Cluster__PublicHostname: "site-b-1" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" @@ -494,6 +559,19 @@ services: Cluster__PublicHostname: "site-b-2" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + # Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps", + # the transport this repo has always used, and every node keeps its ClusterClient wiring built + # and its comm actor registered with the receptionist either way. Flip the whole rig with + # OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild. + # + # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction + # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an + # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in + # silence. Both central nodes are listed so a driver node's client can rotate to whichever one + # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. + MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}" + MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" + MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" diff --git a/docs/Configuration.md b/docs/Configuration.md index 6d7d249c..b92ba78b 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -110,6 +110,29 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only > **`Port` / `PublicHostname` are stored twice.** The node binds from these keys; the fleet's `ClusterNode` row records the same address as `AkkaPort` / `Host` so that **central can dial the node without sharing a gossip ring with it** — which is what [per-cluster mesh Phase 2](plans/2026-07-21-per-cluster-mesh-design.md) needs. Nothing in the schema makes the two agree, so `ClusterNodeAddressReconcilerActor` (admin-role singleton) compares them against live membership and logs an Error on mismatch. If you change `Cluster:Port` or `Cluster:PublicHostname` on a node, **update its `ClusterNode` row too** — a node binding 4054 with a row saying 4053 still gossips fine today, and fails silently in Phase 2. The row's `NodeId` is `host:port` and is also the deploy path's ack identity, so a drifted node's deployment acks stop matching as well. See [`config-db-schema.md` § `ClusterNode`](v2/config-db-schema.md#clusternode). +### `MeshTransport` (central↔node command transport) + +- **Purpose:** selects how central's three command channels (`deployments`, `driver-control`, `alarm-commands`) and the node's `deployment-acks` reply reach the other side — over the mesh-wide DistributedPubSub, or over an Akka `ClusterClient` that does **not** require central and the node to share a gossip ring. The second option is what [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) Phase 6 needs once the meshes split. +- **Options class:** `MeshTransportOptions` (`SectionName = "MeshTransport"`) — `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs`. Bound and validated by `AddOtOpcUaCluster(config)`; `MeshTransportOptionsValidator` runs at `ValidateOnStart`. + +| Key | Type | Default | Meaning | +|---|---|---|---| +| `Mode` | string | `Dps` | `Dps` = DistributedPubSub (the transport this repo has always used). `ClusterClient` = the mesh boundary. Any other value **refuses to start**. | +| `CentralContactPoints` | string[] | `[]` | Node-side only: central node **addresses** (`akka.tcp://@:`). Required under `ClusterClient`, ignored under `Dps`. | +| `ContactRefreshSeconds` | int | `60` | Central-side only: how often central rebuilds its ClusterClient contact set from the `ClusterNode` rows. | + +**Addresses only — no actor-path suffix.** `/system/receptionist` is appended at construction time; a contact carrying `/user/` or `/system/` is rejected at startup, because a doubled suffix produces a path that resolves to nothing and drops every command in silence. + +**It is a dark switch, and both sides are always wired.** The comm actors are spawned and registered with the `ClusterClientReceptionist` on every node in *both* modes, so flipping `Mode` is a config change rather than a redeploy. Under `Dps` the registration is simply idle. + +**Central's contact set comes from the enabled, non-maintenance `ClusterNode` rows** — the same source Phase 1 gave the deploy ack set. A node in `MaintenanceMode` is not dialled, but note that **the contact set does not scope delivery**: a receptionist serves its whole cluster, so `SendToAll` still reaches every registered node exactly as today's DPS broadcast does. The filter is about which receptionists are worth dialling, not about who receives. + +**Buffering is off by design** (`akka.cluster.client.buffer-size = 0`, set in the shipped `akka.conf`). If no receptionist is known, the command is dropped rather than queued. Akka's default of 1000 would replay a `DispatchDeployment` on reconnect — silently, on the client's own timer, at an unbounded remove from the dispatch. Do not "fix" it back to quiet a flaky test. + +> **What this does *not* buy you.** It does not guarantee a node never applies a deployment the DB records as `TimedOut`. A node that missed a dispatch keeps the orphan `Applying` row the coordinator pre-created, and `DriverHostActor.Bootstrap()` replays that row on the node's next restart — live-observed applying a deployment 44 s after it was sealed `TimedOut` (see the [Phase 2 live gate](plans/2026-07-22-mesh-phase2-live-gate.md), finding B). The boot replay is the better-behaved of the two — deliberate, Warning-logged, and bounded to a restart — but it is still a replay. `buffer-size = 0` removes the *silent, timer-driven* one only. + +On the docker-dev rig every node carries these keys already; flip the whole rig with `OTOPCUA_MESH_MODE=ClusterClient` at `docker compose up`. + ### `ConnectionStrings` → `ConfigDb` - **Purpose:** the central Config DB connection string. **Required for every role** — `Program.cs` calls `AddOtOpcUaConfigDb` unconditionally. diff --git a/docs/Redundancy.md b/docs/Redundancy.md index bd8e3f07..c08d35db 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -407,6 +407,37 @@ Net effect: each alarm transition appears **once** on `/alerts` and would histor See [ScriptedAlarms.md](ScriptedAlarms.md) and [AlarmTracking.md](AlarmTracking.md) for the scripted-alarm engine internals. +## Command transport (central↔node) + +Everything above rides one Akka mesh: central publishes on DistributedPubSub, every node +subscribes, and the two ends must be members of the same cluster. Per-cluster mesh **Phase 2** +adds a second transport under `MeshTransport:Mode` (see +[Configuration.md § `MeshTransport`](Configuration.md#meshtransport-centralnode-command-transport)) +that does not need shared membership — the prerequisite for splitting the fleet into one mesh per +application `Cluster`. + +| | `Dps` (default) | `ClusterClient` | +|---|---|---| +| Central → node commands | `Publish` on `deployments` / `driver-control` / `alarm-commands` | `ClusterClient.SendToAll` to `/user/node-communication`, re-emitted on the node's **local** `EventStream` | +| Node → central acks | `Publish` on `deployment-acks` | `ClusterClient.Send` to `/user/central-communication`, forwarded to the deploy-coordinator singleton | +| Requires shared cluster membership | **yes** | no | + +Two properties are load-bearing and easy to break: + +- **Both comm actors are registered per node, NOT as cluster singletons.** A `ClusterClient` rotates + across its contact points and must find a live comm actor at whichever node answers. As a + singleton the boundary would be reachable through exactly one node, and rotation would fail + silently against the others — acks landing or not depending on where the client happened to + settle. `akka.cluster.client.receptionist.role` is deliberately empty for the same reason. +- **Inbound commands go to the node-local `EventStream`, never back onto a DPS topic.** DPS is + mesh-wide and central `SendToAll`s to *every* node, so a DPS re-publish would deliver N copies of + every command to every subscriber. + +Phase 2 changes no redundancy semantics: the Primary gate, ServiceLevel, and the singleton layout +are untouched, and `redundancy-state` itself stays on DPS (it is bidirectional and built from +`Cluster.State`, which makes it genuinely mesh-bound — the hardest remaining dependency before the +meshes can split). + ## Pair-local store (LocalDb — Phases 1 + 2) Independently of the ServiceLevel machinery above, every **driver-role** node keeps a consolidated diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md index 1acff34d..8ccd3ed6 100644 --- a/docs/plans/2026-07-21-per-cluster-mesh-design.md +++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md @@ -68,8 +68,14 @@ Other load-bearing details: caller's Ask times out. *"It keeps the central coordinator stateless with respect to site availability."* A `ConnectionStateChanged` mechanism was built for this and **deleted as dead code**. -- **Every unhandled message replies with a typed failure rather than dropping**, so a central Ask - can never stall on a missing handler. +- ~~**Every unhandled message replies with a typed failure rather than dropping**, so a central Ask + can never stall on a missing handler.~~ **CORRECTED 2026-07-22 (Phase 2 recon).** This overstates + what ScadaBridge does. Neither of their comm actors has a `ReceiveAny` or an `Unhandled` override: + the typed-failure idiom is **per message type**, and it fires only when a *registration* is missing + (no ClusterClient for the site yet, no handler registered for that type). A genuinely unknown + message type still dead-letters exactly as it would anywhere else. Do not build a catch-all failure + reply believing you are copying them. OtOpcUa's Phase 2 comm actors list each type explicitly and + deliberately let an unknown type dead-letter loudly rather than republish it blind. - **Sender preservation is the whole Ask idiom**: `_client.Tell(new ClusterClient.Send(path, msg), Sender)` routes the reply straight back to the waiting Ask, bypassing the comm actor. diff --git a/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md b/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md new file mode 100644 index 00000000..2c26ae5f --- /dev/null +++ b/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md @@ -0,0 +1,1018 @@ +# Per-Cluster Mesh Phase 2 — Comm Actors + ClusterClient Transport + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task. + +**Goal:** Move the three central→node command channels (`deployments`, `driver-control`, `alarm-commands`) and the `deployment-acks` reply channel off DistributedPubSub and onto Akka **ClusterClient**, behind a config flag, so central can command a cluster it does not share a gossip ring with. + +**Architecture:** One receptionist-registered comm actor per side — `/user/central-communication` (on admin nodes) and `/user/node-communication` (on driver nodes), each registered **per node, not as a singleton**, so contact rotation reaches whichever answers. Central's singletons stop touching the mediator: they `Tell` a `MeshCommand` envelope to a node-local router actor that decides DPS or ClusterClient from `MeshTransport:Mode`. Inbound commands land on the node's **EventStream** — genuinely node-local, unlike DPS — and existing subscribers add one `EventStream.Subscribe` line beside their existing DPS `Subscribe`, so the dark switch touches only publishers. + +**Tech Stack:** .NET 10, Akka.NET 1.5.62 (`Akka.Cluster.Tools` already referenced by Cluster, Runtime, and ControlPlane — no new package), xUnit + Shouldly, Akka.TestKit.Xunit2. + +--- + +## Decisions already made (do not re-litigate) + +| Decision | Choice | Why | +|---|---|---| +| Cutover | **Config-flagged dark switch** (`MeshTransport:Mode` = `Dps` default \| `ClusterClient`) | Lets the rig gate A/B the same deploy and roll back with a config change, not a revert. Phase 1's live gate is the argument: the defect only appeared against real infrastructure. | +| Buffering | **`buffer-size = 0`** (drop immediately) | Akka's default buffers 1000 and replays on reconnect. With Phase 1 semantics a deploy recorded `TimedOut` would still be applied minutes later when the node returns — the node runs a config the DB says failed. Fail-fast keeps the DB the single record of truth. **CORRECTED by the live gate ([finding B](2026-07-22-mesh-phase2-live-gate.md)):** this justification was overstated. A node applies a `TimedOut` deployment *anyway*, via `DriverHostActor.Bootstrap()` replaying the orphan `Applying` row on restart (live-observed at +44 s). `buffer-size = 0` removes the *silent, timer-driven* replay, not the boot-time DB one. The decision stands; the rationale narrowed. | +| `alarm-commands` central leg | **In scope** | Same shape as driver-control; leaves no half-migrated command plane. The topic stays alive for the node-local publisher. | + +--- + +## Three corrections to the design doc, found during recon + +These change what "copy ScadaBridge verbatim" means. Fold them into the code comments as you go. + +1. **`docs/plans/2026-07-21-per-cluster-mesh-design.md` §2 is wrong about typed failure replies.** It says *"Every unhandled message replies with a typed failure rather than dropping."* ScadaBridge has **no `ReceiveAny` / `Unhandled` override in either comm actor.** The typed-failure idiom is per-message-type and fires only when a *registration* is missing (no ClusterClient yet, no handler registered). A genuinely unknown message type still dead-letters. Do not build a catch-all failure reply believing it is the sister project's pattern — Task 9 corrects the design doc. + +2. **"No central buffering" was never a setting they wrote.** ScadaBridge has *zero* `akka.cluster.client.*` HOCON; production runs Akka defaults (`buffer-size = 1000`, `reconnect-timeout = off`). Their "no buffering" is app-level only: no ClusterClient entry for a site ⇒ drop + warn. We are choosing `buffer-size = 0` deliberately (Task 1), which they did not. + +3. **`Publish` → `Send` is the wrong substitution; it must be `SendToAll`.** Today's deploy notify is a DPS broadcast that **every** `DriverHostActor` receives — there is no ClusterId or node filter anywhere on the node side (`DriverHostActor.cs:476-485`); scoping happens later, inside the artifact (`DeploymentArtifact.ResolveClusterScope`, `DriverHostActor.cs:1801`). `ClusterClient.Send` delivers to exactly **one** registered actor and would silently deploy to one node of the fleet. + +--- + +## The single-mesh duplicate-delivery trap — read before Task 3 + +Phase 2 ships while the fleet is **still one Akka mesh**. A `ClusterClientReceptionist` serves its whole cluster, so `SendToAll("/user/node-communication")` reaches every registered node-comm actor **in the entire mesh**, regardless of which node's address was used as the contact point. + +Therefore central must create **exactly ONE ClusterClient** in Phase 2, with contact points drawn from all enabled non-maintenance `ClusterNode` rows. Creating one client *per application Cluster* — which is what Phase 6 will want, and what ScadaBridge does — would fan the same command out once per cluster, producing N× duplicate `DispatchDeployment` and N× duplicate `ApplyAck`. + +That is not a bug to fix later; on the single mesh the fleet-wide client is *correct*, and it produces exactly today's DPS fan-out. Phase 6 splits the meshes and converts this to per-cluster clients. Mark the code with `TODO(Phase 6)`. + +--- + +## Deviation from the program plan's exit gate + +`docs/plans/2026-07-22-per-cluster-mesh-program.md` §Phase 2 asks the gate to include *"an Ask timing out cleanly against a stopped node."* + +**There is no cross-boundary Ask in Phase 2, so that gate item cannot be run as written.** Every command being migrated is fire-and-forget on the wire: + +- `DispatchDeployment` — `Tell`; the ack returns later as a separate `ApplyAck` message, not an Ask reply (`ConfigPublishCoordinator.cs:111` subscribes to a topic). +- `RestartDriver` / `ReconnectDriver` — `AdminOperationsActor.cs:397,427` publish, then reply `Ok = true` immediately from the admin node. `Ok` means *dispatched*, not *applied*; the AdminUI string is literally "Restart dispatched". +- `AlarmCommand` — same, `AdminOperationsActor.cs:88,135`. + +The honest equivalents, both of which Task 10's gate runs: + +- **a stopped node** ⇒ the deploy fails at the apply deadline naming that node (Phase 1 semantics), not an Ask timeout; +- **no ClusterClient / no reachable contact** ⇒ the command is dropped with a Warning and never buffered, and the AdminUI still reports "dispatched" (a pre-existing lie, now on a new transport — record it, do not fix it here). + +Task 9 records this deviation in the program plan. + +--- + +## Task 0: `MeshTransportOptions` + validator + +**Classification:** small +**Estimated implement time:** ~4 min +**Parallelizable with:** Task 1 + +**Files:** +- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs` +- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs` +- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs:32-40` (`AddOtOpcUaCluster`) +- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs` + +`ZB.MOM.WW.OtOpcUa.Cluster` is referenced by both ControlPlane (`ControlPlane.csproj:23`) and Runtime, so this is the right home — next to `AkkaClusterOptions`. + +**Step 1: Write the failing tests** + +```csharp +// tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs +using Microsoft.Extensions.Options; +using Shouldly; +using ZB.MOM.WW.OtOpcUa.Cluster; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +public class MeshTransportOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(MeshTransportOptions o) => + new MeshTransportOptionsValidator().Validate(MeshTransportOptions.SectionName, o); + + [Fact] + public void Default_options_are_valid() + { + Validate(new MeshTransportOptions()).Succeeded.ShouldBeTrue(); + } + + [Fact] + public void Unknown_mode_fails() + { + var result = Validate(new MeshTransportOptions { Mode = "grpc" }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("grpc"); + } + + [Fact] + public void ClusterClient_mode_requires_central_contact_points() + { + var result = Validate(new MeshTransportOptions + { + Mode = "ClusterClient", + CentralContactPoints = [], + }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints)); + } + + [Fact] + public void A_contact_point_that_is_not_an_akka_address_fails() + { + var result = Validate(new MeshTransportOptions + { + Mode = "ClusterClient", + CentralContactPoints = ["central-1:4053"], // missing akka.tcp:// scheme + }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("akka.tcp://"); + } + + // The ScadaBridge footgun, ported. Their shipped site template listed the site's OWN + // remoting port as the second central contact — "a permanent failure in the initial-contact + // rotation" (appsettings.Site.json:45). Catch it at boot instead of in an incident. + [Fact] + public void A_contact_point_carrying_an_actor_path_suffix_fails() + { + var result = Validate(new MeshTransportOptions + { + Mode = "ClusterClient", + CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"], + }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("/system/receptionist"); + } + + [Fact] + public void Well_formed_ClusterClient_options_succeed() + { + Validate(new MeshTransportOptions + { + Mode = "ClusterClient", + CentralContactPoints = + [ + "akka.tcp://otopcua@central-1:4053", + "akka.tcp://otopcua@central-2:4053", + ], + }).Succeeded.ShouldBeTrue(); + } +} +``` + +**Step 2: Run to verify they fail** + +Run: `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests --filter "FullyQualifiedName~MeshTransportOptionsValidatorTests"` +Expected: FAIL to compile — `MeshTransportOptions` does not exist. + +**Step 3: Write `MeshTransportOptions.cs`** + +```csharp +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Selects and configures the transport carrying central→node commands and node→central +/// deployment acks (per-cluster mesh Phase 2). +/// +/// +/// +/// The flag is a deliberate dark switch, not a permanent knob: both +/// transports are live in the tree for one phase so the docker-dev rig can run the same +/// deployment over each and compare, and so a bad Phase 2 rolls back with a config change +/// rather than a revert. Phase 6 deletes the DPS branch along with the single mesh. +/// +/// +/// Discovery is deliberately asymmetric, matching the sister project: central discovers +/// nodes from the database (enabled, non-maintenance ClusterNode rows, refreshed +/// on a timer), nodes discover central from this section — static, restart to change. +/// Central's set changes as operators add nodes; central's own address does not. +/// +/// +public sealed class MeshTransportOptions +{ + public const string SectionName = "MeshTransport"; + + public const string ModeDps = "Dps"; + public const string ModeClusterClient = "ClusterClient"; + + /// + /// Dps (default) keeps every command on DistributedPubSub; ClusterClient routes + /// it over the receptionist boundary. Any other value fails the host at startup. + /// + public string Mode { get; set; } = ModeDps; + + /// + /// Akka remoting addresses of the central nodes, e.g. + /// akka.tcp://otopcua@central-1:4053. Node addresses only — the + /// /system/receptionist path is appended at construction time. + /// + /// + /// Each entry MUST be a central node's remoting endpoint, never this node's own port. + /// The sister project shipped a template whose second contact was the site's own port; it is + /// a permanent failure in the initial-contact rotation and is invisible until a command goes + /// missing. rejects the malformed shapes at boot. + /// + public string[] CentralContactPoints { get; set; } = []; + + /// How often central re-reads ClusterNode rows to rebuild its contact points. + public int ContactRefreshSeconds { get; set; } = 60; +} +``` + +**Step 4: Write `MeshTransportOptionsValidator.cs`** + +```csharp +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Fails the host at startup on a shape that would leave +/// commands silently undelivered. Every failure here is one that otherwise surfaces as an +/// absence — a deploy that never arrives, an alarm ack that does nothing — which is the hardest +/// class of fault to attribute in the field. +/// +public sealed class MeshTransportOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, MeshTransportOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var errors = new List(); + + var isDps = string.Equals(options.Mode, MeshTransportOptions.ModeDps, StringComparison.OrdinalIgnoreCase); + var isClusterClient = string.Equals( + options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); + + if (!isDps && !isClusterClient) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.Mode)} is " + + $"'{options.Mode}'. Expected '{MeshTransportOptions.ModeDps}' or " + + $"'{MeshTransportOptions.ModeClusterClient}'."); + } + + if (options.ContactRefreshSeconds <= 0) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.ContactRefreshSeconds)} " + + $"must be > 0 (was {options.ContactRefreshSeconds})."); + } + + if (isClusterClient) + { + if (options.CentralContactPoints.Length == 0) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.CentralContactPoints)} " + + "is empty. Under ClusterClient mode a driver node cannot reach central without at " + + "least one contact point, and its deployment acks would be dropped silently."); + } + + foreach (var cp in options.CentralContactPoints) + { + if (!cp.StartsWith("akka.tcp://", StringComparison.Ordinal)) + { + errors.Add($"Contact point '{cp}' must start with 'akka.tcp://'."); + } + else if (cp.Contains("/user/", StringComparison.Ordinal) + || cp.Contains("/system/receptionist", StringComparison.Ordinal)) + { + errors.Add( + $"Contact point '{cp}' carries an actor-path suffix. Configure the node " + + "address only — '/system/receptionist' is appended at construction time."); + } + } + } + + return errors.Count == 0 + ? ValidateOptionsResult.Success + : ValidateOptionsResult.Fail(string.Join(" ", errors)); + } +} +``` + +**Step 5: Register it** + +In `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs`, inside `AddOtOpcUaCluster`, directly after the existing `AddValidatedOptions` call: + +```csharp + services.AddValidatedOptions( + configuration, MeshTransportOptions.SectionName); +``` + +**Step 6: Run the tests** + +Run: `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests --filter "FullyQualifiedName~MeshTransportOptionsValidatorTests"` +Expected: PASS, 6/6. + +**Step 7: Commit** + +```bash +git add src/Core/ZB.MOM.WW.OtOpcUa.Cluster tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests +git commit -m "feat(mesh): MeshTransportOptions — dark switch between DPS and ClusterClient" +``` + +--- + +## Task 1: Receptionist extension, zero buffering, frame-size logging + +**Classification:** small +**Estimated implement time:** ~4 min +**Parallelizable with:** Task 0 + +**Files:** +- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf:16-33` +- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs` + +**The test must read the effective config off a running `ActorSystem`, never the HOCON text.** This is a settled lesson in this repo (`SplitBrainResolverActivationTests`, and the comment at `ServiceCollectionExtensions.cs:96-99`): several fragments compete and the losing one still reads correctly in the file. + +**Step 1: Write the failing test** + +```csharp +// tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Shouldly; +using ZB.MOM.WW.OtOpcUa.Cluster; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Asserts the ClusterClient / receptionist / frame-size settings that are in force on a REAL +/// ActorSystem built from the shipped base config. Asserting the akka.conf text instead would +/// pass while a competing fragment silently overrode the value. +/// +public class MeshTransportHoconTests : IDisposable +{ + private readonly ActorSystem _sys; + + public MeshTransportHoconTests() + { + var config = Akka.Configuration.ConfigurationFactory + .ParseString("akka.remote.dot-netty.tcp.port = 0\nakka.cluster.seed-nodes = []") + .WithFallback(HoconLoader.LoadBaseConfig()); + _sys = ActorSystem.Create("hocon-probe", config); + } + + [Fact] + public void Cluster_client_buffering_is_disabled() + { + // buffer-size = 0 means "drop immediately if the receptionist is unknown". Akka's default + // of 1000 would replay a DispatchDeployment minutes later, applying a config the DB has + // already recorded as TimedOut. + _sys.Settings.Config.GetInt("akka.cluster.client.buffer-size").ShouldBe(0); + } + + [Fact] + public void Receptionist_serves_every_member_role() + { + // role = "" is what makes "registered per node, not as a singleton" work: every member + // hosts a receptionist, so contact rotation reaches whichever node answers. + _sys.Settings.Config.GetString("akka.cluster.client.receptionist.role").ShouldBe(string.Empty); + } + + [Fact] + public void Oversized_frames_are_logged_rather_than_dropped_silently() + { + // The sister project's most expensive incident: a message over the 128 KB frame limit is + // dropped WITHOUT tearing down the association — heartbeats keep flowing, the node still + // reports healthy, and the only symptom is an absence. log-frame-size-exceeding turns that + // into a log line with the offending size. + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.maximum-frame-size").ShouldBe(128000); + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.log-frame-size-exceeding").ShouldBe(32000); + } + + [Fact] + public void Receptionist_extension_is_registered() + { + // Not auto-started: without the extensions entry the receptionist only materialises if some + // code path happens to call ClusterClientReceptionist.Get(system) first. + _sys.Settings.Config.GetStringList("akka.extensions") + .ShouldContain(s => s.Contains("ClusterClientReceptionistExtensionProvider")); + } + + public void Dispose() => _sys.Terminate().Wait(TimeSpan.FromSeconds(5)); +} +``` + +**Step 2: Run to verify it fails** + +Run: `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests --filter "FullyQualifiedName~MeshTransportHoconTests"` +Expected: FAIL — `buffer-size` reads 1000, `log-frame-size-exceeding` is unset, extension missing. + +**Step 3: Edit `akka.conf`** + +Replace the `extensions` block (lines 16-18) with: + +```hocon + extensions = [ + "Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools" + # Per-cluster mesh Phase 2. NOT auto-started: without this entry the receptionist only + # materialises if something happens to call ClusterClientReceptionist.Get(system) first, + # which makes "is the boundary listening?" depend on startup ordering. + "Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools" + ] +``` + +Add to the `remote.dot-netty.tcp` block (after `port = 4053`): + +```hocon + # Explicit rather than inherited. Both values ARE the Akka defaults for maximum-frame-size; + # what changes is log-frame-size-exceeding, which defaults to `off`. Over the limit the + # transport drops that ONE message without tearing down the association: heartbeats keep + # flowing, the node still reports healthy, and the only symptom is a command that never + # arrived. The sister project lost a deploy path to exactly this and could not attribute + # it — see docs/plans/2026-07-21-per-cluster-mesh-design.md §8. + maximum-frame-size = 128000b + log-frame-size-exceeding = 32000b +``` + +Add a new block inside `akka.cluster` (after the `pub-sub`-adjacent settings, before `singleton`): + +```hocon + # ClusterClient / receptionist — the central↔node command boundary (Phase 2). + client { + # DELIBERATE, and NOT what the sister project runs (they never wrote this section, so + # they inherit buffer-size = 1000). Zero means: if the receptionist is unknown, drop the + # message now. Buffering would replay a DispatchDeployment on reconnect and apply a + # deployment that ConfigPublishCoordinator already sealed as TimedOut — the node would + # then be running a configuration the database says failed. Fail-fast keeps the DB the + # single record of truth, and matches today's DPS behaviour exactly (a node that is down + # misses the publish and self-heals from the DB on restart). + buffer-size = 0 + + # Retry forever rather than self-stopping the client. A central node that is down for an + # hour must not require a driver-node restart to become reachable again. + reconnect-timeout = off + + receptionist { + # Empty = every member hosts a receptionist. This is what makes the comm actors + # "registered per node, not as a singleton" work — contact rotation reaches whichever + # node answers. Do not scope this to a role. + role = "" + } + } +``` + +**Step 4: Run the tests** + +Run: `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests --filter "FullyQualifiedName~MeshTransportHoconTests"` +Expected: PASS, 4/4. + +**Step 5: Sabotage-verify** + +Change `buffer-size = 0` to `buffer-size = 1000`, re-run: `Cluster_client_buffering_is_disabled` must go RED. Restore, re-run, green. + +> **MSBuild mtime trap (cost several cycles in Phase 1):** `akka.conf` is an **embedded resource**. If you restore a file with `mv` or `git checkout` its mtime may land older than the build output and MSBuild will skip recompiling, so the test keeps reading the sabotaged value. After restoring, run `touch src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf` and rebuild. + +**Step 6: Commit** + +```bash +git add src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs +git commit -m "feat(mesh): receptionist extension, zero ClusterClient buffering, frame-size logging" +``` + +--- + +## Task 2: `MeshCommand` envelope + ClusterClient factory + +**Classification:** small +**Estimated implement time:** ~4 min +**Parallelizable with:** none (Task 3 depends on it) + +**Files:** +- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshCommand.cs` +- Create: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshClusterClientFactory.cs` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshClusterClientFactoryTests.cs` + +**Step 1: Write `MeshCommand.cs`** + +```csharp +namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; + +/// +/// A central→node command plus the DistributedPubSub topic it would have been published on. +/// +/// +/// The legacy DPS topic. Carried so the router can publish under MeshTransport:Mode = Dps +/// without a second dispatch table; ignored entirely under ClusterClient mode, where the +/// destination is an actor path rather than a topic. +/// +/// The command itself — never wrapped, so node-side handlers are unchanged. +/// +/// The envelope exists so the admin singletons stop knowing which transport is in force. Before +/// Phase 2 each publisher held its own DistributedPubSub.Get(Context.System).Mediator +/// call, which meant the dark switch would have had to be threaded through four call sites in +/// two actors. It is central-side only and never crosses the wire. +/// +public sealed record MeshCommand(string Topic, object Message); +``` + +**Step 2: Write the factory** + +```csharp +// src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshClusterClientFactory.cs +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +/// Creates the ClusterClient central uses to reach driver nodes. Seam for tests. +public interface IMeshClusterClientFactory +{ + IActorRef Create(ActorSystem system, ImmutableHashSet contacts); +} + +/// +public sealed class DefaultMeshClusterClientFactory : IMeshClusterClientFactory +{ + /// + /// Per-incarnation generation counter. Context.Stop of the previous client is + /// asynchronous — its actor name stays reserved until termination completes — so recreating + /// the same name while handling one message throws . + /// A generation suffix makes every incarnation's name unique by construction. Ported from the + /// sister project, where it was a shipped bug fix rather than foresight. + /// + private long _generation; + + public IActorRef Create(ActorSystem system, ImmutableHashSet contacts) + { + ArgumentNullException.ThrowIfNull(system); + var settings = ClusterClientSettings.Create(system).WithInitialContacts(contacts); + var name = $"mesh-client-{Interlocked.Increment(ref _generation)}"; + return system.ActorOf(ClusterClient.Props(settings), name); + } +} +``` + +**Step 3: Write the test** + +```csharp +// tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshClusterClientFactoryTests.cs +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Configuration; +using Akka.TestKit.Xunit2; +using Shouldly; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +public class MeshClusterClientFactoryTests : TestKit +{ + private static readonly Config TestConfig = ConfigurationFactory.ParseString(@" + akka.actor.provider = cluster + akka.remote.dot-netty.tcp.port = 0 + akka.remote.dot-netty.tcp.hostname = 127.0.0.1") + .WithFallback(ClusterClientReceptionist.DefaultConfig()); + + public MeshClusterClientFactoryTests() : base(TestConfig) { } + + // A real ClusterClient against an unreachable contact is safe in TestKit — it just retries. + [Fact] + public void Recreating_a_client_in_one_pass_does_not_collide_on_the_actor_name() + { + var factory = new DefaultMeshClusterClientFactory(); + var contacts = ImmutableHashSet.Create( + ActorPath.Parse("akka.tcp://otopcua@127.0.0.1:65501/system/receptionist")); + + var first = factory.Create(Sys, contacts); + Sys.Stop(first); + // Stop is async: the name is still reserved here. Without the generation suffix this throws. + var second = factory.Create(Sys, contacts); + + second.Path.Name.ShouldNotBe(first.Path.Name); + } +} +``` + +**Step 4: Run** + +Run: `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests --filter "FullyQualifiedName~MeshClusterClientFactoryTests"` +Expected: PASS. + +**Step 5: Sabotage-verify** — drop the `-{generation}` suffix from the name; the test must go RED with `InvalidActorNameException`. Restore. + +**Step 6: Commit** + +```bash +git add src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication +git commit -m "feat(mesh): MeshCommand envelope + generation-suffixed ClusterClient factory" +``` + +--- + +## Task 3: `CentralCommunicationActor` + +**Classification:** high-risk +**Estimated implement time:** ~5 min +**Parallelizable with:** Task 4 + +**Files:** +- Create: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs` + +Responsibilities, in order of importance: + +1. `Receive` — under `Dps`, `mediator.Tell(new Publish(cmd.Topic, cmd.Message))`; under `ClusterClient`, `client.Tell(new ClusterClient.SendToAll(NodeCommunicationPath, cmd.Message))`. **`SendToAll`, never `Send`** — see the corrections section. +2. `Receive` — inbound from a node's ClusterClient; `Forward` to the `ConfigPublishCoordinatorKey` singleton proxy so `Sender` survives the hop. +3. Contact-point cache — rebuild from enabled non-maintenance `ClusterNode` rows on a `ContactRefreshSeconds` timer and on demand; **exactly one** client (single-mesh trap above). +4. `Receive` — a faulted DB load piped back. Without this handler the refresh fails at Debug level and an operator cannot distinguish "no nodes configured" from "the database is down". + +Key implementation notes to put in the code: + +```csharp + /// Path the node-side comm actor registers under. This string is the wire contract. + public const string NodeCommunicationPath = "/user/node-communication"; + + // TODO(Phase 6): one client per application Cluster. Today ONE client, fleet-wide contacts. + // A ClusterClientReceptionist serves its whole cluster, and the fleet is still a single mesh, + // so SendToAll from ANY contact point reaches every registered node-comm actor in the mesh. + // One client per Cluster would therefore fan each command out once per cluster — N x duplicate + // DispatchDeployment and N x duplicate ApplyAck. On the single mesh the fleet-wide client is + // not a compromise: it reproduces today's DPS fan-out exactly. +``` + +Dropping when unroutable (the app-level "no buffering" decision — Akka's own buffer is already 0 from Task 1): + +```csharp + if (_client is null) + { + _log.Warning( + "No ClusterClient — dropping {MessageType}. No enabled ClusterNode rows produced a " + + "usable contact point at the last refresh; the command is NOT buffered and will " + + "not be retried", + cmd.Message.GetType().Name); + return; + } +``` + +Address parsing — per-row try/catch so one malformed row cannot abort the whole refresh and leave the cache half-built (a shipped sister-project bug, with a regression test that deliberately orders the bad row first): + +```csharp + foreach (var (nodeId, host, akkaPort) in rows) + { + try + { + paths.Add(ActorPath.Parse($"akka.tcp://{systemName}@{host}:{akkaPort}/system/receptionist")); + } + catch (Exception ex) + { + _log.Warning(ex, + "ClusterNode {NodeId} has an unusable address {Host}:{Port}; skipping it in " + + "this refresh (other nodes are unaffected)", nodeId, host, akkaPort); + } + } +``` + +**Tests (all must exist):** + +| Test | Asserts | +|---|---| +| `Dps_mode_publishes_on_the_carried_topic` | mediator sees `Publish(topic, msg)`; no client created | +| `ClusterClient_mode_sends_to_all_not_to_one` | probe receives `ClusterClient.SendToAll` with `Path == "/user/node-communication"` — **assert the type is `SendToAll`, not merely that something was sent** | +| `Exactly_one_client_is_created_for_a_multi_cluster_fleet` | seed `ClusterNode` rows across two `ServerCluster`s; factory invoked once | +| `ApplyAck_is_forwarded_to_the_coordinator_preserving_sender` | coordinator probe receives `ApplyAck` with `LastSender` == the original | +| `A_malformed_node_row_does_not_abort_the_refresh` | bad row ordered **first**; the good row still yields a contact | +| `MeshCommand_with_no_client_is_dropped_with_a_warning` | `EventFilter.Warning(contains: "dropping").ExpectOne(...)` | +| `A_faulted_db_load_logs_a_warning` | `Status.Failure` → Warning, actor still alive | + +Use `AwaitAssert`/`ExpectMsg` with explicit timeouts. **Do not use `Thread.Sleep`** — the sister project's older comm-actor tests do, and it is their documented flake source. + +**Sabotage-verify:** change `SendToAll` to `Send`; `ClusterClient_mode_sends_to_all_not_to_one` must go RED. This is the single most important assertion in the phase — `Send` would deploy to one node of the fleet and every other node would silently keep its old config. + +**Commit** + +```bash +git commit -m "feat(mesh): CentralCommunicationActor — dark-switched fan-out, DB-sourced contacts" +``` + +--- + +## Task 4: `NodeCommunicationActor` + +**Classification:** standard +**Estimated implement time:** ~4 min +**Parallelizable with:** Task 3 + +**Files:** +- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Communication/NodeCommunicationActor.cs` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/NodeCommunicationActorTests.cs` + +**Why inbound commands go to the `EventStream` and not back onto DPS.** Re-publishing an inbound command onto its DPS topic would fan it across the *whole mesh* — and central has already `SendToAll`-ed it to every node — so every subscriber would receive N copies. `Context.System.EventStream` is node-local by construction, which is exactly the needed semantics, needs no registration handshake with actors that are spawned later (`ScriptedAlarmHostActor` is a **child of** `DriverHostActor` and has no registry key), and survives the Phase 6 mesh split unchanged. + +```csharp +/// +/// Node-side end of the central↔node boundary, registered with the +/// ClusterClientReceptionist at /user/node-communication — per node, not as a +/// singleton, so contact rotation reaches whichever node answers. +/// +/// +/// +/// Inbound commands are re-emitted on rather than on their +/// DistributedPubSub topic. DPS is mesh-wide: since central SendToAlls to every node's +/// comm actor, a DPS re-publish would deliver N copies to every subscriber. The EventStream +/// is node-local by construction. +/// +/// +/// Outbound is only. There is no Ask across this boundary in Phase 2 — +/// every migrated command is fire-and-forget, and the deploy ack returns as an independent +/// message the coordinator already subscribes for. +/// +/// +public sealed class NodeCommunicationActor : ReceiveActor +{ + public const string ActorName = "node-communication"; + public const string CentralCommunicationPath = "/user/central-communication"; +``` + +Handlers: `DispatchDeployment`, `RestartDriver`, `ReconnectDriver`, `AlarmCommand` → `Context.System.EventStream.Publish(msg)`; `ApplyAck` → `_centralClient.Tell(new ClusterClient.Send(CentralCommunicationPath, ack))`, or Warning + drop when `_centralClient is null`. + +**Tests:** each inbound type reaches an EventStream subscriber probe exactly once; `ApplyAck` produces a `ClusterClient.Send` (not `SendToAll` — the coordinator is one singleton) with the right path; a null client logs a Warning and drops. + +**Sabotage-verify:** delete the `AlarmCommand` handler — its test must go RED (guards against a handler being added to the plan but not the actor). + +**Commit** + +```bash +git commit -m "feat(mesh): NodeCommunicationActor — inbound commands to the node-local EventStream" +``` + +--- + +## Task 5: Node-side subscribers accept both transports + +**Classification:** standard +**Estimated implement time:** ~3 min +**Parallelizable with:** none + +**Files:** +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs:476-485` and `:2410-2424` +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs:522-530` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostTransportSubscriptionTests.cs` + +**Subscribe to both, unconditionally — no mode plumbing on the node's subscribers.** In `Dps` mode nothing publishes to the EventStream; in `ClusterClient` mode nothing publishes to the topic. Adding both means the dark switch touches **publishers only**, which is what keeps it cheap to flip and cheap to revert. + +In `DriverHostActor.PreStart`, after the existing three `Subscribe` calls: + +```csharp + // Phase 2 dark switch: under MeshTransport:Mode = ClusterClient these arrive from + // NodeCommunicationActor on the node-local EventStream instead of the mesh-wide DPS topic. + // Subscribing to both is safe and deliberate — exactly one of the two ever publishes, so + // the transport flag lives entirely on the publishing side. + Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment)); + Context.System.EventStream.Subscribe(Self, typeof(RestartDriver)); + Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver)); +``` + +Same shape in `ScriptedAlarmHostActor.PreStart` for `typeof(AlarmCommand)`. + +**Rename `_coordinatorOverride` → `_ackRouter`** (field, ctor parameter `coordinator:` → `ackRouter:`, and the `Props` parameter). Under ClusterClient mode this ref is the `NodeCommunicationActor`, not a coordinator, and the old name would actively mislead the next reader. The `SendAck` branch at `:2413` needs no logic change — only the comment: + +```csharp + if (_ackRouter is not null) + { + // Either a direct coordinator handle (tests) or, under MeshTransport:Mode = + // ClusterClient, the NodeCommunicationActor that relays this across the boundary. + _ackRouter.Tell(ack); + } +``` + +Update the call site at `Runtime/ServiceCollectionExtensions.cs:~403` (`coordinator: null`) and every test constructing `DriverHostActor.Props`. + +**Test:** publish a `DispatchDeployment` on the EventStream and assert the driver host acts on it; ditto `AlarmCommand` → `ScriptedAlarmHostActor`. + +**Commit** + +```bash +git commit -m "feat(mesh): node subscribers accept EventStream commands; _coordinatorOverride -> _ackRouter" +``` + +--- + +## Task 6: Wire both sides + pin the actor paths + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** none + +**Files:** +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs` +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs:~380-421` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshCommActorPathTests.cs` + +Central side, in `WithOtOpcUaControlPlaneSingletons` — a plain `WithActors`, **not** a singleton: + +```csharp + // Per node, NOT a cluster singleton: a node's ClusterClient rotates across contact points + // and must find a live comm actor at whichever central node answers. A singleton would be + // reachable only through the one node hosting it, and the rotation would silently fail + // against the other. + builder.WithActors((system, registry, resolver) => + { + var actor = system.ActorOf(CentralCommunicationActor.Props(...), CentralCommunicationActor.ActorName); + ClusterClientReceptionist.Get(system).RegisterService(actor); + registry.Register(actor); + }); +``` + +Node side, in `WithOtOpcUaRuntimeActors`, **after** `driverHost` is spawned: build the node's ClusterClient from `MeshTransport:CentralContactPoints` (only when `Mode = ClusterClient`), spawn `NodeCommunicationActor`, register it with the receptionist, and pass it as `ackRouter:` to `DriverHostActor.Props`. + +Then thread the router into the two singletons (Task 7 uses it): `ConfigPublishCoordinator.Props(dbFactory, applyDeadline, meshRouter)` and `AdminOperationsActor.Props(..., meshRouter)`. + +**Pin the paths.** `/user/central-communication` and `/user/node-communication` are the wire contract and nothing else asserts them — a rename would compile, deploy, and silently deliver nothing: + +```csharp + private async Task AssertActorExists(ActorSystem system, string path) + { + var identity = await system.ActorSelection(path) + .Ask(new Identify(path), TimeSpan.FromSeconds(5)); + identity.Subject.ShouldNotBeNull($"{path} is the ClusterClient wire contract"); + } +``` + +**Sabotage-verify:** rename the actor to `node-comm`; the path test must go RED. + +**Commit** + +```bash +git commit -m "feat(mesh): register comm actors with the receptionist; pin the wire-contract paths" +``` + +--- + +## Task 7: Route the publishers through the router + +**Classification:** standard +**Estimated implement time:** ~4 min +**Parallelizable with:** none + +**Files:** +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs:78-101,175` +- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs:88,135,397,427` +- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTransportTests.cs` + +Five publish sites become one call each. `ConfigPublishCoordinator.cs:175`: + +```csharp + _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg)); +``` + +`AdminOperationsActor.cs:88` / `:135` / `:397` / `:427` likewise, with `AlarmCommandsTopic.Name` / `DriverControlTopic.Name`. + +`_meshRouter` must be **nullable with a mediator fallback**, so the ~12 existing tests that call `ConfigPublishCoordinator.Props(dbFactory)` and `AdminOperationsActor.Props(...)` keep compiling and keep exercising the DPS path: + +```csharp + // Null in tests and on any node wired before Phase 2: fall back to the mediator, which is + // exactly the pre-Phase-2 behaviour. The fallback is not a permanent seam — Phase 6 deletes it + // with the DPS branch. + private void Dispatch(string topic, object message) + { + if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(topic, message)); + else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(topic, message)); + } +``` + +**Test:** with a router probe injected, dispatching a deployment yields `MeshCommand(DeploymentsTopic, DispatchDeployment)` at the probe and **nothing** on the mediator; with no router, the mediator still sees the `Publish`. + +**Sabotage-verify:** make `Dispatch` always take the mediator branch — the router test must go RED. (Phase 1 shipped a test that passed with the production mapping deleted; assume nothing here is load-bearing until you have watched it fail.) + +**Commit** + +```bash +git commit -m "feat(mesh): route deploy, driver-control and alarm commands through the mesh router" +``` + +--- + +## Task 8: Real two-ActorSystem boundary test + +**Classification:** high-risk +**Estimated implement time:** ~5 min +**Parallelizable with:** none + +**Files:** +- Create: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshClusterClientBoundaryTests.cs` + +**This is the deliberate divergence from the sister project, and the most valuable task in the phase.** They have **no** test that puts two real ActorSystems on either side of a ClusterClient — they substitute a 6-line in-process `ClusterClientRelay` that unwraps `ClusterClient.Send` and forwards. That relay tests actor *logic* and cannot fail for any transport reason: not a missing receptionist extension, not `SendToAll` behaving unlike `Send`, not a malformed contact path, not a serialization break. Those are precisely the Phase 2 failure modes. + +Build two genuinely separate systems — same ActorSystem name `otopcua` (required: Akka.Remote address matching means a ClusterClient cannot reach a differently-named system), **non-overlapping seed lists**, each self-seeded so neither joins the other: + +```csharp + // Two clusters, one ActorSystem name, joined ONLY by ClusterClient. If this test ever starts + // passing because the two systems gossiped into one mesh, it is testing nothing — assert the + // separation explicitly. + Cluster.Get(_central).State.Members.Count.ShouldBe(1); + Cluster.Get(_node).State.Members.Count.ShouldBe(1); +``` + +Assertions: + +1. `SendToAll` from central reaches the node's registered comm actor and lands on the node's EventStream. +2. An `ApplyAck` sent from the node reaches central's comm actor. +3. **Falsifiability control** — with the receptionist extension removed from the node's config, (1) must NOT arrive within the timeout. Without this control the test cannot distinguish "the boundary works" from "something else delivered it". + +Use `AwaitAssert` with explicit timeouts throughout; no `Thread.Sleep`. + +> The Phase 1 lesson that earns this task: *"the bad thing didn't happen"* is not evidence the mechanism works. Assertion 3 is what converts assertions 1–2 from a hope into a measurement. + +> **Scope limit — what this test can NOT see:** with exactly one node per side, "registered per +> node" and "registered as a singleton" are indistinguishable by construction, so this test does +> NOT cover the per-node property whose structural assertions Task 6 dropped (both discriminators +> were unfalsifiable — see the dead-end record there). That property's discriminating coverage is +> **live-gate step 8** (stop the OLDEST central; the survivor's comm actor must receive a node ack +> inside the downing window — a singleton comm actor, hosted on the stopped oldest, cannot). Do not +> close the phase with step 8 skipped: it is the honest replacement for the dropped Task 6 test. + +If this test proves irreducibly flaky, **do not delete it and do not weaken it into a relay test** — mark it `[Trait("Category","MeshBoundary")]`, exclude it from the default run, and record the flakiness in the live-gate doc as an open item. A quarantined honest test beats a green vacuous one. + +**Commit** + +```bash +git commit -m "test(mesh): real two-ActorSystem ClusterClient boundary test with falsifiability control" +``` + +--- + +## Task 9: Rig config + docs + +**Classification:** small +**Estimated implement time:** ~5 min +**Parallelizable with:** Task 8 + +**Files:** +- Modify: `docker-dev/docker-compose.yml` (add `MeshTransport__Mode` + `MeshTransport__CentralContactPoints__0/1` to all six nodes; leave `Mode=Dps` so the rig comes up on the old path) +- Modify: `docs/Configuration.md` (new `MeshTransport` section) +- Modify: `docs/Redundancy.md` (the command boundary) +- Modify: `docs/plans/2026-07-21-per-cluster-mesh-design.md` — **correct §2's typed-failure claim** and add a "Phase 2 as shipped" note +- Modify: `docs/plans/2026-07-22-per-cluster-mesh-program.md` — record the no-cross-boundary-Ask gate deviation; flip the Phase 2 row +- Modify: `CLAUDE.md` — a short "Mesh command transport" note beside the Redundancy section + +**Commit** + +```bash +git commit -m "docs(mesh): MeshTransport config, Phase 2 as shipped, design-doc correction" +``` + +--- + +## Task 10: Live gate on docker-dev + +**Classification:** high-risk +**Estimated implement time:** ~5 min (plus rig time) +**Parallelizable with:** none + +**Files:** +- Create: `docs/plans/2026-07-22-mesh-phase2-live-gate.md` + +**Run every step and record the actual output, including anything that fails.** Phase 1's gate found a defect at step 4 that steps 1–3 had convincingly hidden; the value is entirely in running the steps that prove *recovery*, not the ones that prove the happy path. + +| # | Step | Pass condition | +|---|---|---| +| 1 | Rig up with `Mode=Dps`; deploy via `POST :9200/api/deployments` (`X-Api-Key: docker-dev-deploy-key`) | Seals green — baseline on the old transport | +| 2 | Flip all six nodes to `Mode=ClusterClient`, rebuild, redeploy | Seals green; coordinator logs the same ack count | +| 3 | Confirm the transport actually changed | `docker logs` shows the `SendToAll`/receptionist path; **no** `deployments` DPS publish | +| 4 | Stop one node of the site-a pair, deploy | Fails at the apply deadline **naming that node** (Phase 1 semantics), not a hang | +| 5 | Set that node's `MaintenanceMode = 1`, redeploy | Seals green without it | +| 6 | AdminUI Restart + Reconnect on a driver | Reaches the owning node; UI reports "dispatched" | +| 7 | AdminUI alarm ack on `/alerts` | Reaches the owning node's engine | +| 8 | **Contact rotation / the per-node discriminator.** Stop the **OLDEST** central node only (the singleton host — stopping the younger proves nothing, a singleton would also live on the still-up oldest), then immediately restart one driver node so it emits its boot-time ack | The **surviving** central's `central-communication` actor logs receipt of the ack **within seconds — inside the 15 s downing window**, before singleton migration could explain delivery. This is the discriminating test for "registered per node, NOT a singleton" — the property Task 6's structural assertions could not falsify (both dead ends recorded there): a singleton-shaped comm actor was hosted on the node just stopped, so nothing could receive on the survivor until downing + migration (~25 s). Receipt at the comm actor is the pass signal; the onward `Forward` to the coordinator singleton proxy is EXPECTED to buffer until the singleton migrates — do not count that against the step (full recovery is step 10's job) | +| 9 | Stop **both** central nodes, then start a driver node | Node boots; ack is dropped with a Warning and **not** buffered; no crash loop | +| 10 | Restart central, redeploy | Recovers with no operator action beyond the redeploy | +| 11 | Grep every node's logs for frame-size warnings | None (the deploy path is payload-free — this is the canary that stays quiet) | + +Step 8 is the one that proves the `buffer-size = 0` decision. If a command *does* arrive late after central returns, buffering is still on somewhere and the DB is no longer the single record of truth. + +The rig's AdminUI runs with `Security__Auth__DisableLogin=true` — **drive steps 6 and 7 yourself via browser automation at `http://localhost:9200`; do not defer them as needing the user to sign in.** + +**Commit** + +```bash +git commit -m "docs(mesh): Phase 2 live-gate record" +``` + +--- + +## Risks specific to this phase + +- **`SendToAll` vs `Send`** is a one-word difference between "every node deploys" and "one node deploys and the rest silently keep their old config, while the deployment seals green on partial acks". Pinned by a sabotage-verified test in Task 3. +- **The single-mesh duplicate trap** turns "one client per cluster" — the obviously-right-looking shape, and the one Phase 6 wants — into N× delivery today. +- **`buffer-size = 0` is a behaviour choice, not a tuning knob.** If someone later "fixes" it back to Akka's default to make a flaky test pass, they reintroduce apply-after-timeout divergence between node and DB. +- **The AdminUI's `Ok = true` already means "dispatched", not "applied".** Phase 2 moves that lie to a new transport without making it worse; do not let a reviewer believe Phase 2 introduced it. +- **`redundancy-state` is bidirectional and carries two unrelated message types** (`RedundancyStateChanged` central→node, `OpcUaProbeResult` node→node on the same topic). It is explicitly **out of scope** here — but note `RedundancyStateActor` builds its snapshot from `Cluster.State`, so it is genuinely mesh-bound and is the hardest thing standing between here and Phase 6. +- **`alerts` has a node-local subscriber** (`HistorianAdapterActor`), so Phase 5 cannot simply move it to gRPC without keeping a node-local leg. + +--- + +## Follow-ups found during execution (not fixed in Phase 2) + +- **`DefaultMeshClusterClientFactory`'s generation counter is per factory instance, and its clients + are top-level actors.** Production creates one factory per `CentralCommunicationActor` and captures + it in the props closure, so the counter survives a supervision restart and names stay unique — that + part is sound. What is *not* handled: the clients are `system.ActorOf`, not children, so a restart + of the comm actor loses `_client` without stopping it and leaves an orphaned ClusterClient dialling + receptionists forever. Harmless today (nothing sends to it) but it is a leak per restart, and it + will matter in Phase 6 when there is one client per cluster. Found by + `MeshClusterClientBoundaryTests`, which hit the name collision the moment a second factory was + constructed on the same system. diff --git a/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md.tasks.json b/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md.tasks.json new file mode 100644 index 00000000..5d5c8a9a --- /dev/null +++ b/docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md.tasks.json @@ -0,0 +1,141 @@ +{ + "planPath": "docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md", + "note": "Per-cluster mesh Phase 2. Decisions pre-made: config-flagged dark switch (MeshTransport:Mode), buffer-size=0, alarm-commands central leg in scope. Three design-doc corrections and the single-mesh duplicate-delivery trap are documented in the plan header \u2014 read them before Task 3.", + "tasks": [ + { + "id": 0, + "subject": "Task 0: MeshTransportOptions + validator + registration", + "status": "completed", + "classification": "small", + "parallelizableWith": [ + 1 + ], + "commit": "5439f148" + }, + { + "id": 1, + "subject": "Task 1: receptionist extension, buffer-size=0, frame-size logging (effective-config test)", + "status": "completed", + "classification": "small", + "parallelizableWith": [ + 0 + ], + "commit": "7b71a6a3", + "note": "First draft of the buffering test was VACUOUS -- Config.GetInt returns 0 for a missing key, so it passed before the feature existed. Rewritten to assert through ClusterClientSettings/ClusterReceptionistSettings." + }, + { + "id": 2, + "subject": "Task 2: MeshCommand envelope + generation-suffixed ClusterClient factory", + "status": "completed", + "classification": "small", + "blockedBy": [ + 0 + ], + "commit": "0d6669d5", + "note": "Dropped a third test that asserted ClusterClientSettings directly and never touched the factory." + }, + { + "id": 3, + "subject": "Task 3: CentralCommunicationActor \u2014 dark-switched SendToAll fan-out, DB-sourced contacts, ApplyAck forward", + "status": "completed", + "classification": "high-risk", + "blockedBy": [ + 1, + 2 + ], + "parallelizableWith": [ + 4 + ], + "commit": "d5b5cb6e", + "note": "SendToAll sabotage-verified. Two test-harness bugs found: SubscribeAck goes to the SENDER, and EventFilter matches case-INSENSITIVELY (a 'was NOT' filter also caught 'was not created')." + }, + { + "id": 4, + "subject": "Task 4: NodeCommunicationActor \u2014 inbound to node-local EventStream, outbound ApplyAck", + "status": "completed", + "classification": "standard", + "blockedBy": [ + 1, + 2 + ], + "parallelizableWith": [ + 3 + ], + "commit": "915beec1", + "note": "No Ask crosses the boundary in Phase 2, so this actor is far smaller than the sister project's." + }, + { + "id": 5, + "subject": "Task 5: node subscribers accept EventStream too; rename _coordinatorOverride -> _ackRouter", + "status": "completed", + "classification": "standard", + "blockedBy": [ + 4 + ], + "commit": "5cb0e721", + "note": "Two wiring tests added -- unit tests either side of the seam both pass with the subscription deleted. First draft raced (ActorOf returns before PreStart; EventStream publish to nobody is dropped silently); alarm version was vacuous until the warm-up moved outside the assertion window." + }, + { + "id": 6, + "subject": "Task 6: wire both sides, receptionist registration, pin the wire-contract actor paths", + "status": "completed", + "classification": "standard", + "blockedBy": [ + 3, + 5 + ], + "commit": "16d59856", + "note": "Dropped a planned 'not a singleton' assertion: two candidate discriminators were both invalid (a SingletonManager also sits at /user/; /singleton is null even for the known singleton config-publish). A sabotage re-registering via WithSingleton left both green, which exposed them. Property deferred to Task 8." + }, + { + "id": 7, + "subject": "Task 7: route the five publish sites through the mesh router (nullable, mediator fallback)", + "status": "completed", + "classification": "standard", + "blockedBy": [ + 6 + ], + "note": "Sabotage-verified by inverting the branch: all 6 tests red. Comm actor registration moved above the singletons so registry.Get works; an earlier ResolveOne().GetAwaiter().GetResult() blocked inside a props factory." + }, + { + "id": 8, + "subject": "Task 8: real two-ActorSystem ClusterClient boundary test + falsifiability control", + "status": "completed", + "classification": "high-risk", + "blockedBy": [ + 7 + ], + "parallelizableWith": [ + 9 + ], + "commit": "5468b79d", + "note": "Not flaky (3/3 clean repeats; 4 tests, ~16s). Both legs sabotage-verified against production code, and the control was itself falsified by restoring the receptionist. The control necessarily removes RegisterService too -- registering would materialise the extension it is removing. Task 6's dropped per-node/singleton property does NOT land here (one node per side cannot discriminate); it is live-gate step 8, and the MeshCommActorPathTests comment was corrected to say so." + }, + { + "id": 9, + "subject": "Task 9: docker-dev rig config + docs + design-doc correction", + "status": "completed", + "commit": "e4cb9a08", + "note": "Rig Mode is ${OTOPCUA_MESH_MODE:-Dps} rather than a hardcoded Dps, so Task 10 can flip all six nodes at `docker compose up` with no compose edit and no rebuild. Verified via `docker compose config` (6 services resolved).", + "classification": "small", + "blockedBy": [ + 7 + ], + "parallelizableWith": [ + 8 + ] + }, + { + "id": 10, + "subject": "Task 10: live gate on docker-dev (11 steps; step 4 = deploy-seal semantics, step 8 = per-node/contact-rotation discriminator [replaces the dropped Task 6 assertion \u2014 do not skip], step 9 = both-centrals drop behavior)", + "status": "completed", + "note": "PASSED for the transport (steps 1-5, 8, 11 green; flip end-to-end via OTOPCUA_MESH_MODE=ClusterClient). Record: 2026-07-22-mesh-phase2-live-gate.md. Steps 6/7 NOT RUN (no UI surface for driver-control \u2014 finding D, DriverStatusPanel is orphaned; no alarms configured). Step 8's stated log-signal does not exist (HandleApplyAck is silent on success \u2014 finding E); proven instead by both centrals independently logging per-instance ClusterClient creation + rebuild, which a singleton cannot. FOUR findings: A) 10-30s post-restart delivery window is PRE-EXISTING (DPS control failed identically); B) buffer-size=0 rationale OVERSTATED \u2014 a node applies a TimedOut deploy anyway via Bootstrap() orphan-row replay at +44s, corrected in 3 docs; C) stopping BOTH centrals strands non-seed nodes at the membership layer (Phase 7 drill, transport-independent); D) driver Restart/Reconnect UI orphaned since v3 Batch 2. Rig restored to Dps default.", + "classification": "high-risk", + "blockedBy": [ + 8, + 9 + ] + } + ], + "lastUpdated": "2026-07-22T00:00:00Z" +} \ No newline at end of file diff --git a/docs/plans/2026-07-22-mesh-phase2-live-gate.md b/docs/plans/2026-07-22-mesh-phase2-live-gate.md new file mode 100644 index 00000000..b32de27f --- /dev/null +++ b/docs/plans/2026-07-22-mesh-phase2-live-gate.md @@ -0,0 +1,228 @@ +# Per-cluster mesh Phase 2 — live gate record + +**Date:** 2026-07-22 · **Rig:** `docker-dev` (six-node single mesh) · **Branch:** `feat/mesh-phase2` +**Plan:** [`2026-07-22-mesh-phase2-clusterclient-transport.md`](2026-07-22-mesh-phase2-clusterclient-transport.md) + +**Result: PASSED for the transport**, with two gate steps not run (no surface exists to drive them), +one step whose stated pass-signal does not exist in the code, and **four findings** — one of which +corrects a claim this phase's own documentation makes. + +The rig was flipped end-to-end with `OTOPCUA_MESH_MODE=ClusterClient` (no rebuild, no compose edit — +the env indirection added in Task 9 is what made the A/B and the DPS control cheap enough to run). + +--- + +## Result table + +| # | Step | Result | +|---|---|---| +| 1 | Deploy on `Mode=Dps` | **PASS** — `48d0c7c5` Sealed, 6/6 Applied | +| 2 | Flip all six to `ClusterClient`, redeploy | **PASS** — `5db8f6ad` Sealed, 6/6 Applied | +| 3 | Confirm the transport actually changed | **PASS** — positive log evidence, see below | +| 4 | Stop a node, deploy | **PASS** — `59b81248` TimedOut naming `site-b-2:4053`, 5/6 | +| 5 | `MaintenanceMode = 1`, redeploy | **PASS** — `c6cb9812` Sealed, 5 rows, contacts 6→5 | +| 6 | AdminUI Restart + Reconnect | **NOT RUN** — no UI surface exists (finding D) | +| 7 | AdminUI alarm ack | **NOT RUN** — no alarms configured on the rig | +| 8 | Per-node (not singleton) discriminator | **PASS** — via a better discriminator than the plan's | +| 9 | Both centrals down, restart a driver node | **PARTIAL** — node survives; the stated ack-drop cannot occur | +| 10 | Restart central, redeploy | **PASS only after operator action** (finding C) | +| 11 | Frame-size canary | **PASS** — 0 warnings across all six nodes | + +## Step 3 — the transport really changed + +Central, both nodes, at boot under the flag: + +``` +central-1 [22:18:16 INF] Mesh ClusterClient created with 6 contact point(s) +central-2 [22:18:17 INF] Mesh ClusterClient created with 6 contact point(s) +``` + +Node side, with `Serilog__MinimumLevel__Default=Debug` overlaid on `site-a-1` (the rig pins the +default above Debug, so this line is invisible on a stock node): + +``` +site-a-1 [22:10:34 DBG] Mesh: received DispatchDeployment from central; publishing node-locally +``` + +Under `Mode=Dps` the same node logs nothing of the sort and central logs +`Mesh transport is Dps: commands are published on DistributedPubSub and no ClusterClient is created`. + +## Step 5 — the DB-sourced contact set is live + +Setting `MaintenanceMode = 1` on `site-b-2` shrank both centrals' contact sets on the next refresh, +and clearing it restored them — independently, on each central's own 60 s timer: + +``` +central-1 [22:22:16 INF] Mesh contact set changed; replacing the ClusterClient +central-1 [22:22:16 INF] Mesh ClusterClient created with 5 contact point(s) +central-1 [22:23:16 INF] Mesh ClusterClient created with 6 contact point(s) +central-2 [22:22:16 INF] … identical, same timestamps +``` + +No `InvalidActorNameException` on any rebuild — the generation-suffixed client name (Task 2, ported +from the sister project's shipped bug) does its job. + +## Step 8 — the per-node discriminator, replaced with a stronger one + +**The plan's pass-signal does not exist.** It asked for the surviving central's comm actor to *log +receipt* of a node ack inside the downing window. `CentralCommunicationActor.HandleApplyAck` logs +**nothing** on the success path — it only warns when the coordinator is unresolvable. There is no +line to observe. (Recorded as finding E.) + +The evidence above is a **cleaner** discriminator and needs no timing race. `Mesh ClusterClient +created` is emitted from `PreStart`, so it fires **once per live actor instance**. Both centrals +logged it, and both logged their own independent contact-set rebuild at step 5. A +`ClusterSingletonManager` runs the actor on the oldest node only — the other node hosts a manager +that never starts the actor and would log nothing. **Two nodes logging it proves two live instances, +i.e. per-node registration.** The node side is proven the same way: `SendToAll` collected 6/6 acks, +which requires six separately registered node comm actors. + +The failover half was run anyway, and passed: `docker stop central-1` (the oldest, and the singleton +host) → the surviving central accepted a deployment **4 seconds later** and collected acks from all +five live nodes (`e3161c25`; it TimedOut only because `central-1` is itself a configured node that +was down — correct Phase 1 semantics). Graceful stop hands the singleton over immediately, so no +downing wait was involved. + +## Step 9 — the stated failure mode cannot occur + +The plan expected a restarting driver node to emit a boot-time ack that gets dropped with a Warning. +**Driver nodes do not ack on boot.** `DriverHostActor.SendAck` is called only from the apply paths — +a node with no dispatch to answer has nothing to send, so there is no drop to observe. + +What was verified: with **both** centrals stopped, `site-a-1` restarted, came up clean, logged its +association failures against both central addresses, retried, and stayed up — no crash loop. The +node-side ClusterClient's forever-retry (`reconnect-timeout = off`) behaves as configured. + +## Step 11 — the canary stayed quiet + +`grep -ci "frame size|frame-size|maximum-frame"` = **0** on all six nodes, as designed: the deploy +notify carries only `DeploymentId` + `RevisionHash` + `CorrelationId`. + +--- + +## Finding A — a 10–30 s post-restart delivery window, **pre-existing, not Phase 2** + +A deploy issued shortly after a driver node restarts does not reach that node, and the deployment +fails at the 2-minute deadline naming it. Measured against `site-a-1`: + +| Delay after the node logs readiness | Result | +|---|---| +| ~4 s | **missed** (`f738e1de` TimedOut, node row `Applying`) | +| 10 s | **missed** (`0c7a6296` TimedOut) | +| 30 s | applied, sealed | + +**The DPS control fails identically.** Flipping the rig back to `Mode=Dps` and repeating the 10 s +case produced `e85f53c5` → TimedOut, `site-a-1` unacked. So the window is a property of cluster +membership convergence, not of the transport this phase introduces. Recorded rather than fixed. + +> Running that control is the only reason this is a note and not a Phase 2 regression. The +> temptation to skip it — "obviously the new transport broke it" — would have produced a false +> attribution and a wasted investigation. + +## Finding B — **`buffer-size = 0` does not deliver what this phase's docs claim it does** + +The stated rationale, in the plan, `CLAUDE.md`, and `docs/Configuration.md`, is that buffering +*"would replay a `DispatchDeployment` on reconnect and apply a deployment the coordinator already +sealed as `TimedOut`, leaving the node running a configuration the database records as failed."* + +**That outcome happens anyway, by a different route.** Deployment `0c7a6296` timed out at 22:14:30 +with `site-a-1` unacked. The node's row: + +``` +NodeId Status StartedAtUtc AppliedAtUtc +site-a-1:4053 1 2026-07-22 22:12:30.145 2026-07-22 22:15:14.823 +``` + +It applied **44 s after the deployment was sealed TimedOut**, on its next restart. The mechanism is +`DriverHostActor.Bootstrap()`, which reads the node's most recent `NodeDeploymentState`, finds the +orphan `Applying` row the coordinator pre-created at dispatch, and replays it: + +```csharp +case NodeDeploymentStatus.Applying: + _log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying", …); + ApplyAndAck(new DeploymentId(latest.DeploymentId), revision.Value, CorrelationId.NewId()); +``` + +So `buffer-size = 0` removes the **transport-level, timer-driven** replay — a real difference: it is +silent, unbounded in time, and can fire without any node event. The **boot-time DB replay remains**, +and it is the better-behaved of the two (deliberate, Warning-logged, and bounded to a restart). The +decision stands; the *justification* was overstated and the docs are corrected in the same change as +this record. + +## Finding C — stopping **both** centrals strands the non-seed nodes permanently + +After `docker stop` of both centrals followed by `docker start`, the fleet did **not** recover. +Deployments reached 2/6 then 3/6 nodes. The cause is not the transport — the Akka cluster itself had +split: + +``` +Cluster hosts: MEMBERS 3 UP 3 UNREACHABLE 0 + central-1:4053 UP admin,driver leader for admin+driver + central-2:4053 UP admin,driver + site-a-1:4053 UP driver +``` + +The three site nodes that never restarted were **not members at all**, and their own gossip showed +the stale inverse view — `central-2` as `Exiting`, `site-a-1` as `Leaving` — frozen since 22:29: + +``` +Gossip(members = [central-2 status = Exiting, site-a-1 status = Leaving, + site-a-2 Up, site-b-1 Up, site-b-2 Up]) +``` + +A graceful stop puts the centrals into `Leaving`/`Exiting`; the processes died before the removal +handshake completed, leaving the survivors holding members that will never confirm. `auto-down` +does not help — these members are not *Unreachable*, they are stuck mid-`Exiting`. The restarted +centrals formed a fresh cluster (with `site-a-1`, which had also restarted and re-joined through its +seed), and the three stranded nodes had no path back: they believe they are already in a cluster, so +they never re-run the join process. + +**Recovery required restarting the stranded nodes.** After that, the next deploy sealed 6/6 +(`e20836ac`). + +This is transport-independent — membership is below both DPS and ClusterClient, and the set of nodes +that acked matched the cluster view exactly. It is a **downing/rejoin gap of the current single-mesh +topology**, in the same family as the registered outage gap the design doc already records, and it +belongs to the Phase 7 drill work. Under the target per-pair topology (every node a seed of its own +pair) the shape changes, and this drill must be re-run there rather than assumed fixed. + +## Finding D — the AdminUI's driver Restart/Reconnect buttons have no page + +Step 6 could not be run, and not for a rig reason. +`Components/Shared/Drivers/DriverStatusPanel.razor` is the **only** component in the codebase +carrying the `DriverOperator`-gated Reconnect/Restart buttons, and **nothing renders it** — `grep -rn +" +/// Selects and configures the transport carrying central→node commands and node→central +/// deployment acks (per-cluster mesh Phase 2). +/// +/// +/// +/// is a deliberate dark switch, not a permanent knob: both +/// transports live in the tree for one phase so the docker-dev rig can run the same +/// deployment over each and compare, and so a bad Phase 2 rolls back with a config change +/// rather than a revert. Phase 6 deletes the DistributedPubSub branch along with the single +/// mesh that made it viable. +/// +/// +/// Discovery is deliberately asymmetric, matching the sister project. Central +/// discovers nodes from the database — enabled, non-maintenance ClusterNode +/// rows, rebuilt every . Nodes discover central from +/// this section: static, restart to change. The asymmetry follows the rate of change — +/// operators add and retire nodes, but central's own address is part of the deployment. +/// +/// +public sealed class MeshTransportOptions +{ + /// Configuration section name. + public const string SectionName = "MeshTransport"; + + /// Every command stays on DistributedPubSub — the pre-Phase-2 behaviour. + public const string ModeDps = "Dps"; + + /// Commands cross the ClusterClient receptionist boundary. + public const string ModeClusterClient = "ClusterClient"; + + /// + /// (default) or . Any other value fails + /// the host at startup rather than silently falling back — a transport that is neither of + /// these delivers nothing, and the symptom would be commands that vanish. + /// + public string Mode { get; set; } = ModeDps; + + /// + /// Akka remoting addresses of the central nodes, e.g. + /// akka.tcp://otopcua@central-1:4053. Node addresses only — the + /// /system/receptionist path is appended at construction time. + /// + /// + /// Each entry must be a central node's remoting endpoint, never this node's own port. + /// The sister project shipped a site template whose second contact was the site's own + /// remoting port; it is a permanent failure in the initial-contact rotation and stays + /// invisible until a command goes missing. + /// rejects the malformed shapes at boot instead. + /// + public string[] CentralContactPoints { get; set; } = []; + + /// + /// How often central re-reads ClusterNode rows to rebuild its contact points. + /// Refreshing on a timer (rather than only on admin change) means a row edited by any + /// process — migration, direct SQL, a second admin node — is picked up without a restart. + /// + public int ContactRefreshSeconds { get; set; } = 60; +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs new file mode 100644 index 00000000..afcd8779 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptionsValidator.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Fails the host at startup on a shape that would leave +/// commands silently undelivered. +/// +/// +/// Every fault caught here otherwise surfaces as an absence — a deployment that never +/// arrives, an alarm acknowledgement that does nothing, a node whose ack is dropped without a +/// trace. An absence has no stack trace and no failing node to point at, which makes it the +/// most expensive class of misconfiguration to diagnose in the field. Refusing to start is +/// cheaper for everyone. +/// +public sealed class MeshTransportOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, MeshTransportOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var errors = new List(); + + var isDps = string.Equals(options.Mode, MeshTransportOptions.ModeDps, StringComparison.OrdinalIgnoreCase); + var isClusterClient = string.Equals( + options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); + + if (!isDps && !isClusterClient) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.Mode)} is " + + $"'{options.Mode}'. Expected '{MeshTransportOptions.ModeDps}' or " + + $"'{MeshTransportOptions.ModeClusterClient}'."); + } + + if (options.ContactRefreshSeconds <= 0) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.ContactRefreshSeconds)} " + + $"must be greater than 0 (was {options.ContactRefreshSeconds}). Central rebuilds its " + + "ClusterClient contact points on this interval; a non-positive value would leave the " + + "contact set frozen at whatever it held when the node booted."); + } + + // Contact points are only load-bearing under ClusterClient mode. Requiring them under the + // default would make the section mandatory everywhere for a feature that is switched off — + // including on admin-only nodes, which have no reason to carry them at all. + if (isClusterClient) + { + if (options.CentralContactPoints.Length == 0) + { + errors.Add( + $"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.CentralContactPoints)} " + + "is empty. Under ClusterClient mode a driver node cannot reach central without at " + + "least one contact point, and its deployment acks would be dropped silently — the " + + "deployment would then time out naming a node that is running perfectly."); + } + + foreach (var contactPoint in options.CentralContactPoints) + { + if (!contactPoint.StartsWith("akka.tcp://", StringComparison.Ordinal)) + { + errors.Add( + $"Contact point '{contactPoint}' must start with 'akka.tcp://' — it is an Akka " + + "remoting address (akka.tcp://@:), not a bare host:port."); + } + else if (contactPoint.Contains("/user/", StringComparison.Ordinal) + || contactPoint.Contains("/system/", StringComparison.Ordinal)) + { + errors.Add( + $"Contact point '{contactPoint}' carries an actor-path suffix. Configure the " + + "node address only — '/system/receptionist' is appended at construction time, " + + "and a doubled suffix produces a path that never resolves."); + } + } + } + + return errors.Count == 0 + ? ValidateOptionsResult.Success + : ValidateOptionsResult.Fail(string.Join(" ", errors)); + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf index 154de1e8..b7e7a8a7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf @@ -15,6 +15,12 @@ akka { extensions = [ "Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools" + + # Per-cluster mesh Phase 2 — the central<->node command boundary. NOT auto-started: without + # this entry the receptionist materialises only if some code path happens to call + # ClusterClientReceptionist.Get(system) first, which makes "is the boundary listening?" + # depend on startup ordering rather than on configuration. + "Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools" ] actor { @@ -25,6 +31,20 @@ akka { dot-netty.tcp { hostname = "0.0.0.0" port = 4053 + + # FRAME SIZE. maximum-frame-size restates Akka's own default; what actually changes is + # log-frame-size-exceeding, which defaults to `off`. Over the limit the transport drops + # that ONE message without tearing down the association — heartbeats keep flowing, the + # node still reports healthy, and the only symptom is a command that never arrived. The + # sister project lost a deploy path to exactly this and could not attribute it; their + # postmortem also notes the default JSON serializer escapes an already-serialized + # payload a second time, so ~60-70 KB of real JSON is enough to blow 128 KB. + # + # OtOpcUa's deploy notify is payload-free by design (DispatchDeployment carries only + # DeploymentId + RevisionHash + CorrelationId), so this threshold should stay silent. + # It is a canary, and a quiet one is the point. + maximum-frame-size = 128000b + log-frame-size-exceeding = 32000b } transport-failure-detector { heartbeat-interval = 2s @@ -74,6 +94,31 @@ akka { down-removal-margin = 15s run-coordinated-shutdown-when-down = on + # CLUSTERCLIENT — the central<->node command boundary (per-cluster mesh Phase 2). + client { + # DELIBERATE, and NOT what the sister project runs: they never wrote this section, so + # they inherit buffer-size = 1000. Zero means "if the receptionist is unknown, drop the + # message now". Buffering would replay a DispatchDeployment on reconnect and apply a + # deployment ConfigPublishCoordinator had already sealed as TimedOut — the node would + # then be running a configuration the database says failed. Fail-fast keeps the DB the + # single record of truth, and reproduces today's DPS behaviour exactly: a node that is + # down misses the publish and self-heals from the DB on restart. + buffer-size = 0 + + # Retry forever rather than self-stopping the client. Restates Akka's default; stated + # explicitly because the behaviour is load-bearing — a central node down for an hour + # must not require a driver-node restart to become reachable again. + reconnect-timeout = off + + receptionist { + # Empty = every member hosts a receptionist. This is what makes the comm actors + # "registered per node, not as a singleton" work: contact rotation reaches whichever + # node answers. Restates Akka's default; do NOT scope this to a role, or the + # boundary becomes reachable through one node only and rotation silently fails. + role = "" + } + } + singleton { singleton-name = "singleton" } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index 7e5eaa45..1363191e 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -34,6 +34,12 @@ public static class ServiceCollectionExtensions services.AddValidatedOptions( configuration, AkkaClusterOptions.SectionName); + // Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at + // startup for the same reason the cluster options are — a contact point that cannot resolve + // produces silence, not an error, and silence is indistinguishable from "nothing to do". + services.AddValidatedOptions( + configuration, MeshTransportOptions.SectionName); + services.AddSingleton(); return services; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshCommand.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshCommand.cs new file mode 100644 index 00000000..a12ab795 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshCommand.cs @@ -0,0 +1,28 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; + +/// +/// A central→node command, plus the DistributedPubSub topic it would have been published on. +/// +/// +/// The legacy DPS topic. Carried so the router can publish under +/// MeshTransport:Mode = Dps without a second dispatch table; ignored entirely under +/// ClusterClient mode, where the destination is an actor path rather than a topic. +/// +/// +/// The command itself, never wrapped. Node-side handlers see exactly the type they see today, +/// which is what lets the dark switch touch publishers only. +/// +/// +/// +/// This envelope exists so the admin singletons stop knowing which transport is in force. +/// Before per-cluster mesh Phase 2 each publisher held its own +/// DistributedPubSub.Get(Context.System).Mediator call, so the switch would have had +/// to be threaded through five call sites across two actors, and each would have been free +/// to drift. +/// +/// +/// Central-side only — it never crosses the wire, so it carries no serialization contract +/// and needs no additive-evolution discipline. +/// +/// +public sealed record MeshCommand(string Topic, object Message); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshPaths.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshPaths.cs new file mode 100644 index 00000000..308117f1 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Mesh/MeshPaths.cs @@ -0,0 +1,33 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; + +/// +/// Actor paths the two mesh comm actors register under. These strings are the wire +/// contract of the central↔node boundary. +/// +/// +/// +/// Declared once, here, rather than as literals on each side. The deployments / +/// deployment-acks topic names are declared twice — once in +/// ConfigPublishCoordinator and again in DriverHostActor — and agree only by +/// convention; renaming one would compile, deploy, and deliver nothing. A ClusterClient path +/// has the same failure mode and no compiler to catch it, so the constant is shared. +/// +/// +/// Nothing else pins these values, which is why MeshCommActorPathTests boots a real +/// host and Identify-probes both paths. +/// +/// +public static class MeshPaths +{ + /// Central-side comm actor — where a driver node sends its deployment acks. + public const string CentralCommunication = "/user/central-communication"; + + /// Node-side comm actor — where central sends commands. + public const string NodeCommunication = "/user/node-communication"; + + /// Actor name of (the path's final segment). + public const string CentralCommunicationName = "central-communication"; + + /// Actor name of (the path's final segment). + public const string NodeCommunicationName = "node-communication"; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 6766c520..4b000bf4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -4,6 +4,7 @@ using Akka.Event; using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; @@ -26,6 +27,13 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IActorRef _coordinator; private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + + /// + /// Where central→node commands are handed off (per-cluster mesh Phase 2), or + /// to publish on the mediator directly. + /// + private readonly IActorRef? _meshRouter; + private readonly ILoggingAdapter _log = Context.GetLogger(); /// Creates actor props for the admin operations actor. @@ -33,29 +41,38 @@ public sealed class AdminOperationsActor : ReceiveActor /// Reference to the deployment coordinator actor. /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). + /// + /// Central comm actor commands are handed to, or to publish on the + /// DistributedPubSub mediator directly (the pre-Phase-2 behaviour). + /// /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, - TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn) => - Akka.Actor.Props.Create(() => new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode)); + TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, + IActorRef? meshRouter = null) => + Akka.Actor.Props.Create(() => + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. /// Reference to the deployment coordinator actor. /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). + /// Central comm actor, or for the mediator. public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, - TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn) + TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, + IActorRef? meshRouter = null) { _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); _tagConfigValidationMode = tagConfigValidationMode; + _meshRouter = meshRouter; ReceiveAsync(HandleStartDeploymentAsync); ReceiveAsync(HandleTestDriverConnectAsync); @@ -65,6 +82,23 @@ public sealed class AdminOperationsActor : ReceiveActor Receive(HandleShelveAlarm); } + /// + /// Hands a central→node command to the mesh transport router, or publishes it directly when + /// no router is wired. + /// + /// + /// The fallback keeps ~12 existing tests (and any node wired before per-cluster mesh Phase 2) + /// on exactly the pre-Phase-2 behaviour. It is not a permanent seam: Phase 6 deletes it along + /// with the DistributedPubSub branch and the single mesh. + /// + /// The legacy DistributedPubSub topic. + /// The command. + private void Dispatch(string topic, object message) + { + if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(topic, message)); + else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(topic, message)); + } + /// /// AdminUI Acknowledge path. Maps the control-plane command to a /// (Operation = "Acknowledge") and publishes it onto the @@ -85,7 +119,7 @@ public sealed class AdminOperationsActor : ReceiveActor Comment: msg.Comment, UnshelveAtUtc: null); - DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(AlarmCommandsTopic.Name, alarmCmd)); + Dispatch(AlarmCommandsTopic.Name, alarmCmd); _log.Info("AdminOps: Acknowledge published for alarm {AlarmId} by {User}", msg.AlarmId, msg.User); replyTo.Tell(new AcknowledgeAlarmResult(true, null, msg.CorrelationId)); @@ -132,7 +166,7 @@ public sealed class AdminOperationsActor : ReceiveActor Comment: msg.Comment, UnshelveAtUtc: unshelveAt); - DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(AlarmCommandsTopic.Name, alarmCmd)); + Dispatch(AlarmCommandsTopic.Name, alarmCmd); _log.Info("AdminOps: {Operation} published for alarm {AlarmId} by {User}", operation, msg.AlarmId, msg.User); replyTo.Tell(new ShelveAlarmResult(true, null, msg.CorrelationId)); @@ -394,7 +428,7 @@ public sealed class AdminOperationsActor : ReceiveActor { // Broadcast to every DriverHostActor on every node via the driver-control DPS topic. // Only the host that owns the instance will act; others ignore it (id not found in _children). - DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DriverControlTopic.Name, msg)); + Dispatch(DriverControlTopic.Name, msg); await using var db = await _dbFactory.CreateDbContextAsync(); db.ConfigEdits.Add(new ConfigEdit @@ -424,7 +458,7 @@ public sealed class AdminOperationsActor : ReceiveActor try { // Broadcast to every DriverHostActor; only the one owning the instance reacts. - DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DriverControlTopic.Name, msg)); + Dispatch(DriverControlTopic.Name, msg); await using var db = await _dbFactory.CreateDbContextAsync(); db.ConfigEdits.Add(new ConfigEdit diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs new file mode 100644 index 00000000..333087e6 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -0,0 +1,308 @@ +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Cluster.Tools.PublishSubscribe; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +/// +/// Central-side end of the central↔node command boundary (per-cluster mesh Phase 2). Routes +/// every central→node command out — over DistributedPubSub or ClusterClient, per +/// — and forwards inbound s to the +/// deploy coordinator singleton. +/// +/// +/// +/// Registered per node, NOT as a cluster singleton. A driver node's ClusterClient +/// rotates across contact points and must find a live comm actor at whichever central node +/// answers. A singleton would be reachable only through the one node hosting it, and the +/// rotation would fail silently against the other. +/// +/// +/// Exactly one ClusterClient, fleet-wide. See — this is +/// the single most important thing to understand before changing this actor. +/// +/// +public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers +{ + private const string RefreshTimerKey = "mesh-contact-refresh"; + + private readonly IDbContextFactory _dbFactory; + private readonly MeshTransportOptions _options; + private readonly IMeshClusterClientFactory _clientFactory; + private readonly Func _coordinator; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private readonly bool _useClusterClient; + + private IActorRef? _client; + private ImmutableHashSet _currentContacts = ImmutableHashSet.Empty; + private CancellationTokenSource _lifecycle = new(); + + /// Gets the timer scheduler driving the periodic contact refresh. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates the props for this actor. + /// Factory for the config database holding ClusterNode rows. + /// The bound mesh-transport options. + /// Creates the ClusterClient; substituted in tests. + /// + /// Resolves the deploy-coordinator singleton proxy. A rather than + /// an so this actor never depends on registration order at wiring + /// time — it resolves on first use and caches. + /// + /// The props. + public static Props Props( + IDbContextFactory dbFactory, + MeshTransportOptions options, + IMeshClusterClientFactory clientFactory, + Func coordinator) => + Akka.Actor.Props.Create(() => + new CentralCommunicationActor(dbFactory, options, clientFactory, coordinator)); + + /// Initializes a new instance of the class. + /// Factory for the config database. + /// The bound mesh-transport options. + /// Creates the ClusterClient. + /// Lazily resolves the deploy-coordinator singleton proxy. + public CentralCommunicationActor( + IDbContextFactory dbFactory, + MeshTransportOptions options, + IMeshClusterClientFactory clientFactory, + Func coordinator) + { + _dbFactory = dbFactory; + _options = options; + _clientFactory = clientFactory; + _coordinator = coordinator; + _useClusterClient = string.Equals( + options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); + + Receive(HandleMeshCommand); + Receive(HandleApplyAck); + Receive(_ => LoadContactsFromDb()); + Receive(HandleContactsLoaded); + + // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler + // the failure is an unhandled message (debug level only) and the refresh fails silently — + // an operator cannot distinguish "no nodes configured" from "the database is down", and the + // contact set silently freezes at whatever it last held. + Receive(f => _log.Warning( + f.Cause, + "Failed to load ClusterNode contact points; the mesh ClusterClient contact set was NOT " + + "refreshed and may be stale or empty. Commands may be dropped until the next refresh")); + + Receive(_ => { /* DPS subscribe confirmation */ }); + } + + /// + protected override void PreStart() + { + if (!_useClusterClient) + { + _log.Info( + "Mesh transport is {Mode}: commands are published on DistributedPubSub and no " + + "ClusterClient is created", _options.Mode); + return; + } + + Timers.StartPeriodicTimer( + RefreshTimerKey, + new RefreshContacts(), + TimeSpan.Zero, + TimeSpan.FromSeconds(_options.ContactRefreshSeconds)); + } + + /// + protected override void PostStop() + { + _lifecycle.Cancel(); + _lifecycle.Dispose(); + } + + private void HandleMeshCommand(MeshCommand cmd) + { + if (!_useClusterClient) + { + DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(cmd.Topic, cmd.Message)); + return; + } + + if (_client is null) + { + // App-level drop, matching the sister project's decision and this repo's buffer-size = 0. + // The command is NOT queued and will NOT be retried: a deploy fails at the coordinator's + // apply deadline naming the silent nodes, which is a better operator signal than a + // command arriving minutes late against a config the DB already recorded as failed. + _log.Warning( + "No mesh ClusterClient — dropping {MessageType}. The last contact refresh produced " + + "no usable contact point from the enabled, non-maintenance ClusterNode rows; the " + + "command is not buffered and will not be retried", + cmd.Message.GetType().Name); + return; + } + + // SendToAll, NEVER Send. Today's DPS publish reaches EVERY DriverHostActor — there is no + // ClusterId or node filter on the node side at all; scoping happens later, inside the + // artifact (DeploymentArtifact.ResolveClusterScope). ClusterClient.Send delivers to exactly + // ONE registered actor, which would deploy to a single node of the fleet while every other + // node silently kept its old configuration — and the deployment would still seal green if + // the ack set happened to be satisfied. + _client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, cmd.Message)); + } + + private void HandleApplyAck(ApplyAck ack) + { + var coordinator = _coordinator(); + if (coordinator is null) + { + _log.Warning( + "Received ApplyAck for {DeploymentId} from {NodeId} but the deploy coordinator is " + + "not resolvable on this node; the ack is lost and the deployment will time out " + + "naming a node that applied successfully", + ack.DeploymentId, ack.NodeId); + return; + } + + // Forward, not Tell: preserves the original sender across the hop so the coordinator sees + // the node rather than this relay. + coordinator.Forward(ack); + } + + private void LoadContactsFromDb() + { + var self = Self; + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + + var systemName = Context.System.Name; + var dbFactory = _dbFactory; + + // Off the actor thread, result piped back as a message. A synchronous DB read here would + // block every command dispatch behind a slow or unreachable SQL Server. + Task.Run(async () => + { + await using var db = await dbFactory.CreateDbContextAsync(ct).ConfigureAwait(false); + var rows = await db.ClusterNodes + .AsNoTracking() + .Where(n => n.Enabled && !n.MaintenanceMode) + .Select(n => new { n.NodeId, n.Host, n.AkkaPort }) + .ToListAsync(ct) + .ConfigureAwait(false); + + var contacts = new List(rows.Count); + var malformed = new List(); + foreach (var row in rows) + { + var address = $"akka.tcp://{systemName}@{row.Host}:{row.AkkaPort}/system/receptionist"; + + // Parse up front, per row, inside the loop's own guard. A single malformed row must + // not abort the whole refresh and leave the contact set half-built — the sister + // project shipped that bug and its regression test deliberately orders the bad row + // first. + if (ActorPath.TryParse(address, out _)) contacts.Add(address); + else malformed.Add($"{row.NodeId} -> {address}"); + } + + return new ContactsLoaded(contacts, malformed); + }, ct).PipeTo(self); + } + + private void HandleContactsLoaded(ContactsLoaded msg) + { + foreach (var bad in msg.Malformed) + { + _log.Warning( + "ClusterNode {Row} does not yield a usable receptionist address; skipping it in this " + + "refresh (other nodes are unaffected). Check the row's Host and AkkaPort", bad); + } + + var contacts = msg.Contacts.ToImmutableHashSet(); + + if (contacts.IsEmpty) + { + _log.Warning( + "No usable ClusterNode contact points; the mesh ClusterClient was not created and " + + "every central→node command will be dropped until a refresh finds one"); + return; + } + + if (_client is not null && _currentContacts.SetEquals(contacts)) return; + + RebuildClient(contacts); + } + + /// Stops any existing client and creates a replacement for the new contact set. + /// + /// + /// TODO(Phase 6): one client per application Cluster. Today this actor + /// creates exactly one client whose contacts are the whole fleet, and that is not + /// a compromise — it is the only correct shape while the fleet is a single Akka mesh. + /// + /// + /// A ClusterClientReceptionist serves its entire cluster, so + /// SendToAll reaches every registered node-comm actor in the mesh regardless of + /// which node's address was used as the contact point. One client per Cluster would + /// therefore fan each command out once per cluster — N× duplicate + /// DispatchDeployment and N× duplicate ApplyAck. Phase 6 splits the meshes, + /// at which point per-cluster clients become both correct and necessary. + /// + /// + /// Corollary worth stating because it is easy to assume otherwise: the contact set + /// does not scope delivery. Excluding a maintenance-mode node from the contacts does + /// not stop it receiving commands — it still receives them, exactly as it does under + /// today's DPS broadcast. The filter is about which receptionists are worth dialling + /// (a node in maintenance may be switched off), not about who gets the message. + /// + /// + /// The receptionist addresses to dial. + private void RebuildClient(ImmutableHashSet contacts) + { + if (_client is not null) + { + _log.Info("Mesh contact set changed; replacing the ClusterClient"); + Context.Stop(_client); + + // Cleared now: if the create below throws, a stale ref would route commands into a + // stopping actor and they would vanish without a warning. + _client = null; + _currentContacts = ImmutableHashSet.Empty; + } + + try + { + var paths = contacts.Select(ActorPath.Parse).ToImmutableHashSet(); + _client = _clientFactory.Create(Context.System, paths); + _currentContacts = contacts; + _log.Info("Mesh ClusterClient created with {Count} contact point(s)", contacts.Count); + } + catch (Exception ex) + { + _log.Error(ex, + "Failed to create the mesh ClusterClient; central→node commands are dropped until " + + "the next contact refresh succeeds"); + } + } + + /// Internal tick asking for a contact-point refresh. + public sealed record RefreshContacts; + + /// Result of a contact-point load, piped back from the DB read. + /// Receptionist addresses that parsed cleanly. + /// Rows that did not, described for the log. + public sealed record ContactsLoaded( + IReadOnlyList Contacts, + IReadOnlyList Malformed); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshClusterClientFactory.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshClusterClientFactory.cs new file mode 100644 index 00000000..5ea57434 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshClusterClientFactory.cs @@ -0,0 +1,47 @@ +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +/// +/// Creates the ClusterClient central uses to reach driver nodes. Exists as a seam so +/// CentralCommunicationActor can be tested against a TestProbe without standing up +/// a second cluster. +/// +public interface IMeshClusterClientFactory +{ + /// Creates a client for the given receptionist contact points. + /// The actor system to spawn the client in. + /// Receptionist actor paths, one per reachable node. + /// The new client's . + IActorRef Create(ActorSystem system, ImmutableHashSet contacts); +} + +/// +public sealed class DefaultMeshClusterClientFactory : IMeshClusterClientFactory +{ + /// + /// Per-incarnation generation counter, appended to the actor name. + /// + /// + /// Context.Stop of the previous client is asynchronous — its actor name stays + /// reserved until termination completes — so recreating the same name while handling a single + /// message throws . That is exactly what a contact-set + /// change does: stop the old client, create the replacement, both in one handler. A + /// generation suffix makes every incarnation's name unique by construction. Ported from the + /// sister project, where it was a shipped bug fix rather than foresight. + /// + private long _generation; + + /// + public IActorRef Create(ActorSystem system, ImmutableHashSet contacts) + { + ArgumentNullException.ThrowIfNull(system); + ArgumentNullException.ThrowIfNull(contacts); + + var settings = ClusterClientSettings.Create(system).WithInitialContacts(contacts); + var name = $"mesh-client-{Interlocked.Increment(ref _generation)}"; + return system.ActorOf(ClusterClient.Props(settings), name); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index d955dc58..86787656 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -3,6 +3,7 @@ using Akka.Cluster.Tools.PublishSubscribe; using Akka.Event; using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; @@ -60,6 +61,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private readonly IDbContextFactory _dbFactory; private readonly TimeSpan _applyDeadline; + + /// + /// Where central→node commands are handed off (per-cluster mesh Phase 2), or + /// to publish on the mediator directly. + /// + /// + /// Null in tests and on any node wired before Phase 2, where the fallback reproduces the + /// pre-Phase-2 behaviour exactly. The fallback is not a permanent seam — Phase 6 deletes it + /// along with the DistributedPubSub branch and the single mesh. + /// + private readonly IActorRef? _meshRouter; private readonly ILoggingAdapter _log = Context.GetLogger(); // NodeId equality here is case-insensitive (by Value) to match the case-insensitive ClusterId/ // NodeId scoping in DeploymentArtifact.ResolveClusterScope — so an ack from a node whose address @@ -78,22 +90,31 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// The database context factory for accessing configuration data. /// The timeout for waiting for apply acknowledgments from cluster nodes. /// The Akka.NET used to spawn this actor. + /// + /// Central comm actor the dispatch is handed to, or to publish on the + /// DistributedPubSub mediator directly (the pre-Phase-2 behaviour). + /// public static Props Props( IDbContextFactory dbFactory, - TimeSpan? applyDeadline = null) => - Akka.Actor.Props.Create(() => new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline)); + TimeSpan? applyDeadline = null, + IActorRef? meshRouter = null) => + Akka.Actor.Props.Create(() => + new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter)); /// /// Initializes a new instance of the class. /// /// The database context factory for persisting deployment state. /// The timeout for waiting for per-node apply acknowledgments. + /// Central comm actor, or for the mediator. public ConfigPublishCoordinator( IDbContextFactory dbFactory, - TimeSpan applyDeadline) + TimeSpan applyDeadline, + IActorRef? meshRouter = null) { _dbFactory = dbFactory; _applyDeadline = applyDeadline; + _meshRouter = meshRouter; Receive(HandleDispatch); Receive(HandleAck); Receive(HandleDeadline); @@ -172,7 +193,10 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers db.SaveChanges(); } - DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg)); + // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing + // directly, so this actor no longer knows which transport is in force. + if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg)); + else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg)); Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline); if (_expectedAcks.Count == 0) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 5fba90b4..86e1561b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -1,12 +1,17 @@ using Akka.Actor; using Akka.Cluster.Hosting; +using Akka.Cluster.Tools.Client; using Akka.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; @@ -48,12 +53,49 @@ public static class ServiceCollectionExtensions var singletonOptions = new ClusterSingletonOptions { Role = AdminRole }; var proxyOptions = new ClusterSingletonOptions { Role = AdminRole }; + // Per-cluster mesh Phase 2: the central end of the central↔node command boundary. + // + // A plain WithActors, NOT a singleton — and that is the load-bearing part. A driver node's + // ClusterClient rotates across contact points and must find a live comm actor at whichever + // central node answers. As a singleton it would exist on one central node only, and the + // rotation would fail silently against the other: acks would land sometimes, depending on + // which contact the client happened to settle on. + builder.WithActors((system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + var options = resolver.GetService>().Value; + + var actor = system.ActorOf( + CentralCommunicationActor.Props( + dbFactory, + options, + new DefaultMeshClusterClientFactory(), + // Resolved lazily, per message, so this actor never depends on the order in + // which Akka.Hosting materialises the singleton proxy relative to this block. + () => registry.TryGet(out var coordinator) + ? coordinator + : null), + MeshPaths.CentralCommunicationName); + + // Registered unconditionally, in both transport modes. Under Dps nothing dials it, and + // an idle registration costs nothing — whereas registering only under ClusterClient + // would mean flipping the flag on a running fleet requires a central restart before any + // node can ack, turning a config change back into a deployment. + ClusterClientReceptionist.Get(system).RegisterService(actor); + registry.Register(actor); + }); + builder.WithSingleton( ConfigPublishSingletonName, (system, registry, resolver) => { var dbFactory = resolver.GetService>(); - return ConfigPublishCoordinator.Props(dbFactory); + // Registered above, so it is already in the registry — the same + // ordering-by-registration the AdminOperations singleton relies on to resolve the + // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks + // inside a props factory and races the spawn it is waiting for. + var meshRouter = registry.Get(); + return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); }, singletonOptions); @@ -69,7 +111,8 @@ public static class ServiceCollectionExtensions var mode = Enum.TryParse( resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; - return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode); + var meshRouter = registry.Get(); + return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); }, singletonOptions); @@ -123,3 +166,4 @@ public sealed class AuditWriterActorKey { } public sealed class FleetStatusBroadcasterKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } +public sealed class CentralCommunicationActorKey { } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Communication/NodeCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Communication/NodeCommunicationActor.cs new file mode 100644 index 00000000..fe192b2d --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Communication/NodeCommunicationActor.cs @@ -0,0 +1,94 @@ +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Event; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Communication; + +/// +/// Node-side end of the central↔node command boundary (per-cluster mesh Phase 2), registered +/// with the ClusterClientReceptionist at . +/// +/// +/// +/// Registered per node, NOT as a cluster singleton — central's ClusterClient rotates +/// across contact points and must find a live comm actor at whichever node answers. +/// +/// +/// Inbound commands are re-emitted on the node's , not on their +/// DistributedPubSub topic. DPS is mesh-wide, and central SendToAlls to every +/// node's comm actor — so a DPS re-publish would deliver N copies of every command to every +/// subscriber, where N is the node count. The EventStream is node-local by construction, +/// which is exactly the needed semantics. It also avoids a registration handshake with +/// actors spawned later: ScriptedAlarmHostActor is a child of +/// DriverHostActor and has no registry key, so there is no ref to hand in at wiring +/// time. +/// +/// +/// Outbound is only. There is no Ask across this boundary in +/// Phase 2 — every migrated command is fire-and-forget, and the deploy ack returns as an +/// independent message the coordinator already subscribes for. Nothing here needs sender +/// preservation, which is why this actor is far smaller than the sister project's. +/// +/// +public sealed class NodeCommunicationActor : ReceiveActor +{ + private readonly IActorRef? _centralClient; + private readonly ILoggingAdapter _log = Context.GetLogger(); + + /// Creates the props for this actor. + /// + /// ClusterClient dialling central's receptionist, or under + /// MeshTransport:Mode = Dps, where acks still go out over DistributedPubSub from + /// DriverHostActor and this actor never sees them. + /// + /// The props. + public static Props Props(IActorRef? centralClient) => + Akka.Actor.Props.Create(() => new NodeCommunicationActor(centralClient)); + + /// Initializes a new instance of the class. + /// ClusterClient dialling central, or . + public NodeCommunicationActor(IActorRef? centralClient) + { + _centralClient = centralClient; + + // Inbound from central. Each type is listed explicitly rather than caught by a ReceiveAny: + // an unknown type must dead-letter loudly during development rather than be republished + // blind onto the node's EventStream, where it would be silently ignored by every subscriber. + Receive(PublishLocally); + Receive(PublishLocally); + Receive(PublishLocally); + Receive(PublishLocally); + + // Outbound to central. + Receive(HandleApplyAck); + } + + private void PublishLocally(object msg) + { + _log.Debug("Mesh: received {MessageType} from central; publishing node-locally", msg.GetType().Name); + Context.System.EventStream.Publish(msg); + } + + private void HandleApplyAck(ApplyAck ack) + { + if (_centralClient is null) + { + _log.Warning( + "No central ClusterClient — dropping ApplyAck for {DeploymentId}. The ack is not " + + "buffered and will not be retried, so the deployment will time out at central " + + "naming this node even though it applied successfully. Check " + + "MeshTransport:CentralContactPoints", + ack.DeploymentId); + return; + } + + // Send, not SendToAll: the deploy coordinator is a single singleton, and the central comm + // actor forwards to its proxy. SendToAll would deliver one ack per central node, and the + // coordinator would count the same node twice. + _centralClient.Tell(new ClusterClient.Send(MeshPaths.CentralCommunication, ack)); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 5c13c198..7a842744 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -86,7 +86,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers private readonly IDbContextFactory _dbFactory; private readonly CommonsNodeId _localNode; - private readonly IActorRef? _coordinatorOverride; + private readonly IActorRef? _ackRouter; private readonly IDriverFactory _driverFactory; /// Builds the per-driver-instance attached to each @@ -328,7 +328,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// Creates props for a DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. /// The local cluster node identifier. - /// Optional coordinator actor reference for deployment coordination. + /// Where ApplyAcks are sent. A direct coordinator handle in tests; under + /// MeshTransport:Mode = ClusterClient the node's NodeCommunicationActor, which relays + /// across the boundary. Null publishes on the deployment-acks DPS topic. /// Optional driver factory; defaults to null factory if not provided. /// Optional set of roles assigned to the local node. /// Optional actor reference for dependency multiplexing. @@ -364,7 +366,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers public static Props Props( IDbContextFactory dbFactory, CommonsNodeId localNode, - IActorRef? coordinator = null, + IActorRef? ackRouter = null, IDriverFactory? driverFactory = null, IReadOnlySet? localRoles = null, IActorRef? dependencyMux = null, @@ -387,7 +389,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers // the wrong dependency at runtime. New parameters go LAST in all three places — this // signature, the constructor's, and this call — and nothing else moves. Akka.Actor.Props.Create(() => new DriverHostActor( - dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor, + dbFactory, localNode, ackRouter, driverFactory, localRoles, dependencyMux, opcUaPublishActor, healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride, loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider, deploymentArtifactCache, redundancyRoleView, replicationPeerHost)); @@ -395,7 +397,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. /// The local cluster node identifier. - /// Optional coordinator actor reference for deployment coordination. + /// Where ApplyAcks are sent. A direct coordinator handle in tests; under + /// MeshTransport:Mode = ClusterClient the node's NodeCommunicationActor, which relays + /// across the boundary. Null publishes on the deployment-acks DPS topic. /// Optional driver factory; defaults to null factory if not provided. /// Optional set of roles assigned to the local node. /// Optional actor reference for dependency multiplexing. @@ -426,7 +430,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers public DriverHostActor( IDbContextFactory dbFactory, CommonsNodeId localNode, - IActorRef? coordinator, + IActorRef? ackRouter, IDriverFactory? driverFactory = null, IReadOnlySet? localRoles = null, IActorRef? dependencyMux = null, @@ -449,7 +453,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers _replicationPeerHost = replicationPeerHost; _dbFactory = dbFactory; _localNode = localNode; - _coordinatorOverride = coordinator; + _ackRouter = ackRouter; _driverFactory = driverFactory ?? NullDriverFactory.Instance; _invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance; _driverMemberCountProvider = driverMemberCountProvider; @@ -483,6 +487,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers // uses so only the Primary services operator writes (the secondary keeps state warm for failover). _mediator.Tell( new Subscribe(ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RedundancyStateTopic, Self)); + + // Per-cluster mesh Phase 2 dark switch. Under MeshTransport:Mode = ClusterClient these same + // commands arrive from NodeCommunicationActor on the node-local EventStream instead of the + // mesh-wide DPS topic. Subscribing to BOTH unconditionally is deliberate: exactly one of the + // two ever publishes, so the transport flag lives entirely on the publishing side and the + // switch can be flipped (or reverted) without touching any subscriber. + Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment)); + Context.System.EventStream.Subscribe(Self, typeof(RestartDriver)); + Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver)); + // Spawn the VirtualTag host BEFORE Bootstrap so the bootstrap-restore path (which routes // through PushDesiredSubscriptions and Tells ApplyVirtualTags) has a live host to target. SpawnVirtualTagHost(); @@ -2410,15 +2424,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers private void SendAck(DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation) { var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation); - if (_coordinatorOverride is not null) + if (_ackRouter is not null) { - _coordinatorOverride.Tell(ack); + // Either a direct coordinator handle (tests) or, under + // MeshTransport:Mode = ClusterClient, the NodeCommunicationActor that relays this + // across the boundary. Renamed from _coordinatorOverride in per-cluster mesh Phase 2: + // under ClusterClient mode this ref is emphatically NOT a coordinator, and the old + // name would send the next reader looking for one. + _ackRouter.Tell(ack); } else { - // No direct coordinator handle — publish on the dedicated ACK topic. The coordinator - // singleton subscribes there in PreStart so the ACK reaches whichever admin node hosts - // it without an actor-path lookup. + // No router — publish on the dedicated ACK topic. The coordinator singleton subscribes + // there in PreStart so the ACK reaches whichever admin node hosts it without an + // actor-path lookup. DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentAcksTopic, ack)); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index 5c465063..7b538c66 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -528,6 +528,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // RedundancyStateChanged and cache this node's role — OnEngineEmission gates the cluster-wide // alerts publish to the Primary so a 2-node single-cluster deploy doesn't double-emit transitions. _mediator.Tell(new Subscribe(OpcUaPublishActor.RedundancyStateTopic, Self)); + + // Per-cluster mesh Phase 2 dark switch. An AdminUI-originated ack/shelve arrives from + // NodeCommunicationActor on the node-local EventStream under + // MeshTransport:Mode = ClusterClient; the DPS subscribe above still carries the node-LOCAL + // publisher (OtOpcUaServerHostedService, the OPC UA Part 9 method calls), which never + // crosses the boundary and stays on DPS in both modes. Subscribing to both is safe: exactly + // one publisher feeds each path. + Context.System.EventStream.Subscribe(Self, typeof(AlarmCommand)); + base.PreStart(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 9cdbeddb..771e9ae0 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -1,4 +1,6 @@ +using System.Collections.Immutable; using Akka.Actor; +using Akka.Cluster.Tools.Client; using Akka.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -6,6 +8,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Runtime.Communication; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; @@ -37,6 +43,7 @@ public static class ServiceCollectionExtensions public const string OpcUaPublishActorName = "opcua-publish"; public const string PeerProbeSupervisorName = "peer-probe-supervisor"; public const string ContinuousHistorizationRecorderActorName = "continuous-historization-recorder"; + public const string CentralClusterClientName = "central-cluster-client"; /// /// Registers shared runtime services. Currently binds @@ -319,6 +326,40 @@ public static class ServiceCollectionExtensions var mux = system.ActorOf(DependencyMuxActor.Props(), DependencyMuxActorName); registry.Register(mux); + // Per-cluster mesh Phase 2: the node end of the central↔node command boundary. Spawned + // BEFORE DriverHostActor because it is the host's ackRouter under ClusterClient mode. + // It has no dependency of its own on the host — inbound commands travel via the node's + // EventStream — so this ordering costs nothing. + var meshOptions = resolver.GetService>().Value; + var useClusterClient = string.Equals( + meshOptions.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); + + IActorRef? centralClient = null; + if (useClusterClient) + { + // Contacts come from appsettings, not the database — the deliberate asymmetry: + // central's node set changes as operators add and retire nodes, but central's own + // address is part of the deployment. MeshTransportOptionsValidator has already + // rejected an empty or malformed list at boot, so this cannot silently produce a + // client with no contacts. + var contacts = meshOptions.CentralContactPoints + .Select(cp => ActorPath.Parse($"{cp}/system/receptionist")) + .ToImmutableHashSet(); + centralClient = system.ActorOf( + ClusterClient.Props(ClusterClientSettings.Create(system).WithInitialContacts(contacts)), + CentralClusterClientName); + } + + var nodeComm = system.ActorOf( + NodeCommunicationActor.Props(centralClient), + MeshPaths.NodeCommunicationName); + + // Registered with the receptionist in BOTH modes: under Dps nothing dials it, and an + // idle registration costs nothing — whereas registering only under ClusterClient would + // make flipping the flag require a node restart before central could reach it. + ClusterClientReceptionist.Get(system).RegisterService(nodeComm); + registry.Register(nodeComm); + // Continuous-historization recorder — gated on ContinuousHistorization:Enabled AND the // gateway-backed IHistorianValueWriter + the durable IHistorizationOutbox being registered // (the Host registers both ONLY when historization is enabled and the ServerHistorian gateway @@ -398,7 +439,11 @@ public static class ServiceCollectionExtensions registry.Register(peerProbes); var driverHost = system.ActorOf( - DriverHostActor.Props(dbFactory, roleInfo.LocalNode, coordinator: null, + // ackRouter: under ClusterClient mode the node comm actor relays ApplyAcks across + // the boundary; under Dps it stays null and the host publishes on the + // deployment-acks topic exactly as before. + DriverHostActor.Props(dbFactory, roleInfo.LocalNode, + ackRouter: useClusterClient ? nodeComm : null, driverFactory: driverFactory, localRoles: roleInfo.LocalRoles, dependencyMux: mux, opcUaPublishActor: publishActor, @@ -430,6 +475,7 @@ public sealed class DbHealthProbeActorKey { } public sealed class HistorianAdapterActorKey { } public sealed class DependencyMuxActorKey { } public sealed class OpcUaPublishActorKey { } +public sealed class NodeCommunicationActorKey { } /// Marker key for the per-node ContinuousHistorizationRecorder (spawned only when /// ContinuousHistorization:Enabled=true and the gateway value-writer + outbox are registered). diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs new file mode 100644 index 00000000..1960d0c3 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs @@ -0,0 +1,100 @@ +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Configuration; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Asserts the ClusterClient / receptionist / frame-size settings that are actually in +/// force on a real built from the shipped base config. +/// +/// +/// Asserting the akka.conf text instead would pass while a competing HOCON fragment +/// silently overrode the value — the failure mode recorded at length in +/// ServiceCollectionExtensions.BuildDowningHocon and pinned the same way by +/// SplitBrainResolverActivationTests. In this codebase the effective value is the only +/// one worth asserting. +/// +public class MeshTransportHoconTests : IDisposable +{ + private readonly ActorSystem _sys; + + public MeshTransportHoconTests() + { + var config = ConfigurationFactory + .ParseString("akka.remote.dot-netty.tcp.port = 0\nakka.cluster.seed-nodes = []") + .WithFallback(HoconLoader.LoadBaseConfig()); + _sys = ActorSystem.Create("mesh-hocon-probe", config); + } + + [Fact] + public void Cluster_client_buffering_is_disabled() + { + // Asserted through ClusterClientSettings — the object the client is actually built from — + // NOT through Config.GetInt. GetInt returns 0 for a MISSING key, so a config assertion here + // passes identically whether the setting is explicitly 0 or absent with Akka's default of + // 1000 in force. It was green before this feature existed; that is the whole defect class. + // + // buffer-size = 0 means "drop now if the receptionist is unknown". Buffering would replay a + // DispatchDeployment on reconnect and apply a deployment the coordinator already sealed as + // TimedOut, leaving the node running a configuration the database says failed. This is a + // behaviour decision, not a tuning knob. + ClusterClientSettings.Create(_sys).BufferSize.ShouldBe(0); + } + + [Fact] + public void Cluster_client_retries_forever_rather_than_self_stopping() + { + // reconnect-timeout = off surfaces as a null TimeSpan on the settings object. A central node + // down for an hour must not require a driver-node restart to become reachable again. + ClusterClientSettings.Create(_sys).ReconnectTimeout.ShouldBeNull(); + } + + [Fact] + public void Receptionist_serves_every_member_regardless_of_role() + { + // An empty role is what makes "registered per node, not as a singleton" work: every member + // hosts a receptionist, so ClusterClient contact rotation reaches whichever node answers. + // Scoping this to a role would make the boundary reachable through one node only. + ClusterReceptionistSettings.Create(_sys).Role.ShouldBeNull(); + } + + [Fact] + public void Oversized_frames_are_logged_rather_than_dropped_silently() + { + // The sister project's most expensive incident. A message over the frame limit is dropped + // WITHOUT tearing down the association: heartbeats keep flowing, the node still reports + // healthy, and the only symptom is a command that never arrived. log-frame-size-exceeding + // converts that silence into a log line naming the offending size. + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.maximum-frame-size").ShouldBe(128000); + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.log-frame-size-exceeding").ShouldBe(32000); + } + + [Fact] + public void Receptionist_extension_is_registered_at_startup() + { + // Not auto-started. Without this entry the receptionist materialises only if some code path + // happens to call ClusterClientReceptionist.Get(system) first, which makes "is the boundary + // listening?" depend on startup ordering rather than on configuration. + _sys.Settings.Config.GetStringList("akka.extensions") + .ShouldContain(s => s.Contains("ClusterClientReceptionistExtensionProvider")); + } + + [Fact] + public void Distributed_pub_sub_extension_survives_alongside_the_receptionist() + { + // The dark switch keeps both transports live for one phase, and DPS additionally carries + // traffic Phase 2 does not touch at all — node-local alarm-command publishes and the + // node<->node redundancy-state probe leg. Losing it here would break far more than deploys. + _sys.Settings.Config.GetStringList("akka.extensions") + .ShouldContain(s => s.Contains("DistributedPubSubExtensionProvider")); + } + + public void Dispose() + { + _sys.Terminate().Wait(TimeSpan.FromSeconds(5)); + GC.SuppressFinalize(this); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs new file mode 100644 index 00000000..af627a9b --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportOptionsValidatorTests.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Every failure this validator catches otherwise surfaces as an absence — a deployment +/// that never arrives, an alarm ack that does nothing, a node whose ack is dropped without a +/// trace. Absences are the hardest class of fault to attribute in the field, so the whole point +/// of these tests is that the host refuses to start instead. +/// +public class MeshTransportOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(MeshTransportOptions o) => + new MeshTransportOptionsValidator().Validate(MeshTransportOptions.SectionName, o); + + [Fact] + public void Default_options_are_valid() + { + // The default is Dps — the pre-Phase-2 transport. A node that never sets this section must + // behave exactly as it did before, which is the whole premise of the dark switch. + Validate(new MeshTransportOptions()).Succeeded.ShouldBeTrue(); + } + + [Fact] + public void Unknown_mode_fails() + { + var result = Validate(new MeshTransportOptions { Mode = "grpc" }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("grpc"); + } + + [Fact] + public void Mode_matching_is_case_insensitive() + { + Validate(new MeshTransportOptions + { + Mode = "clusterclient", + CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], + }).Succeeded.ShouldBeTrue(); + } + + [Fact] + public void Non_positive_contact_refresh_fails() + { + var result = Validate(new MeshTransportOptions { ContactRefreshSeconds = 0 }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.ContactRefreshSeconds)); + } + + [Fact] + public void ClusterClient_mode_requires_central_contact_points() + { + var result = Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = [], + }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints)); + } + + [Fact] + public void Dps_mode_does_not_require_central_contact_points() + { + // An admin-only node has no reason to carry them, and requiring them under the default mode + // would make the section mandatory everywhere for a feature that is switched off. + Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeDps, + CentralContactPoints = [], + }).Succeeded.ShouldBeTrue(); + } + + [Fact] + public void A_contact_point_that_is_not_an_akka_address_fails() + { + var result = Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = ["central-1:4053"], + }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("akka.tcp://"); + } + + [Fact] + public void A_contact_point_carrying_an_actor_path_suffix_fails() + { + // The sister project's shipped site template listed a full receptionist path, and separately + // listed the site's OWN remoting port as the second contact — "a permanent failure in the + // initial-contact rotation". Both are invisible until a command goes missing. + var result = Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"], + }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("/system/receptionist"); + } + + [Fact] + public void One_bad_contact_point_among_good_ones_still_fails() + { + var result = Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = + [ + "akka.tcp://otopcua@central-1:4053", + "central-2:4053", + ], + }); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("central-2:4053"); + } + + [Fact] + public void Well_formed_ClusterClient_options_succeed() + { + Validate(new MeshTransportOptions + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = + [ + "akka.tcp://otopcua@central-1:4053", + "akka.tcp://otopcua@central-2:4053", + ], + }).Succeeded.ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs new file mode 100644 index 00000000..d2d0fc7a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -0,0 +1,271 @@ +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Cluster.Tools.PublishSubscribe; +using Akka.TestKit; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +public class CentralCommunicationActorTests : ControlPlaneActorTestBase +{ + private const string DeploymentsTopic = "deployments"; + + /// Records every client the actor asks for, and hands back a probe in its place. + private sealed class RecordingClientFactory : IMeshClusterClientFactory + { + private readonly Func _newProbe; + + public RecordingClientFactory(Func newProbe) => _newProbe = newProbe; + + public List> Calls { get; } = []; + + public List Probes { get; } = []; + + public IActorRef Create(ActorSystem system, ImmutableHashSet contacts) + { + Calls.Add(contacts); + var probe = _newProbe(); + Probes.Add(probe); + return probe.Ref; + } + } + + private static MeshTransportOptions ClusterClientMode(int refreshSeconds = 60) => new() + { + Mode = MeshTransportOptions.ModeClusterClient, + CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"], + ContactRefreshSeconds = refreshSeconds, + }; + + private static void SeedCluster( + IDbContextFactory factory, + string clusterId, + params (string NodeId, string Host, int AkkaPort, bool Enabled, bool Maintenance)[] nodes) + { + using var db = factory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, + Name = clusterId, + Enterprise = "ZB", + Site = "Site", + // Warm/Hot require NodeCount = 2 (CK_ServerCluster_RedundancyMode_NodeCount); these + // fixtures only need the row to exist, so use the standalone shape. + RedundancyMode = RedundancyMode.None, + NodeCount = 1, + CreatedBy = "test", + }); + foreach (var n in nodes) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = n.NodeId, + ClusterId = clusterId, + Host = n.Host, + AkkaPort = n.AkkaPort, + ApplicationUri = $"urn:{n.NodeId}", + Enabled = n.Enabled, + MaintenanceMode = n.Maintenance, + CreatedBy = "test", + }); + } + + db.SaveChanges(); + } + + private IActorRef Spawn( + IDbContextFactory factory, + MeshTransportOptions options, + IMeshClusterClientFactory clientFactory, + IActorRef? coordinator = null) => + Sys.ActorOf(CentralCommunicationActor.Props(factory, options, clientFactory, () => coordinator)); + + [Fact] + public void Dps_mode_publishes_on_the_carried_topic_and_creates_no_client() + { + var factory = NewInMemoryDbFactory(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var mediatorProbe = CreateTestProbe(); + // Subscribe on the probe's behalf: SubscribeAck goes to the SENDER, so telling the mediator + // without an explicit sender would ack to TestActor and this wait would hang. + DistributedPubSub.Get(Sys).Mediator.Tell( + new Subscribe(DeploymentsTopic, mediatorProbe.Ref), mediatorProbe.Ref); + mediatorProbe.ExpectMsg(TimeSpan.FromSeconds(5)); + + var actor = Spawn(factory, new MeshTransportOptions(), clients); + var dispatch = new DispatchDeployment( + new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid())); + + actor.Tell(new MeshCommand(DeploymentsTopic, dispatch)); + + mediatorProbe.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(dispatch); + clients.Calls.ShouldBeEmpty(); + } + + [Fact] + public void ClusterClient_mode_sends_to_all_not_to_one() + { + // THE assertion of this phase. ClusterClient.Send delivers to exactly ONE registered actor; + // today's DPS publish reaches EVERY DriverHostActor, and the node side has no ClusterId or + // node filter to compensate. Send would deploy to a single node while every other node + // silently kept its old configuration. + var factory = NewInMemoryDbFactory(); + SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false)); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + + var actor = Spawn(factory, ClusterClientMode(), clients); + AwaitCondition(() => clients.Probes.Count == 1, TimeSpan.FromSeconds(5)); + + var dispatch = new DispatchDeployment( + new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid())); + actor.Tell(new MeshCommand(DeploymentsTopic, dispatch)); + + var sent = clients.Probes[0].ExpectMsg(TimeSpan.FromSeconds(5)); + sent.Path.ShouldBe(MeshPaths.NodeCommunication); + sent.Message.ShouldBe(dispatch); + } + + [Fact] + public void Exactly_one_client_is_created_for_a_multi_cluster_fleet() + { + // A receptionist serves its whole mesh, so one client per Cluster would fan each command out + // once per cluster while the fleet is still a single mesh — N x duplicate deployments. + var factory = NewInMemoryDbFactory(); + SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false)); + SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false)); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + + Spawn(factory, ClusterClientMode(), clients); + AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5)); + + // ...and that one client dials BOTH clusters' nodes. + clients.Calls[0].Count.ShouldBe(2); + // Give a second refresh a chance to (wrongly) create another. + ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + clients.Calls.Count.ShouldBe(1); + } + + [Fact] + public void Disabled_and_maintenance_nodes_are_not_dialled() + { + var factory = NewInMemoryDbFactory(); + SeedCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4053, true, false), + ("host-b:4053", "host-b", 4053, false, false), + ("host-c:4053", "host-c", 4053, true, true)); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + + Spawn(factory, ClusterClientMode(), clients); + AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5)); + + clients.Calls[0].Count.ShouldBe(1); + clients.Calls[0].Single().ToString().ShouldContain("host-a"); + } + + [Fact] + public void A_malformed_node_row_does_not_abort_the_refresh() + { + // The bad row is ordered FIRST on purpose: the failure this guards against is an exception + // that aborts the loop and leaves the contact set half-built. + var factory = NewInMemoryDbFactory(); + SeedCluster( + factory, + "C1", + ("bad", "not a host!!", 4053, true, false), + ("host-a:4053", "host-a", 4053, true, false)); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + + Spawn(factory, ClusterClientMode(), clients); + AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5)); + + clients.Calls[0].ShouldContain(p => p.ToString().Contains("host-a")); + } + + [Fact] + public void A_command_with_no_client_is_dropped_with_a_warning() + { + // No ClusterNode rows at all: nothing to dial, so the command must be dropped loudly rather + // than queued. buffer-size = 0 covers the case where a client exists but cannot connect; + // this covers the case where there is no client at all. + var factory = NewInMemoryDbFactory(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var actor = Spawn(factory, ClusterClientMode(), clients); + + EventFilter.Warning(contains: "dropping").ExpectOne(TimeSpan.FromSeconds(5), () => + actor.Tell(new MeshCommand(DeploymentsTopic, "anything"))); + } + + [Fact] + public void ApplyAck_is_forwarded_to_the_coordinator_preserving_the_sender() + { + var factory = NewInMemoryDbFactory(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var coordinator = CreateTestProbe(); + var node = CreateTestProbe(); + var actor = Spawn(factory, ClusterClientMode(), clients, coordinator.Ref); + + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("host-a:4053"), + ApplyAckOutcome.Applied, + null, + new CorrelationId(Guid.NewGuid())); + + actor.Tell(ack, node.Ref); + + coordinator.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(ack); + // Forward, not Tell: the coordinator must see the node, not this relay. + coordinator.LastSender.ShouldBe(node.Ref); + } + + [Fact] + public void An_unresolvable_coordinator_logs_rather_than_losing_the_ack_silently() + { + var factory = NewInMemoryDbFactory(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var actor = Spawn(factory, ClusterClientMode(), clients, coordinator: null); + + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("host-a:4053"), + ApplyAckOutcome.Applied, + null, + new CorrelationId(Guid.NewGuid())); + + EventFilter.Warning(contains: "not resolvable").ExpectOne(TimeSpan.FromSeconds(5), () => + actor.Tell(ack)); + } + + [Fact] + public void A_faulted_contact_load_warns_and_leaves_the_actor_alive() + { + var factory = NewInMemoryDbFactory(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var actor = Spawn(factory, ClusterClientMode(), clients); + + // Filter on a phrase unique to the DB-failure warning. EventFilter matches case-INSENSITIVELY, + // so a looser "was NOT" also caught this actor's other warning ("the mesh ClusterClient was + // not created") and the assertion failed with two matches instead of one. + EventFilter.Warning(contains: "contact set was NOT refreshed").ExpectOne( + TimeSpan.FromSeconds(5), + () => actor.Tell(new Status.Failure(new InvalidOperationException("db down")))); + + // Still serving: a transient DB outage must not take the command boundary down with it. + actor.Tell(new MeshCommand(DeploymentsTopic, "anything")); + ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshClusterClientFactoryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshClusterClientFactoryTests.cs new file mode 100644 index 00000000..911142b7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshClusterClientFactoryTests.cs @@ -0,0 +1,67 @@ +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Configuration; +using Akka.TestKit.Xunit2; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// Guards the one non-obvious part of client creation: recreating a client with the same name +/// inside a single message handler throws, because Context.Stop is asynchronous and the +/// name stays reserved until termination completes. +/// +public class MeshClusterClientFactoryTests : TestKit +{ + private static readonly Config TestConfig = ConfigurationFactory.ParseString(@" + akka.actor.provider = cluster + akka.remote.dot-netty.tcp.port = 0 + akka.remote.dot-netty.tcp.hostname = 127.0.0.1") + .WithFallback(ClusterClientReceptionist.DefaultConfig()); + + public MeshClusterClientFactoryTests() + : base(TestConfig) + { + } + + // A real ClusterClient pointed at an unreachable contact is safe in TestKit — it simply retries + // the initial-contact rotation and never fails the test. + private static ImmutableHashSet UnreachableContact => ImmutableHashSet.Create( + ActorPath.Parse("akka.tcp://otopcua@127.0.0.1:65501/system/receptionist")); + + [Fact] + public void Recreating_a_client_in_one_pass_does_not_collide_on_the_actor_name() + { + var factory = new DefaultMeshClusterClientFactory(); + + var first = factory.Create(Sys, UnreachableContact); + Sys.Stop(first); + + // Stop is asynchronous: `first`'s name is still reserved right here. Without the generation + // suffix this throws InvalidActorNameException — which is precisely what a contact-set + // change would do in production, where stop-then-recreate happens in one handler. + var second = factory.Create(Sys, UnreachableContact); + + second.Path.Name.ShouldNotBe(first.Path.Name); + } + + [Fact] + public void Every_incarnation_gets_a_distinct_name() + { + var factory = new DefaultMeshClusterClientFactory(); + + var names = Enumerable.Range(0, 5) + .Select(_ => factory.Create(Sys, UnreachableContact).Path.Name) + .ToList(); + + names.Distinct().Count().ShouldBe(5); + } + + // Contact-set propagation is deliberately NOT asserted here: reading initial contacts back + // off a created ClusterClient is not possible, and asserting ClusterClientSettings directly + // would test Akka rather than this factory. It is covered where it can actually fail — the + // real two-ActorSystem boundary test, which only delivers if the contacts reached the client. +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshRouterDispatchTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshRouterDispatchTests.cs new file mode 100644 index 00000000..fdb17d6d --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshRouterDispatchTests.cs @@ -0,0 +1,134 @@ +using Akka.Actor; +using Akka.Cluster.Tools.PublishSubscribe; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// Proves the five central→node publish sites go through the transport router when one is +/// wired, and fall back to the mediator when it is not. +/// +/// +/// The fallback is what keeps the ~12 pre-existing coordinator and admin-operations tests +/// meaningful: they construct these actors without a router and must still exercise the real +/// DistributedPubSub path, not a silently-disabled one. +/// +public class MeshRouterDispatchTests : ControlPlaneActorTestBase +{ + private const string DeploymentsTopic = "deployments"; + + [Fact] + public void Deploy_dispatch_goes_to_the_router_when_one_is_wired() + { + var router = CreateTestProbe(); + var dbFactory = NewInMemoryDbFactory(); + var coordinator = Sys.ActorOf( + ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: router.Ref)); + + var deploymentId = new DeploymentId(Guid.NewGuid()); + coordinator.Tell(new Commons.Messages.Deploy.DispatchDeployment( + deploymentId, new RevisionHash("rev-1"), CorrelationId.NewId())); + + var cmd = router.ExpectMsg(TimeSpan.FromSeconds(5)); + cmd.Topic.ShouldBe(DeploymentsTopic); + cmd.Message.ShouldBeOfType(); + } + + [Fact] + public void Deploy_dispatch_falls_back_to_the_mediator_when_no_router_is_wired() + { + var subscriber = CreateTestProbe(); + DistributedPubSub.Get(Sys).Mediator.Tell( + new Subscribe(DeploymentsTopic, subscriber.Ref), subscriber.Ref); + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory())); + + var deploymentId = new DeploymentId(Guid.NewGuid()); + coordinator.Tell(new Commons.Messages.Deploy.DispatchDeployment( + deploymentId, new RevisionHash("rev-1"), CorrelationId.NewId())); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void Restart_driver_goes_to_the_router_on_the_driver_control_topic() + { + var router = CreateTestProbe(); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + NewInMemoryDbFactory(), + CreateTestProbe().Ref, + Enumerable.Empty(), + TagConfigValidationMode.Warn, + router.Ref)); + + admin.Tell(new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid())); + + var cmd = router.ExpectMsg(TimeSpan.FromSeconds(5)); + cmd.Topic.ShouldBe(DriverControlTopic.Name); + cmd.Message.ShouldBeOfType(); + } + + [Fact] + public void Reconnect_driver_goes_to_the_router_on_the_driver_control_topic() + { + var router = CreateTestProbe(); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + NewInMemoryDbFactory(), + CreateTestProbe().Ref, + Enumerable.Empty(), + TagConfigValidationMode.Warn, + router.Ref)); + + admin.Tell(new ReconnectDriver("C1", "driver-1", "alice", Guid.NewGuid())); + + var cmd = router.ExpectMsg(TimeSpan.FromSeconds(5)); + cmd.Topic.ShouldBe(DriverControlTopic.Name); + cmd.Message.ShouldBeOfType(); + } + + [Fact] + public void Alarm_acknowledge_goes_to_the_router_on_the_alarm_commands_topic() + { + var router = CreateTestProbe(); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + NewInMemoryDbFactory(), + CreateTestProbe().Ref, + Enumerable.Empty(), + TagConfigValidationMode.Warn, + router.Ref)); + + admin.Tell(new AcknowledgeAlarmCommand("alarm-1", "alice", "c", CorrelationId.NewId())); + + var cmd = router.ExpectMsg(TimeSpan.FromSeconds(5)); + cmd.Topic.ShouldBe(AlarmCommandsTopic.Name); + cmd.Message.ShouldBeOfType(); + } + + [Fact] + public void Alarm_shelve_goes_to_the_router_on_the_alarm_commands_topic() + { + var router = CreateTestProbe(); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + NewInMemoryDbFactory(), + CreateTestProbe().Ref, + Enumerable.Empty(), + TagConfigValidationMode.Warn, + router.Ref)); + + admin.Tell(new ShelveAlarmCommand("alarm-1", "alice", ShelveKind.OneShot, null, "c", CorrelationId.NewId())); + + var cmd = router.ExpectMsg(TimeSpan.FromSeconds(5)); + cmd.Topic.ShouldBe(AlarmCommandsTopic.Name); + cmd.Message.ShouldBeOfType(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshClusterClientBoundaryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshClusterClientBoundaryTests.cs new file mode 100644 index 00000000..18592757 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshClusterClientBoundaryTests.cs @@ -0,0 +1,302 @@ +using System.Collections.Immutable; +using Akka.Actor; +using Akka.Cluster; +using Akka.Cluster.Tools.Client; +using Akka.Configuration; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.OtOpcUa.Runtime.Communication; +using AkkaCluster = Akka.Cluster.Cluster; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// Drives a real ClusterClient across two genuinely separate s — +/// the only test in the repo where the mesh boundary carries a message over the wire. +/// +/// +/// +/// The sister project substitutes an in-process relay that unwraps +/// and forwards. That tests actor logic and cannot +/// fail for any transport reason: not a missing receptionist extension, not a service that +/// was never registered, not a malformed contact path, not a serialization break. Those are +/// precisely the Phase 2 failure modes, so they are what this test exercises. +/// +/// +/// Two clusters, one ActorSystem name, joined ONLY by ClusterClient. The shared name +/// is mandatory — Akka.Remote matches on the full address, so a contact path naming +/// otopcua cannot reach a system called anything else. Each system seeds itself and +/// neither lists the other, and +/// asserts the +/// separation outright: if they ever gossiped into one mesh, every delivery below could be +/// plain local messaging and the whole class would be testing nothing. +/// +/// +/// Scope limit. With exactly one node per side, "registered per node" and "registered +/// as a cluster singleton" are indistinguishable by construction, so this class does NOT +/// cover the per-node property whose structural assertions MeshCommActorPathTests +/// dropped. That property's discriminating coverage is live-gate step 8: stop the OLDEST +/// central and the survivor's comm actor must still receive a node ack, which a singleton +/// hosted on the stopped node cannot. +/// +/// +public sealed class MeshClusterClientBoundaryTests : IAsyncLifetime +{ + private const string SharedSystemName = "otopcua"; + private static readonly TimeSpan Arrival = TimeSpan.FromSeconds(30); + + /// + /// How long the falsifiability control waits before concluding nothing arrived. Deliberately + /// shorter than but far longer than a successful delivery takes in + /// practice, so a pass means "the boundary is shut", not "the clock ran out first". + /// + private static readonly TimeSpan Silence = TimeSpan.FromSeconds(10); + + /// + /// One factory for every client this class creates, because the generation counter that keeps + /// client actor names unique is per factory instance. A second factory on the same + /// system restarts at generation 1 and collides with the first factory's client — which is + /// exactly how this test first failed. Production has one instance per + /// CentralCommunicationActor, so the real code path is safe. + /// + private readonly DefaultMeshClusterClientFactory _clientFactory = new(); + + private ActorSystem _central = null!; + private ActorSystem _node = null!; + private IActorRef _nodeComm = null!; + private IActorRef _centralClient = null!; + private TaskCompletionSource _centralInbox = null!; + + /// + public async ValueTask InitializeAsync() + { + _central = StartSelfSeededSystem(withReceptionist: true); + _node = StartSelfSeededSystem(withReceptionist: true); + await Task.WhenAll(AwaitSingleMemberUpAsync(_central), AwaitSingleMemberUpAsync(_node)); + + // Central: the ack landing pad, at the path the node's comm actor sends to. + _centralInbox = NewInbox(); + var centralComm = _central.ActorOf( + CaptureActor.Props(_centralInbox), MeshPaths.CentralCommunicationName); + ClusterClientReceptionist.Get(_central).RegisterService(centralComm); + + // Node: the REAL comm actor, dialling central through the REAL production client factory. + var nodeToCentral = _clientFactory.Create(_node, ReceptionistOf(_central)); + _nodeComm = _node.ActorOf( + NodeCommunicationActor.Props(nodeToCentral), MeshPaths.NodeCommunicationName); + ClusterClientReceptionist.Get(_node).RegisterService(_nodeComm); + + _centralClient = _clientFactory.Create(_central, ReceptionistOf(_node)); + } + + /// + public async ValueTask DisposeAsync() + { + await ShutdownAsync(_central); + await ShutdownAsync(_node); + } + + [Fact] + public void The_two_systems_are_separate_clusters_joined_only_by_the_client() + { + // The load-bearing precondition of every other test here. Two systems sharing a name in one + // process are one gossip accident away from being one cluster, at which point ClusterClient + // is no longer the thing under test. + AkkaCluster.Get(_central).State.Members.Count.ShouldBe(1, + "central must be its own single-member cluster, or the boundary is not being crossed"); + AkkaCluster.Get(_node).State.Members.Count.ShouldBe(1, + "the node must be its own single-member cluster, or the boundary is not being crossed"); + } + + [Fact] + public async Task Central_send_to_all_reaches_the_node_comm_actor_and_lands_on_its_event_stream() + { + var inbox = NewInbox(); + _node.EventStream.Subscribe(_node.ActorOf(CaptureActor.Props(inbox)), typeof(RestartDriver)); + + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + var delivered = await SendUntilDeliveredAsync( + _centralClient, + () => new ClusterClient.SendToAll(MeshPaths.NodeCommunication, command), + inbox, + Arrival); + + delivered.ShouldNotBeNull( + "SendToAll must reach the node's registered comm actor and be re-emitted node-locally; " + + "this is the whole central→node leg of the mesh transport"); + delivered.ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1"); + } + + [Fact] + public async Task A_node_ack_reaches_central_through_the_node_comm_actors_own_client() + { + // Told directly rather than routed from central, so a failure here can only be the outbound + // leg. The inbound leg has its own test above. + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("node-a"), + ApplyAckOutcome.Applied, + null, + CorrelationId.NewId()); + + var delivered = await SendUntilDeliveredAsync( + _nodeComm, () => ack, _centralInbox, Arrival); + + delivered.ShouldNotBeNull( + "the node's ApplyAck must reach central's comm actor, or every deployment times out at " + + "the apply deadline naming nodes that applied successfully"); + delivered.ShouldBeOfType().NodeId.Value.ShouldBe("node-a"); + } + + [Fact] + public async Task Falsifiability_control_a_node_without_the_receptionist_extension_receives_nothing() + { + // Without this, the two tests above cannot distinguish "the ClusterClient boundary works" + // from "something else in the process delivered the message". Same message, same client + // factory, same paths, same comm actor — and it must NOT arrive. + // + // HONEST SCOPE: this removes the receptionist extension AND, unavoidably, the RegisterService + // call — resolving ClusterClientReceptionist.Get to register would materialise the very + // extension whose absence is the control. So it falsifies "delivery happens without a + // receptionist boundary", not the extension line in isolation. Verified falsifiable: giving + // this node the extension and the registration turns the test red, which also proves the send + // below is live rather than silently misaddressed. + var deaf = StartSelfSeededSystem(withReceptionist: false); + try + { + await AwaitSingleMemberUpAsync(deaf); + + var inbox = NewInbox(); + deaf.EventStream.Subscribe(deaf.ActorOf(CaptureActor.Props(inbox)), typeof(RestartDriver)); + + // Deliberately NOT ClusterClientReceptionist.Get(deaf) — resolving the extension would + // materialise the very receptionist whose absence is the control. + var comm = deaf.ActorOf(NodeCommunicationActor.Props(null), MeshPaths.NodeCommunicationName); + comm.ShouldNotBeNull(); + + var client = _clientFactory.Create(_central, ReceptionistOf(deaf)); + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + + var deadline = DateTime.UtcNow + Silence; + while (DateTime.UtcNow < deadline && !inbox.Task.IsCompleted) + { + client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, command)); + await Task.Delay(500, TestContext.Current.CancellationToken); + } + + inbox.Task.IsCompleted.ShouldBeFalse( + "delivery survived removing the receptionist extension, so the two positive tests " + + "above are not measuring the ClusterClient boundary at all"); + } + finally + { + await ShutdownAsync(deaf); + } + } + + /// + /// Starts a single-member cluster on a dynamic port, optionally without the receptionist. + /// + /// + /// drops the receptionist provider from akka.extensions, + /// leaving DistributedPubSub in place so only the one variable changes. + /// + /// The started system. + private static ActorSystem StartSelfSeededSystem(bool withReceptionist) + { + var extensions = withReceptionist + ? string.Empty + : "akka.extensions = [\"Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, " + + "Akka.Cluster.Tools\"]\n"; + + // Port 0 + a post-start self-Join, rather than a seed-nodes list: the address is not known + // until the transport binds, and hard-coding ports makes the test collide with itself under + // parallel execution. + var config = ConfigurationFactory.ParseString( + "akka.remote.dot-netty.tcp.hostname = \"127.0.0.1\"\n" + + "akka.remote.dot-netty.tcp.public-hostname = \"127.0.0.1\"\n" + + "akka.remote.dot-netty.tcp.port = 0\n" + + "akka.cluster.seed-nodes = []\n" + + "akka.loglevel = \"WARNING\"\n" + + extensions) + .WithFallback(ZB.MOM.WW.OtOpcUa.Cluster.HoconLoader.LoadBaseConfig()); + + var system = ActorSystem.Create(SharedSystemName, config); + var cluster = AkkaCluster.Get(system); + cluster.Join(cluster.SelfAddress); + return system; + } + + private static async Task AwaitSingleMemberUpAsync(ActorSystem system) + { + var cluster = AkkaCluster.Get(system); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return; + await Task.Delay(100, TestContext.Current.CancellationToken); + } + + cluster.State.Members.Any(m => m.Status == MemberStatus.Up).ShouldBeTrue( + $"{cluster.SelfAddress} never reached Up as its own seed"); + } + + /// + /// Re-sends until the inbox completes or the deadline passes. + /// + /// + /// A ClusterClient establishes contact asynchronously and buffer-size = 0 means anything + /// sent before that lands is dropped on the floor — by design. A single send would therefore + /// race the handshake and fail intermittently for a reason that is not a defect. Re-sending + /// weakens nothing: the control test re-sends on exactly the same schedule and must still + /// receive nothing. + /// + /// Actor to send to. + /// Builds the message for each attempt. + /// Completion source the receiving capture actor resolves. + /// How long to keep trying. + /// The delivered message, or if none arrived. + private static async Task SendUntilDeliveredAsync( + IActorRef target, + Func message, + TaskCompletionSource inbox, + TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (inbox.Task.IsCompleted) return await inbox.Task; + target.Tell(message()); + await Task.Delay(250, TestContext.Current.CancellationToken); + } + + return inbox.Task.IsCompleted ? await inbox.Task : null; + } + + private static TaskCompletionSource NewInbox() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private static ImmutableHashSet ReceptionistOf(ActorSystem system) => + ImmutableHashSet.Create( + new RootActorPath(AkkaCluster.Get(system).SelfAddress) / "system" / "receptionist"); + + private static async Task ShutdownAsync(ActorSystem? system) + { + if (system is null) return; + await system.Terminate().WaitAsync(TimeSpan.FromSeconds(15)); + } + + /// Completes an inbox with the first message it receives. + private sealed class CaptureActor : ReceiveActor + { + public static Props Props(TaskCompletionSource sink) => + Akka.Actor.Props.Create(() => new CaptureActor(sink)); + + public CaptureActor(TaskCompletionSource sink) => ReceiveAny(m => sink.TrySetResult(m)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshCommActorPathTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshCommActorPathTests.cs new file mode 100644 index 00000000..9751788f --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshCommActorPathTests.cs @@ -0,0 +1,66 @@ +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// Pins the two mesh comm-actor paths against a really booted host. +/// +/// +/// +/// /user/central-communication and /user/node-communication are the wire +/// contract of the central↔node boundary: central sends to a path string, and the node +/// registers under one. Nothing else in the codebase asserts they agree. Renaming either +/// actor would compile, pass every unit test, deploy — and deliver nothing, with no error +/// on either side, because a ClusterClient send to an unregistered path is simply dropped. +/// +/// +/// Booting the real host is the point. A test that spawned the actors itself would pin the +/// constant against itself and prove nothing about the wiring in +/// WithOtOpcUaControlPlaneSingletons / WithOtOpcUaRuntimeActors. +/// +/// +public class MeshCommActorPathTests +{ + private static async Task AssertActorExistsAsync(ActorSystem system, string path) + { + var identity = await system.ActorSelection(path) + .Ask(new Identify(path), TimeSpan.FromSeconds(10)); + + identity.Subject.ShouldNotBeNull( + $"{path} is the ClusterClient wire contract for the central↔node boundary; nothing else " + + "pins it, and a mismatch delivers nothing without erroring on either side"); + } + + [Fact] + public async Task Both_mesh_comm_actors_exist_at_their_contract_paths() + { + await using var harness = await TwoNodeClusterHarness.StartAsync(); + + // Harness nodes carry admin+driver, so BOTH comm actors live on each of them. + await AssertActorExistsAsync(harness.NodeASystem, MeshPaths.CentralCommunication); + await AssertActorExistsAsync(harness.NodeASystem, MeshPaths.NodeCommunication); + await AssertActorExistsAsync(harness.NodeBSystem, MeshPaths.CentralCommunication); + await AssertActorExistsAsync(harness.NodeBSystem, MeshPaths.NodeCommunication); + } + + // NOT TESTED HERE: that the comm actors are registered per node rather than as cluster + // singletons. Two candidate discriminators were tried and BOTH were invalid: + // + // * "the path resolves on both nodes" — a ClusterSingletonManager is also created on every + // node in the role, so this passes under either shape. + // * "no /singleton child exists" — measured against the KNOWN singleton /user/config-publish, + // that child is null there too, so Akka.Hosting does not lay singletons out this way and + // the assertion proves nothing. + // + // A sabotage that re-registered CentralCommunicationActor via WithSingleton left both green, + // which is what exposed them. Rather than ship an assertion I could not make fail, the property + // is left to live-gate step 8 (stop the OLDEST central; the survivor's comm actor must still + // receive a node ack, which a singleton hosted on the stopped node cannot). + // + // NOT MeshClusterClientBoundaryTests: that class proves a send only arrives if the target is + // registered with the receptionist, which is necessary but not discriminating — with one node + // per side, per-node and singleton registration look identical. +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/EventStreamCommandDeliveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/EventStreamCommandDeliveryTests.cs new file mode 100644 index 00000000..c9c66be1 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/EventStreamCommandDeliveryTests.cs @@ -0,0 +1,161 @@ +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Communication; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using Serilog; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication; + +/// +/// Proves the Phase 2 node-side leg joins up: a command handed to +/// reaches the real , +/// not just a test probe. +/// +/// +/// +/// The unit tests either side of this seam both pass with the wiring broken. +/// NodeCommunicationActorTests asserts a probe subscribed to the EventStream receives +/// the message; the driver-host tests drive the actor by direct Tell. Neither notices +/// if DriverHostActor.PreStart never subscribes. That gap is why this file exists. +/// +/// +/// Ordering matters, and it bit this test first time out. ActorOf returns +/// before PreStart has run, and an EventStream.Publish with no subscriber is +/// dropped silently — there is no buffering and no dead-letter. Relaying immediately after +/// spawning therefore fails intermittently. The warm-up below is not ceremony: it is the +/// only available proof that PreStart has completed. (In production the driver host +/// is spawned at startup, long before any deployment, so the same race does not arise.) +/// +/// +public class EventStreamCommandDeliveryTests : RuntimeActorTestBase +{ + private static DispatchDeployment NewDispatch() => new( + new DeploymentId(Guid.NewGuid()), + new RevisionHash("rev-does-not-exist"), + new CorrelationId(Guid.NewGuid())); + + [Fact] + public void A_dispatch_relayed_by_the_comm_actor_reaches_the_real_driver_host() + { + var ackRouter = CreateTestProbe(); + var driverHost = Sys.ActorOf( + DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref), + "driver-host"); + var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication"); + + // Warm-up: a direct Tell whose ack proves PreStart has completed, and with it the + // EventStream subscription. Without this the relay below races the subscription. + var warmup = NewDispatch(); + driverHost.Tell(warmup); + ackRouter.ExpectMsg(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(warmup.DeploymentId); + + // Now the real assertion — relayed exactly as central would send it. The comm actor only + // publishes node-locally, so the host must be subscribed for this to arrive at all. + var relayed = NewDispatch(); + comm.Tell(relayed); + + // The deployment id does not exist, so the host answers Failed. That is the stronger + // assertion, not a weaker one: an ack at all proves the command was received and handled, + // and only PreStart's EventStream subscription can have delivered it. + ackRouter.ExpectMsg(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(relayed.DeploymentId); + } + + [Fact] + public void CONTROL_the_driver_host_acks_a_direct_tell() + { + // Falsifiability control. If the test above fails, this one says which half broke: green + // here means the host is healthy and the EventStream relay is at fault; red here means the + // host would not have acked either way and the relay is not implicated. + var ackRouter = CreateTestProbe(); + var driverHost = Sys.ActorOf( + DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref), + "driver-host-control"); + + var dispatch = NewDispatch(); + driverHost.Tell(dispatch); + + ackRouter.ExpectMsg(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(dispatch.DeploymentId); + } + +} + +/// +/// The alarm leg of the same wiring proof, in its own ActorSystem because the shared +/// harness pins loglevel = WARNING and the only +/// observable signal ScriptedAlarmHostActor gives for an unowned alarm is a Debug line. +/// +public class AlarmCommandEventStreamDeliveryTests : TestKit +{ + private static string DebugLevelHocon => @" +akka { + loglevel = ""DEBUG"" + extensions = [ + ""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools"" + ] + actor { + provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster"" + } + remote.dot-netty.tcp { + hostname = ""127.0.0.1"" + port = 0 + } + cluster { + seed-nodes = [] + roles = [""driver""] + min-nr-of-members = 1 + run-coordinated-shutdown-when-down = off + } +}"; + + public AlarmCommandEventStreamDeliveryTests() + : base(DebugLevelHocon) + { + var cluster = Akka.Cluster.Cluster.Get(Sys); + cluster.Join(cluster.SelfAddress); + AwaitCondition( + () => cluster.State.Members.Any(m => m.Status == Akka.Cluster.MemberStatus.Up), + TimeSpan.FromSeconds(5)); + } + + [Fact] + public void An_alarm_command_relayed_by_the_comm_actor_reaches_the_real_scripted_alarm_host() + { + // Needed for the same reason as the deploy leg: the comm actor's unit test only proves a + // probe receives it, and the alarm-host tests drive the actor by direct Tell. Neither + // notices a missing PreStart subscription. + var upstream = new DependencyMuxTagUpstreamSource(); + var logger = new LoggerConfiguration().CreateLogger(); + var engine = new ScriptedAlarmEngine( + upstream, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + + var host = Sys.ActorOf( + ScriptedAlarmHostActor.Props( + CreateTestProbe().Ref, CreateTestProbe().Ref, upstream, engine, new NodeId("node-A")), + "scripted-alarm-host"); + var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication"); + + var cmd = new AlarmCommand("alarm-unowned", "Acknowledge", "alice", "c", null); + + // Warm-up, OUTSIDE the real assertion: a direct Tell whose debug line proves PreStart has + // completed and the EventStream subscription is registered. An earlier draft did the direct + // Tell INSIDE the assertion window alongside the relay, which made the test vacuous — the + // direct Tell alone produced the log the assertion was waiting for. + EventFilter.Debug(contains: "ignoring AlarmCommand") + .ExpectOne(TimeSpan.FromSeconds(15), () => host.Tell(cmd)); + + // The real assertion: relayed ONLY through the comm actor, which publishes node-locally. + // The host owns no alarms, so it logs and stops — still a positive delivery signal, because + // that line is only reached if the message arrived. + EventFilter.Debug(contains: "ignoring AlarmCommand") + .ExpectOne(TimeSpan.FromSeconds(15), () => comm.Tell(cmd)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/NodeCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/NodeCommunicationActorTests.cs new file mode 100644 index 00000000..7cc06aca --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/NodeCommunicationActorTests.cs @@ -0,0 +1,130 @@ +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Communication; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication; + +public class NodeCommunicationActorTests : RuntimeActorTestBase +{ + private static DispatchDeployment NewDispatch() => new( + new DeploymentId(Guid.NewGuid()), + new RevisionHash("rev-1"), + new CorrelationId(Guid.NewGuid())); + + [Fact] + public void DispatchDeployment_from_central_lands_on_the_node_local_event_stream() + { + var subscriber = CreateTestProbe(); + Sys.EventStream.Subscribe(subscriber.Ref, typeof(DispatchDeployment)); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + var dispatch = NewDispatch(); + actor.Tell(dispatch); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(dispatch); + } + + [Fact] + public void A_command_is_published_exactly_once() + { + // The reason inbound commands do NOT go back onto their DPS topic: central SendToAll-s to + // every node's comm actor, so a mesh-wide republish would deliver N copies to every + // subscriber. One publish per inbound command is the whole point. + var subscriber = CreateTestProbe(); + Sys.EventStream.Subscribe(subscriber.Ref, typeof(DispatchDeployment)); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + actor.Tell(NewDispatch()); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)); + subscriber.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + [Fact] + public void RestartDriver_from_central_lands_on_the_event_stream() + { + var subscriber = CreateTestProbe(); + Sys.EventStream.Subscribe(subscriber.Ref, typeof(RestartDriver)); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + var msg = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + actor.Tell(msg); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(msg); + } + + [Fact] + public void ReconnectDriver_from_central_lands_on_the_event_stream() + { + var subscriber = CreateTestProbe(); + Sys.EventStream.Subscribe(subscriber.Ref, typeof(ReconnectDriver)); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + var msg = new ReconnectDriver("C1", "driver-1", "alice", Guid.NewGuid()); + actor.Tell(msg); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(msg); + } + + [Fact] + public void AlarmCommand_from_central_lands_on_the_event_stream() + { + // ScriptedAlarmHostActor is a CHILD of DriverHostActor with no registry key, so there is no + // ref to hand this actor at wiring time — the EventStream is what makes the alarm-command + // leg possible without a registration handshake. + var subscriber = CreateTestProbe(); + Sys.EventStream.Subscribe(subscriber.Ref, typeof(AlarmCommand)); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + var msg = new AlarmCommand("alarm-1", "Acknowledge", "alice", "comment", null); + actor.Tell(msg); + + subscriber.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(msg); + } + + [Fact] + public void ApplyAck_goes_to_central_as_a_Send_not_a_SendToAll() + { + // Send, not SendToAll: the deploy coordinator is one singleton behind the central comm + // actor's proxy. SendToAll would deliver one ack per central node and the coordinator would + // count this node twice. + var client = CreateTestProbe(); + var actor = Sys.ActorOf(NodeCommunicationActor.Props(client.Ref)); + + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("host-a:4053"), + ApplyAckOutcome.Applied, + null, + new CorrelationId(Guid.NewGuid())); + actor.Tell(ack); + + var sent = client.ExpectMsg(TimeSpan.FromSeconds(5)); + sent.Path.ShouldBe(MeshPaths.CentralCommunication); + sent.Message.ShouldBe(ack); + } + + [Fact] + public void An_ack_with_no_central_client_is_dropped_with_a_warning() + { + var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null)); + + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("host-a:4053"), + ApplyAckOutcome.Applied, + null, + new CorrelationId(Guid.NewGuid())); + + EventFilter.Warning(contains: "dropping ApplyAck").ExpectOne( + TimeSpan.FromSeconds(5), () => actor.Tell(ack)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs index dc03c043..dbd1323e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs @@ -41,7 +41,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase var cache = new StubArtifactCache(cached); var actor = Sys.ActorOf(DriverHostActor.Props( - new ThrowingDbFactory(), TestNode, coordinator: null, + new ThrowingDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: cache)); @@ -70,7 +70,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase var publishProbe = CreateTestProbe(); Sys.ActorOf(DriverHostActor.Props( - new ThrowingDbFactory(), TestNode, coordinator: null, + new ThrowingDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, opcUaPublishActor: publishProbe.Ref, deploymentArtifactCache: new StubArtifactCache(cached))); @@ -85,7 +85,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase { // The pre-cache behaviour, unchanged. A cache miss must not invent a configuration. var actor = Sys.ActorOf(DriverHostActor.Props( - new ThrowingDbFactory(), TestNode, coordinator: null, + new ThrowingDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new StubArtifactCache(current: null))); @@ -100,7 +100,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase { // Admin-only graphs and older harnesses pass null; behaviour is identical to a cache miss. var actor = Sys.ActorOf(DriverHostActor.Props( - new ThrowingDbFactory(), TestNode, coordinator: null, + new ThrowingDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" })); var snapshot = AskDiagnostics(actor); @@ -118,7 +118,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow)); var actor = Sys.ActorOf(DriverHostActor.Props( - NewInMemoryDbFactory(), TestNode, coordinator: null, + NewInMemoryDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: cache)); @@ -140,7 +140,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase deploymentId.ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow)); var actor = Sys.ActorOf(DriverHostActor.Props( - db, TestNode, coordinator: null, + db, TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: cache)); @@ -156,7 +156,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase { // A cache fault must leave the node exactly where it would have been without a cache. var actor = Sys.ActorOf(DriverHostActor.Props( - new ThrowingDbFactory(), TestNode, coordinator: null, + new ThrowingDbFactory(), TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new ThrowingReadCache())); @@ -183,7 +183,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase var cache = new StubArtifactCache(current: null); var actor = Sys.ActorOf(DriverHostActor.Props( - db, TestNode, coordinator: null, + db, TestNode, ackRouter: null, localRoles: new HashSet { "driver" }, deploymentArtifactCache: cache)); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs index 67f8cbbf..0e0ec0ce 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs @@ -183,7 +183,7 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase Sys.ActorOf(DriverHostActor.Props( NewInMemoryDbFactory(), TestNode, - coordinator: null, + ackRouter: null, driverMemberCountProvider: () => driverMemberCount, redundancyRoleView: view, replicationPeerHost: PeerHost)); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxActorTests.cs index 48495808..be6dd343 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxActorTests.cs @@ -139,7 +139,7 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase var hostActor = Sys.ActorOf(DriverHostActor.Props( dbFactory: NewInMemoryDbFactory(), localNode: ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId.Parse("host-1"), - coordinator: hostProbe.Ref, + ackRouter: hostProbe.Ref, dependencyMux: mux)); // Tell the host an AttributeValuePublished — it should fan out to the mux + subscriber.