Compare commits
38 Commits
7654f24dab
...
d01b0695c2
| Author | SHA1 | Date | |
|---|---|---|---|
| d01b0695c2 | |||
| 248e70b425 | |||
| d9de4e3019 | |||
| 44df30236e | |||
| cf6110d02c | |||
| b3210686e2 | |||
| ffbcaa93f2 | |||
| fc568ae0fa | |||
| 23cd6e0118 | |||
| 6d00473866 | |||
| 648f173b18 | |||
| 1a7e3f7ea3 | |||
| 7a8fb5f129 | |||
| 4d88f67c0d | |||
| b28c071bec | |||
| 7d7de482b7 | |||
| 62e8f54f71 | |||
| 9d1e60c00c | |||
| b3cb0f4a54 | |||
| 6eaa81cae7 | |||
| 32055c2238 | |||
| 986b640eea | |||
| e4cb9a08e0 | |||
| 4eca87f04d | |||
| 5468b79d19 | |||
| d246a7e44d | |||
| 68b804c88c | |||
| 2a2e54f083 | |||
| 16d598560a | |||
| 95e4f97529 | |||
| 5cb0e72166 | |||
| 915beec11a | |||
| d5b5cb6ede | |||
| e762ae00f2 | |||
| 0d6669d5ff | |||
| 7b71a6a35d | |||
| 5439f14804 | |||
| 1ec831883c |
@@ -0,0 +1,29 @@
|
||||
# Build context exclusions for docker-dev/Dockerfile, whose `COPY . .` takes the whole repo.
|
||||
#
|
||||
# Without this file the context also carries every project's bin/obj AND .claude/worktrees — agent
|
||||
# worktrees that are themselves full checkouts WITH their own build outputs. That reached ~19 GB and
|
||||
# exhausted the Docker VM's disk mid-COPY, failing the rig build with "no space left on device"
|
||||
# rather than anything a build log would attribute to the repo.
|
||||
#
|
||||
# Nothing excluded here is needed: the build restores and publishes inside the image, and the
|
||||
# migrator stage runs `dotnet ef` against the same restored source.
|
||||
|
||||
# Agent worktrees — full checkouts, build outputs and all.
|
||||
.claude/
|
||||
|
||||
# Local build outputs; the image builds its own.
|
||||
**/bin/
|
||||
**/obj/
|
||||
artifacts/
|
||||
|
||||
# VCS + editor state.
|
||||
.git/
|
||||
.gitignore
|
||||
.vs/
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Docker's own state, and anything the image would never read.
|
||||
**/.DS_Store
|
||||
**/*.user
|
||||
docker-dev/**/*.log
|
||||
@@ -228,7 +228,46 @@ The server supports non-transparent warm/hot redundancy via the `Redundancy` sec
|
||||
|
||||
Assert the **effective** cluster config off a running `ActorSystem` (as `SplitBrainResolverActivationTests` does), never the typed options or the akka.conf text — several HOCON fragments compete and the losing one still reads correctly in the file.
|
||||
|
||||
**Still per-Akka-cluster, not per application `Cluster`.** One Primary is elected across the whole Akka mesh, so a fleet running several application clusters in one mesh (what docker-dev models) gets one Primary among all driver nodes. The fix is the per-cluster mesh workstream: `docs/plans/2026-07-21-per-cluster-mesh-design.md` (Phases 0a/0b/1 done; 2–7 not started).
|
||||
**Still per-Akka-cluster, not per application `Cluster`.** One Primary is elected across the whole Akka mesh, so a fleet running several application clusters in one mesh (what docker-dev models) gets one Primary among all driver nodes. The fix is the per-cluster mesh workstream: `docs/plans/2026-07-21-per-cluster-mesh-design.md` (Phases 0a/0b/1 done; 2 code-complete behind a dark switch; 3–7 not started).
|
||||
|
||||
## Mesh command transport (`MeshTransport`)
|
||||
|
||||
Per-cluster mesh **Phase 2** added a second transport for the three central→node command channels
|
||||
(`deployments`, `driver-control`, `alarm-commands`) and the `deployment-acks` reply, selected by
|
||||
`MeshTransport:Mode`: `Dps` (the DistributedPubSub path this repo has always used, **the default**)
|
||||
or `ClusterClient`, which does **not** require central and the node to share a gossip ring — the
|
||||
prerequisite for splitting the fleet into one mesh per application `Cluster`. Both `CentralCommunicationActor`
|
||||
(`/user/central-communication`) and `NodeCommunicationActor` (`/user/node-communication`) are spawned
|
||||
and registered with the `ClusterClientReceptionist` in **both** modes, so flipping the flag is a config
|
||||
change, not a redeploy. Things worth knowing before touching it:
|
||||
|
||||
- **`SendToAll`, never `Send`.** Today's DPS publish reaches every `DriverHostActor`; there is no
|
||||
ClusterId or node filter on the node side at all (scoping happens later, inside the artifact via
|
||||
`DeploymentArtifact.ResolveClusterScope`). `ClusterClient.Send` delivers to exactly **one** registered
|
||||
actor — it would deploy to a single node while every other node silently kept its old configuration,
|
||||
and the deployment could still seal green.
|
||||
- **Exactly ONE fleet-wide ClusterClient while the fleet is one mesh.** A receptionist serves its whole
|
||||
cluster, so `SendToAll` reaches every registered node-comm actor regardless of which node's address was
|
||||
dialled. One client per application `Cluster` — the obviously-right-looking shape, and what Phase 6
|
||||
wants — would fan each command out once per cluster (N× duplicate dispatch *and* N× duplicate ack).
|
||||
Marked `TODO(Phase 6)` in `CentralCommunicationActor.RebuildClient`. **Corollary:** the contact set does
|
||||
not scope delivery; excluding a `MaintenanceMode` node from the contacts does not stop it receiving.
|
||||
- **Inbound commands re-emit on the node's `EventStream`, not on their DPS topic** — DPS is mesh-wide and
|
||||
central `SendToAll`s to every node, so a DPS re-publish would deliver N copies to every subscriber.
|
||||
Node-side subscribers (`DriverHostActor`, `ScriptedAlarmHostActor`) accept both.
|
||||
- **Comm actors are per-node, NOT singletons.** A client rotates across contact points and must find a live
|
||||
comm actor at whichever node answers; `akka.cluster.client.receptionist.role` is empty for the same reason.
|
||||
- **`buffer-size = 0` is a behaviour decision, not a tuning knob.** Akka's default buffers 1000 and replays
|
||||
on reconnect — silently, on the client's own timer, at an unbounded remove from the dispatch. Do not raise
|
||||
it to quiet a flaky test. **But do not over-claim what it buys:** the Phase 2 live gate observed a node
|
||||
applying a deployment **44 s after it was sealed `TimedOut`** anyway, via `DriverHostActor.Bootstrap()`
|
||||
replaying the orphan `Applying` row on restart. Zero buffering removes the silent replay, not every replay.
|
||||
|
||||
Contact points are node **addresses only** (`akka.tcp://<system>@<host>:<port>`); `/system/receptionist` is
|
||||
appended at construction and `MeshTransportOptionsValidator` refuses to start a node whose contact carries an
|
||||
actor-path suffix. `redundancy-state` stays on DPS in both modes (bidirectional, and built from
|
||||
`Cluster.State`). See `docs/Configuration.md` §`MeshTransport`, `docs/Redundancy.md` §"Command transport",
|
||||
and `docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md`.
|
||||
|
||||
**Phase 1 landed 2026-07-22 and changed a deploy-path behaviour.** `ConfigPublishCoordinator` sources its expected-ack set from **enabled `ClusterNode` rows**, not from `Akka.Cluster.State.Members` filtered by the `driver` role — central must be able to name a deployment's nodes without sharing a gossip ring with them. Consequences worth knowing before touching deploys:
|
||||
|
||||
@@ -237,6 +276,45 @@ Assert the **effective** cluster config off a running `ActorSystem` (as `SplitBr
|
||||
- **There is no such thing as a cluster-scoped deployment.** `Deployment` has no `ClusterId`, `ConfigComposer` always snapshots the whole DB, and `DeploymentArtifact.ResolveClusterScope` is *node-side* self-scoping of a fleet-wide artifact. The phase-1 plan asked for cluster-scope filtering of the ack set; it was dropped as describing a feature that does not exist.
|
||||
- `ClusterNode` gained `AkkaPort` (NOT NULL, 4053) + `GrpcPort` (nullable) as **central's dial targets** for Phases 2/5. They duplicate the node's own `Cluster:Port` / `Cluster:PublicHostname` with no schema-level enforcement, so `ClusterNodeAddressReconcilerActor` (admin singleton) logs an Error on drift. **Phase 2 must revisit it** — once the meshes split, an admin node cannot see site members and every site row would report `EnabledRowNotInCluster` forever.
|
||||
|
||||
## Config source (`ConfigSource` / `ConfigServe`)
|
||||
|
||||
Per-cluster mesh **Phase 3** (config fetch-and-cache) added a second way a **driver** node obtains its
|
||||
deployed configuration, selected by `ConfigSource:Mode`: `Direct` (read `Deployment.ArtifactBlob` from
|
||||
central SQL — **the default**, byte-for-byte today's behaviour) or `FetchAndCache` (fetch the artifact
|
||||
bytes from central over a dedicated-h2c **gRPC stream**, verify `SHA-256 == RevisionHash`, cache in
|
||||
LocalDb, and read **only** the cache — no central-SQL config read). Central (admin role) serves via
|
||||
`DeploymentArtifactService` behind `ConfigServeAuthInterceptor` on `ConfigServe:GrpcListenPort`, gated by
|
||||
a **shared node bearer key** (`ConfigServe:ApiKey == ConfigSource:ApiKey`, `FixedTimeEquals`,
|
||||
fail-closed). Things worth knowing before touching it:
|
||||
|
||||
- **The serve side is wired in both modes** (mapped whenever `ConfigServe:GrpcListenPort > 0` on an
|
||||
admin node), so flipping a node to `FetchAndCache` is a config change, not a redeploy — the same
|
||||
dark-switch discipline as [MeshTransport](#mesh-command-transport-meshtransport).
|
||||
- **`RevisionHash == SHA-256(artifact bytes)` is load-bearing.** `ConfigComposer` computes the revision
|
||||
hash as the lowercase-hex SHA-256 of the exact blob, so the node verifies a fetch against the
|
||||
`RevisionHash` the dispatch already carries — **no hash rides in the proto, and `DispatchDeployment`
|
||||
stays payload-free.** A future change to how the revision hash is derived silently breaks every fetch.
|
||||
- **A null fetch is "no answer," never an empty config (#485).** All endpoints down / `NotFound` / SHA
|
||||
mismatch / zero-length ⇒ the fetcher returns null ⇒ the apply FAILS, the revision does not advance,
|
||||
and the node keeps serving last-known-good. It never tears down to an empty address space. At boot a
|
||||
`FetchAndCache` node restores served state from the LocalDb pointer with **no** central-SQL read; an
|
||||
empty/unreadable cache lands Steady-with-no-revision (the first dispatch fetches), **never Stale**.
|
||||
- **Both pair nodes fetch independently** — no Primary gating of the fetch (same shape as both reading
|
||||
SQL today; `StoreAsync` is idempotent and the cache replicates).
|
||||
- **The dedicated h2c listener coexists with the LocalDb-sync listener.** A fused admin+driver node with
|
||||
both ports set re-binds its existing HTTP surface exactly **once** and adds both dedicated HTTP/2-only
|
||||
ports — getting this wrong either double-binds (crash) or silently unbinds the AdminUI (the Kestrel
|
||||
`Listen*`-discards-`ASPNETCORE_URLS` trap the LocalDb block already warned about).
|
||||
- **Phase boundary: config *reads* only.** A `FetchAndCache` node still writes its `NodeDeploymentState`
|
||||
ack row to central SQL; `DbHealthProbe` and `EfAlarmConditionStateStore` still touch SQL. Cutting those
|
||||
is Phase 4 — resist folding it in, or the "no SQL" scope becomes unprovable-in-parts.
|
||||
|
||||
On the docker-dev rig the central pair carries `ConfigServe` (`:4055`, committed dev key) and stays
|
||||
`Direct`; the four site nodes carry `ConfigSource` (endpoints + matching key) defaulting to `Direct`.
|
||||
Flip only the site nodes with `OTOPCUA_CONFIG_MODE=FetchAndCache` at `docker compose up`. See
|
||||
`docs/Configuration.md` §`ConfigSource` / `ConfigServe`, `docs/Redundancy.md` §"Command transport", and
|
||||
`docs/plans/2026-07-22-mesh-phase3-config-fetch-and-cache.md`.
|
||||
|
||||
## LocalDb pair-local store (Phases 1 + 2)
|
||||
|
||||
Every **driver-role** node keeps a consolidated `ZB.MOM.WW.LocalDb` SQLite database (the retired
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
<PackageVersion Include="Grpc.AspNetCore" Version="2.76.0" />
|
||||
<PackageVersion Include="Grpc.Core.Api" Version="2.76.0" />
|
||||
<PackageVersion Include="Grpc.Net.Client" Version="2.76.0" />
|
||||
<PackageVersion Include="Grpc.Tools" Version="2.76.0" />
|
||||
<PackageVersion Include="libplctag" Version="1.5.2" />
|
||||
<PackageVersion Include="MessagePack" Version="2.5.301" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Authorization" Version="10.0.7" />
|
||||
|
||||
@@ -3,7 +3,12 @@
|
||||
# Compose drives OTOPCUA_ROLES + Cluster:* env per container to differentiate them.
|
||||
# A separate `migrator` stage (below) applies EF migrations once on bring-up.
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
# Build stage is PINNED to linux/amd64. Per-cluster mesh Phase 3 added an in-repo .proto compiled by
|
||||
# Grpc.Tools, whose bundled linux_arm64 `protoc` segfaults (exit 139) on Apple-Silicon Docker — so a
|
||||
# native-arm64 build stage cannot generate the gRPC stubs. `dotnet publish` here is framework-dependent
|
||||
# (portable IL + every RID's native assets), so the amd64-built output runs unchanged on the native
|
||||
# runtime image below; only this one-time build is emulated, runtime stays native/fast.
|
||||
FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN dotnet restore ZB.MOM.WW.OtOpcUa.slnx
|
||||
|
||||
@@ -178,6 +178,14 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "central-1"
|
||||
# Per-cluster mesh Phase 3 (config fetch-and-cache). Central SERVES the deployed-configuration
|
||||
# artifact to FetchAndCache site nodes over a dedicated h2c gRPC listener (:4055), gated by the
|
||||
# shared node key. Central itself is ALWAYS Direct — it IS the SQL source and must never fetch.
|
||||
# ConfigServe:ApiKey must be byte-identical to the site nodes' ConfigSource:ApiKey (fail-closed).
|
||||
# Binding :4055 activates the dedicated-h2c Kestrel takeover on central (it re-binds :9000 too).
|
||||
ConfigServe__GrpcListenPort: "4055"
|
||||
ConfigServe__ApiKey: "configserve-docker-dev-key"
|
||||
ConfigSource__Mode: "Direct"
|
||||
# Both redundancy peers are seeds (#459 / ScadaBridge parity) so a restarted node can
|
||||
# re-join the mesh via EITHER peer, not only central-1.
|
||||
#
|
||||
@@ -190,6 +198,19 @@ services:
|
||||
Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053"
|
||||
Cluster__Roles__0: "admin"
|
||||
Cluster__Roles__1: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345"
|
||||
Security__Jwt__Issuer: "otopcua-dev"
|
||||
Security__Jwt__Audience: "otopcua-dev"
|
||||
@@ -256,6 +277,11 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "central-2"
|
||||
# Per-cluster mesh Phase 3 — serves the artifact to FetchAndCache site nodes (see central-1).
|
||||
# Always Direct; ConfigServe:ApiKey must match the site nodes' ConfigSource:ApiKey.
|
||||
ConfigServe__GrpcListenPort: "4055"
|
||||
ConfigServe__ApiKey: "configserve-docker-dev-key"
|
||||
ConfigSource__Mode: "Direct"
|
||||
# Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. SELF FIRST:
|
||||
# central-2 lists itself as seed-nodes[0], which is what lets it cold-start while central-1
|
||||
# is down (it previously listed central-1 first and simply never came Up in that case).
|
||||
@@ -263,6 +289,19 @@ services:
|
||||
Cluster__SeedNodes__1: "akka.tcp://otopcua@central-1:4053"
|
||||
Cluster__Roles__0: "admin"
|
||||
Cluster__Roles__1: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345"
|
||||
Security__Jwt__Issuer: "otopcua-dev"
|
||||
Security__Jwt__Audience: "otopcua-dev"
|
||||
@@ -324,11 +363,33 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "site-a-1"
|
||||
# Per-cluster mesh Phase 3 dark switch. Default Direct (reads central SQL). Flip the site nodes to
|
||||
# FetchAndCache with `OTOPCUA_CONFIG_MODE=FetchAndCache docker compose up`: they then fetch the
|
||||
# deployed artifact from central over gRPC (endpoints below) and read ONLY the LocalDb cache — no
|
||||
# central SQL config read. Central stays Direct, so this env var moves ONLY the four site nodes.
|
||||
# ConfigSource:ApiKey must equal central's ConfigServe:ApiKey (the interceptor is fail-closed).
|
||||
ConfigSource__Mode: "${OTOPCUA_CONFIG_MODE:-Direct}"
|
||||
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
|
||||
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
|
||||
ConfigSource__ApiKey: "configserve-docker-dev-key"
|
||||
# Site nodes are deliberately NOT seeds — they join the central pair's mesh. The self-first
|
||||
# seed rule is conditional for exactly this reason (it binds only when a node's own address
|
||||
# is in its own seed list), so these configs are exempt rather than broken.
|
||||
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
|
||||
Cluster__Roles__0: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
<<: *secrets-env
|
||||
# Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/
|
||||
# mem_reservation are inherited from the *otopcua-host anchor.
|
||||
@@ -393,8 +454,27 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "site-a-2"
|
||||
# Per-cluster mesh Phase 3 dark switch (see site-a-1). Default Direct; OTOPCUA_CONFIG_MODE=FetchAndCache
|
||||
# flips it to fetch the artifact from central over gRPC and read only the LocalDb cache.
|
||||
ConfigSource__Mode: "${OTOPCUA_CONFIG_MODE:-Direct}"
|
||||
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
|
||||
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
|
||||
ConfigSource__ApiKey: "configserve-docker-dev-key"
|
||||
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
|
||||
Cluster__Roles__0: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
<<: *secrets-env
|
||||
Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
|
||||
Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
|
||||
@@ -451,8 +531,27 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "site-b-1"
|
||||
# Per-cluster mesh Phase 3 dark switch (see site-a-1). Default Direct; OTOPCUA_CONFIG_MODE=FetchAndCache
|
||||
# flips it to fetch the artifact from central over gRPC and read only the LocalDb cache.
|
||||
ConfigSource__Mode: "${OTOPCUA_CONFIG_MODE:-Direct}"
|
||||
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
|
||||
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
|
||||
ConfigSource__ApiKey: "configserve-docker-dev-key"
|
||||
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
|
||||
Cluster__Roles__0: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
<<: *secrets-env
|
||||
Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
|
||||
Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
|
||||
@@ -492,8 +591,27 @@ services:
|
||||
Cluster__Hostname: "0.0.0.0"
|
||||
Cluster__Port: "4053"
|
||||
Cluster__PublicHostname: "site-b-2"
|
||||
# Per-cluster mesh Phase 3 dark switch (see site-a-1). Default Direct; OTOPCUA_CONFIG_MODE=FetchAndCache
|
||||
# flips it to fetch the artifact from central over gRPC and read only the LocalDb cache.
|
||||
ConfigSource__Mode: "${OTOPCUA_CONFIG_MODE:-Direct}"
|
||||
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
|
||||
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
|
||||
ConfigSource__ApiKey: "configserve-docker-dev-key"
|
||||
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
|
||||
Cluster__Roles__0: "driver"
|
||||
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
|
||||
# the transport this repo has always used, and every node keeps its ClusterClient wiring built
|
||||
# and its comm actor registered with the receptionist either way. Flip the whole rig with
|
||||
# OTOPCUA_MESH_MODE=ClusterClient at `docker compose up` — no compose edit, and no rebuild.
|
||||
#
|
||||
# Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction
|
||||
# time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an
|
||||
# actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in
|
||||
# silence. Both central nodes are listed so a driver node's client can rotate to whichever one
|
||||
# is up — that rotation is why the comm actors are per-node and NOT cluster singletons.
|
||||
MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-Dps}"
|
||||
MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053"
|
||||
MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053"
|
||||
<<: *secrets-env
|
||||
Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
|
||||
Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
|
||||
|
||||
@@ -110,6 +110,58 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only
|
||||
|
||||
> **`Port` / `PublicHostname` are stored twice.** The node binds from these keys; the fleet's `ClusterNode` row records the same address as `AkkaPort` / `Host` so that **central can dial the node without sharing a gossip ring with it** — which is what [per-cluster mesh Phase 2](plans/2026-07-21-per-cluster-mesh-design.md) needs. Nothing in the schema makes the two agree, so `ClusterNodeAddressReconcilerActor` (admin-role singleton) compares them against live membership and logs an Error on mismatch. If you change `Cluster:Port` or `Cluster:PublicHostname` on a node, **update its `ClusterNode` row too** — a node binding 4054 with a row saying 4053 still gossips fine today, and fails silently in Phase 2. The row's `NodeId` is `host:port` and is also the deploy path's ack identity, so a drifted node's deployment acks stop matching as well. See [`config-db-schema.md` § `ClusterNode`](v2/config-db-schema.md#clusternode).
|
||||
|
||||
### `MeshTransport` (central↔node command transport)
|
||||
|
||||
- **Purpose:** selects how central's three command channels (`deployments`, `driver-control`, `alarm-commands`) and the node's `deployment-acks` reply reach the other side — over the mesh-wide DistributedPubSub, or over an Akka `ClusterClient` that does **not** require central and the node to share a gossip ring. The second option is what [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) Phase 6 needs once the meshes split.
|
||||
- **Options class:** `MeshTransportOptions` (`SectionName = "MeshTransport"`) — `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/MeshTransportOptions.cs`. Bound and validated by `AddOtOpcUaCluster(config)`; `MeshTransportOptionsValidator` runs at `ValidateOnStart`.
|
||||
|
||||
| Key | Type | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `Mode` | string | `Dps` | `Dps` = DistributedPubSub (the transport this repo has always used). `ClusterClient` = the mesh boundary. Any other value **refuses to start**. |
|
||||
| `CentralContactPoints` | string[] | `[]` | Node-side only: central node **addresses** (`akka.tcp://<system>@<host>:<port>`). Required under `ClusterClient`, ignored under `Dps`. |
|
||||
| `ContactRefreshSeconds` | int | `60` | Central-side only: how often central rebuilds its ClusterClient contact set from the `ClusterNode` rows. |
|
||||
|
||||
**Addresses only — no actor-path suffix.** `/system/receptionist` is appended at construction time; a contact carrying `/user/` or `/system/` is rejected at startup, because a doubled suffix produces a path that resolves to nothing and drops every command in silence.
|
||||
|
||||
**It is a dark switch, and both sides are always wired.** The comm actors are spawned and registered with the `ClusterClientReceptionist` on every node in *both* modes, so flipping `Mode` is a config change rather than a redeploy. Under `Dps` the registration is simply idle.
|
||||
|
||||
**Central's contact set comes from the enabled, non-maintenance `ClusterNode` rows** — the same source Phase 1 gave the deploy ack set. A node in `MaintenanceMode` is not dialled, but note that **the contact set does not scope delivery**: a receptionist serves its whole cluster, so `SendToAll` still reaches every registered node exactly as today's DPS broadcast does. The filter is about which receptionists are worth dialling, not about who receives.
|
||||
|
||||
**Buffering is off by design** (`akka.cluster.client.buffer-size = 0`, set in the shipped `akka.conf`). If no receptionist is known, the command is dropped rather than queued. Akka's default of 1000 would replay a `DispatchDeployment` on reconnect — silently, on the client's own timer, at an unbounded remove from the dispatch. Do not "fix" it back to quiet a flaky test.
|
||||
|
||||
> **What this does *not* buy you.** It does not guarantee a node never applies a deployment the DB records as `TimedOut`. A node that missed a dispatch keeps the orphan `Applying` row the coordinator pre-created, and `DriverHostActor.Bootstrap()` replays that row on the node's next restart — live-observed applying a deployment 44 s after it was sealed `TimedOut` (see the [Phase 2 live gate](plans/2026-07-22-mesh-phase2-live-gate.md), finding B). The boot replay is the better-behaved of the two — deliberate, Warning-logged, and bounded to a restart — but it is still a replay. `buffer-size = 0` removes the *silent, timer-driven* one only.
|
||||
|
||||
On the docker-dev rig every node carries these keys already; flip the whole rig with `OTOPCUA_MESH_MODE=ClusterClient` at `docker compose up`.
|
||||
|
||||
### `ConfigSource` / `ConfigServe` (config fetch-and-cache)
|
||||
|
||||
- **Purpose:** [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) **Phase 3**. `ConfigSource:Mode` selects where a **driver** node reads its deployed configuration: `Direct` (read `Deployment.ArtifactBlob` from central SQL, today's behaviour) or `FetchAndCache` (fetch the artifact bytes from central over a gRPC stream, verify SHA-256, cache in LocalDb, and read **only** the cache — no central-SQL config read). `ConfigServe` is the **central** (admin-role) serve side that a `FetchAndCache` node dials.
|
||||
- **Options classes:** `ConfigSourceOptions` (`SectionName = "ConfigSource"`) + `ConfigServeOptions` (`SectionName = "ConfigServe"`), both in `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/`. Bound by `AddOtOpcUaCluster(config)`; `ConfigSourceOptionsValidator` runs at `ValidateOnStart`.
|
||||
|
||||
**`ConfigSource` (driver-side fetch):**
|
||||
|
||||
| Key | Type | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `Mode` | string | `Direct` | `Direct` = read central SQL (unchanged). `FetchAndCache` = fetch from central over gRPC, read the LocalDb cache. Any other value **refuses to start**. |
|
||||
| `CentralFetchEndpoints` | string[] | `[]` | Central artifact-gRPC base addresses, e.g. `http://central-1:4055` (h2c ⇒ the `http` scheme). Tried in order for failover. **Required** under `FetchAndCache`. |
|
||||
| `ApiKey` | string | `""` | Shared bearer key; must equal central's `ConfigServe:ApiKey`. **Supply via env `ConfigSource__ApiKey` — never commit.** Required under `FetchAndCache`. |
|
||||
| `FetchTimeoutSeconds` | int | `30` | Per-fetch deadline. Must be positive under `FetchAndCache`. |
|
||||
|
||||
**`ConfigServe` (central serve):**
|
||||
|
||||
| Key | Type | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `GrpcListenPort` | int | `0` | Dedicated **h2c-only** Kestrel port for the artifact gRPC service. `0` = disabled (nothing bound). Must differ from the main HTTP port and from `LocalDb:SyncListenPort` (h2c cannot share a cleartext HTTP/1 port). Mapped only on admin-role nodes. |
|
||||
| `ApiKey` | string | `""` | Shared bearer key the serve-side interceptor checks (constant-time, **fail-closed**: an unset key rejects every call). **Supply via env `ConfigServe__ApiKey` — never commit.** |
|
||||
|
||||
**It is a dark switch; the serve side is always wired.** Central maps the `DeploymentArtifactService` behind the `ConfigServeAuthInterceptor` (path-scoped, `FixedTimeEquals`, fail-closed) whenever `ConfigServe:GrpcListenPort > 0`, in both modes — so flipping a node to `FetchAndCache` is a config change, not a redeploy. A `FetchAndCache` node that cannot fetch (all endpoints down, `NotFound`, SHA mismatch) **fails the apply and keeps last-known-good** — a null fetch is "no answer," never an empty configuration (the [#485](plans/2026-07-15-v3-batch4-address-space-plan.md) contract). At boot it restores served state from the LocalDb pointer with no SQL read; an empty or unreadable cache lands Steady-with-no-revision (the first dispatch fetches), never Stale.
|
||||
|
||||
**The listener is dedicated h2c and coexists with the LocalDb-sync listener.** A fused admin+driver node with **both** `ConfigServe:GrpcListenPort` and `LocalDb:SyncListenPort` set re-binds its existing HTTP surface exactly once and adds both dedicated HTTP/2-only ports on top — see the same Kestrel-takeover caveat under [`LocalDb`](#localdb-pair-local-config-cache--driver-role-only).
|
||||
|
||||
**`RevisionHash == SHA-256(artifact bytes)` is load-bearing.** `ConfigComposer` computes the deployment's revision hash as the lowercase-hex SHA-256 of the exact artifact bytes, so the node verifies a fetch against the `RevisionHash` the dispatch already carries — no hash rides in the proto. If a future change makes the revision hash anything else, every fetch fails verification.
|
||||
|
||||
On the docker-dev rig the central pair carries `ConfigServe` (port `4055`, dev key) and stays `Direct`; the four site nodes carry `ConfigSource` (endpoints + matching key) defaulting to `Direct`. Flip only the site nodes with `OTOPCUA_CONFIG_MODE=FetchAndCache` at `docker compose up` — central keeps SQL and models the target topology.
|
||||
|
||||
### `ConnectionStrings` → `ConfigDb`
|
||||
|
||||
- **Purpose:** the central Config DB connection string. **Required for every role** — `Program.cs` calls `AddOtOpcUaConfigDb` unconditionally.
|
||||
|
||||
@@ -407,6 +407,50 @@ Net effect: each alarm transition appears **once** on `/alerts` and would histor
|
||||
|
||||
See [ScriptedAlarms.md](ScriptedAlarms.md) and [AlarmTracking.md](AlarmTracking.md) for the scripted-alarm engine internals.
|
||||
|
||||
## Command transport (central↔node)
|
||||
|
||||
Everything above rides one Akka mesh: central publishes on DistributedPubSub, every node
|
||||
subscribes, and the two ends must be members of the same cluster. Per-cluster mesh **Phase 2**
|
||||
adds a second transport under `MeshTransport:Mode` (see
|
||||
[Configuration.md § `MeshTransport`](Configuration.md#meshtransport-centralnode-command-transport))
|
||||
that does not need shared membership — the prerequisite for splitting the fleet into one mesh per
|
||||
application `Cluster`.
|
||||
|
||||
| | `Dps` (default) | `ClusterClient` |
|
||||
|---|---|---|
|
||||
| Central → node commands | `Publish` on `deployments` / `driver-control` / `alarm-commands` | `ClusterClient.SendToAll` to `/user/node-communication`, re-emitted on the node's **local** `EventStream` |
|
||||
| Node → central acks | `Publish` on `deployment-acks` | `ClusterClient.Send` to `/user/central-communication`, forwarded to the deploy-coordinator singleton |
|
||||
| Requires shared cluster membership | **yes** | no |
|
||||
|
||||
Two properties are load-bearing and easy to break:
|
||||
|
||||
- **Both comm actors are registered per node, NOT as cluster singletons.** A `ClusterClient` rotates
|
||||
across its contact points and must find a live comm actor at whichever node answers. As a
|
||||
singleton the boundary would be reachable through exactly one node, and rotation would fail
|
||||
silently against the others — acks landing or not depending on where the client happened to
|
||||
settle. `akka.cluster.client.receptionist.role` is deliberately empty for the same reason.
|
||||
- **Inbound commands go to the node-local `EventStream`, never back onto a DPS topic.** DPS is
|
||||
mesh-wide and central `SendToAll`s to *every* node, so a DPS re-publish would deliver N copies of
|
||||
every command to every subscriber.
|
||||
|
||||
Phase 2 changes no redundancy semantics: the Primary gate, ServiceLevel, and the singleton layout
|
||||
are untouched, and `redundancy-state` itself stays on DPS (it is bidirectional and built from
|
||||
`Cluster.State`, which makes it genuinely mesh-bound — the hardest remaining dependency before the
|
||||
meshes can split).
|
||||
|
||||
**Config bytes travel out-of-band (Phase 3).** Commands stay on one of the two transports above, but
|
||||
the *configuration artifact itself* does not ride any Akka message — `DispatchDeployment` is
|
||||
payload-free (`DeploymentId` + `RevisionHash`). Under `ConfigSource:Mode = FetchAndCache` (see
|
||||
[Configuration.md § `ConfigSource`](Configuration.md#configsource--configserve-config-fetch-and-cache))
|
||||
a driver node fetches the bytes from central over a dedicated h2c **gRPC stream**, verifies
|
||||
`SHA-256 == RevisionHash`, and reads only its LocalDb cache — no central-SQL config read. **Both pair
|
||||
nodes fetch independently**, exactly as both read SQL today: there is no Primary gating of the fetch.
|
||||
That is safe because `StoreAsync` is idempotent and the cache replicates (pair replication carries the
|
||||
artifact to a node that missed its own fetch), and it keeps config delivery decoupled from redundancy
|
||||
timing. Central serves the bytes from its SQL row; a `FetchAndCache` node that cannot reach central
|
||||
fails the apply and keeps last-known-good (it never tears down to an empty address space), and it boots
|
||||
its served state from the LocalDb pointer with no central dependency at all.
|
||||
|
||||
## Pair-local store (LocalDb — Phases 1 + 2)
|
||||
|
||||
Independently of the ServiceLevel machinery above, every **driver-role** node keeps a consolidated
|
||||
|
||||
@@ -68,8 +68,14 @@ Other load-bearing details:
|
||||
caller's Ask times out. *"It keeps the central coordinator stateless with respect to site
|
||||
availability."* A `ConnectionStateChanged` mechanism was built for this and **deleted as dead
|
||||
code**.
|
||||
- **Every unhandled message replies with a typed failure rather than dropping**, so a central Ask
|
||||
can never stall on a missing handler.
|
||||
- ~~**Every unhandled message replies with a typed failure rather than dropping**, so a central Ask
|
||||
can never stall on a missing handler.~~ **CORRECTED 2026-07-22 (Phase 2 recon).** This overstates
|
||||
what ScadaBridge does. Neither of their comm actors has a `ReceiveAny` or an `Unhandled` override:
|
||||
the typed-failure idiom is **per message type**, and it fires only when a *registration* is missing
|
||||
(no ClusterClient for the site yet, no handler registered for that type). A genuinely unknown
|
||||
message type still dead-letters exactly as it would anywhere else. Do not build a catch-all failure
|
||||
reply believing you are copying them. OtOpcUa's Phase 2 comm actors list each type explicitly and
|
||||
deliberately let an unknown type dead-letter loudly rather than republish it blind.
|
||||
- **Sender preservation is the whole Ask idiom**: `_client.Tell(new ClusterClient.Send(path, msg),
|
||||
Sender)` routes the reply straight back to the waiting Ask, bypassing the comm actor.
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-07-22-mesh-phase2-clusterclient-transport.md",
|
||||
"note": "Per-cluster mesh Phase 2. Decisions pre-made: config-flagged dark switch (MeshTransport:Mode), buffer-size=0, alarm-commands central leg in scope. Three design-doc corrections and the single-mesh duplicate-delivery trap are documented in the plan header \u2014 read them before Task 3.",
|
||||
"tasks": [
|
||||
{
|
||||
"id": 0,
|
||||
"subject": "Task 0: MeshTransportOptions + validator + registration",
|
||||
"status": "completed",
|
||||
"classification": "small",
|
||||
"parallelizableWith": [
|
||||
1
|
||||
],
|
||||
"commit": "5439f148"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"subject": "Task 1: receptionist extension, buffer-size=0, frame-size logging (effective-config test)",
|
||||
"status": "completed",
|
||||
"classification": "small",
|
||||
"parallelizableWith": [
|
||||
0
|
||||
],
|
||||
"commit": "7b71a6a3",
|
||||
"note": "First draft of the buffering test was VACUOUS -- Config.GetInt returns 0 for a missing key, so it passed before the feature existed. Rewritten to assert through ClusterClientSettings/ClusterReceptionistSettings."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"subject": "Task 2: MeshCommand envelope + generation-suffixed ClusterClient factory",
|
||||
"status": "completed",
|
||||
"classification": "small",
|
||||
"blockedBy": [
|
||||
0
|
||||
],
|
||||
"commit": "0d6669d5",
|
||||
"note": "Dropped a third test that asserted ClusterClientSettings directly and never touched the factory."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"subject": "Task 3: CentralCommunicationActor \u2014 dark-switched SendToAll fan-out, DB-sourced contacts, ApplyAck forward",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"parallelizableWith": [
|
||||
4
|
||||
],
|
||||
"commit": "d5b5cb6e",
|
||||
"note": "SendToAll sabotage-verified. Two test-harness bugs found: SubscribeAck goes to the SENDER, and EventFilter matches case-INSENSITIVELY (a 'was NOT' filter also caught 'was not created')."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"subject": "Task 4: NodeCommunicationActor \u2014 inbound to node-local EventStream, outbound ApplyAck",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"parallelizableWith": [
|
||||
3
|
||||
],
|
||||
"commit": "915beec1",
|
||||
"note": "No Ask crosses the boundary in Phase 2, so this actor is far smaller than the sister project's."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"subject": "Task 5: node subscribers accept EventStream too; rename _coordinatorOverride -> _ackRouter",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
4
|
||||
],
|
||||
"commit": "5cb0e721",
|
||||
"note": "Two wiring tests added -- unit tests either side of the seam both pass with the subscription deleted. First draft raced (ActorOf returns before PreStart; EventStream publish to nobody is dropped silently); alarm version was vacuous until the warm-up moved outside the assertion window."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"subject": "Task 6: wire both sides, receptionist registration, pin the wire-contract actor paths",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
3,
|
||||
5
|
||||
],
|
||||
"commit": "16d59856",
|
||||
"note": "Dropped a planned 'not a singleton' assertion: two candidate discriminators were both invalid (a SingletonManager also sits at /user/<name>; /singleton is null even for the known singleton config-publish). A sabotage re-registering via WithSingleton left both green, which exposed them. Property deferred to Task 8."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"subject": "Task 7: route the five publish sites through the mesh router (nullable, mediator fallback)",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
6
|
||||
],
|
||||
"note": "Sabotage-verified by inverting the branch: all 6 tests red. Comm actor registration moved above the singletons so registry.Get works; an earlier ResolveOne().GetAwaiter().GetResult() blocked inside a props factory."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"subject": "Task 8: real two-ActorSystem ClusterClient boundary test + falsifiability control",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
7
|
||||
],
|
||||
"parallelizableWith": [
|
||||
9
|
||||
],
|
||||
"commit": "5468b79d",
|
||||
"note": "Not flaky (3/3 clean repeats; 4 tests, ~16s). Both legs sabotage-verified against production code, and the control was itself falsified by restoring the receptionist. The control necessarily removes RegisterService too -- registering would materialise the extension it is removing. Task 6's dropped per-node/singleton property does NOT land here (one node per side cannot discriminate); it is live-gate step 8, and the MeshCommActorPathTests comment was corrected to say so."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"subject": "Task 9: docker-dev rig config + docs + design-doc correction",
|
||||
"status": "completed",
|
||||
"commit": "e4cb9a08",
|
||||
"note": "Rig Mode is ${OTOPCUA_MESH_MODE:-Dps} rather than a hardcoded Dps, so Task 10 can flip all six nodes at `docker compose up` with no compose edit and no rebuild. Verified via `docker compose config` (6 services resolved).",
|
||||
"classification": "small",
|
||||
"blockedBy": [
|
||||
7
|
||||
],
|
||||
"parallelizableWith": [
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"subject": "Task 10: live gate on docker-dev (11 steps; step 4 = deploy-seal semantics, step 8 = per-node/contact-rotation discriminator [replaces the dropped Task 6 assertion \u2014 do not skip], step 9 = both-centrals drop behavior)",
|
||||
"status": "completed",
|
||||
"note": "PASSED for the transport (steps 1-5, 8, 11 green; flip end-to-end via OTOPCUA_MESH_MODE=ClusterClient). Record: 2026-07-22-mesh-phase2-live-gate.md. Steps 6/7 NOT RUN (no UI surface for driver-control \u2014 finding D, DriverStatusPanel is orphaned; no alarms configured). Step 8's stated log-signal does not exist (HandleApplyAck is silent on success \u2014 finding E); proven instead by both centrals independently logging per-instance ClusterClient creation + rebuild, which a singleton cannot. FOUR findings: A) 10-30s post-restart delivery window is PRE-EXISTING (DPS control failed identically); B) buffer-size=0 rationale OVERSTATED \u2014 a node applies a TimedOut deploy anyway via Bootstrap() orphan-row replay at +44s, corrected in 3 docs; C) stopping BOTH centrals strands non-seed nodes at the membership layer (Phase 7 drill, transport-independent); D) driver Restart/Reconnect UI orphaned since v3 Batch 2. Rig restored to Dps default.",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
8,
|
||||
9
|
||||
]
|
||||
}
|
||||
],
|
||||
"lastUpdated": "2026-07-22T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
# Per-cluster mesh Phase 2 — live gate record
|
||||
|
||||
**Date:** 2026-07-22 · **Rig:** `docker-dev` (six-node single mesh) · **Branch:** `feat/mesh-phase2`
|
||||
**Plan:** [`2026-07-22-mesh-phase2-clusterclient-transport.md`](2026-07-22-mesh-phase2-clusterclient-transport.md)
|
||||
|
||||
**Result: PASSED for the transport**, with two gate steps not run (no surface exists to drive them),
|
||||
one step whose stated pass-signal does not exist in the code, and **four findings** — one of which
|
||||
corrects a claim this phase's own documentation makes.
|
||||
|
||||
The rig was flipped end-to-end with `OTOPCUA_MESH_MODE=ClusterClient` (no rebuild, no compose edit —
|
||||
the env indirection added in Task 9 is what made the A/B and the DPS control cheap enough to run).
|
||||
|
||||
---
|
||||
|
||||
## Result table
|
||||
|
||||
| # | Step | Result |
|
||||
|---|---|---|
|
||||
| 1 | Deploy on `Mode=Dps` | **PASS** — `48d0c7c5` Sealed, 6/6 Applied |
|
||||
| 2 | Flip all six to `ClusterClient`, redeploy | **PASS** — `5db8f6ad` Sealed, 6/6 Applied |
|
||||
| 3 | Confirm the transport actually changed | **PASS** — positive log evidence, see below |
|
||||
| 4 | Stop a node, deploy | **PASS** — `59b81248` TimedOut naming `site-b-2:4053`, 5/6 |
|
||||
| 5 | `MaintenanceMode = 1`, redeploy | **PASS** — `c6cb9812` Sealed, 5 rows, contacts 6→5 |
|
||||
| 6 | AdminUI Restart + Reconnect | **NOT RUN** — no UI surface exists (finding D) |
|
||||
| 7 | AdminUI alarm ack | **NOT RUN** — no alarms configured on the rig |
|
||||
| 8 | Per-node (not singleton) discriminator | **PASS** — via a better discriminator than the plan's |
|
||||
| 9 | Both centrals down, restart a driver node | **PARTIAL** — node survives; the stated ack-drop cannot occur |
|
||||
| 10 | Restart central, redeploy | **PASS only after operator action** (finding C) |
|
||||
| 11 | Frame-size canary | **PASS** — 0 warnings across all six nodes |
|
||||
|
||||
## Step 3 — the transport really changed
|
||||
|
||||
Central, both nodes, at boot under the flag:
|
||||
|
||||
```
|
||||
central-1 [22:18:16 INF] Mesh ClusterClient created with 6 contact point(s)
|
||||
central-2 [22:18:17 INF] Mesh ClusterClient created with 6 contact point(s)
|
||||
```
|
||||
|
||||
Node side, with `Serilog__MinimumLevel__Default=Debug` overlaid on `site-a-1` (the rig pins the
|
||||
default above Debug, so this line is invisible on a stock node):
|
||||
|
||||
```
|
||||
site-a-1 [22:10:34 DBG] Mesh: received DispatchDeployment from central; publishing node-locally
|
||||
```
|
||||
|
||||
Under `Mode=Dps` the same node logs nothing of the sort and central logs
|
||||
`Mesh transport is Dps: commands are published on DistributedPubSub and no ClusterClient is created`.
|
||||
|
||||
## Step 5 — the DB-sourced contact set is live
|
||||
|
||||
Setting `MaintenanceMode = 1` on `site-b-2` shrank both centrals' contact sets on the next refresh,
|
||||
and clearing it restored them — independently, on each central's own 60 s timer:
|
||||
|
||||
```
|
||||
central-1 [22:22:16 INF] Mesh contact set changed; replacing the ClusterClient
|
||||
central-1 [22:22:16 INF] Mesh ClusterClient created with 5 contact point(s)
|
||||
central-1 [22:23:16 INF] Mesh ClusterClient created with 6 contact point(s)
|
||||
central-2 [22:22:16 INF] … identical, same timestamps
|
||||
```
|
||||
|
||||
No `InvalidActorNameException` on any rebuild — the generation-suffixed client name (Task 2, ported
|
||||
from the sister project's shipped bug) does its job.
|
||||
|
||||
## Step 8 — the per-node discriminator, replaced with a stronger one
|
||||
|
||||
**The plan's pass-signal does not exist.** It asked for the surviving central's comm actor to *log
|
||||
receipt* of a node ack inside the downing window. `CentralCommunicationActor.HandleApplyAck` logs
|
||||
**nothing** on the success path — it only warns when the coordinator is unresolvable. There is no
|
||||
line to observe. (Recorded as finding E.)
|
||||
|
||||
The evidence above is a **cleaner** discriminator and needs no timing race. `Mesh ClusterClient
|
||||
created` is emitted from `PreStart`, so it fires **once per live actor instance**. Both centrals
|
||||
logged it, and both logged their own independent contact-set rebuild at step 5. A
|
||||
`ClusterSingletonManager` runs the actor on the oldest node only — the other node hosts a manager
|
||||
that never starts the actor and would log nothing. **Two nodes logging it proves two live instances,
|
||||
i.e. per-node registration.** The node side is proven the same way: `SendToAll` collected 6/6 acks,
|
||||
which requires six separately registered node comm actors.
|
||||
|
||||
The failover half was run anyway, and passed: `docker stop central-1` (the oldest, and the singleton
|
||||
host) → the surviving central accepted a deployment **4 seconds later** and collected acks from all
|
||||
five live nodes (`e3161c25`; it TimedOut only because `central-1` is itself a configured node that
|
||||
was down — correct Phase 1 semantics). Graceful stop hands the singleton over immediately, so no
|
||||
downing wait was involved.
|
||||
|
||||
## Step 9 — the stated failure mode cannot occur
|
||||
|
||||
The plan expected a restarting driver node to emit a boot-time ack that gets dropped with a Warning.
|
||||
**Driver nodes do not ack on boot.** `DriverHostActor.SendAck` is called only from the apply paths —
|
||||
a node with no dispatch to answer has nothing to send, so there is no drop to observe.
|
||||
|
||||
What was verified: with **both** centrals stopped, `site-a-1` restarted, came up clean, logged its
|
||||
association failures against both central addresses, retried, and stayed up — no crash loop. The
|
||||
node-side ClusterClient's forever-retry (`reconnect-timeout = off`) behaves as configured.
|
||||
|
||||
## Step 11 — the canary stayed quiet
|
||||
|
||||
`grep -ci "frame size|frame-size|maximum-frame"` = **0** on all six nodes, as designed: the deploy
|
||||
notify carries only `DeploymentId` + `RevisionHash` + `CorrelationId`.
|
||||
|
||||
---
|
||||
|
||||
## Finding A — a 10–30 s post-restart delivery window, **pre-existing, not Phase 2**
|
||||
|
||||
A deploy issued shortly after a driver node restarts does not reach that node, and the deployment
|
||||
fails at the 2-minute deadline naming it. Measured against `site-a-1`:
|
||||
|
||||
| Delay after the node logs readiness | Result |
|
||||
|---|---|
|
||||
| ~4 s | **missed** (`f738e1de` TimedOut, node row `Applying`) |
|
||||
| 10 s | **missed** (`0c7a6296` TimedOut) |
|
||||
| 30 s | applied, sealed |
|
||||
|
||||
**The DPS control fails identically.** Flipping the rig back to `Mode=Dps` and repeating the 10 s
|
||||
case produced `e85f53c5` → TimedOut, `site-a-1` unacked. So the window is a property of cluster
|
||||
membership convergence, not of the transport this phase introduces. Recorded rather than fixed.
|
||||
|
||||
> Running that control is the only reason this is a note and not a Phase 2 regression. The
|
||||
> temptation to skip it — "obviously the new transport broke it" — would have produced a false
|
||||
> attribution and a wasted investigation.
|
||||
|
||||
## Finding B — **`buffer-size = 0` does not deliver what this phase's docs claim it does**
|
||||
|
||||
The stated rationale, in the plan, `CLAUDE.md`, and `docs/Configuration.md`, is that buffering
|
||||
*"would replay a `DispatchDeployment` on reconnect and apply a deployment the coordinator already
|
||||
sealed as `TimedOut`, leaving the node running a configuration the database records as failed."*
|
||||
|
||||
**That outcome happens anyway, by a different route.** Deployment `0c7a6296` timed out at 22:14:30
|
||||
with `site-a-1` unacked. The node's row:
|
||||
|
||||
```
|
||||
NodeId Status StartedAtUtc AppliedAtUtc
|
||||
site-a-1:4053 1 2026-07-22 22:12:30.145 2026-07-22 22:15:14.823
|
||||
```
|
||||
|
||||
It applied **44 s after the deployment was sealed TimedOut**, on its next restart. The mechanism is
|
||||
`DriverHostActor.Bootstrap()`, which reads the node's most recent `NodeDeploymentState`, finds the
|
||||
orphan `Applying` row the coordinator pre-created at dispatch, and replays it:
|
||||
|
||||
```csharp
|
||||
case NodeDeploymentStatus.Applying:
|
||||
_log.Warning("DriverHost {Node}: found orphan Applying row for deployment {Id}; replaying", …);
|
||||
ApplyAndAck(new DeploymentId(latest.DeploymentId), revision.Value, CorrelationId.NewId());
|
||||
```
|
||||
|
||||
So `buffer-size = 0` removes the **transport-level, timer-driven** replay — a real difference: it is
|
||||
silent, unbounded in time, and can fire without any node event. The **boot-time DB replay remains**,
|
||||
and it is the better-behaved of the two (deliberate, Warning-logged, and bounded to a restart). The
|
||||
decision stands; the *justification* was overstated and the docs are corrected in the same change as
|
||||
this record.
|
||||
|
||||
## Finding C — stopping **both** centrals strands the non-seed nodes permanently
|
||||
|
||||
After `docker stop` of both centrals followed by `docker start`, the fleet did **not** recover.
|
||||
Deployments reached 2/6 then 3/6 nodes. The cause is not the transport — the Akka cluster itself had
|
||||
split:
|
||||
|
||||
```
|
||||
Cluster hosts: MEMBERS 3 UP 3 UNREACHABLE 0
|
||||
central-1:4053 UP admin,driver leader for admin+driver
|
||||
central-2:4053 UP admin,driver
|
||||
site-a-1:4053 UP driver
|
||||
```
|
||||
|
||||
The three site nodes that never restarted were **not members at all**, and their own gossip showed
|
||||
the stale inverse view — `central-2` as `Exiting`, `site-a-1` as `Leaving` — frozen since 22:29:
|
||||
|
||||
```
|
||||
Gossip(members = [central-2 status = Exiting, site-a-1 status = Leaving,
|
||||
site-a-2 Up, site-b-1 Up, site-b-2 Up])
|
||||
```
|
||||
|
||||
A graceful stop puts the centrals into `Leaving`/`Exiting`; the processes died before the removal
|
||||
handshake completed, leaving the survivors holding members that will never confirm. `auto-down`
|
||||
does not help — these members are not *Unreachable*, they are stuck mid-`Exiting`. The restarted
|
||||
centrals formed a fresh cluster (with `site-a-1`, which had also restarted and re-joined through its
|
||||
seed), and the three stranded nodes had no path back: they believe they are already in a cluster, so
|
||||
they never re-run the join process.
|
||||
|
||||
**Recovery required restarting the stranded nodes.** After that, the next deploy sealed 6/6
|
||||
(`e20836ac`).
|
||||
|
||||
This is transport-independent — membership is below both DPS and ClusterClient, and the set of nodes
|
||||
that acked matched the cluster view exactly. It is a **downing/rejoin gap of the current single-mesh
|
||||
topology**, in the same family as the registered outage gap the design doc already records, and it
|
||||
belongs to the Phase 7 drill work. Under the target per-pair topology (every node a seed of its own
|
||||
pair) the shape changes, and this drill must be re-run there rather than assumed fixed.
|
||||
|
||||
## Finding D — the AdminUI's driver Restart/Reconnect buttons have no page
|
||||
|
||||
Step 6 could not be run, and not for a rig reason.
|
||||
`Components/Shared/Drivers/DriverStatusPanel.razor` is the **only** component in the codebase
|
||||
carrying the `DriverOperator`-gated Reconnect/Restart buttons, and **nothing renders it** — `grep -rn
|
||||
"<DriverStatusPanel"` across `src/` returns nothing, and no `@page` route under `Components/Pages/`
|
||||
hosts it. It was orphaned when v3 Batch 2 retired the routed `/clusters/{id}/drivers` flow.
|
||||
|
||||
So the `driver-control` channel has no operator surface **on either transport** — this is a
|
||||
pre-existing v3 regression that Phase 2 merely surfaced. Filed as a follow-up; not fixed here.
|
||||
|
||||
**Residual risk for the channel:** central-side dispatch is sabotage-verified by
|
||||
`MeshRouterDispatchTests` (all four `AdminOperationsActor` call sites go through one `Dispatch`
|
||||
helper); node-side `EventStream` receipt by `DriverHostActor` is proven **live** by this gate, since
|
||||
`DispatchDeployment` arrives through the identical `PublishLocally` path. What is unproven live is
|
||||
only the composition for those two message types.
|
||||
|
||||
Step 7 (alarm ack) was likewise not run: the rig has no scripted alarms configured
|
||||
(`ScriptedAlarmHost: applying (enabled=0/0, depRefs=0)`) and `/alerts` is empty, so there is nothing
|
||||
to acknowledge. Authoring a scripted alarm end-to-end was judged out of proportion to the residual
|
||||
risk above. `ScriptedAlarmHostActor`'s `AlarmCommand` EventStream subscription therefore rests on its
|
||||
(sabotage-verified) unit test.
|
||||
|
||||
## Finding E — the central comm actor is silent on the success path
|
||||
|
||||
`HandleApplyAck` logs only on failure. During this gate that made "did the ack reach the surviving
|
||||
central?" unanswerable from logs, which is exactly the question an operator asks during a failover.
|
||||
A Debug line on ack receipt would have made step 8 directly observable. Not added during the gate —
|
||||
adding code to make a gate step pass is the wrong order — but recorded as a follow-up.
|
||||
|
||||
---
|
||||
|
||||
## What this gate did not cover
|
||||
|
||||
- **Steps 6 and 7** — see findings D and E; no surface exists for driver-control, no alarms for ack.
|
||||
- **The `alarm-commands` node-side leg** under a real transport, for the reason above.
|
||||
- **Split meshes.** Everything here ran on the single mesh. The contact set was fleet-wide (6 points,
|
||||
one client), which is the correct and only shape until Phase 6 splits the meshes.
|
||||
- **A network partition** (as opposed to process stops). `auto-down`'s dual-active trade is untested
|
||||
here and remains Phase 7's.
|
||||
@@ -0,0 +1,473 @@
|
||||
# Per-Cluster Mesh Phase 3 — Config Fetch-and-Cache from Central
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** In a config-flagged `FetchAndCache` mode, a driver node obtains its deployed configuration by **fetching the artifact bytes from central over a gRPC stream**, caching them in LocalDb, and reading **all** config from that cache — so the driver never reads `Deployment.ArtifactBlob` from central SQL. Default `Direct` mode is byte-for-byte today's behaviour.
|
||||
|
||||
**Architecture:** Central (admin role) hosts a new gRPC service `DeploymentArtifactService.Fetch(deployment_id) → stream ArtifactChunk`, on a **dedicated h2c Kestrel listener** (the LocalDb-sync listener pattern), gated by a **shared bearer key** (fail-closed, `FixedTimeEquals`, path-scoped interceptor). A driver node in `FetchAndCache` mode, on `DispatchDeployment`, streams the bytes, verifies `SHA-256(bytes) == RevisionHash`, writes them through the existing `IDeploymentArtifactCache`, and applies from the cache; every config read (`ReconcileDrivers`, `PushDesiredSubscriptions`, address-space rebuild, `Bootstrap` recovery, Stale recovery) is redirected to the cache. The #485 "empty/unreadable bytes ⇒ no answer, keep last-known-good, fail the apply" guard is carried onto the fetch and cache-read paths.
|
||||
|
||||
**Tech Stack:** .NET 10, Akka.NET 1.5.62, `Grpc.AspNetCore` (server, already referenced) + **`Grpc.Tools` + a first in-repo `.proto`** (new — the repo has consumed packaged gRPC clients but never compiled a proto locally), `Grpc.Net.Client` (node client), the Phase-1 `ZB.MOM.WW.LocalDb` cache (`IDeploymentArtifactCache` / `LocalDbDeploymentArtifactCache`), xUnit + Shouldly + Akka.TestKit.
|
||||
|
||||
---
|
||||
|
||||
## Decisions already made (do not re-litigate)
|
||||
|
||||
| Decision | Choice | Consequence for this plan |
|
||||
|---|---|---|
|
||||
| Fetch transport | **gRPC fetch RPC** (user chose this over token-gated HTTP) | Introduces the repo's first `.proto` + `Grpc.Tools` codegen (Task 1); a new central-side h2c listener (Task 3); a generated client on the node (Task 4). |
|
||||
| Fetch auth | **Shared node key** (user chose this over a per-deployment token) | A single bearer secret (`ConfigServe:ApiKey` == `ConfigSource:ApiKey`), `FixedTimeEquals`, fail-closed. **No migration, no token lifecycle, and `DispatchDeployment` is UNCHANGED** — the node fetches by deployment id and authenticates with the shared key. |
|
||||
| Cutover | **Config-flagged dark switch** (`ConfigSource:Mode` = `Direct` default \| `FetchAndCache`), per node | Matches Phase 2. Rig comes up on `Direct`; the live gate flips only the **site** nodes to `FetchAndCache` (central keeps SQL, models the target). Rollback is a config change. |
|
||||
| Both nodes fetch | **Yes — no Primary gating of the fetch** | Today both pair nodes read central SQL independently (no gating); `FetchAndCache` keeps that shape. `IDeploymentArtifactCache.StoreAsync` is idempotent (`IsAlreadyCachedAsync`), and pair replication means a node that missed its fetch can still get the bytes from the peer. Primary-only fetch is a possible later optimization, explicitly out of scope. |
|
||||
|
||||
## Five facts the recon nailed down (read before Task 5)
|
||||
|
||||
1. **`RevisionHash == SHA-256(artifact bytes).`** `ConfigComposer.SnapshotAndFlattenAsync` (`src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs:57-58`) computes the revision hash as SHA-256-hex-lowercase over the exact serialized blob — the **same** value the cache stores as `deployment_pointer.artifact_sha256`. So the node can verify a fetch against the `RevisionHash` already carried in `DispatchDeployment`; **no hash field is needed in the proto.**
|
||||
|
||||
2. **The driver reads central SQL on exactly these config paths, all reading `Deployment.ArtifactBlob` (never re-composing from raw rows):** `Bootstrap()` (`DriverHostActor.cs:584-603`, reads `NodeDeploymentStates` + `Deployments.RevisionHash`), `ReconcileDrivers` (`1717-1768`, reads at `1722-1726`), `PushDesiredSubscriptions` (`1880-1898`, reads at `1885-1889`), `TryRecoverFromStale` (`2363-2390`, reads at `2367-2373`), and the delegated `OpcUaPublishActor.HandleRebuild` → `LoadArtifact`/`LoadLatestArtifact` (`OpcUaPublishActor.cs:475-510`) which fires **only when `ApplyAndAck` passes no blob** (`DriverHostActor.cs:1667`). Phase 3 redirects these in `FetchAndCache` mode.
|
||||
|
||||
3. **The cache is already complete and correct.** `IDeploymentArtifactCache` / `LocalDbDeploymentArtifactCache` (`src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Deployment/`) does chunked base64 (128 KiB), SHA-256-verified reassembly, newest-2 retention, and replicates. It is written by `CacheAppliedArtifact` (`DriverHostActor.cs:1796-1835`) after a successful apply, and read only by `TryBootFromCache` (`671-714`) in the `Bootstrap()` SQL-unreachable catch. **Phase 3 changes WHEN it is written (before apply, from the fetch) and WHEN it is read (always, in `FetchAndCache`).** The `_isRunningFromCache` flag (field `85`, set at `698`, cleared at `1676`) already exists.
|
||||
|
||||
4. **`DispatchDeployment` is payload-free by design** (`DeploymentId` + `RevisionHash` + `CorrelationId`) so it never approaches the Akka frame limit. **Phase 3 keeps it payload-free** — the shared-key decision means no token rides in it, and the bytes travel out-of-band over gRPC. Do not add the artifact to any Akka message.
|
||||
|
||||
5. **The #485 guard is at every byte-parsing seam and must be carried onto the new paths:** `ReconcileDrivers` returns `null` on `blob.Length == 0` (`1735-1747`) → `ApplyAndAck` treats null as an apply **failure** (does not advance `_currentRevision`, writes `Failed`, sends a Failed ack — so a re-dispatch of the same revision actually retries, `1639-1659`); `CacheAppliedArtifact` skips empty (`1801-1808`); `PushDesiredSubscriptionsFromArtifact` skips empty (`1908-1920`); `OpcUaPublishActor.HandleRebuild` returns on `{ Length: 0 }` keeping the served address space (`372-391`); the cache's `ReassembleAsync` returns null on any chunk-count/base64/SHA/timestamp failure. **A zero-byte or SHA-mismatched fetch is "no answer," never "a config with no drivers."**
|
||||
|
||||
---
|
||||
|
||||
## The dark switch, precisely
|
||||
|
||||
`ConfigSourceOptions.Mode` selects the driver's config source:
|
||||
|
||||
- **`Direct`** (default): every path reads central SQL exactly as today. The new fetcher is not invoked. Byte-for-byte no behaviour change; the whole phase is inert.
|
||||
- **`FetchAndCache`**: on `DispatchDeployment`, fetch bytes from central over gRPC → verify SHA-256 == `RevisionHash` → `StoreAsync` into the cache → apply **from the fetched bytes in hand** (passed through to every consumer, so `OpcUaPublishActor` never reads SQL). `Bootstrap()` recovers the last-applied deployment from the LocalDb **pointer**, not `NodeDeploymentStates`. No driver-side `Deployment.ArtifactBlob` read occurs.
|
||||
|
||||
The central-side serve components (gRPC service, listener, interceptor) are wired **unconditionally on admin-role nodes** whenever `ConfigServe:GrpcListenPort > 0`, in both modes — mirroring Phase 2's "register in both modes so flipping the flag is not a redeploy." An idle listener costs nothing.
|
||||
|
||||
**Explicitly deferred to Phase 4 (do NOT touch here):** the `NodeDeploymentState` **write** (`UpsertNodeDeploymentState`, `DriverHostActor.cs:2392-2422`, still writes the ack row to central SQL), `DbHealthProbeActor`, `EfAlarmConditionStateStore`, and removing the driver-role ConfigDb connection string. Phase 3 removes the config-blob **reads** only. A `FetchAndCache` node still writes its ack row to SQL; that is Phase 4's cut.
|
||||
|
||||
---
|
||||
|
||||
## Task 0: `ConfigSourceOptions` + `ConfigServeOptions` + validators + registration
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 1
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ConfigSourceOptions.cs`
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ConfigSourceOptionsValidator.cs`
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs` (register both in `AddOtOpcUaCluster`, beside the `MeshTransportOptions` registration)
|
||||
- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ConfigSourceOptionsValidatorTests.cs`
|
||||
|
||||
**Design.** One options class holds both the node (fetch) and central (serve) surfaces, keyed `ConfigSource` and `ConfigServe`. Model after `MeshTransportOptions` / `MeshTransportOptionsValidator` (same directory) exactly — the validator fails host start on a `FetchAndCache` shape that would leave the node unable to fetch (empty endpoints, empty key), because every such fault otherwise surfaces as a silent absence (a deploy that never applies).
|
||||
|
||||
```csharp
|
||||
// ConfigSourceOptions.cs
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>Node-side config source selection (per-cluster mesh Phase 3).</summary>
|
||||
public sealed class ConfigSourceOptions
|
||||
{
|
||||
public const string SectionName = "ConfigSource";
|
||||
public const string ModeDirect = "Direct";
|
||||
public const string ModeFetchAndCache = "FetchAndCache";
|
||||
|
||||
/// <summary><see cref="ModeDirect"/> (read central SQL, today's behaviour) or
|
||||
/// <see cref="ModeFetchAndCache"/> (fetch from central over gRPC, read LocalDb).</summary>
|
||||
public string Mode { get; set; } = ModeDirect;
|
||||
|
||||
/// <summary>Central artifact-gRPC base addresses, e.g. <c>http://central-1:4055</c>. h2c (http
|
||||
/// scheme). Required and tried in order (failover) under <see cref="ModeFetchAndCache"/>.</summary>
|
||||
public string[] CentralFetchEndpoints { get; set; } = [];
|
||||
|
||||
/// <summary>Shared bearer key; must equal central's <see cref="ConfigServeOptions.ApiKey"/>.
|
||||
/// Supply via env <c>ConfigSource__ApiKey</c>; never commit. Required under FetchAndCache.</summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Per-fetch deadline. Non-positive rejected under FetchAndCache.</summary>
|
||||
public int FetchTimeoutSeconds { get; set; } = 30;
|
||||
}
|
||||
|
||||
/// <summary>Central-side artifact-serve surface (per-cluster mesh Phase 3).</summary>
|
||||
public sealed class ConfigServeOptions
|
||||
{
|
||||
public const string SectionName = "ConfigServe";
|
||||
|
||||
/// <summary>Dedicated h2c listener port for the artifact gRPC service. <c>0</c> = disabled
|
||||
/// (nothing bound). Must differ from the main HTTP port and from LocalDb:SyncListenPort.</summary>
|
||||
public int GrpcListenPort { get; set; }
|
||||
|
||||
/// <summary>Shared bearer key the interceptor checks (FixedTimeEquals, fail-closed). Supply via
|
||||
/// env <c>ConfigServe__ApiKey</c>; never commit.</summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
}
|
||||
```
|
||||
|
||||
The validator (mirror `MeshTransportOptionsValidator`): reject an unknown `Mode`; under `FetchAndCache` reject empty `CentralFetchEndpoints`, any endpoint not starting `http://` or `https://`, empty `ApiKey`, and non-positive `FetchTimeoutSeconds`. `ConfigServeOptions` needs no cross-field validation (a `0` port is a legitimate "disabled").
|
||||
|
||||
**Step 1: Write the failing test** — `ConfigSourceOptionsValidatorTests` with cases: default (`Direct`, empty everything) → Success; unknown mode → Fail; `FetchAndCache` + empty endpoints → Fail; + non-`http(s)` endpoint → Fail; + empty key → Fail; + `FetchTimeoutSeconds = 0` → Fail; fully-populated `FetchAndCache` → Success. (No implicit usings in Cluster.Tests — `using Xunit;`.)
|
||||
|
||||
**Step 2:** `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests --filter "FullyQualifiedName~ConfigSourceOptionsValidatorTests"` → FAIL (types missing).
|
||||
|
||||
**Step 3:** Write the two options classes + the validator; register in `AddOtOpcUaCluster`:
|
||||
```csharp
|
||||
services.AddValidatedOptions<ConfigSourceOptions, ConfigSourceOptionsValidator>(
|
||||
configuration, ConfigSourceOptions.SectionName);
|
||||
services.Configure<ConfigServeOptions>(configuration.GetSection(ConfigServeOptions.SectionName));
|
||||
```
|
||||
|
||||
**Step 4:** Re-run → PASS. **Sabotage-check:** flip one validator branch (e.g. accept empty endpoints) → its test reddens.
|
||||
|
||||
**Step 5: Commit** `feat(mesh): ConfigSource/ConfigServe options + validator (Phase 3 dark switch)`
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `deployment_artifact.v1` proto + Grpc.Tools codegen
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 0
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/deployment_artifact.proto`
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj` (add `Grpc.Tools` + `Google.Protobuf` + `Grpc.Core.Api` package refs and a `<Protobuf>` item; `GrpcServices="Both"` so both server base + client stub generate)
|
||||
- Modify: `Directory.Packages.props` (pin `Grpc.Tools`, `Google.Protobuf`, `Grpc.Core.Api` if not already pinned — check first)
|
||||
- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests/DeploymentArtifactProtoTests.cs` (a compile-touch test that references the generated `DeploymentArtifactService.DeploymentArtifactServiceBase` type and `FetchRequest`, proving codegen ran)
|
||||
|
||||
**Why Commons.** Both the central server (in Host/AdminUI) and the node client (in Runtime) need the generated types, and Commons is already referenced by every server project. This is the repo's **first** locally-compiled proto — there is no existing `<Protobuf>` item to copy, so wire it from scratch.
|
||||
|
||||
```proto
|
||||
syntax = "proto3";
|
||||
package deployment_artifact.v1;
|
||||
option csharp_namespace = "ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1";
|
||||
|
||||
// Central serves the deployed-configuration artifact bytes to a driver node (Phase 3).
|
||||
service DeploymentArtifactService {
|
||||
// Streams the artifact for a sealed deployment as ordered chunks. NotFound if the id is
|
||||
// unknown or not sealed. The client verifies SHA-256(reassembled) == the RevisionHash it
|
||||
// already holds, so no hash travels here.
|
||||
rpc Fetch(FetchRequest) returns (stream ArtifactChunk);
|
||||
}
|
||||
message FetchRequest { string deployment_id = 1; }
|
||||
message ArtifactChunk { bytes data = 1; }
|
||||
```
|
||||
|
||||
`.csproj` addition (mirror the packaging note the LocalDb refs use):
|
||||
```xml
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Grpc.Tools" PrivateAssets="all" />
|
||||
<PackageReference Include="Google.Protobuf" />
|
||||
<PackageReference Include="Grpc.Core.Api" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Protobuf Include="Protos\deployment_artifact.proto" GrpcServices="Both" />
|
||||
</ItemGroup>
|
||||
```
|
||||
|
||||
**Step 1:** Write the compile-touch test referencing `DeploymentArtifactService.DeploymentArtifactServiceBase` and `new FetchRequest { DeploymentId = "x" }`.
|
||||
**Step 2:** `dotnet build src/Core/ZB.MOM.WW.OtOpcUa.Commons` → FAIL (no proto/codegen).
|
||||
**Step 3:** Add the packages, the proto, the `<Protobuf>` item. `dotnet restore` then `dotnet build src/Core/ZB.MOM.WW.OtOpcUa.Commons`.
|
||||
**Step 4:** `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests --filter "FullyQualifiedName~DeploymentArtifactProtoTests"` → PASS. Confirm generated types resolve (`obj/.../DeploymentArtifactGrpc.cs`).
|
||||
**Step 5: Commit** `feat(mesh): deployment_artifact.v1 proto + first in-repo Grpc.Tools codegen`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Central-side `DeploymentArtifactService` (streams the blob, existence-hiding)
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 4
|
||||
**Blocked by:** Task 1
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Grpc/DeploymentArtifactService.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Grpc/DeploymentArtifactServiceTests.cs`
|
||||
|
||||
**Design.** Subclass the generated `DeploymentArtifactServiceBase`. `Fetch` reads the `Deployment` row by id from `OtOpcUaConfigDbContext` (admin node has the connection); if the row is missing OR `Status != Sealed` OR `ArtifactBlob.Length == 0`, throw `RpcException(new Status(StatusCode.NotFound, "…"))` (existence-hiding + #485: never stream zero bytes as if valid). Otherwise stream the blob in ≤128 KiB `ArtifactChunk`s via `responseStream.WriteAsync`. Lives in AdminUI (where the other admin-only endpoints + the config DB context live).
|
||||
|
||||
```csharp
|
||||
public sealed class DeploymentArtifactService : DeploymentArtifactServiceBase
|
||||
{
|
||||
private const int ChunkSize = 128 * 1024; // matches LocalDbDeploymentArtifactCache.ChunkSize
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly ILogger<DeploymentArtifactService> _log;
|
||||
// ctor injects both
|
||||
|
||||
public override async Task Fetch(
|
||||
FetchRequest request, IServerStreamWriter<ArtifactChunk> responseStream, ServerCallContext context)
|
||||
{
|
||||
if (!Guid.TryParse(request.DeploymentId, out var id))
|
||||
throw new RpcException(new Status(StatusCode.NotFound, "unknown deployment"));
|
||||
|
||||
await using var db = await _dbFactory.CreateDbContextAsync(context.CancellationToken);
|
||||
var row = await db.Deployments.AsNoTracking()
|
||||
.Where(d => d.DeploymentId == id)
|
||||
.Select(d => new { d.Status, d.ArtifactBlob })
|
||||
.FirstOrDefaultAsync(context.CancellationToken);
|
||||
|
||||
// Existence-hiding + #485: unknown / not-sealed / empty are one indistinguishable NotFound.
|
||||
if (row is null || row.Status != DeploymentStatus.Sealed || row.ArtifactBlob.Length == 0)
|
||||
throw new RpcException(new Status(StatusCode.NotFound, "unknown deployment"));
|
||||
|
||||
for (var offset = 0; offset < row.ArtifactBlob.Length; offset += ChunkSize)
|
||||
{
|
||||
var len = Math.Min(ChunkSize, row.ArtifactBlob.Length - offset);
|
||||
await responseStream.WriteAsync(
|
||||
new ArtifactChunk { Data = ByteString.CopyFrom(row.ArtifactBlob, offset, len) },
|
||||
context.CancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
(Confirm the sealed-status enum member name from `DeploymentStatus`; the recon showed `Deployment.Status` and the deploy sets `Sealed`.)
|
||||
|
||||
**Step 1: Write the failing test** — an in-memory `OtOpcUaConfigDbContext` (the AdminUI.Tests pattern) seeded with one sealed deployment; a fake `IServerStreamWriter<ArtifactChunk>` collecting writes; assert: (a) a sealed non-empty blob streams back and reassembles byte-equal, chunk-bounded at 128 KiB (seed a ~300 KB blob → 3 chunks); (b) an unknown id throws `RpcException` NotFound; (c) a non-sealed row throws NotFound; (d) a zero-length blob throws NotFound (the #485 serve-side guard).
|
||||
**Step 2:** run → FAIL.
|
||||
**Step 3:** implement.
|
||||
**Step 4:** run → PASS. **Sabotage:** drop the `row.Status != Sealed` check → the non-sealed test reddens; drop the `ArtifactBlob.Length == 0` check → the zero-length test reddens.
|
||||
**Step 5: Commit** `feat(mesh): central DeploymentArtifactService — stream the sealed artifact, NotFound-hide the rest`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Central serve auth interceptor + dedicated h2c listener wiring
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none
|
||||
**Blocked by:** Task 0, Task 2
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ConfigServeAuthInterceptor.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs` (register the interceptor with `AddGrpc`; add the dedicated h2c listener block mirroring the LocalDb sync block at `407-476`; `MapGrpcService<DeploymentArtifactService>()` inside a `hasAdmin && configServeGrpcPort > 0` guard beside `514`)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ConfigServeAuthInterceptorTests.cs`
|
||||
|
||||
**Interceptor** — mirror `LocalDbSyncAuthInterceptor` exactly, scoped by service path `"/deployment_artifact.v1.DeploymentArtifactService/"`, fail-closed (no `ConfigServe:ApiKey` ⇒ reject ALL with `Unauthenticated`), `Authorization: Bearer <key>`, `FixedTimeEquals`. Calls to other services pass through untouched (so it can share `AddGrpc` with the LocalDb sync interceptor).
|
||||
|
||||
**Listener** — the artifact service is h2c and cannot share the cleartext HTTP/1 port (same reason the LocalDb sync listener is dedicated). Add, right after the LocalDb sync listener block:
|
||||
```csharp
|
||||
var configServeGrpcPort = builder.Configuration.GetValue<int>("ConfigServe:GrpcListenPort");
|
||||
if (hasAdmin && configServeGrpcPort > 0)
|
||||
{
|
||||
// Same dedicated-h2c-listener dance as the LocalDb sync listener above: h2c can't negotiate on a
|
||||
// cleartext Http1AndHttp2 port, so bind a dedicated HTTP/2-only port and re-apply existing bindings.
|
||||
// (Refactor the LocalDb block's existing-binding computation into a shared local if both run — a
|
||||
// node that is admin+driver with BOTH ports set must re-apply existing bindings once and add both.)
|
||||
builder.WebHost.ConfigureKestrel(kestrel =>
|
||||
kestrel.ListenAnyIP(configServeGrpcPort, o => o.Protocols = HttpProtocols.Http2));
|
||||
Log.Information("Config-serve artifact gRPC listener bound on :{Port} (h2c).", configServeGrpcPort);
|
||||
}
|
||||
```
|
||||
> **Load-bearing interaction with the LocalDb listener.** A fused central node is admin **and** driver; on the rig it may have `LocalDb:SyncListenPort` set too. `ConfigureKestrel` is additive across calls, but the LocalDb block re-applies `existingBindings` (URLS/HTTP_PORTS) and this block must NOT clobber that. Implement by computing `existingBindings` once and applying it once, then adding whichever of the two dedicated h2c ports are configured. The test below asserts the AdminUI HTTP port still answers when both dedicated ports are set.
|
||||
|
||||
`MapGrpcService`:
|
||||
```csharp
|
||||
if (hasAdmin && configServeGrpcPort > 0)
|
||||
app.MapGrpcService<DeploymentArtifactService>();
|
||||
```
|
||||
|
||||
**Step 1: Write the failing test** — boot the Host (or a minimal `WebApplicationFactory` with `hasAdmin`, `ConfigServe:GrpcListenPort` set, `ConfigServe:ApiKey=k`, a seeded sealed deployment): a `Grpc.Net.Client` channel to the dedicated port with `Authorization: Bearer k` fetches + reassembles the blob; **no header** ⇒ `Unauthenticated`; **wrong key** ⇒ `Unauthenticated`; and the AdminUI HTTP surface (`/health/active`) still returns 200 (listener coexistence). Use `TestContext.Current.CancellationToken`.
|
||||
**Step 2:** run → FAIL.
|
||||
**Step 3:** implement interceptor + Program.cs wiring.
|
||||
**Step 4:** run → PASS. **Sabotage:** make the interceptor return without checking when the key is unset (i.e. break fail-closed) → the no-header test reddens; break the listener coexistence (apply only this block's binding) → the `/health/active` assertion reddens.
|
||||
**Step 5: Commit** `feat(mesh): config-serve auth interceptor + dedicated h2c listener (fail-closed bearer)`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Node-side `IDeploymentArtifactFetcher` + gRPC client (SHA-verified, failover)
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 2
|
||||
**Blocked by:** Task 1
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Deployment/IDeploymentArtifactFetcher.cs`
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Deployment/GrpcDeploymentArtifactFetcher.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Deployment/GrpcDeploymentArtifactFetcherTests.cs`
|
||||
|
||||
**Design.**
|
||||
```csharp
|
||||
public interface IDeploymentArtifactFetcher
|
||||
{
|
||||
/// <summary>Fetches + reassembles the artifact for <paramref name="deploymentId"/>, verifying
|
||||
/// SHA-256(bytes) == <paramref name="expectedRevisionHash"/>. Returns the bytes, or <see
|
||||
/// langword="null"/> on any failure (all endpoints unreachable, NotFound, SHA mismatch,
|
||||
/// zero-length) — a null is "no answer," NEVER an empty config (#485).</summary>
|
||||
Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct);
|
||||
}
|
||||
```
|
||||
`GrpcDeploymentArtifactFetcher` takes `IOptions<ConfigSourceOptions>`. For each endpoint in order: open a `GrpcChannel` (h2c: `new HttpClientHandler` / `GrpcChannelOptions` with the `http://` address), attach `Authorization: Bearer {ApiKey}` via a `CallCredentials` or a request header on the call, stream `Fetch`, accumulate `chunk.Data` into a buffer, then `SHA-256(buffer)` hex-lowercase and compare to `expectedRevisionHash`. On success return the bytes. On `RpcException` (NotFound / Unavailable / deadline), log and try the next endpoint. If all fail, or the reassembled buffer is empty, or the SHA mismatches, return `null` (and log a Warning naming the reason). Deadline = `FetchTimeoutSeconds`.
|
||||
|
||||
> **Why return null, not throw.** The caller (Task 5) treats null identically to `ReconcileDrivers` returning null today: apply failure, keep last-known-good, do not advance the revision. Throwing would risk an unhandled actor message; a typed "no bytes" keeps the #485 contract explicit.
|
||||
|
||||
**Seam for the test.** So the test needs no real second Host, make the gRPC call site injectable: a `Func<string /*endpoint*/, DeploymentArtifactService.DeploymentArtifactServiceClient>` factory defaulted to the real channel builder, overridable in tests with an in-memory client backed by a fake that streams canned chunks. (This mirrors Phase 2's `IMeshClusterClientFactory` seam.)
|
||||
|
||||
**Step 1: Write the failing test** — with a fake client factory: (a) a client streaming chunks whose reassembly hashes to `expectedRevisionHash` → returns the exact bytes; (b) chunks whose hash ≠ expected → returns null; (c) an empty stream → returns null; (d) first endpoint throws `RpcException(Unavailable)`, second streams good bytes → returns the bytes (failover); (e) all endpoints throw → null. Assert the failover case actually consulted endpoint 2 (record calls).
|
||||
**Step 2:** run → FAIL.
|
||||
**Step 3:** implement.
|
||||
**Step 4:** run → PASS. **Sabotage:** drop the SHA-256 comparison → the mismatch test reddens; break failover (return after the first endpoint's exception) → the failover test reddens.
|
||||
**Step 5: Commit** `feat(mesh): node gRPC artifact fetcher — SHA-verified reassembly, endpoint failover, null-on-any-failure`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Wire fetch-and-cache into the apply path (FetchAndCache mode)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none
|
||||
**Blocked by:** Task 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs` (add `_configSourceMode` + `_artifactFetcher` fields + ctor params + both `Props` overloads; a new `FetchThenApply` path used from `HandleDispatchFromSteady` / `ApplyAndAck` when `FetchAndCache`)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs` (resolve `IOptions<ConfigSourceOptions>` + register `IDeploymentArtifactFetcher` in the `hasDriver` branch; pass both into `DriverHostActor.Props`)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/...` (DI: `services.AddSingleton<IDeploymentArtifactFetcher, GrpcDeploymentArtifactFetcher>()` in the driver branch — place beside the `IDeploymentArtifactCache` registration)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorFetchAndCacheTests.cs`
|
||||
|
||||
**Design.** In `FetchAndCache` mode, on a `DispatchDeployment` whose revision differs from `_currentRevision`:
|
||||
1. `bytes = await _artifactFetcher.FetchAsync(deploymentId, revisionHash, ct)` (message-loop-safe: run via `PipeTo` to self as a `FetchResult(deploymentId, revisionHash, correlation, bytes?)`, NOT a blocking await inside the receive).
|
||||
2. On `bytes == null` → **apply failure**: do not advance `_currentRevision`, `UpsertNodeDeploymentState(Failed)`, `SendAck(Failed)` — identical to today's `ReconcileDrivers`-returned-null path (`1639-1659`). The node keeps serving last-known-good. A re-dispatch retries (revision still not current).
|
||||
3. On success → `StoreAsync(clusterId, deploymentId, revisionHash, bytes)` into the cache, then apply **from `bytes` in hand**: `ReconcileDriversFromBlob(bytes)`, `RebuildAddressSpace(correlation, deploymentId, bytes)` (pass the blob so `OpcUaPublishActor` never reads SQL), `PushDesiredSubscriptionsFromArtifact(bytes)`, `UpsertNodeDeploymentState(Applied)`, `SendAck(Applied)`, `_isRunningFromCache = false`, set `_currentRevision`.
|
||||
|
||||
**Refactor note — do NOT duplicate reconcile logic.** `ReconcileDrivers(DeploymentId)` today reads the blob itself (`1722-1726`). Extract the post-read body into `ReconcileDriversFromBlob(byte[] blob)` (the existing #485 `blob.Length == 0` guard at `1735-1747` moves into it verbatim), and have the `Direct`-mode `ReconcileDrivers` read-then-call it. Then `FetchThenApply` calls `ReconcileDriversFromBlob(bytes)` directly. Same for `PushDesiredSubscriptions` → it already has `PushDesiredSubscriptionsFromArtifact(blob)` (`1908`); reuse it.
|
||||
|
||||
**Idempotency:** before fetching, check the cache — if `IsAlreadyCached(deploymentId, revisionHash)` (add a cheap `GetCurrentAsync` compare, or a new `bool ContainsAsync`), skip the fetch and apply from cache. This makes a re-dispatch of an already-applied revision a no-op fetch (and covers "peer already replicated it to us").
|
||||
|
||||
**Step 1: Write the failing test** — a `DriverHostActor` in `FetchAndCache` mode with a fake `IDeploymentArtifactFetcher` + a real in-memory `LocalDbDeploymentArtifactCache` (the `DriverHostActorArtifactCacheTests` harness): (a) dispatch → fetcher returns good bytes → node applies, cache written, ack Applied, `_currentRevision` advances, and it **never touched the ConfigDb factory** (inject a throwing `IDbContextFactory` to prove no SQL read on the config path — note the `NodeDeploymentState` write is Phase 4, so use a factory that permits the ack write but fails `Deployments` reads, or assert via a spy that `Deployments` was never queried); (b) fetcher returns null → ack Failed, `_currentRevision` unchanged, children kept; (c) re-dispatch of the applied revision → no fetch (idempotent), immediate Applied ack.
|
||||
**Step 2:** run → FAIL.
|
||||
**Step 3:** implement.
|
||||
**Step 4:** run → PASS. **Sabotage:** make the null-bytes branch advance `_currentRevision` anyway → test (b) reddens (this is the #485 apply-failure contract); remove the idempotency check → test (c) reddens.
|
||||
**Step 5: Commit** `feat(mesh): FetchAndCache apply path — fetch→cache→apply-from-bytes, null=apply-failure`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Redirect all config reads to LocalDb; bootstrap from the pointer
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none
|
||||
**Blocked by:** Task 5
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs` (`Bootstrap`, `TryRecoverFromStale`, and any residual read seams under `FetchAndCache`)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorFetchAndCacheBootstrapTests.cs`
|
||||
|
||||
**Design.** In `FetchAndCache` mode:
|
||||
- **`Bootstrap()`** does NOT read `NodeDeploymentStates`/`Deployments` from SQL. Instead: `cached = await _cache.GetCurrentUnkeyedAsync()`. If non-null → `_currentRevision = RevisionHash.Parse(cached.RevisionHash)`, `Become(Steady)`, `ApplyCachedArtifact(new DeploymentId(cached.DeploymentId), cached.Artifact)` (the existing cache-boot path at `725-748`, which reconciles + rebuilds + resubscribes from the blob and does **not** read SQL or re-ack). If null (fresh node, empty cache) → `Become(Steady)` with no revision; the first `DispatchDeployment` triggers a fetch. On a cache-read exception → log and `Become(Steady)` (no revision) rather than `Stale` (Stale meant "SQL down"; in FetchAndCache there is no SQL config read to be down).
|
||||
- **`TryRecoverFromStale()`** is unreachable in `FetchAndCache` (nothing enters Stale from a config read). Leave it for `Direct` mode; guard its entry so `FetchAndCache` never schedules it.
|
||||
- **`OpcUaPublishActor`** is untouched: `FetchThenApply` and `ApplyCachedArtifact` both pass the blob to `RebuildAddressSpace`, so `LoadArtifact`/`LoadLatestArtifact` (the SQL reads) are never hit in `FetchAndCache`. Add an assertion-comment; no code change there.
|
||||
|
||||
**The one SQL touch that remains (by design, Phase 4's to cut):** `UpsertNodeDeploymentState` still writes the ack row. That is a write, not a config read, and the phase boundary is explicit.
|
||||
|
||||
**Step 1: Write the failing test** — `FetchAndCache` mode, throwing `IDbContextFactory` for `Deployments` reads: (a) a cache pre-seeded with an applied artifact → `Bootstrap` restores served state (drivers reconciled, address space rebuilt) with **no** `Deployments` read; (b) an empty cache → `Bootstrap` lands in Steady-no-revision without error and a subsequent dispatch fetches; (c) a cache-read failure → Steady-no-revision, not Stale. Reuse the `DriverHostActorBootFromCacheTests` harness.
|
||||
**Step 2:** run → FAIL.
|
||||
**Step 3:** implement the mode-gated bootstrap branch.
|
||||
**Step 4:** run → PASS. **Sabotage:** in `FetchAndCache`, make `Bootstrap` fall through to the SQL read → test (a)'s throwing-factory makes it red.
|
||||
**Step 5: Commit** `feat(mesh): FetchAndCache bootstrap from the LocalDb pointer; no driver-side config SQL read`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: #485 guard coverage on the new fetch + cache-read paths
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none
|
||||
**Blocked by:** Task 6
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs` (only if a seam is found unguarded)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorFetchAndCacheUnreadableTests.cs`
|
||||
|
||||
This task adds **no new mechanism** — it proves the #485 contract holds on every `FetchAndCache` seam, and adds a guard only where a test exposes a gap. The contract: zero-length / SHA-mismatched / unreadable ⇒ no answer, keep last-known-good, fail the apply, never tear down to an empty address space.
|
||||
|
||||
**Step 1: Write the failing tests** (expect some to pass already — that is fine; they are the regression net):
|
||||
- Fetcher returns `null` (all endpoints down) mid-steady-state → address space + drivers + subscriptions UNCHANGED, ack Failed, `_currentRevision` unchanged. (Covered by Task 5(b); re-assert the address-space-kept angle explicitly.)
|
||||
- Central serves a **zero-length** blob (shouldn't happen — Task 2 NotFound-hides it — but prove the client also refuses): fetcher gets an empty stream → `null` → apply failure.
|
||||
- Cache holds a **truncated/corrupt** artifact at boot (drop a chunk row / corrupt base64) → `ReassembleAsync` returns null → `Bootstrap` treats it as empty cache (Steady-no-revision), does NOT apply a partial config.
|
||||
- SHA mismatch (server bytes don't match the dispatched `RevisionHash`) → fetcher `null` → apply failure, last-known-good kept.
|
||||
**Step 2:** run → the corrupt-cache and any unguarded case FAIL if a gap exists; otherwise all green (regression net).
|
||||
**Step 3:** add a guard only where a test is red.
|
||||
**Step 4:** run → PASS. **Sabotage:** temporarily make `ReassembleAsync` return partial bytes on a missing chunk → the corrupt-cache test reddens (proving the guard, not the mock, is what holds the line).
|
||||
**Step 5: Commit** `test(mesh): #485 empty/corrupt/mismatch coverage on the FetchAndCache paths`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Real two-Host gRPC fetch boundary test + falsifiability control
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 9
|
||||
**Blocked by:** Task 3, Task 4
|
||||
|
||||
**Files:**
|
||||
- Create: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DeploymentArtifactFetchBoundaryTests.cs`
|
||||
|
||||
The Phase-2 lesson applied to Phase 3: an in-memory fake client proves fetcher **logic**, not that a real gRPC stream crosses a real Kestrel h2c listener through the real interceptor. Stand a real central Host (admin, `ConfigServe:GrpcListenPort` set, `ConfigServe:ApiKey=k`, a seeded sealed deployment) and drive the real `GrpcDeploymentArtifactFetcher` against it.
|
||||
|
||||
Assertions:
|
||||
1. A fetch with the right key + a >256 KB blob reassembles byte-equal and verifies against the deployment's `RevisionHash`.
|
||||
2. **Falsifiability control** — the same fetch with a **wrong key** returns `null` (interceptor rejects; without this control, assertion 1 cannot distinguish "the boundary works" from "something else supplied the bytes"). Verify the control is real by confirming the right-key fetch on the *same* server succeeds.
|
||||
3. An unknown deployment id → `null` (NotFound-hidden).
|
||||
4. Endpoint failover: give the fetcher `[dead-port, real-port]` → it still returns the bytes.
|
||||
|
||||
If irreducibly flaky, quarantine with `[Trait("Category","ArtifactBoundary")]` — do **not** weaken to the in-memory fake (that is Task 4's job).
|
||||
|
||||
**Step 5: Commit** `test(mesh): real two-Host gRPC artifact-fetch boundary test with wrong-key control`
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Rig config + docs
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 8
|
||||
**Blocked by:** Task 5
|
||||
|
||||
**Files:**
|
||||
- Modify: `docker-dev/docker-compose.yml` — central-1/central-2 get `ConfigServe__GrpcListenPort` (e.g. `4055`) + `ConfigServe__ApiKey` (dev value, the committed-dev-secret exception); all six get `ConfigSource__Mode: "${OTOPCUA_CONFIG_MODE:-Direct}"`; site nodes get `ConfigSource__CentralFetchEndpoints__0/1: http://central-1:4055 / http://central-2:4055` + a matching `ConfigSource__ApiKey`. Leave `Mode=Direct` so the rig comes up unchanged; expose the flip via `OTOPCUA_CONFIG_MODE=FetchAndCache`.
|
||||
- Modify: `docs/Configuration.md` — new `ConfigSource` + `ConfigServe` sections (the dark switch, the shared-key auth, the h2c dedicated-port requirement, the env-only key rule).
|
||||
- Modify: `docs/Redundancy.md` — extend the "Command transport" section: config bytes travel out-of-band over the artifact gRPC stream, both pair nodes fetch independently (idempotent + replicated), central serves from SQL.
|
||||
- Modify: `docs/plans/2026-07-22-per-cluster-mesh-program.md` — flip the Phase 3 row + section to reflect the gRPC/shared-key choices.
|
||||
- Modify: `CLAUDE.md` — a short "Config source (Phase 3)" note beside the Mesh command transport note.
|
||||
|
||||
**Step 5: Commit** `docs(mesh): ConfigSource/ConfigServe config, rig flip switch, Phase 3 as shipped`
|
||||
|
||||
---
|
||||
|
||||
## Task 10: Live gate on docker-dev
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min (plus rig time)
|
||||
**Parallelizable with:** none
|
||||
**Blocked by:** Task 8, Task 9
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/plans/2026-07-22-mesh-phase3-live-gate.md`
|
||||
|
||||
Flip only the **site** nodes to `FetchAndCache` (`OTOPCUA_CONFIG_MODE=FetchAndCache` scoped to site services, or a compose override) so central keeps SQL and the rig models the target topology (site nodes have no SQL). Central stays `Direct`. Rig AdminUI login is disabled — drive UI steps via browser automation at `http://localhost:9200`; deploy via `POST /api/deployments` with `X-Api-Key: docker-dev-deploy-key`.
|
||||
|
||||
| # | Step | Pass condition |
|
||||
|---|---|---|
|
||||
| 1 | Rig up all-`Direct`; deploy | Seals green — baseline |
|
||||
| 2 | Flip site nodes to `FetchAndCache`, restart them, deploy | Seals green; **site node logs show a gRPC fetch + cache write + apply-from-bytes**, and **no `Deployments.ArtifactBlob` SQL read on the site side** |
|
||||
| 3 | Confirm central served it | central logs the `DeploymentArtifactService.Fetch` call; site logs the SHA-256 verify pass |
|
||||
| 4 | **Stop central SQL, deploy** (the phase's headline gate) | The site fetch fails (central can't read SQL to serve) → site logs apply-failure, **keeps serving last-known-good**, ack Failed → deploy TimedOut naming the site nodes; **no crash, no empty address space** (#485) |
|
||||
| 5 | Restart SQL, redeploy | Fetch lands, seals green — retry semantics on the new path |
|
||||
| 6 | Restart a site node with a warm cache, central up | Boots from the LocalDb pointer, restores served state, **no `Deployments` read** |
|
||||
| 7 | Restart a site node with central DOWN | Boots last-known-good from cache; no crash loop (config path needs no central at boot) |
|
||||
| 8 | Corrupt one cached artifact chunk on a site node, restart it | Reassembly rejects it → boots Steady-no-revision (empty-cache path), does NOT apply a partial config; next deploy re-fetches |
|
||||
| 9 | Wrong `ConfigSource__ApiKey` on one site node, deploy | That node's fetch is rejected (Unauthenticated) → apply-failure + Warning; the other site node (right key) applies |
|
||||
| 10 | Both site nodes fetch the same deploy | Idempotent — the second store is a no-op (`IsAlreadyCached`); pair replication carries the artifact even to a node that missed its own fetch |
|
||||
|
||||
Step 4 is the headline: it proves central-SQL independence at the driver **and** the #485 last-known-good contract on the fetch path simultaneously. Record actual output for every step, especially anything that fails.
|
||||
|
||||
**Step 5: Commit** `docs(mesh): Phase 3 live-gate record`
|
||||
|
||||
---
|
||||
|
||||
## Risks specific to this phase
|
||||
|
||||
- **First in-repo proto.** The `Grpc.Tools` codegen path (Task 1) is new here; a restore/build ordering or `Directory.Packages.props` pin miss will surface as CS-missing-type, not a proto error. Build Commons in isolation first.
|
||||
- **The dedicated-h2c-listener interaction.** A fused central node with BOTH the LocalDb sync port and the ConfigServe port set must re-apply existing HTTP bindings exactly once; getting this wrong silently moves the AdminUI off its port (the exact failure the LocalDb block's comments warn about). Task 3's coexistence assertion is the guard.
|
||||
- **`RevisionHash == SHA-256(blob)` is load-bearing.** If a future change makes the revision hash something other than the raw-bytes SHA, the fetch verification breaks silently (every fetch → null → every deploy fails). Task 4's mismatch test pins the equality; note it in `ConfigComposer` if touched.
|
||||
- **Blocking in the actor loop.** The fetch is I/O; it MUST run via `PipeTo(self)`, never an awaited call inside a receive (would freeze the mailbox for `FetchTimeoutSeconds`). Task 5.
|
||||
- **Phase-boundary discipline.** Phase 3 removes config **reads** only. The `NodeDeploymentState` write, `DbHealthProbe`, and `EfAlarmConditionStateStore` still touch SQL by design — resist folding Phase 4 in, or the live gate's "no SQL" scope becomes unprovable-in-parts.
|
||||
- **Both-nodes-fetch is deliberate.** Do not gate the fetch on the Primary role to "save a fetch"; it couples config delivery to redundancy timing and diverges from today's both-read behaviour. Idempotency + replication already make the second fetch cheap.
|
||||
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-07-22-mesh-phase3-config-fetch-and-cache.md",
|
||||
"note": "Per-cluster mesh Phase 3. Decisions pre-made by the user: gRPC fetch RPC (NOT token-gated HTTP); shared node key (NOT per-deployment token, so no migration and DispatchDeployment is UNCHANGED); config-flagged dark switch ConfigSource:Mode=Direct default; both pair nodes fetch (no Primary gating). Five recon facts + the phase-boundary rule (Phase 3 = config READS only; NodeDeploymentState write / DbHealthProbe / EfAlarmConditionStateStore stay for Phase 4) are in the plan header. RevisionHash == SHA-256(blob) is load-bearing for fetch verification.",
|
||||
"tasks": [
|
||||
{
|
||||
"id": 0,
|
||||
"subject": "Task 0: ConfigSource/ConfigServe options + validator + registration",
|
||||
"status": "completed",
|
||||
"classification": "small",
|
||||
"parallelizableWith": [
|
||||
1
|
||||
],
|
||||
"commit": "9d1e60c0",
|
||||
"note": "One sabotage (accept empty endpoints) reddened its test; file was untracked so restored by hand not git checkout."
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"subject": "Task 1: deployment_artifact.v1 proto + first in-repo Grpc.Tools codegen",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"parallelizableWith": [
|
||||
0
|
||||
],
|
||||
"commit": "62e8f54f",
|
||||
"note": "First locally-compiled proto in the repo. Grpc.Tools 2.76.0 pinned (aligned with the other Grpc 2.76.0 refs); GrpcServices=Both. Commons builds clean, codegen verified."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"subject": "Task 2: central DeploymentArtifactService \u2014 stream sealed artifact, NotFound-hide the rest",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
1
|
||||
],
|
||||
"parallelizableWith": [
|
||||
4
|
||||
],
|
||||
"commit": "7d7de482",
|
||||
"note": "Impl RENAMED to DeploymentArtifactGrpcService \u2014 the generated container class is itself named DeploymentArtifactService and its nested base collided. Aliased the base. Sabotage dropping the sealed+empty guard reddened 2 tests."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"subject": "Task 3: config-serve auth interceptor + dedicated h2c listener wiring",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"commit": "4d88f67c",
|
||||
"note": "Merged the LocalDb-sync + ConfigServe listener blocks into ONE that re-applies the existing HTTP surface exactly once and adds both dedicated h2c ports; a fused admin+driver node double-binding the HTTP port throws 'address already in use' (sabotage-proven). AddGrpc moved out of the hasDriver block to hasDriver||hasAdmin with both path-scoped interceptors. StatusCode.Unauthenticated (LocalDb uses PermissionDenied). Coexistence proven by a minimal-WebApplication shape test, mirroring LocalDbSyncListenerTests; real end-to-end fetch is Task 8."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"subject": "Task 4: node gRPC artifact fetcher \u2014 SHA-verified reassembly, endpoint failover, null-on-any-failure",
|
||||
"status": "completed",
|
||||
"classification": "standard",
|
||||
"blockedBy": [
|
||||
1
|
||||
],
|
||||
"parallelizableWith": [
|
||||
2
|
||||
],
|
||||
"commit": "7a8fb5f1",
|
||||
"note": "Uniform invariant: return the first endpoint's bytes that reassemble AND verify (SHA-256 lowercase-hex == dispatched RevisionHash); else null. RpcException / empty-stream / hash-mismatch all -> try-next, never throw. Func<endpoint,client> seam keeps gRPC out of unit tests (fake subclasses the generated client's protected ctor). Namespace is .DeploymentCache NOT .Deployment (else it shadows Configuration.Entities.Deployment across the whole test assembly via enclosing-namespace lookup) \u2014 same trap LocalDbDeploymentArtifactCache already dodged. Added Grpc.Net.Client to Runtime.csproj. xUnit v2 here: CancellationToken.None, not TestContext.Current."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"subject": "Task 5: FetchAndCache apply path \u2014 fetch->cache->apply-from-bytes, null=apply-failure",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
4
|
||||
],
|
||||
"commit": "648f173b",
|
||||
"note": "DispatchDeployment (rev differs) in FetchAndCache -> BeginFetchAndCacheApply PipeTo(Self) FetchedForApply(bytes?), never a blocking await in a receive. HandleFetchedForApply applies synchronously mirroring ApplyAndAck: null bytes = FAILURE (keep last-known-good, don't advance rev, ack Failed); good bytes reconcile+rebuild+push FROM the bytes (OpcUaPublishActor never reads SQL) then cache. A faulted fetch task maps to null via PipeTo failure:. Extracted ReconcileDriversFromBlob (Direct/FetchAndCache/boot all share it; #485 empty guard moved in); ApplyCachedArtifact now reuses it. Props gains fetchAndCacheMode+artifactFetcher LAST. Idempotent re-dispatch is the existing _currentRevision short-circuit (no separate cache-contains check added; peer-replication-missed-fetch is covered by Task 6 boot + StoreAsync idempotency). Sabotages: advance-rev-on-null reddens test b; drop _currentRevision guard reddens test c."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"subject": "Task 6: FetchAndCache bootstrap from the LocalDb pointer; no driver-side config SQL read",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
5
|
||||
],
|
||||
"commit": "6d004738",
|
||||
"note": "Bootstrap() branches to BootstrapFromCache() in FetchAndCache: GetCurrentUnkeyedAsync -> set _currentRevision -> ApplyCachedArtifact (no SQL, no re-ack). _isRunningFromCache STAYS false (normal op, not degraded). Empty/unreadable cache -> Steady-no-revision (first dispatch fetches), NEVER Stale (Stale = 'SQL down, retry'; FetchAndCache has no config SQL read to recover). TryRecoverFromStale gains a defensive FetchAndCache early-return (unreachable anyway \u2014 retry-db timer only starts in Stale). KEY test lever: UpsertNodeDeploymentState SWALLOWS db failures, so a FetchAndCache dispatch runs end-to-end under a ThrowingDbFactory \u2014 the ack proves Steady-not-Stale AND no-SQL-at-boot in one shot. NodeDiagnosticsSnapshot lives in Commons.Interfaces not Messages.Fleet."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"subject": "Task 7: #485 empty/corrupt/mismatch coverage on the FetchAndCache paths",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
6
|
||||
],
|
||||
"commit": "fc568ae0",
|
||||
"note": "Regression net, no new mechanism \u2014 all green (no gaps to guard). Pins the NEGATIVE on failure paths: (1) null fetch mid-steady-state -> no RebuildAddressSpace for the failed rev, revision + served state kept (publishProbe.ExpectNoMsg is the discriminator); (2) zero-length blob handed to the actor fails via ReconcileDriversFromBlob's own #485 guard (independent of the fetcher null contract); (3) corrupt cache at boot = GetCurrentUnkeyedAsync null -> Steady-no-revision, no rebuild at boot (no partial apply). Removing the #485 empty guard reddens the zero-length test. Corrupt-cache uses a stub-returns-null (the real ReassembleAsync-null-on-corrupt is LocalDbDeploymentArtifactCacheTests' job)."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"subject": "Task 8: real two-Host gRPC artifact-fetch boundary test + wrong-key falsifiability control",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"parallelizableWith": [
|
||||
9
|
||||
],
|
||||
"commit": "ffbcaa93",
|
||||
"note": "Minimal real Host (WebApplication + h2c-only Kestrel + real ConfigServeAuthInterceptor + real DeploymentArtifactGrpcService + in-memory EF sealed deployment) driven by the REAL GrpcDeploymentArtifactFetcher. 4 green: right-key >256KB reassembles byte-equal + verifies; wrong-key null WHILE right-key succeeds on the SAME server (falsifiability control); unknown-id null; [dead-port,real-port] failover. h2c GrpcChannel.ForAddress(http://) works out of the box in .NET 10 \u2014 no AppContext switch needed. async can't have out params -> StartServerAsync returns a tuple. Tagged [Trait Category=ArtifactBoundary]."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"subject": "Task 9: docker-dev rig config + docs + program-plan flip",
|
||||
"status": "completed",
|
||||
"classification": "small",
|
||||
"blockedBy": [
|
||||
5
|
||||
],
|
||||
"parallelizableWith": [
|
||||
8
|
||||
],
|
||||
"commit": "cf6110d0",
|
||||
"note": "Centrals: ConfigServe :4055 + committed dev key, ConfigSource:Mode=Direct (they never fetch). Sites: ConfigSource endpoints (both centrals) + matching key, Mode=${OTOPCUA_CONFIG_MODE:-Direct}. One env var flips ONLY the sites because centrals hard-code Direct. Docs: Configuration.md ConfigSource/ConfigServe section, Redundancy.md out-of-band note, program plan Phase 3 CODE-COMPLETE + tracking row, CLAUDE.md Config-source section."
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"subject": "Task 10: live gate on docker-dev (site nodes -> FetchAndCache; step 4 = stop central SQL mid-fetch = the headline gate)",
|
||||
"status": "completed",
|
||||
"classification": "high-risk",
|
||||
"blockedBy": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"commit": "(gate doc) 2026-07-22-mesh-phase3-live-gate.md",
|
||||
"note": "PASSED \u2014 and CAUGHT A REAL DEFECT every offline test missed: DeploymentArtifactGrpcService gated on Status==Sealed, but central seals only AFTER all nodes ack, and a FetchAndCache node can't ack until it fetches => seal-needs-ack->ack-needs-fetch->fetch-needs-sealed DEADLOCK. Fetch reached central + passed the interceptor, got clean NotFound, #485 kept last-known-good, deploy never sealed. Fixed by dropping the Sealed gate (serve any non-empty blob; Direct reads the same AwaitingApplyAcks row). Task 2 test flipped to non-sealed-with-blob-is-served. After fix: fetch+verify+apply+seal green 6/6. HEADLINE step 4 PASS: SQL stopped, sites keep serving, restarted site boots last-applied config from the LocalDb pointer, no crash, never Stale. Findings: Kestrel takeover+AdminUI coexist live on central (:9000+:4055); transient Direct SQL-reconnect blip on central-1 is #486 not a regression; failover to central-2 works; deploy POST needs Content-Type:application/json (else 404). Build: Dockerfile build stage pinned to linux/amd64 \u2014 Grpc.Tools linux_arm64 protoc segfaults (139) on Apple-Silicon Docker; portable-IL publish keeps runtime native arm64. Steps 8/9 folded into Task7/Task8/Task3 real coverage rather than slow emulated recreate."
|
||||
}
|
||||
],
|
||||
"lastUpdated": "2026-07-23T01:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
# Per-cluster mesh Phase 3 — live gate record
|
||||
|
||||
**Date:** 2026-07-23 · **Rig:** `docker-dev` (six-node single mesh) · **Branch:** `feat/mesh-phase3`
|
||||
**Plan:** [`2026-07-22-mesh-phase3-config-fetch-and-cache.md`](2026-07-22-mesh-phase3-config-fetch-and-cache.md)
|
||||
|
||||
**Result: PASSED** — the FetchAndCache config path works end-to-end on the real rig, and the gate
|
||||
**caught one real defect** (a deploy-sealing deadlock) that every offline test had missed. The defect
|
||||
was fixed and pinned before the gate was re-run green. Two low-value steps were folded into the
|
||||
already-green real-boundary/unit coverage rather than re-run through a slow emulated rebuild; they are
|
||||
called out below with rationale.
|
||||
|
||||
Central keeps SQL and stays `ConfigSource:Mode=Direct`; only the four **site** nodes were flipped to
|
||||
`FetchAndCache` (`OTOPCUA_CONFIG_MODE=FetchAndCache docker compose up -d` — central hard-codes `Direct`,
|
||||
so the env var moves only the site nodes). Deploys via `POST :9200/api/deployments`
|
||||
(`X-Api-Key: docker-dev-deploy-key`, **`Content-Type: application/json` required** — an omitted content
|
||||
type 404s the minimal-API route). Revision bumps by renaming the single seeded `UnsArea` row.
|
||||
|
||||
---
|
||||
|
||||
## Result table
|
||||
|
||||
| # | Step | Result |
|
||||
|---|---|---|
|
||||
| 1 | Rig up all-`Direct`; deploy | **PASS** — `36e5cdd2` Sealed (Status 2), 6/6 Applied — baseline |
|
||||
| 2 | Flip site nodes to `FetchAndCache`, restart, deploy | **PASS after the fix** — `59399bd8` Sealed, 6/6 Applied; site logs *"Fetched + verified artifact … from central-1:4055 (13387 bytes)"* → *"applied fetched deployment"* |
|
||||
| 3 | Confirm central served it | **PASS** — central logs *"Served artifact for 59399bd8… (13387 bytes, 1 chunks)"*; site logs the *verified* (SHA-256-checked) reassembly |
|
||||
| 4 | **Stop central SQL** — driver independence + #485 | **PASS (headline)** — site nodes keep serving (`restarts=0`); a restarted site-b-1 boots its **last-applied** config (`59399bd8` / rev `00ac18b7`) from the LocalDb pointer with SQL down, no crash, never Stale |
|
||||
| 5 | Restart SQL, redeploy | **PASS** — retry lands green (`d6f71cc2` Sealed, 6/6) after one transient Direct-mode blip on central-1 (see finding B) |
|
||||
| 6 | Restart a site node, central up — boot from pointer | **PASS** — every FetchAndCache boot logs *"restored served state … from the LocalDb pointer"*, no `Deployment` read |
|
||||
| 7 | Restart a site node with central DOWN | **PASS** — folded into step 4 (site-b-1 restarted while SQL was stopped) |
|
||||
| 8 | Corrupt a cached chunk → boot Steady-no-revision | **NOT RE-RUN on rig** — covered by `DriverHostActorFetchAndCacheUnreadableTests` (corrupt cache ⇒ `GetCurrentUnkeyedAsync` null ⇒ Steady-no-revision, no partial apply). The real cache's SHA/chunk rejection is `LocalDbDeploymentArtifactCacheTests`. |
|
||||
| 9 | Wrong `ConfigSource:ApiKey` on one site → fetch rejected | **NOT RE-RUN on rig** — the real mechanism is proven by `DeploymentArtifactFetchBoundaryTests` (real Kestrel h2c + real interceptor + wrong key ⇒ null) and `ConfigServeAuthInterceptorTests` (fail-closed); the actor's response to a null fetch is step 4's evidence |
|
||||
| 10 | Both pair nodes fetch the same deploy | **PASS** — both site-a and both site-b nodes fetched independently in step 2; central logged a `Served artifact` per fetch; `StoreAsync` is idempotent |
|
||||
|
||||
---
|
||||
|
||||
## The defect the gate caught — deploy-sealing deadlock (fixed)
|
||||
|
||||
**Symptom.** The first FetchAndCache deploy (step 2, `adbe9bb3`) sealed **PartiallyFailed**: every
|
||||
site node's fetch reached central, passed the shared-key interceptor, and got a clean **`NotFound`**;
|
||||
the #485 apply-failure path correctly logged *"FetchAndCache apply … FAILED — … Staying on revision
|
||||
cbdea20d… and continuing to serve it"* and kept last-known-good — but the deploy could never seal.
|
||||
|
||||
**Root cause.** `DeploymentArtifactGrpcService` gated on `Status == Sealed`. But central seals a
|
||||
deployment **only after every node acks Applied**, and a FetchAndCache node **cannot ack until it has
|
||||
fetched** the artifact:
|
||||
|
||||
> seal-needs-ack → ack-needs-fetch → **fetch-needs-sealed** → deadlock.
|
||||
|
||||
Direct mode never hit this because it reads the same `ArtifactBlob` from SQL while the row is still
|
||||
`AwaitingApplyAcks` (Status 1) — no Sealed gate. Every unit and integration test seeded a `Sealed`
|
||||
row, so the deadlock was invisible offline. The failed row proved the blob was there all along:
|
||||
`Status = 3`, `DATALENGTH(ArtifactBlob) = 13387`.
|
||||
|
||||
**Fix** (`fix(mesh): serve the artifact regardless of deployment status`). Drop the Sealed gate —
|
||||
serve any deployment whose blob is non-empty; unknown-id / empty-blob still collapse to `NotFound`
|
||||
(existence-hiding + #485). Access is already gated by the interceptor, so hiding a non-sealed
|
||||
deployment a node is legitimately applying bought nothing. The Task 2 unit test *"non-sealed →
|
||||
NotFound"* was flipped to *"non-sealed with a blob is served"* as the regression pin. After the fix,
|
||||
step 2 re-ran green (`59399bd8` Sealed, 6/6).
|
||||
|
||||
**Lesson.** "Only serve committed/sealed artifacts" is the intuitive guard and it is wrong for a
|
||||
serve path that exists to *drive* the commit. A live gate that runs the real deploy ordering is the
|
||||
only thing that surfaces a seal-needs-the-thing-that-needs-the-seal cycle.
|
||||
|
||||
---
|
||||
|
||||
## Findings
|
||||
|
||||
- **A — the Kestrel takeover now runs on central, live, and coexists.** Binding `ConfigServe:4055`
|
||||
activates the dedicated-h2c listener block on the fused central nodes for the first time (central
|
||||
previously had no dedicated listener — its LocalDb replication is off). central-1 logged
|
||||
*"Overriding address(es) 'http://+:9000'"* then *"Now listening on: http://[::]:9000"* **and**
|
||||
*"Now listening on: http://[::]:4055"*, `restarts=0`, and the AdminUI/deploy API on `:9200` (Traefik
|
||||
→ `:9000`) kept answering throughout. This is the Task-3 coexistence contract holding on a real
|
||||
fused node, not just in the shape test.
|
||||
- **B — a transient Direct-mode SQL-reconnect blip is not a Phase 3 regression.** In step 5, the deploy
|
||||
immediately after SQL restarted sealed PartiallyFailed with **central-1** (Direct) failing its blob
|
||||
read (*"ConfigDb unreachable"*) — its EF pool still held broken connections. The #485 guard did
|
||||
exactly the right thing (apply-failure, keep last-known-good, ack Failed), and a redeploy sealed
|
||||
green. This is the pre-existing #486 behaviour firing on a reconnect edge, unrelated to FetchAndCache.
|
||||
- **C — endpoint failover works live.** site-b-1's step-5 fetch succeeded *from central-2*
|
||||
(`Fetched + verified … from http://central-2:4055`) — the fetcher rotated past central-1 to the
|
||||
second configured endpoint exactly as `GrpcDeploymentArtifactFetcher` intends.
|
||||
- **D — `Content-Type: application/json` is mandatory on the deploy POST.** Without it the minimal-API
|
||||
route 404s (not 415). The baseline call carried it; a later batch of retries dropped it and 404'd
|
||||
five times before the header was restored. Rig-operational, not a Phase 3 concern, but recorded so
|
||||
the next operator does not chase a phantom outage.
|
||||
|
||||
---
|
||||
|
||||
## Build note (rig enablement)
|
||||
|
||||
The rig image could not build until the Dockerfile's **build stage was pinned to `linux/amd64`**
|
||||
(`fix(docker-dev): pin the build stage to linux/amd64`). Phase 3 added the repo's first in-repo
|
||||
`.proto`, and `Grpc.Tools`' bundled `linux_arm64` `protoc` **segfaults (exit 139)** on Apple-Silicon
|
||||
Docker. `dotnet publish` is framework-dependent (portable IL + every RID's native assets), so pinning
|
||||
only the build stage leaves the runtime image native arm64; just the one-time build is emulated.
|
||||
@@ -99,7 +99,35 @@ this removes the coordinator's one genuinely mesh-bound dependency).
|
||||
set proven DB-sourced (test: a `ClusterNode` row present but node down → deploy reports that node
|
||||
missing; a node up but row absent → its ack is not expected).
|
||||
|
||||
### Phase 2 — Comm actors + ClusterClient transport
|
||||
### Phase 2 — Comm actors + ClusterClient transport — **DONE 2026-07-22 (live gate PASSED)**
|
||||
Shipped per [`2026-07-22-mesh-phase2-clusterclient-transport.md`](2026-07-22-mesh-phase2-clusterclient-transport.md)
|
||||
as a **dark switch**: `MeshTransport:Mode` defaults to `Dps`, so nothing changes on any deployment
|
||||
until the flag flips. The live gate ([`2026-07-22-mesh-phase2-live-gate.md`](2026-07-22-mesh-phase2-live-gate.md))
|
||||
flipped the whole rig to `ClusterClient` and passed for the transport, with four findings — most
|
||||
notably that `buffer-size = 0`'s stated rationale was overstated (a node applies a `TimedOut`
|
||||
deployment anyway via boot-time orphan-row replay) and that stopping both centrals strands non-seed
|
||||
nodes at the membership layer (a Phase 7 drill item, transport-independent). Both comm actors are spawned and receptionist-registered in **both** modes, so
|
||||
the flip is a config change rather than a redeploy. Three scope corrections landed against the
|
||||
design doc (that plan's header carries them in full):
|
||||
|
||||
- the node path is `/user/node-communication`, not `/user/cluster-communication` (there is no
|
||||
per-cluster scoping until Phase 6);
|
||||
- the substitution for `Publish` is **`SendToAll`**, not `Send` — `Send` reaches exactly one node and
|
||||
would deploy to one member of the fleet while the rest silently kept their old config;
|
||||
- central creates **exactly one fleet-wide ClusterClient**. On a single mesh a receptionist serves
|
||||
its whole cluster, so a client per application `Cluster` would fan every command out once per
|
||||
cluster. Per-cluster clients become correct only when Phase 6 splits the meshes (`TODO(Phase 6)`
|
||||
in `CentralCommunicationActor.RebuildClient`).
|
||||
|
||||
**Exit-gate deviation — "an Ask timing out cleanly against a stopped node" cannot be run as written.**
|
||||
There is no cross-boundary Ask in Phase 2: every migrated command is fire-and-forget on the wire.
|
||||
`DispatchDeployment` is a `Tell` whose ack returns later as an independent `ApplyAck`;
|
||||
`RestartDriver` / `ReconnectDriver` / `AlarmCommand` reply `Ok = true` from the admin node the moment
|
||||
they are dispatched (`Ok` has always meant *dispatched*, not *applied* — the AdminUI string is
|
||||
literally "Restart dispatched"). The honest equivalents, both in the live gate: a **stopped node**
|
||||
fails the deploy at the apply deadline naming it (Phase 1 semantics), and **no reachable contact**
|
||||
drops the command with a Warning and never buffers it.
|
||||
|
||||
**Scope:** one receptionist-registered actor per side (`/user/central-communication`,
|
||||
`/user/cluster-communication`), registered **per node, not as a singleton** (contact rotation);
|
||||
ClusterClient central → cluster carrying deploy notify + acks + driver-control; the ScadaBridge
|
||||
@@ -113,16 +141,30 @@ admin-change refresh), clusters know central from appsettings (static, restart t
|
||||
flow over ClusterClient (DPS paths deleted or dark-switched), including an Ask timing out cleanly
|
||||
against a stopped node.
|
||||
|
||||
### Phase 3 — Config fetch-and-cache from central
|
||||
**Scope:** the deploy artifact is served **by central** (transport per its own phase plan — the
|
||||
ScadaBridge analogue is token-gated HTTP; decide HTTP vs a gRPC fetch RPC when planning) and
|
||||
cached in LocalDb; driver nodes read config exclusively from LocalDb (boot-from-cache becomes the
|
||||
normal path — design §6.1); chunking, SHA-256 verify, newest-2 retention, pair replication carry
|
||||
over unchanged. **This phase changes a running data path — it gets its own live gate** (design §7
|
||||
note): deploy lands on a driver pair with central SQL stopped mid-fetch → retry lands; #485
|
||||
last-known-good semantics re-proven on the new path.
|
||||
**Exit gate:** live gate green on the rig; a driver node with an empty LocalDb and reachable
|
||||
central boots into the current config; with central down it boots last-known-good.
|
||||
### Phase 3 — Config fetch-and-cache from central — **CODE-COMPLETE 2026-07-23 (Tasks 0–8; live gate Task 10 pending)**
|
||||
**Decisions taken:** the deploy artifact is served **by central over a gRPC fetch RPC**
|
||||
(`DeploymentArtifactService.Fetch`, streamed chunks — chosen over token-gated HTTP), authenticated by
|
||||
a **shared node bearer key** (`ConfigServe:ApiKey == ConfigSource:ApiKey`, `FixedTimeEquals`,
|
||||
fail-closed — chosen over a per-deployment token, so **no migration** and `DispatchDeployment` stays
|
||||
payload-free). Cutover is a **config-flagged dark switch** `ConfigSource:Mode` (`Direct` default |
|
||||
`FetchAndCache`), per node. Plan + task ledger:
|
||||
[`docs/plans/2026-07-22-mesh-phase3-config-fetch-and-cache.md`](2026-07-22-mesh-phase3-config-fetch-and-cache.md).
|
||||
**Shipped (all green, sabotage-checked):** `ConfigSource`/`ConfigServe` options + validator; the repo's
|
||||
first in-repo `.proto` + `Grpc.Tools` codegen; central `DeploymentArtifactGrpcService` (streams the
|
||||
sealed blob, NotFound-hides unknown/not-sealed/empty); `ConfigServeAuthInterceptor` + a dedicated h2c
|
||||
Kestrel listener merged with the LocalDb-sync listener (re-binds the existing surface once, adds both
|
||||
dedicated ports); node `GrpcDeploymentArtifactFetcher` (SHA-verified reassembly, endpoint failover,
|
||||
null-on-any-failure); the `FetchAndCache` apply path (`PipeTo(Self)`, fetch→cache→apply-from-bytes,
|
||||
null = apply-failure keeping last-known-good) and bootstrap-from-the-LocalDb-pointer (no driver-side
|
||||
config SQL read); #485 coverage; and a **real two-Host gRPC boundary test** (right-key reassembles,
|
||||
wrong-key control returns null, failover). Chunking, SHA-256 verify, newest-2 retention, pair
|
||||
replication carry over unchanged. **Phase boundary held:** config **reads** only — the
|
||||
`NodeDeploymentState` ack-row write, `DbHealthProbe`, `EfAlarmConditionStateStore` stay for Phase 4.
|
||||
**Remaining (Tasks 9–10):** rig config + docs (done) and the live gate — deploy on a site pair flipped
|
||||
to `FetchAndCache` with central SQL stopped mid-fetch → retry lands; #485 last-known-good re-proven on
|
||||
the new path.
|
||||
**Exit gate:** live gate green on the rig; a driver node with an empty LocalDb and reachable central
|
||||
boots into the current config; with central down it boots last-known-good.
|
||||
|
||||
### Phase 4 — Cut the driver-side ConfigDb connection
|
||||
**Scope (design §6.1 audit table):** re-home `EfAlarmConditionStateStore` to LocalDb (pair-local
|
||||
@@ -198,10 +240,10 @@ resource sizing on the site VMs should be checked once both products run the ful
|
||||
|---|---|
|
||||
| 0a downing strategy | DONE 2026-07-21 (live gate → Phase 7) |
|
||||
| 0b oldest-Up election | DONE 2026-07-21 |
|
||||
| Prereq: selfform-fallback + manual-failover plan | plan written 2026-07-22, not executed |
|
||||
| 1 ClusterNode columns + DB ack set | **detailed plan ALREADY WRITTEN** (`2026-07-21-per-cluster-mesh-phase1.md`, from the design session — do NOT write a new one), not executed. ⚠ Carries a decision gate: DB-derived expected-ack set changes deploy-seal semantics (a stopped-but-enabled node now FAILS the deploy at the apply deadline instead of sealing green without it; escape hatch = `ClusterNode.Enabled=0`) — confirm with the owner before its Task 3. |
|
||||
| 2 ClusterClient transport | not started |
|
||||
| 3 fetch-and-cache | not started |
|
||||
| Prereq: selfform-fallback + manual-failover plan | **DONE 2026-07-22**, merged `a78425ea`. Shipped *not* as the planned `SelfFormAfter` watchdog — the live gate caught that islanding a manually-failed-over node — but as self-first seed ordering + `AkkaClusterOptionsValidator`. Phase 6's seed-ordering scope is therefore already discharged. |
|
||||
| 1 ClusterNode columns + DB ack set | **DONE 2026-07-22**, merged `7654f24d`. Deploy-seal semantics changed as flagged, but the escape hatch is **`ClusterNode.MaintenanceMode`**, not `Enabled=0` — the live gate found `Enabled=0` is rejected by `DraftValidator.ValidateClusterTopology` on any 2-node pair. Gate record: `2026-07-22-mesh-phase1-live-gate.md`. |
|
||||
| 2 ClusterClient transport | **DONE 2026-07-22** (live gate PASSED, shipped dark) — shipped dark (`MeshTransport:Mode=Dps` default), both comm actors registered in both modes. Corrections: `SendToAll` not `Send`, one fleet-wide client not one per cluster, `/user/node-communication` not `/user/cluster-communication`. The gate's "Ask timing out" item does not exist to test — no cross-boundary Ask in this phase; see the phase section. |
|
||||
| 3 fetch-and-cache | **CODE-COMPLETE 2026-07-23** (Tasks 0–8 merged to the phase branch, all green + sabotage-checked; live gate Task 10 pending). gRPC fetch RPC + shared node key; dark switch `ConfigSource:Mode` (Direct default). Rig flip: `OTOPCUA_CONFIG_MODE=FetchAndCache` on the site nodes. See `2026-07-22-mesh-phase3-config-fetch-and-cache.md`. |
|
||||
| 4 cut driver ConfigDb | not started |
|
||||
| 5 gRPC telemetry | not started |
|
||||
| 6 mesh partition + co-location | not started |
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
"planPath": "docs/plans/2026-07-22-per-cluster-mesh-program.md",
|
||||
"note": "PROGRAM plan: each task = write that phase's detailed plan (writing-plans), execute it (executing-plans), run its exit gate, update the tracking tables. Prereq task 0 is the separate selfform-fallback plan in this repo.",
|
||||
"tasks": [
|
||||
{"id": 0, "subject": "Prereq: execute 2026-07-22-selfform-fallback-and-manual-failover.md (7 tasks)", "status": "pending"},
|
||||
{"id": 1, "subject": "Phase 1: ClusterNode Akka+gRPC address columns; ConfigPublishCoordinator ack set from DB", "status": "pending"},
|
||||
{"id": 2, "subject": "Phase 2: comm actors + receptionist + ClusterClient transport (deploy notify/acks, driver-control)", "status": "pending", "blockedBy": [1]},
|
||||
{"id": 0, "subject": "Prereq: execute 2026-07-22-selfform-fallback-and-manual-failover.md (7 tasks)", "status": "completed", "note": "Shipped as self-first seed ordering + AkkaClusterOptionsValidator, NOT the planned SelfFormAfter watchdog (live gate caught it islanding a failed-over node). Merged a78425ea."},
|
||||
{"id": 1, "subject": "Phase 1: ClusterNode Akka+gRPC address columns; ConfigPublishCoordinator ack set from DB", "status": "completed", "note": "Merged 7654f24d. Escape hatch is ClusterNode.MaintenanceMode, not Enabled=0."},
|
||||
{"id": 2, "subject": "Phase 2: comm actors + receptionist + ClusterClient transport (deploy notify/acks, driver-control)", "status": "in_progress", "blockedBy": [1]},
|
||||
{"id": 3, "subject": "Phase 3: config fetch-and-cache from central; LocalDb steady-state config store (live gate)", "status": "pending", "blockedBy": [2]},
|
||||
{"id": 4, "subject": "Phase 4: cut driver-side ConfigDb — EfAlarmConditionStateStore to LocalDb, DbHealthProbeActor ServiceLevel input, OpcUaPublishActor audit (live gate)", "status": "pending", "blockedBy": [3]},
|
||||
{"id": 5, "subject": "Phase 5: gRPC oneof stream contract; migrate 7 observability topics; AdminUI reconnect story", "status": "pending", "blockedBy": [2]},
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Node-side selection of where a driver reads its deployed configuration (per-cluster mesh
|
||||
/// Phase 3). The dark switch that lets a driver node stop reading <c>Deployment.ArtifactBlob</c>
|
||||
/// from central SQL and instead fetch the artifact from central over gRPC and read it from the
|
||||
/// pair-local LocalDb cache.
|
||||
/// </summary>
|
||||
public sealed class ConfigSourceOptions
|
||||
{
|
||||
/// <summary>Configuration section name.</summary>
|
||||
public const string SectionName = "ConfigSource";
|
||||
|
||||
/// <summary>Read config directly from central SQL — today's behaviour, and the default.</summary>
|
||||
public const string ModeDirect = "Direct";
|
||||
|
||||
/// <summary>Fetch the artifact from central over gRPC, cache it in LocalDb, and read from there.</summary>
|
||||
public const string ModeFetchAndCache = "FetchAndCache";
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ModeDirect"/> (default) or <see cref="ModeFetchAndCache"/>. Any other value
|
||||
/// fails host start.
|
||||
/// </summary>
|
||||
public string Mode { get; set; } = ModeDirect;
|
||||
|
||||
/// <summary>
|
||||
/// Central artifact-gRPC base addresses, e.g. <c>http://central-1:4055</c> (h2c ⇒ the
|
||||
/// <c>http</c> scheme). Tried in order for failover. Required under
|
||||
/// <see cref="ModeFetchAndCache"/>; ignored under <see cref="ModeDirect"/>.
|
||||
/// </summary>
|
||||
public string[] CentralFetchEndpoints { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Shared bearer key; must equal central's <see cref="ConfigServeOptions.ApiKey"/>. Supply via
|
||||
/// the environment (<c>ConfigSource__ApiKey</c>) — never commit it. Required under
|
||||
/// <see cref="ModeFetchAndCache"/>.
|
||||
/// </summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Per-fetch deadline in seconds. Must be positive under <see cref="ModeFetchAndCache"/>.</summary>
|
||||
public int FetchTimeoutSeconds { get; set; } = 30;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Central-side artifact-serve surface (per-cluster mesh Phase 3). Configures the dedicated h2c
|
||||
/// listener + shared bearer key the <c>DeploymentArtifactService</c> is exposed behind. Only
|
||||
/// admin-role nodes (which hold the central SQL connection) serve.
|
||||
/// </summary>
|
||||
public sealed class ConfigServeOptions
|
||||
{
|
||||
/// <summary>Configuration section name.</summary>
|
||||
public const string SectionName = "ConfigServe";
|
||||
|
||||
/// <summary>
|
||||
/// Dedicated HTTP/2-only (h2c) listener port for the artifact gRPC service. <c>0</c> (the
|
||||
/// default) disables it — nothing is bound. Must differ from the main HTTP port and from
|
||||
/// <c>LocalDb:SyncListenPort</c> (h2c cannot share a cleartext HTTP/1 port).
|
||||
/// </summary>
|
||||
public int GrpcListenPort { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Shared bearer key the serve-side interceptor checks (constant-time, fail-closed: an unset
|
||||
/// key rejects every call). Supply via the environment (<c>ConfigServe__ApiKey</c>) — never
|
||||
/// commit it.
|
||||
/// </summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Fails the host at startup on a <see cref="ConfigSourceOptions"/> shape that would leave a
|
||||
/// driver node unable to fetch its configuration.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Like <see cref="MeshTransportOptionsValidator"/>, every fault caught here otherwise surfaces as
|
||||
/// an <i>absence</i> — a deployment that never applies because the node has no endpoint to fetch
|
||||
/// from or no key to authenticate with. An absence has no stack trace and no failing node to point
|
||||
/// at; refusing to start is cheaper to diagnose.
|
||||
/// </remarks>
|
||||
public sealed class ConfigSourceOptionsValidator : IValidateOptions<ConfigSourceOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public ValidateOptionsResult Validate(string? name, ConfigSourceOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var errors = new List<string>();
|
||||
|
||||
var isDirect = string.Equals(options.Mode, ConfigSourceOptions.ModeDirect, StringComparison.OrdinalIgnoreCase);
|
||||
var isFetch = string.Equals(
|
||||
options.Mode, ConfigSourceOptions.ModeFetchAndCache, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (!isDirect && !isFetch)
|
||||
{
|
||||
errors.Add(
|
||||
$"{ConfigSourceOptions.SectionName}:{nameof(ConfigSourceOptions.Mode)} is "
|
||||
+ $"'{options.Mode}'. Expected '{ConfigSourceOptions.ModeDirect}' or "
|
||||
+ $"'{ConfigSourceOptions.ModeFetchAndCache}'.");
|
||||
}
|
||||
|
||||
// The fetch surface is only load-bearing under FetchAndCache. Requiring it under the default
|
||||
// would make the section mandatory everywhere for a feature that is switched off — including on
|
||||
// admin-only nodes, which never fetch.
|
||||
if (isFetch)
|
||||
{
|
||||
if (options.CentralFetchEndpoints.Length == 0)
|
||||
{
|
||||
errors.Add(
|
||||
$"{ConfigSourceOptions.SectionName}:{nameof(ConfigSourceOptions.CentralFetchEndpoints)} "
|
||||
+ "is empty. Under FetchAndCache a driver node cannot reach central to fetch its "
|
||||
+ "configuration, and every deployment would time out naming a node that is running "
|
||||
+ "perfectly.");
|
||||
}
|
||||
|
||||
foreach (var endpoint in options.CentralFetchEndpoints)
|
||||
{
|
||||
if (!endpoint.StartsWith("http://", StringComparison.OrdinalIgnoreCase)
|
||||
&& !endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
errors.Add(
|
||||
$"Central fetch endpoint '{endpoint}' must start with 'http://' (h2c) or "
|
||||
+ "'https://' — it is a gRPC base address (scheme://host:port), not a bare "
|
||||
+ "host:port.");
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(options.ApiKey))
|
||||
{
|
||||
errors.Add(
|
||||
$"{ConfigSourceOptions.SectionName}:{nameof(ConfigSourceOptions.ApiKey)} is empty. "
|
||||
+ "Under FetchAndCache the shared bearer key is the whole authentication boundary to "
|
||||
+ "central's artifact service; without it every fetch is rejected. Supply it via the "
|
||||
+ "environment (ConfigSource__ApiKey).");
|
||||
}
|
||||
|
||||
if (options.FetchTimeoutSeconds <= 0)
|
||||
{
|
||||
errors.Add(
|
||||
$"{ConfigSourceOptions.SectionName}:{nameof(ConfigSourceOptions.FetchTimeoutSeconds)} "
|
||||
+ $"must be greater than 0 (was {options.FetchTimeoutSeconds}).");
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Count == 0
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(string.Join(" ", errors));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Selects and configures the transport carrying central→node commands and node→central
|
||||
/// deployment acks (per-cluster mesh Phase 2).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="Mode"/> is a deliberate <b>dark switch</b>, not a permanent knob: both
|
||||
/// transports live in the tree for one phase so the docker-dev rig can run the same
|
||||
/// deployment over each and compare, and so a bad Phase 2 rolls back with a config change
|
||||
/// rather than a revert. Phase 6 deletes the DistributedPubSub branch along with the single
|
||||
/// mesh that made it viable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Discovery is deliberately asymmetric</b>, matching the sister project. Central
|
||||
/// discovers nodes <i>from the database</i> — enabled, non-maintenance <c>ClusterNode</c>
|
||||
/// rows, rebuilt every <see cref="ContactRefreshSeconds"/>. Nodes discover central <i>from
|
||||
/// this section</i>: static, restart to change. The asymmetry follows the rate of change —
|
||||
/// operators add and retire nodes, but central's own address is part of the deployment.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class MeshTransportOptions
|
||||
{
|
||||
/// <summary>Configuration section name.</summary>
|
||||
public const string SectionName = "MeshTransport";
|
||||
|
||||
/// <summary>Every command stays on DistributedPubSub — the pre-Phase-2 behaviour.</summary>
|
||||
public const string ModeDps = "Dps";
|
||||
|
||||
/// <summary>Commands cross the ClusterClient receptionist boundary.</summary>
|
||||
public const string ModeClusterClient = "ClusterClient";
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ModeDps"/> (default) or <see cref="ModeClusterClient"/>. Any other value fails
|
||||
/// the host at startup rather than silently falling back — a transport that is neither of
|
||||
/// these delivers nothing, and the symptom would be commands that vanish.
|
||||
/// </summary>
|
||||
public string Mode { get; set; } = ModeDps;
|
||||
|
||||
/// <summary>
|
||||
/// Akka remoting addresses of the central nodes, e.g.
|
||||
/// <c>akka.tcp://otopcua@central-1:4053</c>. <b>Node addresses only</b> — the
|
||||
/// <c>/system/receptionist</c> path is appended at construction time.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each entry must be a <i>central</i> node's remoting endpoint, never this node's own port.
|
||||
/// The sister project shipped a site template whose second contact was the site's own
|
||||
/// remoting port; it is a permanent failure in the initial-contact rotation and stays
|
||||
/// invisible until a command goes missing. <see cref="MeshTransportOptionsValidator"/>
|
||||
/// rejects the malformed shapes at boot instead.
|
||||
/// </remarks>
|
||||
public string[] CentralContactPoints { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// How often central re-reads <c>ClusterNode</c> rows to rebuild its contact points.
|
||||
/// Refreshing on a timer (rather than only on admin change) means a row edited by any
|
||||
/// process — migration, direct SQL, a second admin node — is picked up without a restart.
|
||||
/// </summary>
|
||||
public int ContactRefreshSeconds { get; set; } = 60;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Fails the host at startup on a <see cref="MeshTransportOptions"/> shape that would leave
|
||||
/// commands silently undelivered.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every fault caught here otherwise surfaces as an <i>absence</i> — a deployment that never
|
||||
/// arrives, an alarm acknowledgement that does nothing, a node whose ack is dropped without a
|
||||
/// trace. An absence has no stack trace and no failing node to point at, which makes it the
|
||||
/// most expensive class of misconfiguration to diagnose in the field. Refusing to start is
|
||||
/// cheaper for everyone.
|
||||
/// </remarks>
|
||||
public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTransportOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public ValidateOptionsResult Validate(string? name, MeshTransportOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var errors = new List<string>();
|
||||
|
||||
var isDps = string.Equals(options.Mode, MeshTransportOptions.ModeDps, StringComparison.OrdinalIgnoreCase);
|
||||
var isClusterClient = string.Equals(
|
||||
options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (!isDps && !isClusterClient)
|
||||
{
|
||||
errors.Add(
|
||||
$"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.Mode)} is "
|
||||
+ $"'{options.Mode}'. Expected '{MeshTransportOptions.ModeDps}' or "
|
||||
+ $"'{MeshTransportOptions.ModeClusterClient}'.");
|
||||
}
|
||||
|
||||
if (options.ContactRefreshSeconds <= 0)
|
||||
{
|
||||
errors.Add(
|
||||
$"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.ContactRefreshSeconds)} "
|
||||
+ $"must be greater than 0 (was {options.ContactRefreshSeconds}). Central rebuilds its "
|
||||
+ "ClusterClient contact points on this interval; a non-positive value would leave the "
|
||||
+ "contact set frozen at whatever it held when the node booted.");
|
||||
}
|
||||
|
||||
// Contact points are only load-bearing under ClusterClient mode. Requiring them under the
|
||||
// default would make the section mandatory everywhere for a feature that is switched off —
|
||||
// including on admin-only nodes, which have no reason to carry them at all.
|
||||
if (isClusterClient)
|
||||
{
|
||||
if (options.CentralContactPoints.Length == 0)
|
||||
{
|
||||
errors.Add(
|
||||
$"{MeshTransportOptions.SectionName}:{nameof(MeshTransportOptions.CentralContactPoints)} "
|
||||
+ "is empty. Under ClusterClient mode a driver node cannot reach central without at "
|
||||
+ "least one contact point, and its deployment acks would be dropped silently — the "
|
||||
+ "deployment would then time out naming a node that is running perfectly.");
|
||||
}
|
||||
|
||||
foreach (var contactPoint in options.CentralContactPoints)
|
||||
{
|
||||
if (!contactPoint.StartsWith("akka.tcp://", StringComparison.Ordinal))
|
||||
{
|
||||
errors.Add(
|
||||
$"Contact point '{contactPoint}' must start with 'akka.tcp://' — it is an Akka "
|
||||
+ "remoting address (akka.tcp://<system>@<host>:<port>), not a bare host:port.");
|
||||
}
|
||||
else if (contactPoint.Contains("/user/", StringComparison.Ordinal)
|
||||
|| contactPoint.Contains("/system/", StringComparison.Ordinal))
|
||||
{
|
||||
errors.Add(
|
||||
$"Contact point '{contactPoint}' carries an actor-path suffix. Configure the "
|
||||
+ "node address only — '/system/receptionist' is appended at construction time, "
|
||||
+ "and a doubled suffix produces a path that never resolves.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Count == 0
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(string.Join(" ", errors));
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,12 @@ akka {
|
||||
|
||||
extensions = [
|
||||
"Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools"
|
||||
|
||||
# Per-cluster mesh Phase 2 — the central<->node command boundary. NOT auto-started: without
|
||||
# this entry the receptionist materialises only if some code path happens to call
|
||||
# ClusterClientReceptionist.Get(system) first, which makes "is the boundary listening?"
|
||||
# depend on startup ordering rather than on configuration.
|
||||
"Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools"
|
||||
]
|
||||
|
||||
actor {
|
||||
@@ -25,6 +31,20 @@ akka {
|
||||
dot-netty.tcp {
|
||||
hostname = "0.0.0.0"
|
||||
port = 4053
|
||||
|
||||
# FRAME SIZE. maximum-frame-size restates Akka's own default; what actually changes is
|
||||
# log-frame-size-exceeding, which defaults to `off`. Over the limit the transport drops
|
||||
# that ONE message without tearing down the association — heartbeats keep flowing, the
|
||||
# node still reports healthy, and the only symptom is a command that never arrived. The
|
||||
# sister project lost a deploy path to exactly this and could not attribute it; their
|
||||
# postmortem also notes the default JSON serializer escapes an already-serialized
|
||||
# payload a second time, so ~60-70 KB of real JSON is enough to blow 128 KB.
|
||||
#
|
||||
# OtOpcUa's deploy notify is payload-free by design (DispatchDeployment carries only
|
||||
# DeploymentId + RevisionHash + CorrelationId), so this threshold should stay silent.
|
||||
# It is a canary, and a quiet one is the point.
|
||||
maximum-frame-size = 128000b
|
||||
log-frame-size-exceeding = 32000b
|
||||
}
|
||||
transport-failure-detector {
|
||||
heartbeat-interval = 2s
|
||||
@@ -74,6 +94,31 @@ akka {
|
||||
down-removal-margin = 15s
|
||||
run-coordinated-shutdown-when-down = on
|
||||
|
||||
# CLUSTERCLIENT — the central<->node command boundary (per-cluster mesh Phase 2).
|
||||
client {
|
||||
# DELIBERATE, and NOT what the sister project runs: they never wrote this section, so
|
||||
# they inherit buffer-size = 1000. Zero means "if the receptionist is unknown, drop the
|
||||
# message now". Buffering would replay a DispatchDeployment on reconnect and apply a
|
||||
# deployment ConfigPublishCoordinator had already sealed as TimedOut — the node would
|
||||
# then be running a configuration the database says failed. Fail-fast keeps the DB the
|
||||
# single record of truth, and reproduces today's DPS behaviour exactly: a node that is
|
||||
# down misses the publish and self-heals from the DB on restart.
|
||||
buffer-size = 0
|
||||
|
||||
# Retry forever rather than self-stopping the client. Restates Akka's default; stated
|
||||
# explicitly because the behaviour is load-bearing — a central node down for an hour
|
||||
# must not require a driver-node restart to become reachable again.
|
||||
reconnect-timeout = off
|
||||
|
||||
receptionist {
|
||||
# Empty = every member hosts a receptionist. This is what makes the comm actors
|
||||
# "registered per node, not as a singleton" work: contact rotation reaches whichever
|
||||
# node answers. Restates Akka's default; do NOT scope this to a role, or the
|
||||
# boundary becomes reachable through one node only and rotation silently fails.
|
||||
role = ""
|
||||
}
|
||||
}
|
||||
|
||||
singleton {
|
||||
singleton-name = "singleton"
|
||||
}
|
||||
|
||||
@@ -34,6 +34,20 @@ public static class ServiceCollectionExtensions
|
||||
services.AddValidatedOptions<AkkaClusterOptions, AkkaClusterOptionsValidator>(
|
||||
configuration, AkkaClusterOptions.SectionName);
|
||||
|
||||
// Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at
|
||||
// startup for the same reason the cluster options are — a contact point that cannot resolve
|
||||
// produces silence, not an error, and silence is indistinguishable from "nothing to do".
|
||||
services.AddValidatedOptions<MeshTransportOptions, MeshTransportOptionsValidator>(
|
||||
configuration, MeshTransportOptions.SectionName);
|
||||
|
||||
// Per-cluster mesh Phase 3: where a driver reads its config (Direct SQL, or fetch-and-cache
|
||||
// over gRPC). Validated at startup for the same reason — a FetchAndCache node with no endpoint
|
||||
// or no key produces silence, not an error. ConfigServe (the central serve surface) needs no
|
||||
// cross-field validation: a 0 port is a legitimate "disabled", so a plain Configure suffices.
|
||||
services.AddValidatedOptions<ConfigSourceOptions, ConfigSourceOptionsValidator>(
|
||||
configuration, ConfigSourceOptions.SectionName);
|
||||
services.Configure<ConfigServeOptions>(configuration.GetSection(ConfigServeOptions.SectionName));
|
||||
|
||||
services.AddSingleton<IClusterRoleInfo, ClusterRoleInfo>();
|
||||
|
||||
return services;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
|
||||
/// <summary>
|
||||
/// A central→node command, plus the DistributedPubSub topic it would have been published on.
|
||||
/// </summary>
|
||||
/// <param name="Topic">
|
||||
/// The legacy DPS topic. Carried so the router can publish under
|
||||
/// <c>MeshTransport:Mode = Dps</c> without a second dispatch table; ignored entirely under
|
||||
/// <c>ClusterClient</c> mode, where the destination is an actor path rather than a topic.
|
||||
/// </param>
|
||||
/// <param name="Message">
|
||||
/// The command itself, never wrapped. Node-side handlers see exactly the type they see today,
|
||||
/// which is what lets the dark switch touch publishers only.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This envelope exists so the admin singletons stop knowing which transport is in force.
|
||||
/// Before per-cluster mesh Phase 2 each publisher held its own
|
||||
/// <c>DistributedPubSub.Get(Context.System).Mediator</c> call, so the switch would have had
|
||||
/// to be threaded through five call sites across two actors, and each would have been free
|
||||
/// to drift.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Central-side only — it never crosses the wire, so it carries no serialization contract
|
||||
/// and needs no additive-evolution discipline.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed record MeshCommand(string Topic, object Message);
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
|
||||
/// <summary>
|
||||
/// Actor paths the two mesh comm actors register under. <b>These strings are the wire
|
||||
/// contract</b> of the central↔node boundary.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Declared once, here, rather than as literals on each side. The <c>deployments</c> /
|
||||
/// <c>deployment-acks</c> topic names are declared twice — once in
|
||||
/// <c>ConfigPublishCoordinator</c> and again in <c>DriverHostActor</c> — and agree only by
|
||||
/// convention; renaming one would compile, deploy, and deliver nothing. A ClusterClient path
|
||||
/// has the same failure mode and no compiler to catch it, so the constant is shared.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Nothing else pins these values, which is why <c>MeshCommActorPathTests</c> boots a real
|
||||
/// host and <c>Identify</c>-probes both paths.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class MeshPaths
|
||||
{
|
||||
/// <summary>Central-side comm actor — where a driver node sends its deployment acks.</summary>
|
||||
public const string CentralCommunication = "/user/central-communication";
|
||||
|
||||
/// <summary>Node-side comm actor — where central sends commands.</summary>
|
||||
public const string NodeCommunication = "/user/node-communication";
|
||||
|
||||
/// <summary>Actor name of <see cref="CentralCommunication"/> (the path's final segment).</summary>
|
||||
public const string CentralCommunicationName = "central-communication";
|
||||
|
||||
/// <summary>Actor name of <see cref="NodeCommunication"/> (the path's final segment).</summary>
|
||||
public const string NodeCommunicationName = "node-communication";
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package deployment_artifact.v1;
|
||||
|
||||
option csharp_namespace = "ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1";
|
||||
|
||||
// Central serves the deployed-configuration artifact bytes to a driver node (per-cluster mesh
|
||||
// Phase 3). Only admin-role nodes — which hold the central SQL connection — serve; driver nodes in
|
||||
// FetchAndCache mode are the clients. The bytes travel here, out-of-band, precisely so they never
|
||||
// ride an Akka message and hit the frame-size limit.
|
||||
service DeploymentArtifactService {
|
||||
// Streams the artifact for a sealed deployment as ordered chunks. NotFound when the id is
|
||||
// unknown, the deployment is not sealed, or the blob is empty (existence-hiding + the #485
|
||||
// "empty bytes are never an empty config" rule). The client verifies
|
||||
// SHA-256(reassembled) == the RevisionHash it already holds from DispatchDeployment, so no hash
|
||||
// is carried here.
|
||||
rpc Fetch(FetchRequest) returns (stream ArtifactChunk);
|
||||
}
|
||||
|
||||
message FetchRequest {
|
||||
string deployment_id = 1;
|
||||
}
|
||||
|
||||
message ArtifactChunk {
|
||||
bytes data = 1;
|
||||
}
|
||||
@@ -11,6 +11,24 @@
|
||||
<PackageReference Include="ZB.MOM.WW.Audit"/>
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Per-cluster mesh Phase 3: the first locally-compiled proto in the repo (all other gRPC here
|
||||
is packaged clients). Grpc.Tools generates both the server base and the client stub from
|
||||
deployment_artifact.proto; Google.Protobuf + Grpc.Core.Api carry the generated code's runtime
|
||||
types. Placed in Commons so both the central server (Host/AdminUI) and the node client
|
||||
(Runtime) get the types from one reference. -->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Google.Protobuf"/>
|
||||
<PackageReference Include="Grpc.Core.Api"/>
|
||||
<PackageReference Include="Grpc.Tools">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Protobuf Include="Protos\deployment_artifact.proto" GrpcServices="Both"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ZB.MOM.WW.OtOpcUa.Core.Abstractions\ZB.MOM.WW.OtOpcUa.Core.Abstractions.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
// The generated service container is itself named DeploymentArtifactService; alias its nested server
|
||||
// base so this impl can keep the natural name without colliding with the generated type.
|
||||
using GeneratedServiceBase =
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1.DeploymentArtifactService.DeploymentArtifactServiceBase;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side artifact serve (per-cluster mesh Phase 3). Streams a deployment's
|
||||
/// <c>ArtifactBlob</c> to a driver node fetching in <c>FetchAndCache</c> mode. Registered only on
|
||||
/// admin-role nodes (which hold the central SQL connection) and only when
|
||||
/// <c>ConfigServe:GrpcListenPort > 0</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Serves any deployment that has a non-empty blob — status is NOT gated.</b> A node
|
||||
/// fetches the artifact <em>during apply</em>, before the deployment is sealed: central seals
|
||||
/// only once every node has acked Applied, and a <c>FetchAndCache</c> node cannot ack until it
|
||||
/// has fetched. Gating on <c>Sealed</c> would deadlock every such deploy (seal-needs-ack →
|
||||
/// ack-needs-fetch → fetch-needs-sealed), and it matches Direct mode, which reads the same
|
||||
/// <c>ArtifactBlob</c> from SQL while the row is still <c>AwaitingApplyAcks</c>. (Caught on the
|
||||
/// Phase 3 live gate.) Access is already gated by the shared-key interceptor, so there is no
|
||||
/// reason to hide a non-sealed deployment.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>NotFound is indistinguishable across two cases</b> — unknown id and an empty blob. The
|
||||
/// first is existence-hiding; the second is the #485 rule (a zero-length artifact is "no
|
||||
/// answer," never "a configuration with no drivers"). Streaming empty bytes as if valid is
|
||||
/// exactly what that defect class forbids.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DeploymentArtifactGrpcService : GeneratedServiceBase
|
||||
{
|
||||
/// <summary>Chunk size, matching <c>LocalDbDeploymentArtifactCache.ChunkSize</c> (128 KiB).</summary>
|
||||
private const int ChunkSize = 128 * 1024;
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly ILogger<DeploymentArtifactGrpcService> _log;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="DeploymentArtifactGrpcService"/> class.</summary>
|
||||
/// <param name="dbFactory">Factory for the central config database holding the artifact bytes.</param>
|
||||
/// <param name="log">Logger.</param>
|
||||
public DeploymentArtifactGrpcService(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
ILogger<DeploymentArtifactGrpcService> log)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task Fetch(
|
||||
FetchRequest request,
|
||||
IServerStreamWriter<ArtifactChunk> responseStream,
|
||||
ServerCallContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(responseStream);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
if (!Guid.TryParse(request.DeploymentId, out var id))
|
||||
throw new RpcException(new Status(StatusCode.NotFound, "unknown deployment"));
|
||||
|
||||
await using var db = await _dbFactory.CreateDbContextAsync(context.CancellationToken);
|
||||
var row = await db.Deployments
|
||||
.AsNoTracking()
|
||||
.Where(d => d.DeploymentId == id)
|
||||
.Select(d => new { d.ArtifactBlob })
|
||||
.FirstOrDefaultAsync(context.CancellationToken);
|
||||
|
||||
// Existence-hiding + #485: unknown id / empty blob collapse to one NotFound. Status is NOT
|
||||
// gated — a node fetches during apply, before the deployment is sealed (see the class remarks).
|
||||
if (row is null || row.ArtifactBlob.Length == 0)
|
||||
{
|
||||
_log.LogDebug(
|
||||
"Artifact fetch for {DeploymentId} -> NotFound (found={Found}, bytes={Bytes})",
|
||||
request.DeploymentId, row is not null, row?.ArtifactBlob.Length ?? 0);
|
||||
throw new RpcException(new Status(StatusCode.NotFound, "unknown deployment"));
|
||||
}
|
||||
|
||||
for (var offset = 0; offset < row.ArtifactBlob.Length; offset += ChunkSize)
|
||||
{
|
||||
var len = Math.Min(ChunkSize, row.ArtifactBlob.Length - offset);
|
||||
await responseStream.WriteAsync(
|
||||
new ArtifactChunk { Data = ByteString.CopyFrom(row.ArtifactBlob, offset, len) },
|
||||
context.CancellationToken);
|
||||
}
|
||||
|
||||
_log.LogInformation(
|
||||
"Served artifact for {DeploymentId} ({Bytes} bytes, {Chunks} chunks)",
|
||||
request.DeploymentId, row.ArtifactBlob.Length,
|
||||
(row.ArtifactBlob.Length + ChunkSize - 1) / ChunkSize);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
@@ -26,6 +27,13 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
private readonly IActorRef _coordinator;
|
||||
private readonly IReadOnlyDictionary<string, IDriverProbe> _probesByType;
|
||||
private readonly TagConfigValidationMode _tagConfigValidationMode;
|
||||
|
||||
/// <summary>
|
||||
/// Where central→node commands are handed off (per-cluster mesh Phase 2), or
|
||||
/// <see langword="null"/> to publish on the mediator directly.
|
||||
/// </summary>
|
||||
private readonly IActorRef? _meshRouter;
|
||||
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <summary>Creates actor props for the admin operations actor.</summary>
|
||||
@@ -33,29 +41,38 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
/// <param name="coordinator">Reference to the deployment coordinator actor.</param>
|
||||
/// <param name="probes">Driver probes registered in DI; keyed by DriverType (case-insensitive).</param>
|
||||
/// <param name="tagConfigValidationMode">Deploy-gate TagConfig strictness mode (Warn default | Error opt-in).</param>
|
||||
/// <param name="meshRouter">
|
||||
/// Central comm actor commands are handed to, or <see langword="null"/> to publish on the
|
||||
/// DistributedPubSub mediator directly (the pre-Phase-2 behaviour).
|
||||
/// </param>
|
||||
/// <returns>Props configured to create an AdminOperationsActor.</returns>
|
||||
public static Props Props(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
IActorRef coordinator,
|
||||
IEnumerable<IDriverProbe> probes,
|
||||
TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn) =>
|
||||
Akka.Actor.Props.Create(() => new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode));
|
||||
TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
|
||||
IActorRef? meshRouter = null) =>
|
||||
Akka.Actor.Props.Create(() =>
|
||||
new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter));
|
||||
|
||||
/// <summary>Initializes a new instance of the AdminOperationsActor.</summary>
|
||||
/// <param name="dbFactory">Factory for creating config database contexts.</param>
|
||||
/// <param name="coordinator">Reference to the deployment coordinator actor.</param>
|
||||
/// <param name="probes">Driver probes registered in DI; keyed by DriverType (case-insensitive).</param>
|
||||
/// <param name="tagConfigValidationMode">Deploy-gate TagConfig strictness mode (Warn default | Error opt-in).</param>
|
||||
/// <param name="meshRouter">Central comm actor, or <see langword="null"/> for the mediator.</param>
|
||||
public AdminOperationsActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
IActorRef coordinator,
|
||||
IEnumerable<IDriverProbe> probes,
|
||||
TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn)
|
||||
TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
|
||||
IActorRef? meshRouter = null)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_coordinator = coordinator;
|
||||
_probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase);
|
||||
_tagConfigValidationMode = tagConfigValidationMode;
|
||||
_meshRouter = meshRouter;
|
||||
|
||||
ReceiveAsync<StartDeployment>(HandleStartDeploymentAsync);
|
||||
ReceiveAsync<TestDriverConnect>(HandleTestDriverConnectAsync);
|
||||
@@ -65,6 +82,23 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
Receive<ShelveAlarmCommand>(HandleShelveAlarm);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hands a central→node command to the mesh transport router, or publishes it directly when
|
||||
/// no router is wired.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The fallback keeps ~12 existing tests (and any node wired before per-cluster mesh Phase 2)
|
||||
/// on exactly the pre-Phase-2 behaviour. It is not a permanent seam: Phase 6 deletes it along
|
||||
/// with the DistributedPubSub branch and the single mesh.
|
||||
/// </remarks>
|
||||
/// <param name="topic">The legacy DistributedPubSub topic.</param>
|
||||
/// <param name="message">The command.</param>
|
||||
private void Dispatch(string topic, object message)
|
||||
{
|
||||
if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(topic, message));
|
||||
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(topic, message));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// AdminUI Acknowledge path. Maps the control-plane command to a
|
||||
/// <see cref="AlarmCommand"/> (<c>Operation = "Acknowledge"</c>) and publishes it onto the
|
||||
@@ -85,7 +119,7 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
Comment: msg.Comment,
|
||||
UnshelveAtUtc: null);
|
||||
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(AlarmCommandsTopic.Name, alarmCmd));
|
||||
Dispatch(AlarmCommandsTopic.Name, alarmCmd);
|
||||
|
||||
_log.Info("AdminOps: Acknowledge published for alarm {AlarmId} by {User}", msg.AlarmId, msg.User);
|
||||
replyTo.Tell(new AcknowledgeAlarmResult(true, null, msg.CorrelationId));
|
||||
@@ -132,7 +166,7 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
Comment: msg.Comment,
|
||||
UnshelveAtUtc: unshelveAt);
|
||||
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(AlarmCommandsTopic.Name, alarmCmd));
|
||||
Dispatch(AlarmCommandsTopic.Name, alarmCmd);
|
||||
|
||||
_log.Info("AdminOps: {Operation} published for alarm {AlarmId} by {User}", operation, msg.AlarmId, msg.User);
|
||||
replyTo.Tell(new ShelveAlarmResult(true, null, msg.CorrelationId));
|
||||
@@ -394,7 +428,7 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
{
|
||||
// Broadcast to every DriverHostActor on every node via the driver-control DPS topic.
|
||||
// Only the host that owns the instance will act; others ignore it (id not found in _children).
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DriverControlTopic.Name, msg));
|
||||
Dispatch(DriverControlTopic.Name, msg);
|
||||
|
||||
await using var db = await _dbFactory.CreateDbContextAsync();
|
||||
db.ConfigEdits.Add(new ConfigEdit
|
||||
@@ -424,7 +458,7 @@ public sealed class AdminOperationsActor : ReceiveActor
|
||||
try
|
||||
{
|
||||
// Broadcast to every DriverHostActor; only the one owning the instance reacts.
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DriverControlTopic.Name, msg));
|
||||
Dispatch(DriverControlTopic.Name, msg);
|
||||
|
||||
await using var db = await _dbFactory.CreateDbContextAsync();
|
||||
db.ConfigEdits.Add(new ConfigEdit
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side end of the central↔node command boundary (per-cluster mesh Phase 2). Routes
|
||||
/// every central→node command out — over DistributedPubSub or ClusterClient, per
|
||||
/// <see cref="MeshTransportOptions.Mode"/> — and forwards inbound <see cref="ApplyAck"/>s to the
|
||||
/// deploy coordinator singleton.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Registered per node, NOT as a cluster singleton.</b> A driver node's ClusterClient
|
||||
/// rotates across contact points and must find a live comm actor at whichever central node
|
||||
/// answers. A singleton would be reachable only through the one node hosting it, and the
|
||||
/// rotation would fail silently against the other.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Exactly one ClusterClient, fleet-wide.</b> See <see cref="RebuildClient"/> — this is
|
||||
/// the single most important thing to understand before changing this actor.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
private const string RefreshTimerKey = "mesh-contact-refresh";
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly MeshTransportOptions _options;
|
||||
private readonly IMeshClusterClientFactory _clientFactory;
|
||||
private readonly Func<IActorRef?> _coordinator;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private readonly bool _useClusterClient;
|
||||
|
||||
private IActorRef? _client;
|
||||
private ImmutableHashSet<string> _currentContacts = ImmutableHashSet<string>.Empty;
|
||||
private CancellationTokenSource _lifecycle = new();
|
||||
|
||||
/// <summary>Gets the timer scheduler driving the periodic contact refresh.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>Creates the props for this actor.</summary>
|
||||
/// <param name="dbFactory">Factory for the config database holding <c>ClusterNode</c> rows.</param>
|
||||
/// <param name="options">The bound mesh-transport options.</param>
|
||||
/// <param name="clientFactory">Creates the ClusterClient; substituted in tests.</param>
|
||||
/// <param name="coordinator">
|
||||
/// Resolves the deploy-coordinator singleton proxy. A <see cref="Func{TResult}"/> rather than
|
||||
/// an <see cref="IActorRef"/> so this actor never depends on registration order at wiring
|
||||
/// time — it resolves on first use and caches.
|
||||
/// </param>
|
||||
/// <returns>The props.</returns>
|
||||
public static Props Props(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
MeshTransportOptions options,
|
||||
IMeshClusterClientFactory clientFactory,
|
||||
Func<IActorRef?> coordinator) =>
|
||||
Akka.Actor.Props.Create(() =>
|
||||
new CentralCommunicationActor(dbFactory, options, clientFactory, coordinator));
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="CentralCommunicationActor"/> class.</summary>
|
||||
/// <param name="dbFactory">Factory for the config database.</param>
|
||||
/// <param name="options">The bound mesh-transport options.</param>
|
||||
/// <param name="clientFactory">Creates the ClusterClient.</param>
|
||||
/// <param name="coordinator">Lazily resolves the deploy-coordinator singleton proxy.</param>
|
||||
public CentralCommunicationActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
MeshTransportOptions options,
|
||||
IMeshClusterClientFactory clientFactory,
|
||||
Func<IActorRef?> coordinator)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_options = options;
|
||||
_clientFactory = clientFactory;
|
||||
_coordinator = coordinator;
|
||||
_useClusterClient = string.Equals(
|
||||
options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
Receive<MeshCommand>(HandleMeshCommand);
|
||||
Receive<ApplyAck>(HandleApplyAck);
|
||||
Receive<RefreshContacts>(_ => LoadContactsFromDb());
|
||||
Receive<ContactsLoaded>(HandleContactsLoaded);
|
||||
|
||||
// A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler
|
||||
// the failure is an unhandled message (debug level only) and the refresh fails silently —
|
||||
// an operator cannot distinguish "no nodes configured" from "the database is down", and the
|
||||
// contact set silently freezes at whatever it last held.
|
||||
Receive<Status.Failure>(f => _log.Warning(
|
||||
f.Cause,
|
||||
"Failed to load ClusterNode contact points; the mesh ClusterClient contact set was NOT "
|
||||
+ "refreshed and may be stale or empty. Commands may be dropped until the next refresh"));
|
||||
|
||||
Receive<SubscribeAck>(_ => { /* DPS subscribe confirmation */ });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
if (!_useClusterClient)
|
||||
{
|
||||
_log.Info(
|
||||
"Mesh transport is {Mode}: commands are published on DistributedPubSub and no "
|
||||
+ "ClusterClient is created", _options.Mode);
|
||||
return;
|
||||
}
|
||||
|
||||
Timers.StartPeriodicTimer(
|
||||
RefreshTimerKey,
|
||||
new RefreshContacts(),
|
||||
TimeSpan.Zero,
|
||||
TimeSpan.FromSeconds(_options.ContactRefreshSeconds));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
_lifecycle.Cancel();
|
||||
_lifecycle.Dispose();
|
||||
}
|
||||
|
||||
private void HandleMeshCommand(MeshCommand cmd)
|
||||
{
|
||||
if (!_useClusterClient)
|
||||
{
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(cmd.Topic, cmd.Message));
|
||||
return;
|
||||
}
|
||||
|
||||
if (_client is null)
|
||||
{
|
||||
// App-level drop, matching the sister project's decision and this repo's buffer-size = 0.
|
||||
// The command is NOT queued and will NOT be retried: a deploy fails at the coordinator's
|
||||
// apply deadline naming the silent nodes, which is a better operator signal than a
|
||||
// command arriving minutes late against a config the DB already recorded as failed.
|
||||
_log.Warning(
|
||||
"No mesh ClusterClient — dropping {MessageType}. The last contact refresh produced "
|
||||
+ "no usable contact point from the enabled, non-maintenance ClusterNode rows; the "
|
||||
+ "command is not buffered and will not be retried",
|
||||
cmd.Message.GetType().Name);
|
||||
return;
|
||||
}
|
||||
|
||||
// SendToAll, NEVER Send. Today's DPS publish reaches EVERY DriverHostActor — there is no
|
||||
// ClusterId or node filter on the node side at all; scoping happens later, inside the
|
||||
// artifact (DeploymentArtifact.ResolveClusterScope). ClusterClient.Send delivers to exactly
|
||||
// ONE registered actor, which would deploy to a single node of the fleet while every other
|
||||
// node silently kept its old configuration — and the deployment would still seal green if
|
||||
// the ack set happened to be satisfied.
|
||||
_client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, cmd.Message));
|
||||
}
|
||||
|
||||
private void HandleApplyAck(ApplyAck ack)
|
||||
{
|
||||
var coordinator = _coordinator();
|
||||
if (coordinator is null)
|
||||
{
|
||||
_log.Warning(
|
||||
"Received ApplyAck for {DeploymentId} from {NodeId} but the deploy coordinator is "
|
||||
+ "not resolvable on this node; the ack is lost and the deployment will time out "
|
||||
+ "naming a node that applied successfully",
|
||||
ack.DeploymentId, ack.NodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Forward, not Tell: preserves the original sender across the hop so the coordinator sees
|
||||
// the node rather than this relay.
|
||||
coordinator.Forward(ack);
|
||||
}
|
||||
|
||||
private void LoadContactsFromDb()
|
||||
{
|
||||
var self = Self;
|
||||
CancellationToken ct;
|
||||
try
|
||||
{
|
||||
ct = _lifecycle.Token;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return; // Stopping.
|
||||
}
|
||||
|
||||
var systemName = Context.System.Name;
|
||||
var dbFactory = _dbFactory;
|
||||
|
||||
// Off the actor thread, result piped back as a message. A synchronous DB read here would
|
||||
// block every command dispatch behind a slow or unreachable SQL Server.
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await using var db = await dbFactory.CreateDbContextAsync(ct).ConfigureAwait(false);
|
||||
var rows = await db.ClusterNodes
|
||||
.AsNoTracking()
|
||||
.Where(n => n.Enabled && !n.MaintenanceMode)
|
||||
.Select(n => new { n.NodeId, n.Host, n.AkkaPort })
|
||||
.ToListAsync(ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var contacts = new List<string>(rows.Count);
|
||||
var malformed = new List<string>();
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var address = $"akka.tcp://{systemName}@{row.Host}:{row.AkkaPort}/system/receptionist";
|
||||
|
||||
// Parse up front, per row, inside the loop's own guard. A single malformed row must
|
||||
// not abort the whole refresh and leave the contact set half-built — the sister
|
||||
// project shipped that bug and its regression test deliberately orders the bad row
|
||||
// first.
|
||||
if (ActorPath.TryParse(address, out _)) contacts.Add(address);
|
||||
else malformed.Add($"{row.NodeId} -> {address}");
|
||||
}
|
||||
|
||||
return new ContactsLoaded(contacts, malformed);
|
||||
}, ct).PipeTo(self);
|
||||
}
|
||||
|
||||
private void HandleContactsLoaded(ContactsLoaded msg)
|
||||
{
|
||||
foreach (var bad in msg.Malformed)
|
||||
{
|
||||
_log.Warning(
|
||||
"ClusterNode {Row} does not yield a usable receptionist address; skipping it in this "
|
||||
+ "refresh (other nodes are unaffected). Check the row's Host and AkkaPort", bad);
|
||||
}
|
||||
|
||||
var contacts = msg.Contacts.ToImmutableHashSet();
|
||||
|
||||
if (contacts.IsEmpty)
|
||||
{
|
||||
_log.Warning(
|
||||
"No usable ClusterNode contact points; the mesh ClusterClient was not created and "
|
||||
+ "every central→node command will be dropped until a refresh finds one");
|
||||
return;
|
||||
}
|
||||
|
||||
if (_client is not null && _currentContacts.SetEquals(contacts)) return;
|
||||
|
||||
RebuildClient(contacts);
|
||||
}
|
||||
|
||||
/// <summary>Stops any existing client and creates a replacement for the new contact set.</summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>TODO(Phase 6): one client per application <c>Cluster</c>.</b> Today this actor
|
||||
/// creates exactly <b>one</b> client whose contacts are the whole fleet, and that is not
|
||||
/// a compromise — it is the only correct shape while the fleet is a single Akka mesh.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A <c>ClusterClientReceptionist</c> serves its entire cluster, so
|
||||
/// <c>SendToAll</c> reaches every registered node-comm actor in the mesh <i>regardless of
|
||||
/// which node's address was used as the contact point</i>. One client per Cluster would
|
||||
/// therefore fan each command out once per cluster — N× duplicate
|
||||
/// <c>DispatchDeployment</c> and N× duplicate <c>ApplyAck</c>. Phase 6 splits the meshes,
|
||||
/// at which point per-cluster clients become both correct and necessary.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Corollary worth stating because it is easy to assume otherwise: <b>the contact set
|
||||
/// does not scope delivery.</b> Excluding a maintenance-mode node from the contacts does
|
||||
/// not stop it receiving commands — it still receives them, exactly as it does under
|
||||
/// today's DPS broadcast. The filter is about which receptionists are worth <i>dialling</i>
|
||||
/// (a node in maintenance may be switched off), not about who gets the message.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="contacts">The receptionist addresses to dial.</param>
|
||||
private void RebuildClient(ImmutableHashSet<string> contacts)
|
||||
{
|
||||
if (_client is not null)
|
||||
{
|
||||
_log.Info("Mesh contact set changed; replacing the ClusterClient");
|
||||
Context.Stop(_client);
|
||||
|
||||
// Cleared now: if the create below throws, a stale ref would route commands into a
|
||||
// stopping actor and they would vanish without a warning.
|
||||
_client = null;
|
||||
_currentContacts = ImmutableHashSet<string>.Empty;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var paths = contacts.Select(ActorPath.Parse).ToImmutableHashSet();
|
||||
_client = _clientFactory.Create(Context.System, paths);
|
||||
_currentContacts = contacts;
|
||||
_log.Info("Mesh ClusterClient created with {Count} contact point(s)", contacts.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Error(ex,
|
||||
"Failed to create the mesh ClusterClient; central→node commands are dropped until "
|
||||
+ "the next contact refresh succeeds");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Internal tick asking for a contact-point refresh.</summary>
|
||||
public sealed record RefreshContacts;
|
||||
|
||||
/// <summary>Result of a contact-point load, piped back from the DB read.</summary>
|
||||
/// <param name="Contacts">Receptionist addresses that parsed cleanly.</param>
|
||||
/// <param name="Malformed">Rows that did not, described for the log.</param>
|
||||
public sealed record ContactsLoaded(
|
||||
IReadOnlyList<string> Contacts,
|
||||
IReadOnlyList<string> Malformed);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Creates the <c>ClusterClient</c> central uses to reach driver nodes. Exists as a seam so
|
||||
/// <c>CentralCommunicationActor</c> can be tested against a <c>TestProbe</c> without standing up
|
||||
/// a second cluster.
|
||||
/// </summary>
|
||||
public interface IMeshClusterClientFactory
|
||||
{
|
||||
/// <summary>Creates a client for the given receptionist contact points.</summary>
|
||||
/// <param name="system">The actor system to spawn the client in.</param>
|
||||
/// <param name="contacts">Receptionist actor paths, one per reachable node.</param>
|
||||
/// <returns>The new client's <see cref="IActorRef"/>.</returns>
|
||||
IActorRef Create(ActorSystem system, ImmutableHashSet<ActorPath> contacts);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public sealed class DefaultMeshClusterClientFactory : IMeshClusterClientFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-incarnation generation counter, appended to the actor name.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <c>Context.Stop</c> of the previous client is <b>asynchronous</b> — its actor name stays
|
||||
/// reserved until termination completes — so recreating the same name while handling a single
|
||||
/// message throws <see cref="InvalidActorNameException"/>. That is exactly what a contact-set
|
||||
/// change does: stop the old client, create the replacement, both in one handler. A
|
||||
/// generation suffix makes every incarnation's name unique by construction. Ported from the
|
||||
/// sister project, where it was a shipped bug fix rather than foresight.
|
||||
/// </remarks>
|
||||
private long _generation;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IActorRef Create(ActorSystem system, ImmutableHashSet<ActorPath> contacts)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(system);
|
||||
ArgumentNullException.ThrowIfNull(contacts);
|
||||
|
||||
var settings = ClusterClientSettings.Create(system).WithInitialContacts(contacts);
|
||||
var name = $"mesh-client-{Interlocked.Increment(ref _generation)}";
|
||||
return system.ActorOf(ClusterClient.Props(settings), name);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
@@ -60,6 +61,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly TimeSpan _applyDeadline;
|
||||
|
||||
/// <summary>
|
||||
/// Where central→node commands are handed off (per-cluster mesh Phase 2), or
|
||||
/// <see langword="null"/> to publish on the mediator directly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Null in tests and on any node wired before Phase 2, where the fallback reproduces the
|
||||
/// pre-Phase-2 behaviour exactly. The fallback is not a permanent seam — Phase 6 deletes it
|
||||
/// along with the DistributedPubSub branch and the single mesh.
|
||||
/// </remarks>
|
||||
private readonly IActorRef? _meshRouter;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
// NodeId equality here is case-insensitive (by Value) to match the case-insensitive ClusterId/
|
||||
// NodeId scoping in DeploymentArtifact.ResolveClusterScope — so an ack from a node whose address
|
||||
@@ -78,22 +90,31 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
|
||||
/// <param name="dbFactory">The database context factory for accessing configuration data.</param>
|
||||
/// <param name="applyDeadline">The timeout for waiting for apply acknowledgments from cluster nodes.</param>
|
||||
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
|
||||
/// <param name="meshRouter">
|
||||
/// Central comm actor the dispatch is handed to, or <see langword="null"/> to publish on the
|
||||
/// DistributedPubSub mediator directly (the pre-Phase-2 behaviour).
|
||||
/// </param>
|
||||
public static Props Props(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
TimeSpan? applyDeadline = null) =>
|
||||
Akka.Actor.Props.Create(() => new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline));
|
||||
TimeSpan? applyDeadline = null,
|
||||
IActorRef? meshRouter = null) =>
|
||||
Akka.Actor.Props.Create(() =>
|
||||
new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter));
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ConfigPublishCoordinator"/> class.
|
||||
/// </summary>
|
||||
/// <param name="dbFactory">The database context factory for persisting deployment state.</param>
|
||||
/// <param name="applyDeadline">The timeout for waiting for per-node apply acknowledgments.</param>
|
||||
/// <param name="meshRouter">Central comm actor, or <see langword="null"/> for the mediator.</param>
|
||||
public ConfigPublishCoordinator(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
TimeSpan applyDeadline)
|
||||
TimeSpan applyDeadline,
|
||||
IActorRef? meshRouter = null)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_applyDeadline = applyDeadline;
|
||||
_meshRouter = meshRouter;
|
||||
Receive<DispatchDeployment>(HandleDispatch);
|
||||
Receive<ApplyAck>(HandleAck);
|
||||
Receive<DeadlineElapsed>(HandleDeadline);
|
||||
@@ -172,7 +193,10 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg));
|
||||
// Per-cluster mesh Phase 2: hand off to the transport router rather than publishing
|
||||
// directly, so this actor no longer knows which transport is in force.
|
||||
if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg));
|
||||
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg));
|
||||
Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline);
|
||||
|
||||
if (_expectedAcks.Count == 0)
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Hosting;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Hosting;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
@@ -48,12 +53,49 @@ public static class ServiceCollectionExtensions
|
||||
var singletonOptions = new ClusterSingletonOptions { Role = AdminRole };
|
||||
var proxyOptions = new ClusterSingletonOptions { Role = AdminRole };
|
||||
|
||||
// Per-cluster mesh Phase 2: the central end of the central↔node command boundary.
|
||||
//
|
||||
// A plain WithActors, NOT a singleton — and that is the load-bearing part. A driver node's
|
||||
// ClusterClient rotates across contact points and must find a live comm actor at whichever
|
||||
// central node answers. As a singleton it would exist on one central node only, and the
|
||||
// rotation would fail silently against the other: acks would land sometimes, depending on
|
||||
// which contact the client happened to settle on.
|
||||
builder.WithActors((system, registry, resolver) =>
|
||||
{
|
||||
var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
||||
var options = resolver.GetService<IOptions<MeshTransportOptions>>().Value;
|
||||
|
||||
var actor = system.ActorOf(
|
||||
CentralCommunicationActor.Props(
|
||||
dbFactory,
|
||||
options,
|
||||
new DefaultMeshClusterClientFactory(),
|
||||
// Resolved lazily, per message, so this actor never depends on the order in
|
||||
// which Akka.Hosting materialises the singleton proxy relative to this block.
|
||||
() => registry.TryGet<ConfigPublishCoordinatorKey>(out var coordinator)
|
||||
? coordinator
|
||||
: null),
|
||||
MeshPaths.CentralCommunicationName);
|
||||
|
||||
// Registered unconditionally, in both transport modes. Under Dps nothing dials it, and
|
||||
// an idle registration costs nothing — whereas registering only under ClusterClient
|
||||
// would mean flipping the flag on a running fleet requires a central restart before any
|
||||
// node can ack, turning a config change back into a deployment.
|
||||
ClusterClientReceptionist.Get(system).RegisterService(actor);
|
||||
registry.Register<CentralCommunicationActorKey>(actor);
|
||||
});
|
||||
|
||||
builder.WithSingleton<ConfigPublishCoordinatorKey>(
|
||||
ConfigPublishSingletonName,
|
||||
(system, registry, resolver) =>
|
||||
{
|
||||
var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
||||
return ConfigPublishCoordinator.Props(dbFactory);
|
||||
// Registered above, so it is already in the registry — the same
|
||||
// ordering-by-registration the AdminOperations singleton relies on to resolve the
|
||||
// coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks
|
||||
// inside a props factory and races the spawn it is waiting for.
|
||||
var meshRouter = registry.Get<CentralCommunicationActorKey>();
|
||||
return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter);
|
||||
},
|
||||
singletonOptions);
|
||||
|
||||
@@ -69,7 +111,8 @@ public static class ServiceCollectionExtensions
|
||||
var mode = Enum.TryParse<TagConfigValidationMode>(
|
||||
resolver.GetService<IConfiguration>()?["Deployment:TagConfigValidationMode"],
|
||||
ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn;
|
||||
return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode);
|
||||
var meshRouter = registry.Get<CentralCommunicationActorKey>();
|
||||
return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter);
|
||||
},
|
||||
singletonOptions);
|
||||
|
||||
@@ -123,3 +166,4 @@ public sealed class AuditWriterActorKey { }
|
||||
public sealed class FleetStatusBroadcasterKey { }
|
||||
public sealed class RedundancyStateActorKey { }
|
||||
public sealed class ClusterNodeAddressReconcilerKey { }
|
||||
public sealed class CentralCommunicationActorKey { }
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Grpc.Core;
|
||||
using Grpc.Core.Interceptors;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Gates the central config-serve artifact endpoint (per-cluster mesh Phase 3). The
|
||||
/// <c>DeploymentArtifactService</c> streams a driver node the exact configuration it will boot
|
||||
/// on; anything able to reach central's serve port could otherwise pull the whole deployed config
|
||||
/// out-of-band. This interceptor is the ONLY inbound auth on that endpoint.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Scoped by method path.</b> Only calls under
|
||||
/// <c>/deployment_artifact.v1.DeploymentArtifactService/</c> are gated; every other method on
|
||||
/// the shared gRPC pipeline (notably the LocalDb sync service) passes through untouched, so
|
||||
/// this can share one <c>AddGrpc</c> registration with <see cref="LocalDbSyncAuthInterceptor"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Fail-closed.</b> With no <c>ConfigServe:ApiKey</c> configured, NO fetch is accepted,
|
||||
/// authenticated or not. Treating "no key" as "no auth required" would expose the deployed
|
||||
/// configuration on exactly the default shape most nodes ship with. The node dialing in must
|
||||
/// present the same <c>ConfigSource:ApiKey</c> (the shared node key), so a key typo stops
|
||||
/// config delivery outright rather than degrading to unauthenticated.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Comparison is <see cref="CryptographicOperations.FixedTimeEquals"/> over UTF-8 bytes, so a
|
||||
/// wrong key cannot be recovered byte-by-byte from response timing. Length differences are
|
||||
/// unavoidably observable and are not sensitive.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ConfigServeAuthInterceptor : Interceptor
|
||||
{
|
||||
private const string ServicePrefix = "/deployment_artifact.v1.DeploymentArtifactService/";
|
||||
private const string AuthorizationHeader = "authorization";
|
||||
private const string BearerPrefix = "Bearer ";
|
||||
|
||||
private readonly IOptions<ConfigServeOptions> _options;
|
||||
private readonly ILogger<ConfigServeAuthInterceptor> _logger;
|
||||
|
||||
/// <summary>Creates the interceptor.</summary>
|
||||
/// <param name="options">Config-serve options; <c>ApiKey</c> is the expected bearer token.</param>
|
||||
/// <param name="logger">Logger for denial diagnostics.</param>
|
||||
public ConfigServeAuthInterceptor(
|
||||
IOptions<ConfigServeOptions> options,
|
||||
ILogger<ConfigServeAuthInterceptor> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
|
||||
TRequest request,
|
||||
ServerCallContext context,
|
||||
UnaryServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(request, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task DuplexStreamingServerHandler<TRequest, TResponse>(
|
||||
IAsyncStreamReader<TRequest> requestStream,
|
||||
IServerStreamWriter<TResponse> responseStream,
|
||||
ServerCallContext context,
|
||||
DuplexStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(requestStream, responseStream, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
|
||||
IAsyncStreamReader<TRequest> requestStream,
|
||||
ServerCallContext context,
|
||||
ClientStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(requestStream, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task ServerStreamingServerHandler<TRequest, TResponse>(
|
||||
TRequest request,
|
||||
IServerStreamWriter<TResponse> responseStream,
|
||||
ServerCallContext context,
|
||||
ServerStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(request, responseStream, context);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Throws <see cref="RpcException"/> with <see cref="StatusCode.Unauthenticated"/> if this is
|
||||
/// a config-serve call that does not carry the configured bearer token. Non-serve calls return
|
||||
/// immediately.
|
||||
/// </summary>
|
||||
private void Authorize(ServerCallContext context)
|
||||
{
|
||||
if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
|
||||
return;
|
||||
|
||||
var expected = _options.Value.ApiKey;
|
||||
if (string.IsNullOrEmpty(expected))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rejected a config-serve call to {Method}: no ConfigServe:ApiKey is configured, so the " +
|
||||
"artifact endpoint is closed. Configure the shared node key on central and every node.",
|
||||
context.Method);
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.Unauthenticated,
|
||||
"Config-serve is not accepting connections: no API key is configured on this node."));
|
||||
}
|
||||
|
||||
var presented = ExtractBearerToken(context.RequestHeaders);
|
||||
if (presented is null || !FixedTimeEquals(presented, expected))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rejected a config-serve call to {Method}: {Reason}.",
|
||||
context.Method,
|
||||
presented is null ? "no bearer token presented" : "bearer token did not match");
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.Unauthenticated,
|
||||
"Config-serve authentication failed."));
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractBearerToken(Metadata headers)
|
||||
{
|
||||
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
|
||||
// Metadata in a test behaves the same as a real request.
|
||||
foreach (var entry in headers)
|
||||
{
|
||||
if (!string.Equals(entry.Key, AuthorizationHeader, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var value = entry.Value;
|
||||
if (value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
return value[BearerPrefix.Length..];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool FixedTimeEquals(string presented, string expected)
|
||||
=> CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using Serilog.Events;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Components;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
@@ -29,6 +30,7 @@ using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
@@ -185,11 +187,15 @@ if (hasDriver)
|
||||
// (initiator) / LocalDb:SyncListenPort (listener) are set. See LocalDbRegistration.
|
||||
builder.Services.AddOtOpcUaLocalDb(builder.Configuration);
|
||||
|
||||
// gRPC server plumbing for the passive sync endpoint mapped below. Registered only under
|
||||
// hasDriver so admin-only nodes expose no sync surface at all. The interceptor is the ONLY
|
||||
// inbound auth on that endpoint — the replication library's LocalDbSyncService verifies
|
||||
// nothing — and it fail-closes when no ApiKey is configured.
|
||||
builder.Services.AddGrpc(o => o.Interceptors.Add<LocalDbSyncAuthInterceptor>());
|
||||
// Node-side gRPC artifact fetcher (per-cluster mesh Phase 3). Resolved + used by DriverHostActor
|
||||
// only under ConfigSource:Mode = FetchAndCache; idle (never invoked) in the default Direct mode, so
|
||||
// registering it unconditionally on driver nodes is harmless. IDisposable — DI disposes its cached
|
||||
// h2c channels at shutdown.
|
||||
builder.Services.AddSingleton<IDeploymentArtifactFetcher, GrpcDeploymentArtifactFetcher>();
|
||||
|
||||
// gRPC server plumbing (AddGrpc + the two path-scoped auth interceptors) is registered once,
|
||||
// below, for hasDriver || hasAdmin — the LocalDb passive sync endpoint (driver) and the
|
||||
// ConfigServe artifact endpoint (admin) share one pipeline.
|
||||
|
||||
// Config-gated server-side HistoryRead backend. When the ServerHistorian section is enabled this
|
||||
// overrides the NullHistorianDataSource default from AddOtOpcUaRuntime (last registration wins) with
|
||||
@@ -389,23 +395,46 @@ builder.Services.AddOtOpcUaSecrets(builder.Configuration);
|
||||
builder.Services.AddOtOpcUaHealth();
|
||||
builder.Services.AddOtOpcUaObservability(builder.Configuration);
|
||||
|
||||
// gRPC server plumbing shared by two endpoints: the LocalDb passive sync endpoint (driver-role,
|
||||
// mapped below on hasDriver && syncListenPort > 0) and the ConfigServe artifact endpoint
|
||||
// (admin-role, mapped on hasAdmin && configServeGrpcPort > 0). Each interceptor is scoped strictly
|
||||
// to its own service path, so both can share one pipeline and each is a harmless pass-through when
|
||||
// its service is unmapped. Both fail-closed with no ApiKey configured — the interceptor is the ONLY
|
||||
// inbound auth on either endpoint (the libraries verify nothing). Registered whenever either role is
|
||||
// present so an admin-only node still serves config and a driver-only node still gates sync.
|
||||
if (hasDriver || hasAdmin)
|
||||
{
|
||||
builder.Services.AddGrpc(o =>
|
||||
{
|
||||
if (hasDriver)
|
||||
o.Interceptors.Add<LocalDbSyncAuthInterceptor>();
|
||||
if (hasAdmin)
|
||||
o.Interceptors.Add<ConfigServeAuthInterceptor>();
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// LocalDb sync listener (default-OFF).
|
||||
// Dedicated h2c listeners (both default-OFF): the LocalDb sync endpoint (driver) and the
|
||||
// ConfigServe artifact endpoint (admin, per-cluster mesh Phase 3).
|
||||
//
|
||||
// DANGER: any explicit Kestrel Listen* call makes Kestrel IGNORE ASPNETCORE_URLS/urls ENTIRELY
|
||||
// (it logs "Overriding address(es)"). The host has no ConfigureKestrel today and binds solely via
|
||||
// that configuration, so adding a listener naively would silently unbind the AdminUI + deploy API
|
||||
// behind Traefik. Everything the host was already asked to serve is therefore re-bound explicitly
|
||||
// in the same block.
|
||||
// in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have BOTH
|
||||
// dedicated ports set, and re-applying the existing surface per-port would double-bind it and throw
|
||||
// "address already in use". So the existing surface is computed and applied once, then whichever of
|
||||
// the two dedicated ports are configured are added on top.
|
||||
//
|
||||
// The listener is HTTP/2-ONLY on purpose: the sync client speaks prior-knowledge h2c, which a
|
||||
// cleartext Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence a
|
||||
// dedicated port rather than multiplexing onto the main one.
|
||||
// Each listener is HTTP/2-ONLY on purpose: both clients speak prior-knowledge h2c, which a cleartext
|
||||
// Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence dedicated ports
|
||||
// rather than multiplexing onto the main one.
|
||||
//
|
||||
// When LocalDb:SyncListenPort is 0 (the default) none of this runs and URL binding is untouched.
|
||||
// When both ports are 0 (the default) none of this runs and URL binding is untouched.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
var syncListenPort = LocalDbRegistration.SyncListenPort(builder.Configuration);
|
||||
if (hasDriver && syncListenPort > 0)
|
||||
var syncListenPort = hasDriver ? LocalDbRegistration.SyncListenPort(builder.Configuration) : 0;
|
||||
var configServeGrpcPort = hasAdmin ? builder.Configuration.GetValue<int>("ConfigServe:GrpcListenPort") : 0;
|
||||
if (syncListenPort > 0 || configServeGrpcPort > 0)
|
||||
{
|
||||
// Mirror Kestrel's own source precedence: URLS wins; else the HTTP_PORTS/HTTPS_PORTS bare-port
|
||||
// vars; else Kestrel's localhost:5000 default. Missing the HTTP_PORTS leg is not academic — the
|
||||
@@ -428,8 +457,8 @@ if (hasDriver && syncListenPort > 0)
|
||||
];
|
||||
if (existingBindings.Count > 0)
|
||||
Log.Information(
|
||||
"LocalDb sync listener enabled; re-binding the HTTP_PORTS surface ({HttpPorts}/{HttpsPorts}) " +
|
||||
"alongside the sync port.", httpPorts, httpsPorts);
|
||||
"Dedicated h2c listener(s) enabled; re-binding the HTTP_PORTS surface ({HttpPorts}/{HttpsPorts}) " +
|
||||
"alongside the dedicated port(s).", httpPorts, httpsPorts);
|
||||
}
|
||||
|
||||
// A driver-only Windows-service node gets NO URLS and NO *_PORTS (Install-Services.ps1 sets URLS
|
||||
@@ -440,38 +469,48 @@ if (hasDriver && syncListenPort > 0)
|
||||
{
|
||||
existingBindings = [new KestrelHttpBinding("localhost", 5000, IsSecure: false)];
|
||||
Log.Information(
|
||||
"LocalDb sync listener enabled with no URLs configured; re-binding Kestrel's default " +
|
||||
"http://localhost:5000 alongside the sync port.");
|
||||
"Dedicated h2c listener(s) enabled with no URLs configured; re-binding Kestrel's default " +
|
||||
"http://localhost:5000 alongside the dedicated port(s).");
|
||||
}
|
||||
|
||||
// Re-binding an https endpoint would need its certificate configuration replayed too, which
|
||||
// this host does not model (TLS is terminated at Traefik). Rather than silently serve it
|
||||
// without TLS — or drop it — refuse to take over Kestrel at all and leave the existing
|
||||
// surface exactly as it was. The operator gets a loud, actionable error instead of an
|
||||
// AdminUI that stopped answering.
|
||||
// AdminUI that stopped answering. Disables BOTH dedicated ports (either would force the same
|
||||
// Kestrel takeover).
|
||||
if (existingBindings.Any(b => b.IsSecure))
|
||||
{
|
||||
Log.Error(
|
||||
"LocalDb:SyncListenPort is set to {Port} but this host serves HTTPS endpoint(s) ({Urls}). " +
|
||||
"Binding the sync listener requires re-binding every existing endpoint explicitly, and the " +
|
||||
"HTTPS certificate configuration cannot be replayed safely. The sync listener is DISABLED; " +
|
||||
"terminate TLS upstream (as the docker-dev rig does) or leave replication off on this node.",
|
||||
syncListenPort, configuredUrls);
|
||||
"A dedicated h2c listener is requested (LocalDb:SyncListenPort={SyncPort}, " +
|
||||
"ConfigServe:GrpcListenPort={ConfigServePort}) but this host serves HTTPS endpoint(s) ({Urls}). " +
|
||||
"Binding a dedicated listener requires re-binding every existing endpoint explicitly, and the " +
|
||||
"HTTPS certificate configuration cannot be replayed safely. BOTH dedicated listeners are DISABLED; " +
|
||||
"terminate TLS upstream (as the docker-dev rig does) or leave these features off on this node.",
|
||||
syncListenPort, configServeGrpcPort, configuredUrls);
|
||||
syncListenPort = 0;
|
||||
configServeGrpcPort = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Capture locals so the ConfigureKestrel closure does not observe later reassignment.
|
||||
var syncPortToBind = syncListenPort;
|
||||
var configServePortToBind = configServeGrpcPort;
|
||||
builder.WebHost.ConfigureKestrel(kestrel =>
|
||||
{
|
||||
foreach (var binding in existingBindings)
|
||||
binding.Apply(kestrel);
|
||||
|
||||
kestrel.ListenAnyIP(syncListenPort, o => o.Protocols = HttpProtocols.Http2);
|
||||
if (syncPortToBind > 0)
|
||||
kestrel.ListenAnyIP(syncPortToBind, o => o.Protocols = HttpProtocols.Http2);
|
||||
if (configServePortToBind > 0)
|
||||
kestrel.ListenAnyIP(configServePortToBind, o => o.Protocols = HttpProtocols.Http2);
|
||||
});
|
||||
|
||||
Log.Information(
|
||||
"LocalDb sync listener bound on :{SyncPort} (h2c); re-bound existing endpoint(s) {Urls}.",
|
||||
syncListenPort, configuredUrls ?? "http://localhost:5000 (Kestrel default)");
|
||||
"Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}); " +
|
||||
"re-bound existing endpoint(s) {Urls}.",
|
||||
syncListenPort, configServeGrpcPort, configuredUrls ?? "http://localhost:5000 (Kestrel default)");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -514,6 +553,14 @@ if (hasDriver && syncListenPort > 0)
|
||||
app.MapZbLocalDbSync();
|
||||
}
|
||||
|
||||
// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on admin-role
|
||||
// nodes (which hold the central SQL connection) and only when a dedicated h2c port is bound. The
|
||||
// ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes with no ConfigServe:ApiKey.
|
||||
if (hasAdmin && configServeGrpcPort > 0)
|
||||
{
|
||||
app.MapGrpcService<DeploymentArtifactGrpcService>();
|
||||
}
|
||||
|
||||
app.MapOtOpcUaHealth();
|
||||
app.MapOtOpcUaMetrics();
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Node-side end of the central↔node command boundary (per-cluster mesh Phase 2), registered
|
||||
/// with the <c>ClusterClientReceptionist</c> at <see cref="MeshPaths.NodeCommunication"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Registered per node, NOT as a cluster singleton</b> — central's ClusterClient rotates
|
||||
/// across contact points and must find a live comm actor at whichever node answers.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Inbound commands are re-emitted on the node's <see cref="EventStream"/>, not on their
|
||||
/// DistributedPubSub topic.</b> DPS is mesh-wide, and central <c>SendToAll</c>s to every
|
||||
/// node's comm actor — so a DPS re-publish would deliver N copies of every command to every
|
||||
/// subscriber, where N is the node count. The EventStream is node-local by construction,
|
||||
/// which is exactly the needed semantics. It also avoids a registration handshake with
|
||||
/// actors spawned later: <c>ScriptedAlarmHostActor</c> is a <i>child of</i>
|
||||
/// <c>DriverHostActor</c> and has no registry key, so there is no ref to hand in at wiring
|
||||
/// time.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Outbound is <see cref="ApplyAck"/> only.</b> There is no Ask across this boundary in
|
||||
/// Phase 2 — every migrated command is fire-and-forget, and the deploy ack returns as an
|
||||
/// independent message the coordinator already subscribes for. Nothing here needs sender
|
||||
/// preservation, which is why this actor is far smaller than the sister project's.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class NodeCommunicationActor : ReceiveActor
|
||||
{
|
||||
private readonly IActorRef? _centralClient;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <summary>Creates the props for this actor.</summary>
|
||||
/// <param name="centralClient">
|
||||
/// ClusterClient dialling central's receptionist, or <see langword="null"/> under
|
||||
/// <c>MeshTransport:Mode = Dps</c>, where acks still go out over DistributedPubSub from
|
||||
/// <c>DriverHostActor</c> and this actor never sees them.
|
||||
/// </param>
|
||||
/// <returns>The props.</returns>
|
||||
public static Props Props(IActorRef? centralClient) =>
|
||||
Akka.Actor.Props.Create(() => new NodeCommunicationActor(centralClient));
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="NodeCommunicationActor"/> class.</summary>
|
||||
/// <param name="centralClient">ClusterClient dialling central, or <see langword="null"/>.</param>
|
||||
public NodeCommunicationActor(IActorRef? centralClient)
|
||||
{
|
||||
_centralClient = centralClient;
|
||||
|
||||
// Inbound from central. Each type is listed explicitly rather than caught by a ReceiveAny:
|
||||
// an unknown type must dead-letter loudly during development rather than be republished
|
||||
// blind onto the node's EventStream, where it would be silently ignored by every subscriber.
|
||||
Receive<DispatchDeployment>(PublishLocally);
|
||||
Receive<RestartDriver>(PublishLocally);
|
||||
Receive<ReconnectDriver>(PublishLocally);
|
||||
Receive<AlarmCommand>(PublishLocally);
|
||||
|
||||
// Outbound to central.
|
||||
Receive<ApplyAck>(HandleApplyAck);
|
||||
}
|
||||
|
||||
private void PublishLocally(object msg)
|
||||
{
|
||||
_log.Debug("Mesh: received {MessageType} from central; publishing node-locally", msg.GetType().Name);
|
||||
Context.System.EventStream.Publish(msg);
|
||||
}
|
||||
|
||||
private void HandleApplyAck(ApplyAck ack)
|
||||
{
|
||||
if (_centralClient is null)
|
||||
{
|
||||
_log.Warning(
|
||||
"No central ClusterClient — dropping ApplyAck for {DeploymentId}. The ack is not "
|
||||
+ "buffered and will not be retried, so the deployment will time out at central "
|
||||
+ "naming this node even though it applied successfully. Check "
|
||||
+ "MeshTransport:CentralContactPoints",
|
||||
ack.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Send, not SendToAll: the deploy coordinator is a single singleton, and the central comm
|
||||
// actor forwards to its proxy. SendToAll would deliver one ack per central node, and the
|
||||
// coordinator would count the same node twice.
|
||||
_centralClient.Tell(new ClusterClient.Send(MeshPaths.CentralCommunication, ack));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
|
||||
/// <summary>
|
||||
/// Streams the deployed-configuration artifact from central over gRPC and verifies it against the
|
||||
/// revision hash the dispatch already carries (per-cluster mesh Phase 3, <c>FetchAndCache</c>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>One invariant.</b> Return the first endpoint's bytes that both reassemble AND verify
|
||||
/// (<c>SHA-256(bytes)</c> lowercase-hex == the expected revision hash); otherwise
|
||||
/// <see langword="null"/>. Every per-endpoint problem — an <see cref="RpcException"/>
|
||||
/// (Unavailable / NotFound / deadline), a zero-length stream, or a hash mismatch — is treated
|
||||
/// uniformly as "this endpoint did not supply valid bytes," logged, and the next endpoint is
|
||||
/// tried. A hash mismatch is tried-next rather than fatal because a partitioned central could
|
||||
/// legitimately serve stale bytes from one node and current bytes from another; when every
|
||||
/// central shares one SQL store (the common case) the retry is merely cheap and harmless.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why <see langword="null"/>, not throw.</b> The caller treats <see langword="null"/>
|
||||
/// identically to today's "reconcile returned null": an apply failure that keeps
|
||||
/// last-known-good and does not advance the revision. A typed "no bytes" keeps the #485
|
||||
/// contract explicit and cannot escape as an unhandled actor message.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GrpcDeploymentArtifactFetcher : IDeploymentArtifactFetcher, IDisposable
|
||||
{
|
||||
private readonly ConfigSourceOptions _options;
|
||||
private readonly ILogger<GrpcDeploymentArtifactFetcher> _log;
|
||||
private readonly Func<string, DeploymentArtifactService.DeploymentArtifactServiceClient> _clientFactory;
|
||||
private readonly ConcurrentDictionary<string, GrpcChannel> _channels = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="GrpcDeploymentArtifactFetcher"/> class.</summary>
|
||||
/// <param name="options">Config-source options; supplies the endpoints, shared key, and deadline.</param>
|
||||
/// <param name="log">Logger.</param>
|
||||
/// <param name="clientFactory">
|
||||
/// Seam for tests: builds the generated client for an endpoint. Defaults to a real h2c
|
||||
/// <see cref="GrpcChannel"/> (one cached per endpoint).
|
||||
/// </param>
|
||||
public GrpcDeploymentArtifactFetcher(
|
||||
IOptions<ConfigSourceOptions> options,
|
||||
ILogger<GrpcDeploymentArtifactFetcher> log,
|
||||
Func<string, DeploymentArtifactService.DeploymentArtifactServiceClient>? clientFactory = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(log);
|
||||
|
||||
_options = options.Value;
|
||||
_log = log;
|
||||
_clientFactory = clientFactory ?? DefaultClientFactory;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(deploymentId);
|
||||
ArgumentException.ThrowIfNullOrEmpty(expectedRevisionHash);
|
||||
|
||||
var endpoints = _options.CentralFetchEndpoints;
|
||||
if (endpoints.Length == 0)
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Artifact fetch for {DeploymentId} skipped: no ConfigSource:CentralFetchEndpoints configured.",
|
||||
deploymentId);
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var endpoint in endpoints)
|
||||
{
|
||||
try
|
||||
{
|
||||
var bytes = await FetchFromEndpointAsync(endpoint, deploymentId, ct);
|
||||
if (bytes.Length == 0)
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Artifact fetch for {DeploymentId} from {Endpoint} returned zero bytes; trying next endpoint.",
|
||||
deploymentId, endpoint);
|
||||
continue;
|
||||
}
|
||||
|
||||
var actualHash = Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
if (!string.Equals(actualHash, expectedRevisionHash, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Artifact fetch for {DeploymentId} from {Endpoint} failed SHA-256 verification " +
|
||||
"(expected {Expected}, got {Actual}); trying next endpoint.",
|
||||
deploymentId, endpoint, expectedRevisionHash, actualHash);
|
||||
continue;
|
||||
}
|
||||
|
||||
_log.LogInformation(
|
||||
"Fetched + verified artifact for {DeploymentId} from {Endpoint} ({Bytes} bytes).",
|
||||
deploymentId, endpoint, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
catch (RpcException ex)
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Artifact fetch for {DeploymentId} from {Endpoint} failed ({Status}); trying next endpoint.",
|
||||
deploymentId, endpoint, ex.StatusCode);
|
||||
}
|
||||
}
|
||||
|
||||
_log.LogWarning(
|
||||
"Artifact fetch for {DeploymentId} failed on ALL {Count} endpoint(s); returning no bytes " +
|
||||
"(apply failure — last-known-good is kept).",
|
||||
deploymentId, endpoints.Length);
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<byte[]> FetchFromEndpointAsync(string endpoint, string deploymentId, CancellationToken ct)
|
||||
{
|
||||
var client = _clientFactory(endpoint);
|
||||
var headers = new Metadata { { "authorization", $"Bearer {_options.ApiKey}" } };
|
||||
|
||||
using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
if (_options.FetchTimeoutSeconds > 0)
|
||||
deadlineCts.CancelAfter(TimeSpan.FromSeconds(_options.FetchTimeoutSeconds));
|
||||
|
||||
using var call = client.Fetch(
|
||||
new FetchRequest { DeploymentId = deploymentId }, headers, cancellationToken: deadlineCts.Token);
|
||||
|
||||
using var buffer = new MemoryStream();
|
||||
await foreach (var chunk in call.ResponseStream.ReadAllAsync(deadlineCts.Token))
|
||||
chunk.Data.WriteTo(buffer);
|
||||
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
private DeploymentArtifactService.DeploymentArtifactServiceClient DefaultClientFactory(string endpoint)
|
||||
{
|
||||
// h2c: GrpcChannel over an http:// address negotiates prior-knowledge HTTP/2. One channel per
|
||||
// endpoint, cached and reused (creating a channel per call would leak sockets).
|
||||
var channel = _channels.GetOrAdd(endpoint, ep => GrpcChannel.ForAddress(ep));
|
||||
return new DeploymentArtifactService.DeploymentArtifactServiceClient(channel);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var channel in _channels.Values)
|
||||
channel.Dispose();
|
||||
_channels.Clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
|
||||
/// <summary>
|
||||
/// Fetches the deployed-configuration artifact bytes from central over gRPC (per-cluster mesh
|
||||
/// Phase 3, <c>FetchAndCache</c> mode). The node-side counterpart of central's
|
||||
/// <c>DeploymentArtifactService</c>.
|
||||
/// </summary>
|
||||
public interface IDeploymentArtifactFetcher
|
||||
{
|
||||
/// <summary>
|
||||
/// Fetches and reassembles the artifact for <paramref name="deploymentId"/> from the first
|
||||
/// configured central endpoint that answers, verifying that
|
||||
/// <c>SHA-256(reassembled bytes)</c> (lowercase hex) equals
|
||||
/// <paramref name="expectedRevisionHash"/> — the hash already carried in the dispatch.
|
||||
/// </summary>
|
||||
/// <param name="deploymentId">The deployment id to fetch.</param>
|
||||
/// <param name="expectedRevisionHash">
|
||||
/// The lowercase-hex SHA-256 the reassembled bytes must match (<c>Deployment.RevisionHash</c>).
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>
|
||||
/// The verified artifact bytes, or <see langword="null"/> on ANY failure — every configured
|
||||
/// endpoint unreachable/NotFound, a zero-length stream, or a SHA-256 mismatch. A
|
||||
/// <see langword="null"/> is "no answer," NEVER an empty configuration: the caller treats it as
|
||||
/// an apply failure and keeps last-known-good (the #485 contract).
|
||||
/// </returns>
|
||||
Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct);
|
||||
}
|
||||
@@ -72,6 +72,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private readonly IDeploymentArtifactCache? _deploymentArtifactCache;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3. When <see langword="true"/> (<c>ConfigSource:Mode =
|
||||
/// FetchAndCache</c>) this node obtains its config by fetching the artifact bytes from central
|
||||
/// over gRPC and applying them from the LocalDb cache, reading NO config from central SQL.
|
||||
/// When <see langword="false"/> (default <c>Direct</c>) every path reads central SQL as before.
|
||||
/// </summary>
|
||||
private readonly bool _fetchAndCacheMode;
|
||||
|
||||
/// <summary>
|
||||
/// Node-side gRPC artifact fetcher, non-null only under <see cref="_fetchAndCacheMode"/>.
|
||||
/// </summary>
|
||||
private readonly IDeploymentArtifactFetcher? _artifactFetcher;
|
||||
|
||||
/// <summary>
|
||||
/// True once this node has booted a cached artifact because central SQL was unreachable.
|
||||
/// </summary>
|
||||
@@ -86,7 +99,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly CommonsNodeId _localNode;
|
||||
private readonly IActorRef? _coordinatorOverride;
|
||||
private readonly IActorRef? _ackRouter;
|
||||
private readonly IDriverFactory _driverFactory;
|
||||
|
||||
/// <summary>Builds the per-driver-instance <see cref="IDriverCapabilityInvoker"/> attached to each
|
||||
@@ -328,7 +341,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
/// <summary>Creates props for a DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
/// <param name="localNode">The local cluster node identifier.</param>
|
||||
/// <param name="coordinator">Optional coordinator actor reference for deployment coordination.</param>
|
||||
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
|
||||
/// <c>MeshTransport:Mode = ClusterClient</c> the node's <c>NodeCommunicationActor</c>, which relays
|
||||
/// across the boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
|
||||
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
|
||||
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
|
||||
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
|
||||
@@ -364,7 +379,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
public static Props Props(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
IActorRef? coordinator = null,
|
||||
IActorRef? ackRouter = null,
|
||||
IDriverFactory? driverFactory = null,
|
||||
IReadOnlySet<string>? localRoles = null,
|
||||
IActorRef? dependencyMux = null,
|
||||
@@ -380,22 +395,26 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
IDeploymentArtifactCache? deploymentArtifactCache = null,
|
||||
IRedundancyRoleView? redundancyRoleView = null,
|
||||
string? replicationPeerHost = null) =>
|
||||
string? replicationPeerHost = null,
|
||||
bool fetchAndCacheMode = false,
|
||||
IDeploymentArtifactFetcher? artifactFetcher = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
// the wrong dependency at runtime. New parameters go LAST in all three places — this
|
||||
// signature, the constructor's, and this call — and nothing else moves.
|
||||
Akka.Actor.Props.Create(() => new DriverHostActor(
|
||||
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
|
||||
dbFactory, localNode, ackRouter, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
|
||||
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
|
||||
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost));
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
/// <param name="localNode">The local cluster node identifier.</param>
|
||||
/// <param name="coordinator">Optional coordinator actor reference for deployment coordination.</param>
|
||||
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
|
||||
/// <c>MeshTransport:Mode = ClusterClient</c> the node's <c>NodeCommunicationActor</c>, which relays
|
||||
/// across the boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
|
||||
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
|
||||
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
|
||||
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
|
||||
@@ -426,7 +445,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
public DriverHostActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
IActorRef? coordinator,
|
||||
IActorRef? ackRouter,
|
||||
IDriverFactory? driverFactory = null,
|
||||
IReadOnlySet<string>? localRoles = null,
|
||||
IActorRef? dependencyMux = null,
|
||||
@@ -442,14 +461,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
IDeploymentArtifactCache? deploymentArtifactCache = null,
|
||||
IRedundancyRoleView? redundancyRoleView = null,
|
||||
string? replicationPeerHost = null)
|
||||
string? replicationPeerHost = null,
|
||||
bool fetchAndCacheMode = false,
|
||||
IDeploymentArtifactFetcher? artifactFetcher = null)
|
||||
{
|
||||
_deploymentArtifactCache = deploymentArtifactCache;
|
||||
_redundancyRoleView = redundancyRoleView;
|
||||
_replicationPeerHost = replicationPeerHost;
|
||||
_fetchAndCacheMode = fetchAndCacheMode;
|
||||
_artifactFetcher = artifactFetcher;
|
||||
_dbFactory = dbFactory;
|
||||
_localNode = localNode;
|
||||
_coordinatorOverride = coordinator;
|
||||
_ackRouter = ackRouter;
|
||||
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
|
||||
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
|
||||
_driverMemberCountProvider = driverMemberCountProvider;
|
||||
@@ -483,6 +506,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
// uses so only the Primary services operator writes (the secondary keeps state warm for failover).
|
||||
_mediator.Tell(
|
||||
new Subscribe(ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RedundancyStateTopic, Self));
|
||||
|
||||
// Per-cluster mesh Phase 2 dark switch. Under MeshTransport:Mode = ClusterClient these same
|
||||
// commands arrive from NodeCommunicationActor on the node-local EventStream instead of the
|
||||
// mesh-wide DPS topic. Subscribing to BOTH unconditionally is deliberate: exactly one of the
|
||||
// two ever publishes, so the transport flag lives entirely on the publishing side and the
|
||||
// switch can be flipped (or reverted) without touching any subscriber.
|
||||
Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment));
|
||||
Context.System.EventStream.Subscribe(Self, typeof(RestartDriver));
|
||||
Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver));
|
||||
|
||||
// Spawn the VirtualTag host BEFORE Bootstrap so the bootstrap-restore path (which routes
|
||||
// through PushDesiredSubscriptions and Tells ApplyVirtualTags) has a live host to target.
|
||||
SpawnVirtualTagHost();
|
||||
@@ -562,6 +595,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
|
||||
private void Bootstrap()
|
||||
{
|
||||
// Per-cluster mesh Phase 3: a FetchAndCache node NEVER reads central SQL for config, at boot
|
||||
// or otherwise. It restores its served state from the LocalDb pointer instead.
|
||||
if (_fetchAndCacheMode)
|
||||
{
|
||||
BootstrapFromCache();
|
||||
return;
|
||||
}
|
||||
|
||||
// Read the most-recent NodeDeploymentState for this node; if it's Applied, jump
|
||||
// to Steady with that revision. If Applying (orphan from a crash), discard and replay.
|
||||
// If the DB is unreachable, fall back to Stale and start the reconnect loop.
|
||||
@@ -626,6 +667,60 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 boot for <c>FetchAndCache</c> mode. Restores served state from the
|
||||
/// LocalDb pointer without any central-SQL read.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Distinct from <see cref="TryBootFromCache"/> (the Direct-mode SQL-unreachable fallback):
|
||||
/// reading from the cache is <b>normal</b> operation here, not a degraded state, so
|
||||
/// <see cref="_isRunningFromCache"/> stays <see langword="false"/> — the node can still
|
||||
/// fetch new deployments. An empty or unreadable cache lands Steady-with-no-revision (the
|
||||
/// first dispatch fetches), never <c>Stale</c>: Stale means "central SQL is down, retry it",
|
||||
/// and FetchAndCache has no config SQL read to recover.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private void BootstrapFromCache()
|
||||
{
|
||||
try
|
||||
{
|
||||
var cached = _deploymentArtifactCache?.GetCurrentUnkeyedAsync().GetAwaiter().GetResult();
|
||||
if (cached is null)
|
||||
{
|
||||
_log.Info(
|
||||
"DriverHost {Node}: FetchAndCache boot — the LocalDb cache is empty; entering Steady " +
|
||||
"(no revision). The first dispatch will fetch this node's configuration from central.",
|
||||
_localNode);
|
||||
Become(Steady);
|
||||
return;
|
||||
}
|
||||
|
||||
var deploymentId = DeploymentId.Parse(cached.DeploymentId);
|
||||
var revision = RevisionHash.Parse(cached.RevisionHash);
|
||||
|
||||
_currentRevision = revision;
|
||||
Become(Steady);
|
||||
// Restore drivers + address space + subscriptions from the cached bytes — no SQL read, no
|
||||
// re-ack (the deployment is already Applied). Same in-hand-bytes apply as the Direct-mode
|
||||
// cache boot, but this is the steady-state config source, not an outage fallback.
|
||||
ApplyCachedArtifact(deploymentId, cached.Artifact);
|
||||
|
||||
_log.Info(
|
||||
"DriverHost {Node}: FetchAndCache boot — restored served state for deployment {Id} " +
|
||||
"(rev {Rev}) from the LocalDb pointer.",
|
||||
_localNode, deploymentId, revision);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex,
|
||||
"DriverHost {Node}: FetchAndCache boot — failed to read the LocalDb pointer; entering " +
|
||||
"Steady (no revision). The next dispatch will fetch.",
|
||||
_localNode);
|
||||
Become(Steady);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Last-resort boot path: apply the artifact cached by a previous successful deploy (or
|
||||
/// replicated from this node's pair peer) when central SQL cannot be reached.
|
||||
@@ -712,16 +807,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
var correlation = CorrelationId.NewId();
|
||||
|
||||
var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
|
||||
var snapshots = _children.ToDictionary(
|
||||
kv => kv.Key,
|
||||
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
|
||||
StringComparer.Ordinal);
|
||||
var plan = DriverSpawnPlanner.Compute(snapshots, specs);
|
||||
|
||||
foreach (var id in plan.ToStop) StopChild(id);
|
||||
foreach (var spec in plan.ToApplyDelta) ApplyChildDelta(spec);
|
||||
foreach (var spec in plan.ToSpawn) SpawnChild(spec);
|
||||
ReconcileDriversFromBlob(deploymentId, blob);
|
||||
|
||||
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
|
||||
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
|
||||
@@ -767,6 +853,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
_localNode, msg.DeploymentId, _applyingDeploymentId);
|
||||
Self.Forward(msg); // re-deliver after we transition back
|
||||
});
|
||||
// The FetchAndCache fetch pipes its result back here (BeginFetchAndCacheApply ran in Steady,
|
||||
// then Become(Applying)); completing the apply transitions back to Steady.
|
||||
Receive<FetchedForApply>(HandleFetchedForApply);
|
||||
Receive<GetDiagnostics>(HandleGetDiagnostics);
|
||||
Receive<DriverInstanceActor.AttributeValuePublished>(ForwardToMux);
|
||||
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
|
||||
@@ -1601,9 +1690,132 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.CorrelationId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_fetchAndCacheMode)
|
||||
{
|
||||
BeginFetchAndCacheApply(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
|
||||
return;
|
||||
}
|
||||
|
||||
ApplyAndAck(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (<c>FetchAndCache</c>). Kicks off a gRPC artifact fetch and pipes
|
||||
/// the result back to <see cref="Self"/> as a <see cref="FetchedForApply"/> — the fetch is I/O
|
||||
/// and MUST NOT block the mailbox for the fetch deadline, so it never runs as an awaited call
|
||||
/// inside a receive. The apply itself completes synchronously on the actor thread in
|
||||
/// <see cref="HandleFetchedForApply"/>, mirroring <see cref="ApplyAndAck"/>.
|
||||
/// </summary>
|
||||
private void BeginFetchAndCacheApply(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
|
||||
{
|
||||
_applyingDeploymentId = deploymentId;
|
||||
Become(Applying);
|
||||
|
||||
// Persist Applying row (idempotent on PK) — same crash-boundary marker as ApplyAndAck.
|
||||
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applying, failureReason: null);
|
||||
|
||||
if (_artifactFetcher is null)
|
||||
{
|
||||
// Misconfiguration: FetchAndCache with no fetcher wired. Fail the apply rather than fetch
|
||||
// nothing forever; the node keeps serving last-known-good.
|
||||
const string reason = "FetchAndCache mode is set but no artifact fetcher is configured on this node";
|
||||
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Failed, reason);
|
||||
SendAck(deploymentId, ApplyAckOutcome.Failed, reason, correlation);
|
||||
_log.Error("DriverHost {Node}: cannot apply {Id} — {Reason}", _localNode, deploymentId, reason);
|
||||
_applyingDeploymentId = null;
|
||||
Become(Steady);
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Debug("DriverHost {Node}: fetching artifact for {Id} (rev {Rev}) from central over gRPC",
|
||||
_localNode, deploymentId, revision);
|
||||
|
||||
// A faulted fetch task maps to null bytes — the #485 apply-failure path, exactly like an
|
||||
// all-endpoints-down fetch — so a fault can never escape as an unhandled actor message.
|
||||
_artifactFetcher.FetchAsync(deploymentId.ToString(), revision.Value, CancellationToken.None)
|
||||
.PipeTo(
|
||||
Self,
|
||||
success: bytes => new FetchedForApply(deploymentId, revision, correlation, bytes),
|
||||
failure: _ => new FetchedForApply(deploymentId, revision, correlation, Bytes: null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes a <c>FetchAndCache</c> apply once the fetched bytes are in hand. Null bytes are an
|
||||
/// apply FAILURE (keep last-known-good, do not advance the revision — #485); non-null bytes are
|
||||
/// applied from hand (no ConfigDb read) and cached.
|
||||
/// </summary>
|
||||
private void HandleFetchedForApply(FetchedForApply msg)
|
||||
{
|
||||
// Ignore a result for anything but the in-flight apply (a stale/duplicate pipe-back).
|
||||
if (_applyingDeploymentId is null || msg.DeploymentId != _applyingDeploymentId.Value)
|
||||
{
|
||||
_log.Debug("DriverHost {Node}: dropping stale fetch result for {Id} (in-flight={Cur})",
|
||||
_localNode, msg.DeploymentId, _applyingDeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
using var span = OtOpcUaTelemetry.StartDeployApplySpan(msg.DeploymentId.ToString());
|
||||
span?.SetTag("otopcua.node_id", _localNode.ToString());
|
||||
span?.SetTag("otopcua.revision", msg.Revision.ToString());
|
||||
span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
try
|
||||
{
|
||||
// #485: null bytes = "no answer" (every endpoint unreachable / NotFound / SHA mismatch).
|
||||
// Fail the apply and stay on the current revision so a re-dispatch actually retries; keep
|
||||
// serving the last-known-good address space, drivers and subscriptions throughout.
|
||||
var appliedBlob = ReconcileDriversFromBlob(msg.DeploymentId, msg.Bytes);
|
||||
if (appliedBlob is null)
|
||||
{
|
||||
const string reason =
|
||||
"the deployment artifact could not be fetched from central (all endpoints unreachable, NotFound, or SHA-256 mismatch); nothing was applied";
|
||||
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, reason);
|
||||
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, reason, msg.Correlation);
|
||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "reject"));
|
||||
span?.SetStatus(ActivityStatusCode.Error, reason);
|
||||
_log.Error(
|
||||
"DriverHost {Node}: FetchAndCache apply of {Id} FAILED — {Reason}. Staying on revision {Rev} and continuing to serve it; re-dispatch once central is reachable",
|
||||
_localNode, msg.DeploymentId, reason, _currentRevision);
|
||||
return;
|
||||
}
|
||||
|
||||
_currentRevision = msg.Revision;
|
||||
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
|
||||
SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.Correlation);
|
||||
// Rebuild the address space + push subscriptions FROM the fetched bytes so OpcUaPublishActor
|
||||
// never reads central SQL (the whole point of FetchAndCache).
|
||||
_opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(
|
||||
msg.Correlation, msg.DeploymentId, appliedBlob));
|
||||
PushDesiredSubscriptionsFromArtifact(msg.DeploymentId, appliedBlob);
|
||||
// Cache the fetched bytes as this node's last-known-good (idempotent store; skips empty).
|
||||
CacheAppliedArtifact(msg.DeploymentId, msg.Revision, appliedBlob);
|
||||
_isRunningFromCache = false;
|
||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "ack"));
|
||||
_log.Info("DriverHost {Node}: applied fetched deployment {Id} (rev {Rev}, children={Count})",
|
||||
_localNode, msg.DeploymentId, msg.Revision, _children.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, ex.Message);
|
||||
SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, ex.Message, msg.Correlation);
|
||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "reject"));
|
||||
span?.SetStatus(ActivityStatusCode.Error, ex.Message);
|
||||
_log.Error(ex, "DriverHost {Node}: FetchAndCache apply of {Id} failed", _localNode, msg.DeploymentId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
OtOpcUaTelemetry.DeploymentApplyDurationSec.Record(sw.Elapsed.TotalSeconds);
|
||||
_applyingDeploymentId = null;
|
||||
Become(Steady);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Pipe-back carrier for a <c>FetchAndCache</c> fetch result (null bytes = no answer).</summary>
|
||||
private sealed record FetchedForApply(
|
||||
DeploymentId DeploymentId, RevisionHash Revision, CorrelationId Correlation, byte[]? Bytes);
|
||||
|
||||
private void ApplyAndAck(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
|
||||
{
|
||||
_applyingDeploymentId = deploymentId;
|
||||
@@ -1718,6 +1930,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
return null;
|
||||
}
|
||||
|
||||
return ReconcileDriversFromBlob(deploymentId, blob);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The blob-processing half of <see cref="ReconcileDrivers"/>, split out so the
|
||||
/// <c>FetchAndCache</c> apply path (which already holds the fetched bytes) and the
|
||||
/// boot-from-cache path can reconcile without a ConfigDb read.
|
||||
/// </summary>
|
||||
/// <returns>The reconciled blob, or <see langword="null"/> when it was empty (#485).</returns>
|
||||
private byte[]? ReconcileDriversFromBlob(DeploymentId deploymentId, byte[]? blob)
|
||||
{
|
||||
blob ??= Array.Empty<byte>();
|
||||
|
||||
// Issue #485 — no bytes is no answer, not "a configuration with no drivers". Parsing zero bytes
|
||||
// yields zero specs, and DriverSpawnPlanner then plans EVERY running child for StopChild: a missing
|
||||
// deployment row (or a half-written one) silently takes this node's entire field I/O down while the
|
||||
@@ -2348,6 +2573,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
|
||||
private void TryRecoverFromStale()
|
||||
{
|
||||
// Defensive: FetchAndCache never enters Stale (BootstrapFromCache always Becomes Steady, so
|
||||
// the retry-db timer that drives this is never started), and its recovery must NOT read
|
||||
// central SQL. If we somehow land here, leave Steady rather than re-reading Deployments.
|
||||
if (_fetchAndCacheMode)
|
||||
{
|
||||
Timers.Cancel("retry-db");
|
||||
Become(Steady);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var db = _dbFactory.CreateDbContext();
|
||||
@@ -2410,15 +2645,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
private void SendAck(DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation)
|
||||
{
|
||||
var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation);
|
||||
if (_coordinatorOverride is not null)
|
||||
if (_ackRouter is not null)
|
||||
{
|
||||
_coordinatorOverride.Tell(ack);
|
||||
// Either a direct coordinator handle (tests) or, under
|
||||
// MeshTransport:Mode = ClusterClient, the NodeCommunicationActor that relays this
|
||||
// across the boundary. Renamed from _coordinatorOverride in per-cluster mesh Phase 2:
|
||||
// under ClusterClient mode this ref is emphatically NOT a coordinator, and the old
|
||||
// name would send the next reader looking for one.
|
||||
_ackRouter.Tell(ack);
|
||||
}
|
||||
else
|
||||
{
|
||||
// No direct coordinator handle — publish on the dedicated ACK topic. The coordinator
|
||||
// singleton subscribes there in PreStart so the ACK reaches whichever admin node hosts
|
||||
// it without an actor-path lookup.
|
||||
// No router — publish on the dedicated ACK topic. The coordinator singleton subscribes
|
||||
// there in PreStart so the ACK reaches whichever admin node hosts it without an
|
||||
// actor-path lookup.
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentAcksTopic, ack));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -528,6 +528,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
// RedundancyStateChanged and cache this node's role — OnEngineEmission gates the cluster-wide
|
||||
// alerts publish to the Primary so a 2-node single-cluster deploy doesn't double-emit transitions.
|
||||
_mediator.Tell(new Subscribe(OpcUaPublishActor.RedundancyStateTopic, Self));
|
||||
|
||||
// Per-cluster mesh Phase 2 dark switch. An AdminUI-originated ack/shelve arrives from
|
||||
// NodeCommunicationActor on the node-local EventStream under
|
||||
// MeshTransport:Mode = ClusterClient; the DPS subscribe above still carries the node-LOCAL
|
||||
// publisher (OtOpcUaServerHostedService, the OPC UA Part 9 method calls), which never
|
||||
// crosses the boundary and stays on DPS in both modes. Subscribing to both is safe: exactly
|
||||
// one publisher feeds each path.
|
||||
Context.System.EventStream.Subscribe(Self, typeof(AlarmCommand));
|
||||
|
||||
base.PreStart();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Hosting;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
@@ -6,6 +8,10 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
@@ -37,6 +43,7 @@ public static class ServiceCollectionExtensions
|
||||
public const string OpcUaPublishActorName = "opcua-publish";
|
||||
public const string PeerProbeSupervisorName = "peer-probe-supervisor";
|
||||
public const string ContinuousHistorizationRecorderActorName = "continuous-historization-recorder";
|
||||
public const string CentralClusterClientName = "central-cluster-client";
|
||||
|
||||
/// <summary>
|
||||
/// Registers shared runtime services. Currently binds <see cref="IAlarmHistorianSink"/>
|
||||
@@ -275,6 +282,16 @@ public static class ServiceCollectionExtensions
|
||||
// harnesses) rather than given a null-object, so DriverHostActor skips caching outright
|
||||
// instead of pretending to cache into a sink that drops everything.
|
||||
var deploymentArtifactCache = resolver.GetService<IDeploymentArtifactCache>();
|
||||
// Per-cluster mesh Phase 3: ConfigSource:Mode selects where this driver reads config.
|
||||
// FetchAndCache pulls the artifact from central over gRPC and reads only the LocalDb cache;
|
||||
// Direct (default) reads central SQL as before. The fetcher is resolved only in FetchAndCache
|
||||
// mode (registered by the Host under hasDriver); its absence there is a misconfiguration the
|
||||
// actor fails the apply on rather than fetching nothing forever.
|
||||
var configSourceOptions =
|
||||
resolver.GetService<IOptions<ConfigSourceOptions>>()?.Value ?? new ConfigSourceOptions();
|
||||
var fetchAndCacheMode = string.Equals(
|
||||
configSourceOptions.Mode, ConfigSourceOptions.ModeFetchAndCache, StringComparison.OrdinalIgnoreCase);
|
||||
var artifactFetcher = fetchAndCacheMode ? resolver.GetService<IDeploymentArtifactFetcher>() : null;
|
||||
// Where this actor publishes its Primary-gate verdict for the alarm store-and-forward
|
||||
// drain, which runs on a timer and so cannot read RedundancyStateChanged itself.
|
||||
// Registered by AddAlarmHistorian; absent when no durable sink is configured, in which
|
||||
@@ -319,6 +336,40 @@ public static class ServiceCollectionExtensions
|
||||
var mux = system.ActorOf(DependencyMuxActor.Props(), DependencyMuxActorName);
|
||||
registry.Register<DependencyMuxActorKey>(mux);
|
||||
|
||||
// Per-cluster mesh Phase 2: the node end of the central↔node command boundary. Spawned
|
||||
// BEFORE DriverHostActor because it is the host's ackRouter under ClusterClient mode.
|
||||
// It has no dependency of its own on the host — inbound commands travel via the node's
|
||||
// EventStream — so this ordering costs nothing.
|
||||
var meshOptions = resolver.GetService<IOptions<MeshTransportOptions>>().Value;
|
||||
var useClusterClient = string.Equals(
|
||||
meshOptions.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
IActorRef? centralClient = null;
|
||||
if (useClusterClient)
|
||||
{
|
||||
// Contacts come from appsettings, not the database — the deliberate asymmetry:
|
||||
// central's node set changes as operators add and retire nodes, but central's own
|
||||
// address is part of the deployment. MeshTransportOptionsValidator has already
|
||||
// rejected an empty or malformed list at boot, so this cannot silently produce a
|
||||
// client with no contacts.
|
||||
var contacts = meshOptions.CentralContactPoints
|
||||
.Select(cp => ActorPath.Parse($"{cp}/system/receptionist"))
|
||||
.ToImmutableHashSet();
|
||||
centralClient = system.ActorOf(
|
||||
ClusterClient.Props(ClusterClientSettings.Create(system).WithInitialContacts(contacts)),
|
||||
CentralClusterClientName);
|
||||
}
|
||||
|
||||
var nodeComm = system.ActorOf(
|
||||
NodeCommunicationActor.Props(centralClient),
|
||||
MeshPaths.NodeCommunicationName);
|
||||
|
||||
// Registered with the receptionist in BOTH modes: under Dps nothing dials it, and an
|
||||
// idle registration costs nothing — whereas registering only under ClusterClient would
|
||||
// make flipping the flag require a node restart before central could reach it.
|
||||
ClusterClientReceptionist.Get(system).RegisterService(nodeComm);
|
||||
registry.Register<NodeCommunicationActorKey>(nodeComm);
|
||||
|
||||
// Continuous-historization recorder — gated on ContinuousHistorization:Enabled AND the
|
||||
// gateway-backed IHistorianValueWriter + the durable IHistorizationOutbox being registered
|
||||
// (the Host registers both ONLY when historization is enabled and the ServerHistorian gateway
|
||||
@@ -398,7 +449,11 @@ public static class ServiceCollectionExtensions
|
||||
registry.Register<PeerProbeSupervisorKey>(peerProbes);
|
||||
|
||||
var driverHost = system.ActorOf(
|
||||
DriverHostActor.Props(dbFactory, roleInfo.LocalNode, coordinator: null,
|
||||
// ackRouter: under ClusterClient mode the node comm actor relays ApplyAcks across
|
||||
// the boundary; under Dps it stays null and the host publishes on the
|
||||
// deployment-acks topic exactly as before.
|
||||
DriverHostActor.Props(dbFactory, roleInfo.LocalNode,
|
||||
ackRouter: useClusterClient ? nodeComm : null,
|
||||
driverFactory: driverFactory, localRoles: roleInfo.LocalRoles,
|
||||
dependencyMux: mux,
|
||||
opcUaPublishActor: publishActor,
|
||||
@@ -410,7 +465,9 @@ public static class ServiceCollectionExtensions
|
||||
invokerFactory: invokerFactory,
|
||||
deploymentArtifactCache: deploymentArtifactCache,
|
||||
redundancyRoleView: redundancyRoleView,
|
||||
replicationPeerHost: replicationPeerHost),
|
||||
replicationPeerHost: replicationPeerHost,
|
||||
fetchAndCacheMode: fetchAndCacheMode,
|
||||
artifactFetcher: artifactFetcher),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
@@ -430,6 +487,7 @@ public sealed class DbHealthProbeActorKey { }
|
||||
public sealed class HistorianAdapterActorKey { }
|
||||
public sealed class DependencyMuxActorKey { }
|
||||
public sealed class OpcUaPublishActorKey { }
|
||||
public sealed class NodeCommunicationActorKey { }
|
||||
|
||||
/// <summary>Marker key for the per-node ContinuousHistorizationRecorder (spawned only when
|
||||
/// <c>ContinuousHistorization:Enabled=true</c> and the gateway value-writer + outbox are registered).</summary>
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka.Hosting"/>
|
||||
<PackageReference Include="Akka.Cluster.Tools"/>
|
||||
<!-- Node-side gRPC client that fetches the deployed-configuration artifact from central
|
||||
(per-cluster mesh Phase 3, FetchAndCache mode). The generated stub lives in Commons. -->
|
||||
<PackageReference Include="Grpc.Net.Client"/>
|
||||
<!-- Core LocalDb only (ILocalDb + the connection/transaction seam) — the deployment-artifact
|
||||
cache lives here, but the sync engine and its gRPC endpoint are the Host's concern. -->
|
||||
<PackageReference Include="ZB.MOM.WW.LocalDb" />
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A <c>FetchAndCache</c> node with no endpoint or no key does not error — it simply never
|
||||
/// applies a deployment, and the deploy times out naming a node that is otherwise healthy. These
|
||||
/// tests make that misconfiguration a loud host-start failure instead.
|
||||
/// </summary>
|
||||
public class ConfigSourceOptionsValidatorTests
|
||||
{
|
||||
private static ValidateOptionsResult Validate(ConfigSourceOptions o) =>
|
||||
new ConfigSourceOptionsValidator().Validate(ConfigSourceOptions.SectionName, o);
|
||||
|
||||
private static ConfigSourceOptions ValidFetch() => new()
|
||||
{
|
||||
Mode = ConfigSourceOptions.ModeFetchAndCache,
|
||||
CentralFetchEndpoints = ["http://central-1:4055", "http://central-2:4055"],
|
||||
ApiKey = "shared-key",
|
||||
FetchTimeoutSeconds = 30,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Default_options_are_valid()
|
||||
{
|
||||
// The default is Direct with everything empty — the pre-Phase-3 behaviour. A node that never
|
||||
// sets this section must validate, which is the premise of the dark switch.
|
||||
Validate(new ConfigSourceOptions()).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_mode_fails()
|
||||
{
|
||||
var result = Validate(new ConfigSourceOptions { Mode = "cache" });
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("cache");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Mode_matching_is_case_insensitive()
|
||||
{
|
||||
Validate(new ConfigSourceOptions
|
||||
{
|
||||
Mode = "fetchandcache",
|
||||
CentralFetchEndpoints = ["http://central-1:4055"],
|
||||
ApiKey = "k",
|
||||
}).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchAndCache_with_no_endpoints_fails()
|
||||
{
|
||||
var o = ValidFetch();
|
||||
o.CentralFetchEndpoints = [];
|
||||
|
||||
var result = Validate(o);
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain(nameof(ConfigSourceOptions.CentralFetchEndpoints));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchAndCache_with_a_non_http_endpoint_fails()
|
||||
{
|
||||
var o = ValidFetch();
|
||||
o.CentralFetchEndpoints = ["central-1:4055"];
|
||||
|
||||
var result = Validate(o);
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("central-1:4055");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchAndCache_with_an_empty_key_fails()
|
||||
{
|
||||
var o = ValidFetch();
|
||||
o.ApiKey = "";
|
||||
|
||||
var result = Validate(o);
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain(nameof(ConfigSourceOptions.ApiKey));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchAndCache_with_a_non_positive_timeout_fails()
|
||||
{
|
||||
var o = ValidFetch();
|
||||
o.FetchTimeoutSeconds = 0;
|
||||
|
||||
var result = Validate(o);
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain(nameof(ConfigSourceOptions.FetchTimeoutSeconds));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_fully_populated_FetchAndCache_config_is_valid()
|
||||
{
|
||||
Validate(ValidFetch()).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Direct_ignores_the_empty_fetch_surface()
|
||||
{
|
||||
// Under Direct the endpoints/key are never read, so their emptiness must not fail validation —
|
||||
// otherwise every admin-only and every Direct node would be forced to carry fetch config.
|
||||
Validate(new ConfigSourceOptions
|
||||
{
|
||||
Mode = ConfigSourceOptions.ModeDirect,
|
||||
CentralFetchEndpoints = [],
|
||||
ApiKey = "",
|
||||
FetchTimeoutSeconds = 0,
|
||||
}).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Configuration;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Asserts the ClusterClient / receptionist / frame-size settings that are actually <b>in
|
||||
/// force</b> on a real <see cref="ActorSystem"/> built from the shipped base config.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Asserting the <c>akka.conf</c> text instead would pass while a competing HOCON fragment
|
||||
/// silently overrode the value — the failure mode recorded at length in
|
||||
/// <c>ServiceCollectionExtensions.BuildDowningHocon</c> and pinned the same way by
|
||||
/// <c>SplitBrainResolverActivationTests</c>. In this codebase the effective value is the only
|
||||
/// one worth asserting.
|
||||
/// </remarks>
|
||||
public class MeshTransportHoconTests : IDisposable
|
||||
{
|
||||
private readonly ActorSystem _sys;
|
||||
|
||||
public MeshTransportHoconTests()
|
||||
{
|
||||
var config = ConfigurationFactory
|
||||
.ParseString("akka.remote.dot-netty.tcp.port = 0\nakka.cluster.seed-nodes = []")
|
||||
.WithFallback(HoconLoader.LoadBaseConfig());
|
||||
_sys = ActorSystem.Create("mesh-hocon-probe", config);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cluster_client_buffering_is_disabled()
|
||||
{
|
||||
// Asserted through ClusterClientSettings — the object the client is actually built from —
|
||||
// NOT through Config.GetInt. GetInt returns 0 for a MISSING key, so a config assertion here
|
||||
// passes identically whether the setting is explicitly 0 or absent with Akka's default of
|
||||
// 1000 in force. It was green before this feature existed; that is the whole defect class.
|
||||
//
|
||||
// buffer-size = 0 means "drop now if the receptionist is unknown". Buffering would replay a
|
||||
// DispatchDeployment on reconnect and apply a deployment the coordinator already sealed as
|
||||
// TimedOut, leaving the node running a configuration the database says failed. This is a
|
||||
// behaviour decision, not a tuning knob.
|
||||
ClusterClientSettings.Create(_sys).BufferSize.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cluster_client_retries_forever_rather_than_self_stopping()
|
||||
{
|
||||
// reconnect-timeout = off surfaces as a null TimeSpan on the settings object. A central node
|
||||
// down for an hour must not require a driver-node restart to become reachable again.
|
||||
ClusterClientSettings.Create(_sys).ReconnectTimeout.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Receptionist_serves_every_member_regardless_of_role()
|
||||
{
|
||||
// An empty role is what makes "registered per node, not as a singleton" work: every member
|
||||
// hosts a receptionist, so ClusterClient contact rotation reaches whichever node answers.
|
||||
// Scoping this to a role would make the boundary reachable through one node only.
|
||||
ClusterReceptionistSettings.Create(_sys).Role.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oversized_frames_are_logged_rather_than_dropped_silently()
|
||||
{
|
||||
// The sister project's most expensive incident. A message over the frame limit is dropped
|
||||
// WITHOUT tearing down the association: heartbeats keep flowing, the node still reports
|
||||
// healthy, and the only symptom is a command that never arrived. log-frame-size-exceeding
|
||||
// converts that silence into a log line naming the offending size.
|
||||
_sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.maximum-frame-size").ShouldBe(128000);
|
||||
_sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.log-frame-size-exceeding").ShouldBe(32000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Receptionist_extension_is_registered_at_startup()
|
||||
{
|
||||
// Not auto-started. Without this entry the receptionist materialises only if some code path
|
||||
// happens to call ClusterClientReceptionist.Get(system) first, which makes "is the boundary
|
||||
// listening?" depend on startup ordering rather than on configuration.
|
||||
_sys.Settings.Config.GetStringList("akka.extensions")
|
||||
.ShouldContain(s => s.Contains("ClusterClientReceptionistExtensionProvider"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Distributed_pub_sub_extension_survives_alongside_the_receptionist()
|
||||
{
|
||||
// The dark switch keeps both transports live for one phase, and DPS additionally carries
|
||||
// traffic Phase 2 does not touch at all — node-local alarm-command publishes and the
|
||||
// node<->node redundancy-state probe leg. Losing it here would break far more than deploys.
|
||||
_sys.Settings.Config.GetStringList("akka.extensions")
|
||||
.ShouldContain(s => s.Contains("DistributedPubSubExtensionProvider"));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_sys.Terminate().Wait(TimeSpan.FromSeconds(5));
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Every failure this validator catches otherwise surfaces as an <i>absence</i> — a deployment
|
||||
/// that never arrives, an alarm ack that does nothing, a node whose ack is dropped without a
|
||||
/// trace. Absences are the hardest class of fault to attribute in the field, so the whole point
|
||||
/// of these tests is that the host refuses to start instead.
|
||||
/// </summary>
|
||||
public class MeshTransportOptionsValidatorTests
|
||||
{
|
||||
private static ValidateOptionsResult Validate(MeshTransportOptions o) =>
|
||||
new MeshTransportOptionsValidator().Validate(MeshTransportOptions.SectionName, o);
|
||||
|
||||
[Fact]
|
||||
public void Default_options_are_valid()
|
||||
{
|
||||
// The default is Dps — the pre-Phase-2 transport. A node that never sets this section must
|
||||
// behave exactly as it did before, which is the whole premise of the dark switch.
|
||||
Validate(new MeshTransportOptions()).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_mode_fails()
|
||||
{
|
||||
var result = Validate(new MeshTransportOptions { Mode = "grpc" });
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("grpc");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Mode_matching_is_case_insensitive()
|
||||
{
|
||||
Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = "clusterclient",
|
||||
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
||||
}).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Non_positive_contact_refresh_fails()
|
||||
{
|
||||
var result = Validate(new MeshTransportOptions { ContactRefreshSeconds = 0 });
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.ContactRefreshSeconds));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClusterClient_mode_requires_central_contact_points()
|
||||
{
|
||||
var result = Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints = [],
|
||||
});
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dps_mode_does_not_require_central_contact_points()
|
||||
{
|
||||
// An admin-only node has no reason to carry them, and requiring them under the default mode
|
||||
// would make the section mandatory everywhere for a feature that is switched off.
|
||||
Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeDps,
|
||||
CentralContactPoints = [],
|
||||
}).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_contact_point_that_is_not_an_akka_address_fails()
|
||||
{
|
||||
var result = Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints = ["central-1:4053"],
|
||||
});
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("akka.tcp://");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_contact_point_carrying_an_actor_path_suffix_fails()
|
||||
{
|
||||
// The sister project's shipped site template listed a full receptionist path, and separately
|
||||
// listed the site's OWN remoting port as the second contact — "a permanent failure in the
|
||||
// initial-contact rotation". Both are invisible until a command goes missing.
|
||||
var result = Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"],
|
||||
});
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("/system/receptionist");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void One_bad_contact_point_among_good_ones_still_fails()
|
||||
{
|
||||
var result = Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints =
|
||||
[
|
||||
"akka.tcp://otopcua@central-1:4053",
|
||||
"central-2:4053",
|
||||
],
|
||||
});
|
||||
|
||||
result.Failed.ShouldBeTrue();
|
||||
result.FailureMessage.ShouldContain("central-2:4053");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Well_formed_ClusterClient_options_succeed()
|
||||
{
|
||||
Validate(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints =
|
||||
[
|
||||
"akka.tcp://otopcua@central-1:4053",
|
||||
"akka.tcp://otopcua@central-2:4053",
|
||||
],
|
||||
}).Succeeded.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A compile-touch test: it exists so that the repo's first locally-compiled proto
|
||||
/// (<c>deployment_artifact.proto</c>) is proven to generate both the client stub and the server
|
||||
/// base and the message types. If Grpc.Tools codegen regresses, this file stops compiling —
|
||||
/// which, under TreatWarningsAsErrors, breaks the build loudly rather than silently.
|
||||
/// </summary>
|
||||
public class DeploymentArtifactProtoTests
|
||||
{
|
||||
[Fact]
|
||||
public void The_request_message_carries_the_deployment_id()
|
||||
{
|
||||
var request = new FetchRequest { DeploymentId = "abc" };
|
||||
|
||||
request.DeploymentId.ShouldBe("abc");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_chunk_message_carries_bytes()
|
||||
{
|
||||
var chunk = new ArtifactChunk { Data = Google.Protobuf.ByteString.CopyFromUtf8("x") };
|
||||
|
||||
chunk.Data.Length.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_service_base_and_client_types_generate()
|
||||
{
|
||||
// Referencing these types is the whole assertion — GrpcServices="Both" must emit both.
|
||||
typeof(DeploymentArtifactService.DeploymentArtifactServiceBase).ShouldNotBeNull();
|
||||
typeof(DeploymentArtifactService.DeploymentArtifactServiceClient).ShouldNotBeNull();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// The serve side of Phase 3: streams a sealed deployment's bytes, and collapses unknown /
|
||||
/// not-sealed / empty into one indistinguishable NotFound (existence-hiding + #485).
|
||||
/// </summary>
|
||||
public class DeploymentArtifactServiceTests
|
||||
{
|
||||
private static IDbContextFactory<OtOpcUaConfigDbContext> Factory(string name) => new NamedFactory(name);
|
||||
|
||||
private static DeploymentArtifactGrpcService Service(string dbName) =>
|
||||
new(Factory(dbName), NullLogger<DeploymentArtifactGrpcService>.Instance);
|
||||
|
||||
private static Guid Seed(string dbName, DeploymentStatus status, byte[] blob)
|
||||
{
|
||||
var id = Guid.NewGuid();
|
||||
using var db = new OtOpcUaConfigDbContext(
|
||||
new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseInMemoryDatabase(dbName).Options);
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id,
|
||||
RevisionHash = new string('a', 64),
|
||||
Status = status,
|
||||
CreatedBy = "test",
|
||||
ArtifactBlob = blob,
|
||||
});
|
||||
db.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_sealed_non_empty_blob_streams_back_byte_equal_and_chunk_bounded()
|
||||
{
|
||||
var dbName = $"artsvc-{Guid.NewGuid():N}";
|
||||
// ~300 KB > 2 * 128 KiB -> exactly 3 chunks.
|
||||
var blob = new byte[300 * 1024];
|
||||
for (var i = 0; i < blob.Length; i++) blob[i] = (byte)(i % 251);
|
||||
var id = Seed(dbName, DeploymentStatus.Sealed, blob);
|
||||
|
||||
var writer = new CollectingStreamWriter();
|
||||
await Service(dbName).Fetch(
|
||||
new FetchRequest { DeploymentId = id.ToString() }, writer, new FakeServerCallContext());
|
||||
|
||||
writer.Chunks.Count.ShouldBe(3);
|
||||
writer.Chunks[0].Data.Length.ShouldBe(128 * 1024);
|
||||
writer.Chunks[1].Data.Length.ShouldBe(128 * 1024);
|
||||
writer.Reassembled().ShouldBe(blob);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unknown_id_throws_NotFound()
|
||||
{
|
||||
var dbName = $"artsvc-{Guid.NewGuid():N}";
|
||||
Seed(dbName, DeploymentStatus.Sealed, [1, 2, 3]);
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Service(dbName).Fetch(
|
||||
new FetchRequest { DeploymentId = Guid.NewGuid().ToString() },
|
||||
new CollectingStreamWriter(), new FakeServerCallContext()));
|
||||
|
||||
ex.StatusCode.ShouldBe(StatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_non_sealed_deployment_with_a_blob_is_served()
|
||||
{
|
||||
// A node fetches DURING apply, before the deployment is sealed (seal-needs-ack →
|
||||
// ack-needs-fetch). Gating on Sealed would deadlock every FetchAndCache deploy. So a
|
||||
// non-sealed deployment that has a blob must stream — matching Direct mode reading the same
|
||||
// AwaitingApplyAcks row from SQL. (Regression for the Phase 3 live-gate deadlock.)
|
||||
var dbName = $"artsvc-{Guid.NewGuid():N}";
|
||||
var blob = new byte[] { 1, 2, 3 };
|
||||
var id = Seed(dbName, DeploymentStatus.AwaitingApplyAcks, blob);
|
||||
|
||||
var writer = new CollectingStreamWriter();
|
||||
await Service(dbName).Fetch(
|
||||
new FetchRequest { DeploymentId = id.ToString() }, writer, new FakeServerCallContext());
|
||||
|
||||
writer.Reassembled().ShouldBe(blob);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_length_blob_throws_NotFound()
|
||||
{
|
||||
// The serve-side #485 guard: never stream empty bytes as if they were a valid empty config.
|
||||
var dbName = $"artsvc-{Guid.NewGuid():N}";
|
||||
var id = Seed(dbName, DeploymentStatus.Sealed, []);
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Service(dbName).Fetch(
|
||||
new FetchRequest { DeploymentId = id.ToString() },
|
||||
new CollectingStreamWriter(), new FakeServerCallContext()));
|
||||
|
||||
ex.StatusCode.ShouldBe(StatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_non_guid_id_throws_NotFound()
|
||||
{
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Service($"artsvc-{Guid.NewGuid():N}").Fetch(
|
||||
new FetchRequest { DeploymentId = "not-a-guid" },
|
||||
new CollectingStreamWriter(), new FakeServerCallContext()));
|
||||
|
||||
ex.StatusCode.ShouldBe(StatusCode.NotFound);
|
||||
}
|
||||
|
||||
private sealed class CollectingStreamWriter : IServerStreamWriter<ArtifactChunk>
|
||||
{
|
||||
public List<ArtifactChunk> Chunks { get; } = [];
|
||||
public WriteOptions? WriteOptions { get; set; }
|
||||
|
||||
public Task WriteAsync(ArtifactChunk message)
|
||||
{
|
||||
Chunks.Add(message);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public byte[] Reassembled()
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
foreach (var c in Chunks) c.Data.WriteTo(ms);
|
||||
return ms.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NamedFactory(string name) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() =>
|
||||
new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseInMemoryDatabase(name).Options);
|
||||
|
||||
public Task<OtOpcUaConfigDbContext> CreateDbContextAsync(CancellationToken ct = default) =>
|
||||
Task.FromResult(CreateDbContext());
|
||||
}
|
||||
|
||||
/// <summary>Minimal <see cref="ServerCallContext"/> — the service reads only the cancellation token.</summary>
|
||||
private sealed class FakeServerCallContext : ServerCallContext
|
||||
{
|
||||
protected override string MethodCore => "/deployment_artifact.v1.DeploymentArtifactService/Fetch";
|
||||
protected override string HostCore => "localhost";
|
||||
protected override string PeerCore => "ipv4:127.0.0.1:0";
|
||||
protected override DateTime DeadlineCore => DateTime.MaxValue;
|
||||
protected override Metadata RequestHeadersCore => [];
|
||||
protected override CancellationToken CancellationTokenCore => CancellationToken.None;
|
||||
protected override Metadata ResponseTrailersCore { get; } = [];
|
||||
protected override Status StatusCore { get; set; }
|
||||
protected override WriteOptions? WriteOptionsCore { get; set; }
|
||||
protected override AuthContext AuthContextCore => new(null, new Dictionary<string, List<AuthProperty>>());
|
||||
|
||||
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.TestKit;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
||||
|
||||
public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
{
|
||||
private const string DeploymentsTopic = "deployments";
|
||||
|
||||
/// <summary>Records every client the actor asks for, and hands back a probe in its place.</summary>
|
||||
private sealed class RecordingClientFactory : IMeshClusterClientFactory
|
||||
{
|
||||
private readonly Func<TestProbe> _newProbe;
|
||||
|
||||
public RecordingClientFactory(Func<TestProbe> newProbe) => _newProbe = newProbe;
|
||||
|
||||
public List<ImmutableHashSet<ActorPath>> Calls { get; } = [];
|
||||
|
||||
public List<TestProbe> Probes { get; } = [];
|
||||
|
||||
public IActorRef Create(ActorSystem system, ImmutableHashSet<ActorPath> contacts)
|
||||
{
|
||||
Calls.Add(contacts);
|
||||
var probe = _newProbe();
|
||||
Probes.Add(probe);
|
||||
return probe.Ref;
|
||||
}
|
||||
}
|
||||
|
||||
private static MeshTransportOptions ClusterClientMode(int refreshSeconds = 60) => new()
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeClusterClient,
|
||||
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
||||
ContactRefreshSeconds = refreshSeconds,
|
||||
};
|
||||
|
||||
private static void SeedCluster(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
||||
string clusterId,
|
||||
params (string NodeId, string Host, int AkkaPort, bool Enabled, bool Maintenance)[] nodes)
|
||||
{
|
||||
using var db = factory.CreateDbContext();
|
||||
db.ServerClusters.Add(new ServerCluster
|
||||
{
|
||||
ClusterId = clusterId,
|
||||
Name = clusterId,
|
||||
Enterprise = "ZB",
|
||||
Site = "Site",
|
||||
// Warm/Hot require NodeCount = 2 (CK_ServerCluster_RedundancyMode_NodeCount); these
|
||||
// fixtures only need the row to exist, so use the standalone shape.
|
||||
RedundancyMode = RedundancyMode.None,
|
||||
NodeCount = 1,
|
||||
CreatedBy = "test",
|
||||
});
|
||||
foreach (var n in nodes)
|
||||
{
|
||||
db.ClusterNodes.Add(new ClusterNode
|
||||
{
|
||||
NodeId = n.NodeId,
|
||||
ClusterId = clusterId,
|
||||
Host = n.Host,
|
||||
AkkaPort = n.AkkaPort,
|
||||
ApplicationUri = $"urn:{n.NodeId}",
|
||||
Enabled = n.Enabled,
|
||||
MaintenanceMode = n.Maintenance,
|
||||
CreatedBy = "test",
|
||||
});
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
private IActorRef Spawn(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
||||
MeshTransportOptions options,
|
||||
IMeshClusterClientFactory clientFactory,
|
||||
IActorRef? coordinator = null) =>
|
||||
Sys.ActorOf(CentralCommunicationActor.Props(factory, options, clientFactory, () => coordinator));
|
||||
|
||||
[Fact]
|
||||
public void Dps_mode_publishes_on_the_carried_topic_and_creates_no_client()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
var mediatorProbe = CreateTestProbe();
|
||||
// Subscribe on the probe's behalf: SubscribeAck goes to the SENDER, so telling the mediator
|
||||
// without an explicit sender would ack to TestActor and this wait would hang.
|
||||
DistributedPubSub.Get(Sys).Mediator.Tell(
|
||||
new Subscribe(DeploymentsTopic, mediatorProbe.Ref), mediatorProbe.Ref);
|
||||
mediatorProbe.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
||||
|
||||
var actor = Spawn(factory, new MeshTransportOptions(), clients);
|
||||
var dispatch = new DispatchDeployment(
|
||||
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
|
||||
|
||||
mediatorProbe.ExpectMsg<DispatchDeployment>(TimeSpan.FromSeconds(5)).ShouldBe(dispatch);
|
||||
clients.Calls.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClusterClient_mode_sends_to_all_not_to_one()
|
||||
{
|
||||
// THE assertion of this phase. ClusterClient.Send delivers to exactly ONE registered actor;
|
||||
// today's DPS publish reaches EVERY DriverHostActor, and the node side has no ClusterId or
|
||||
// node filter to compensate. Send would deploy to a single node while every other node
|
||||
// silently kept its old configuration.
|
||||
var factory = NewInMemoryDbFactory();
|
||||
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
|
||||
var actor = Spawn(factory, ClusterClientMode(), clients);
|
||||
AwaitCondition(() => clients.Probes.Count == 1, TimeSpan.FromSeconds(5));
|
||||
|
||||
var dispatch = new DispatchDeployment(
|
||||
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
||||
actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
|
||||
|
||||
var sent = clients.Probes[0].ExpectMsg<ClusterClient.SendToAll>(TimeSpan.FromSeconds(5));
|
||||
sent.Path.ShouldBe(MeshPaths.NodeCommunication);
|
||||
sent.Message.ShouldBe(dispatch);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Exactly_one_client_is_created_for_a_multi_cluster_fleet()
|
||||
{
|
||||
// A receptionist serves its whole mesh, so one client per Cluster would fan each command out
|
||||
// once per cluster while the fleet is still a single mesh — N x duplicate deployments.
|
||||
var factory = NewInMemoryDbFactory();
|
||||
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
||||
SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false));
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
|
||||
Spawn(factory, ClusterClientMode(), clients);
|
||||
AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
||||
|
||||
// ...and that one client dials BOTH clusters' nodes.
|
||||
clients.Calls[0].Count.ShouldBe(2);
|
||||
// Give a second refresh a chance to (wrongly) create another.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
clients.Calls.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disabled_and_maintenance_nodes_are_not_dialled()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
SeedCluster(
|
||||
factory,
|
||||
"C1",
|
||||
("host-a:4053", "host-a", 4053, true, false),
|
||||
("host-b:4053", "host-b", 4053, false, false),
|
||||
("host-c:4053", "host-c", 4053, true, true));
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
|
||||
Spawn(factory, ClusterClientMode(), clients);
|
||||
AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
||||
|
||||
clients.Calls[0].Count.ShouldBe(1);
|
||||
clients.Calls[0].Single().ToString().ShouldContain("host-a");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_malformed_node_row_does_not_abort_the_refresh()
|
||||
{
|
||||
// The bad row is ordered FIRST on purpose: the failure this guards against is an exception
|
||||
// that aborts the loop and leaves the contact set half-built.
|
||||
var factory = NewInMemoryDbFactory();
|
||||
SeedCluster(
|
||||
factory,
|
||||
"C1",
|
||||
("bad", "not a host!!", 4053, true, false),
|
||||
("host-a:4053", "host-a", 4053, true, false));
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
|
||||
Spawn(factory, ClusterClientMode(), clients);
|
||||
AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
||||
|
||||
clients.Calls[0].ShouldContain(p => p.ToString().Contains("host-a"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_command_with_no_client_is_dropped_with_a_warning()
|
||||
{
|
||||
// No ClusterNode rows at all: nothing to dial, so the command must be dropped loudly rather
|
||||
// than queued. buffer-size = 0 covers the case where a client exists but cannot connect;
|
||||
// this covers the case where there is no client at all.
|
||||
var factory = NewInMemoryDbFactory();
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
var actor = Spawn(factory, ClusterClientMode(), clients);
|
||||
|
||||
EventFilter.Warning(contains: "dropping").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
||||
actor.Tell(new MeshCommand(DeploymentsTopic, "anything")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplyAck_is_forwarded_to_the_coordinator_preserving_the_sender()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
var coordinator = CreateTestProbe();
|
||||
var node = CreateTestProbe();
|
||||
var actor = Spawn(factory, ClusterClientMode(), clients, coordinator.Ref);
|
||||
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("host-a:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
actor.Tell(ack, node.Ref);
|
||||
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).ShouldBe(ack);
|
||||
// Forward, not Tell: the coordinator must see the node, not this relay.
|
||||
coordinator.LastSender.ShouldBe(node.Ref);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unresolvable_coordinator_logs_rather_than_losing_the_ack_silently()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
var actor = Spawn(factory, ClusterClientMode(), clients, coordinator: null);
|
||||
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("host-a:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
EventFilter.Warning(contains: "not resolvable").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
||||
actor.Tell(ack));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_faulted_contact_load_warns_and_leaves_the_actor_alive()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
||||
var actor = Spawn(factory, ClusterClientMode(), clients);
|
||||
|
||||
// Filter on a phrase unique to the DB-failure warning. EventFilter matches case-INSENSITIVELY,
|
||||
// so a looser "was NOT" also caught this actor's other warning ("the mesh ClusterClient was
|
||||
// not created") and the assertion failed with two matches instead of one.
|
||||
EventFilter.Warning(contains: "contact set was NOT refreshed").ExpectOne(
|
||||
TimeSpan.FromSeconds(5),
|
||||
() => actor.Tell(new Status.Failure(new InvalidOperationException("db down"))));
|
||||
|
||||
// Still serving: a transient DB outage must not take the command boundary down with it.
|
||||
actor.Tell(new MeshCommand(DeploymentsTopic, "anything"));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
}
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Configuration;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Guards the one non-obvious part of client creation: recreating a client with the same name
|
||||
/// inside a single message handler throws, because <c>Context.Stop</c> is asynchronous and the
|
||||
/// name stays reserved until termination completes.
|
||||
/// </summary>
|
||||
public class MeshClusterClientFactoryTests : TestKit
|
||||
{
|
||||
private static readonly Config TestConfig = ConfigurationFactory.ParseString(@"
|
||||
akka.actor.provider = cluster
|
||||
akka.remote.dot-netty.tcp.port = 0
|
||||
akka.remote.dot-netty.tcp.hostname = 127.0.0.1")
|
||||
.WithFallback(ClusterClientReceptionist.DefaultConfig());
|
||||
|
||||
public MeshClusterClientFactoryTests()
|
||||
: base(TestConfig)
|
||||
{
|
||||
}
|
||||
|
||||
// A real ClusterClient pointed at an unreachable contact is safe in TestKit — it simply retries
|
||||
// the initial-contact rotation and never fails the test.
|
||||
private static ImmutableHashSet<ActorPath> UnreachableContact => ImmutableHashSet.Create(
|
||||
ActorPath.Parse("akka.tcp://otopcua@127.0.0.1:65501/system/receptionist"));
|
||||
|
||||
[Fact]
|
||||
public void Recreating_a_client_in_one_pass_does_not_collide_on_the_actor_name()
|
||||
{
|
||||
var factory = new DefaultMeshClusterClientFactory();
|
||||
|
||||
var first = factory.Create(Sys, UnreachableContact);
|
||||
Sys.Stop(first);
|
||||
|
||||
// Stop is asynchronous: `first`'s name is still reserved right here. Without the generation
|
||||
// suffix this throws InvalidActorNameException — which is precisely what a contact-set
|
||||
// change would do in production, where stop-then-recreate happens in one handler.
|
||||
var second = factory.Create(Sys, UnreachableContact);
|
||||
|
||||
second.Path.Name.ShouldNotBe(first.Path.Name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_incarnation_gets_a_distinct_name()
|
||||
{
|
||||
var factory = new DefaultMeshClusterClientFactory();
|
||||
|
||||
var names = Enumerable.Range(0, 5)
|
||||
.Select(_ => factory.Create(Sys, UnreachableContact).Path.Name)
|
||||
.ToList();
|
||||
|
||||
names.Distinct().Count().ShouldBe(5);
|
||||
}
|
||||
|
||||
// Contact-set propagation is deliberately NOT asserted here: reading initial contacts back
|
||||
// off a created ClusterClient is not possible, and asserting ClusterClientSettings directly
|
||||
// would test Akka rather than this factory. It is covered where it can actually fail — the
|
||||
// real two-ActorSystem boundary test, which only delivers if the contacts reached the client.
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Proves the five central→node publish sites go through the transport router when one is
|
||||
/// wired, and fall back to the mediator when it is not.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The fallback is what keeps the ~12 pre-existing coordinator and admin-operations tests
|
||||
/// meaningful: they construct these actors without a router and must still exercise the real
|
||||
/// DistributedPubSub path, not a silently-disabled one.
|
||||
/// </remarks>
|
||||
public class MeshRouterDispatchTests : ControlPlaneActorTestBase
|
||||
{
|
||||
private const string DeploymentsTopic = "deployments";
|
||||
|
||||
[Fact]
|
||||
public void Deploy_dispatch_goes_to_the_router_when_one_is_wired()
|
||||
{
|
||||
var router = CreateTestProbe();
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
var coordinator = Sys.ActorOf(
|
||||
ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: router.Ref));
|
||||
|
||||
var deploymentId = new DeploymentId(Guid.NewGuid());
|
||||
coordinator.Tell(new Commons.Messages.Deploy.DispatchDeployment(
|
||||
deploymentId, new RevisionHash("rev-1"), CorrelationId.NewId()));
|
||||
|
||||
var cmd = router.ExpectMsg<MeshCommand>(TimeSpan.FromSeconds(5));
|
||||
cmd.Topic.ShouldBe(DeploymentsTopic);
|
||||
cmd.Message.ShouldBeOfType<Commons.Messages.Deploy.DispatchDeployment>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Deploy_dispatch_falls_back_to_the_mediator_when_no_router_is_wired()
|
||||
{
|
||||
var subscriber = CreateTestProbe();
|
||||
DistributedPubSub.Get(Sys).Mediator.Tell(
|
||||
new Subscribe(DeploymentsTopic, subscriber.Ref), subscriber.Ref);
|
||||
subscriber.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory()));
|
||||
|
||||
var deploymentId = new DeploymentId(Guid.NewGuid());
|
||||
coordinator.Tell(new Commons.Messages.Deploy.DispatchDeployment(
|
||||
deploymentId, new RevisionHash("rev-1"), CorrelationId.NewId()));
|
||||
|
||||
subscriber.ExpectMsg<Commons.Messages.Deploy.DispatchDeployment>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Restart_driver_goes_to_the_router_on_the_driver_control_topic()
|
||||
{
|
||||
var router = CreateTestProbe();
|
||||
var admin = Sys.ActorOf(AdminOperationsActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
CreateTestProbe().Ref,
|
||||
Enumerable.Empty<IDriverProbe>(),
|
||||
TagConfigValidationMode.Warn,
|
||||
router.Ref));
|
||||
|
||||
admin.Tell(new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()));
|
||||
|
||||
var cmd = router.ExpectMsg<MeshCommand>(TimeSpan.FromSeconds(5));
|
||||
cmd.Topic.ShouldBe(DriverControlTopic.Name);
|
||||
cmd.Message.ShouldBeOfType<RestartDriver>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconnect_driver_goes_to_the_router_on_the_driver_control_topic()
|
||||
{
|
||||
var router = CreateTestProbe();
|
||||
var admin = Sys.ActorOf(AdminOperationsActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
CreateTestProbe().Ref,
|
||||
Enumerable.Empty<IDriverProbe>(),
|
||||
TagConfigValidationMode.Warn,
|
||||
router.Ref));
|
||||
|
||||
admin.Tell(new ReconnectDriver("C1", "driver-1", "alice", Guid.NewGuid()));
|
||||
|
||||
var cmd = router.ExpectMsg<MeshCommand>(TimeSpan.FromSeconds(5));
|
||||
cmd.Topic.ShouldBe(DriverControlTopic.Name);
|
||||
cmd.Message.ShouldBeOfType<ReconnectDriver>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Alarm_acknowledge_goes_to_the_router_on_the_alarm_commands_topic()
|
||||
{
|
||||
var router = CreateTestProbe();
|
||||
var admin = Sys.ActorOf(AdminOperationsActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
CreateTestProbe().Ref,
|
||||
Enumerable.Empty<IDriverProbe>(),
|
||||
TagConfigValidationMode.Warn,
|
||||
router.Ref));
|
||||
|
||||
admin.Tell(new AcknowledgeAlarmCommand("alarm-1", "alice", "c", CorrelationId.NewId()));
|
||||
|
||||
var cmd = router.ExpectMsg<MeshCommand>(TimeSpan.FromSeconds(5));
|
||||
cmd.Topic.ShouldBe(AlarmCommandsTopic.Name);
|
||||
cmd.Message.ShouldBeOfType<AlarmCommand>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Alarm_shelve_goes_to_the_router_on_the_alarm_commands_topic()
|
||||
{
|
||||
var router = CreateTestProbe();
|
||||
var admin = Sys.ActorOf(AdminOperationsActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
CreateTestProbe().Ref,
|
||||
Enumerable.Empty<IDriverProbe>(),
|
||||
TagConfigValidationMode.Warn,
|
||||
router.Ref));
|
||||
|
||||
admin.Tell(new ShelveAlarmCommand("alarm-1", "alice", ShelveKind.OneShot, null, "c", CorrelationId.NewId()));
|
||||
|
||||
var cmd = router.ExpectMsg<MeshCommand>(TimeSpan.FromSeconds(5));
|
||||
cmd.Topic.ShouldBe(AlarmCommandsTopic.Name);
|
||||
cmd.Message.ShouldBeOfType<AlarmCommand>();
|
||||
}
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 3) — the config-serve artifact endpoint's inbound gate.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The <c>DeploymentArtifactService</c> streams a driver node the exact configuration it will
|
||||
/// boot on. Anything able to reach central's serve port could otherwise pull the whole deployed
|
||||
/// config, so the endpoint is gated by a shared bearer key, fail-closed, constant-time — the
|
||||
/// same contract as <see cref="LocalDbSyncAuthInterceptor"/>, scoped to a different service path.
|
||||
/// </remarks>
|
||||
public sealed class ConfigServeAuthInterceptorTests
|
||||
{
|
||||
private const string FetchMethod = "/deployment_artifact.v1.DeploymentArtifactService/Fetch";
|
||||
private const string OtherMethod = "/localdb_sync.v1.LocalDbSync/Sync";
|
||||
|
||||
private static ConfigServeAuthInterceptor CreateInterceptor(string? apiKey)
|
||||
=> new(
|
||||
Options.Create(new ConfigServeOptions { ApiKey = apiKey ?? string.Empty }),
|
||||
NullLogger<ConfigServeAuthInterceptor>.Instance);
|
||||
|
||||
private static ServerCallContext CreateContext(string method, string? authorizationHeader)
|
||||
{
|
||||
var headers = new Metadata();
|
||||
if (authorizationHeader is not null)
|
||||
headers.Add("authorization", authorizationHeader);
|
||||
|
||||
return new FakeServerCallContext(method, headers);
|
||||
}
|
||||
|
||||
/// <summary>Invokes the interceptor's server-streaming path (how <c>Fetch</c> actually runs).</summary>
|
||||
private static Task Invoke(ConfigServeAuthInterceptor interceptor, ServerCallContext context)
|
||||
=> interceptor.ServerStreamingServerHandler<string, string>(
|
||||
"request", responseStream: null!, context, (_, _, _) => Task.CompletedTask);
|
||||
|
||||
[Fact]
|
||||
public async Task NonServeMethod_PassesThrough_EvenWithNoKeyConfigured()
|
||||
{
|
||||
// The interceptor shares the AddGrpc pipeline with the LocalDb sync interceptor, so it sees
|
||||
// every call. It must be scoped strictly to the artifact service — a sync call is not its
|
||||
// business.
|
||||
var interceptor = CreateInterceptor(apiKey: null);
|
||||
var context = CreateContext(OtherMethod, authorizationHeader: null);
|
||||
|
||||
await Invoke(interceptor, context); // no throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchMethod_WithNoKeyConfigured_IsDenied_EvenWithABearerToken()
|
||||
{
|
||||
// Fail-closed. "No key configured" is the DEFAULT — treating it as "no auth required" would
|
||||
// expose the deployed config on exactly the shape most nodes ship with. A token must not help.
|
||||
var interceptor = CreateInterceptor(apiKey: null);
|
||||
var context = CreateContext(FetchMethod, "Bearer anything-at-all");
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchMethod_WithNoBearerToken_IsDenied()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(FetchMethod, authorizationHeader: null);
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchMethod_WithWrongBearerToken_IsDenied()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(FetchMethod, "Bearer the-wrong-key");
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchMethod_WithCorrectBearerToken_PassesThrough()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(FetchMethod, "Bearer the-shared-key");
|
||||
|
||||
await Invoke(interceptor, context); // no throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchMethod_TokenComparison_IsNotAPrefixMatch()
|
||||
{
|
||||
// A prefix/StartsWith comparison would accept a truncated key and make the secret
|
||||
// recoverable one character at a time.
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(FetchMethod, "Bearer the-shared-ke");
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnaryPath_IsAlsoGated()
|
||||
{
|
||||
// Fetch is server-streaming today, but gating only that path would leave any future unary
|
||||
// method on the same service open. Gate every handler shape.
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(FetchMethod, "Bearer the-wrong-key");
|
||||
|
||||
var ex = await Should.ThrowAsync<RpcException>(() =>
|
||||
interceptor.UnaryServerHandler<string, string>(
|
||||
"request", context, (_, _) => Task.FromResult("ok")));
|
||||
|
||||
ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal <see cref="ServerCallContext"/> carrying just a method name and request headers —
|
||||
/// the only two things the interceptor reads. Hand-rolled because
|
||||
/// <c>Grpc.Core.Testing.TestServerCallContext</c> ships only in the retired native package.
|
||||
/// </summary>
|
||||
private sealed class FakeServerCallContext(string method, Metadata requestHeaders)
|
||||
: ServerCallContext
|
||||
{
|
||||
protected override string MethodCore => method;
|
||||
protected override string HostCore => "localhost";
|
||||
protected override string PeerCore => "ipv4:127.0.0.1:12345";
|
||||
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
|
||||
protected override Metadata RequestHeadersCore => requestHeaders;
|
||||
protected override CancellationToken CancellationTokenCore => CancellationToken.None;
|
||||
protected override Metadata ResponseTrailersCore { get; } = [];
|
||||
protected override Status StatusCore { get; set; }
|
||||
protected override WriteOptions? WriteOptionsCore { get; set; }
|
||||
|
||||
protected override AuthContext AuthContextCore { get; } =
|
||||
new(null, new Dictionary<string, List<AuthProperty>>());
|
||||
|
||||
protected override ContextPropagationToken CreatePropagationTokenCore(
|
||||
ContextPropagationOptions? options)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System.Net;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 3) — the coexistence of the ConfigServe artifact h2c listener
|
||||
/// with the LocalDb-sync h2c listener on a fused admin+driver node.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why this is the load-bearing test.</b> A fused central node can carry BOTH dedicated
|
||||
/// h2c ports (LocalDb sync + ConfigServe). Any explicit Kestrel <c>Listen*</c> makes Kestrel
|
||||
/// discard <c>ASPNETCORE_URLS</c> wholesale, so the existing HTTP surface must be re-bound
|
||||
/// exactly ONCE alongside the two dedicated ports. Re-binding it twice (one block per h2c
|
||||
/// port) throws "address already in use"; re-binding it zero times silently unbinds the
|
||||
/// AdminUI behind Traefik. This test pins that all three answer simultaneously.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Like <see cref="LocalDbSyncListenerTests"/>, this drives a minimal <c>WebApplication</c>
|
||||
/// shaped exactly like <c>Program.cs</c>'s merged listener block rather than booting the real
|
||||
/// host (which needs SQL, an Akka mesh and LDAP). What is under test is the Kestrel binding
|
||||
/// contract, fully reproduced here; the real end-to-end fetch is the Task 8 boundary test.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ConfigServeListenerTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task BothDedicatedH2cPorts_AndTheHttpSurface_AnswerSimultaneously()
|
||||
{
|
||||
var httpPort = GetFreePort();
|
||||
var syncPort = GetFreePort();
|
||||
var configServePort = GetFreePort();
|
||||
|
||||
var builder = WebApplication.CreateBuilder();
|
||||
builder.Environment.ApplicationName = typeof(ConfigServeListenerTests).Assembly.GetName().Name!;
|
||||
builder.Services.AddGrpc();
|
||||
|
||||
// Exactly the shape Program.cs's merged block applies: compute the existing surface ONCE,
|
||||
// apply it ONCE, then add each configured dedicated h2c port.
|
||||
var existingBindings = KestrelHttpBinding.Parse($"http://localhost:{httpPort}");
|
||||
builder.WebHost.ConfigureKestrel(kestrel =>
|
||||
{
|
||||
foreach (var binding in existingBindings)
|
||||
binding.Apply(kestrel);
|
||||
|
||||
kestrel.ListenAnyIP(syncPort, o => o.Protocols = HttpProtocols.Http2);
|
||||
kestrel.ListenAnyIP(configServePort, o => o.Protocols = HttpProtocols.Http2);
|
||||
});
|
||||
|
||||
await using var app = builder.Build();
|
||||
app.MapGet("/healthz", () => "ok");
|
||||
await app.StartAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
// The re-bound HTTP/1.1 surface still answers (the AdminUI/Traefik guarantee).
|
||||
using var http1 = new HttpClient();
|
||||
(await http1.GetStringAsync(
|
||||
$"http://localhost:{httpPort}/healthz", TestContext.Current.CancellationToken))
|
||||
.ShouldBe("ok");
|
||||
|
||||
// Both dedicated ports negotiate prior-knowledge h2c. A 404 is a fine result — it proves
|
||||
// the HTTP/2 connection was established and routed, which is all that is in question.
|
||||
using var http2 = new HttpClient
|
||||
{
|
||||
DefaultRequestVersion = HttpVersion.Version20,
|
||||
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact,
|
||||
};
|
||||
|
||||
using var syncResponse = await http2.GetAsync(
|
||||
$"http://localhost:{syncPort}/", TestContext.Current.CancellationToken);
|
||||
syncResponse.Version.ShouldBe(HttpVersion.Version20);
|
||||
|
||||
using var configServeResponse = await http2.GetAsync(
|
||||
$"http://localhost:{configServePort}/", TestContext.Current.CancellationToken);
|
||||
configServeResponse.Version.ShouldBe(HttpVersion.Version20);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await app.StopAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static int GetFreePort()
|
||||
{
|
||||
using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
return port;
|
||||
}
|
||||
}
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 8) — the REAL gRPC artifact-fetch boundary. An in-memory fake
|
||||
/// client (Task 4) proves the fetcher's logic; this proves a real stream crosses a real Kestrel
|
||||
/// h2c listener through the real <see cref="ConfigServeAuthInterceptor"/> into the real
|
||||
/// <see cref="DeploymentArtifactGrpcService"/> and back through the real
|
||||
/// <see cref="GrpcDeploymentArtifactFetcher"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The Phase-2 lesson applied: a passing fake client cannot distinguish "the boundary works" from
|
||||
/// "something else supplied the bytes". The wrong-key control (assertion 2) is the falsifier — it
|
||||
/// must return null on the SAME server where the right key succeeds.
|
||||
/// </remarks>
|
||||
[Trait("Category", "ArtifactBoundary")]
|
||||
public sealed class DeploymentArtifactFetchBoundaryTests
|
||||
{
|
||||
private const string ApiKey = "the-shared-node-key";
|
||||
|
||||
[Fact]
|
||||
public async Task RightKey_FetchesAndReassembles_ByteEqual_AndVerifiesAgainstTheRevisionHash()
|
||||
{
|
||||
// >256 KB so the stream really chunks (3 × 128 KiB).
|
||||
var blob = RandomBlob(300 * 1024);
|
||||
var revisionHash = Sha256Hex(blob);
|
||||
|
||||
var (server, deploymentId, port) = await StartServerAsync(revisionHash, blob);
|
||||
await using var _ = server;
|
||||
|
||||
var fetcher = Fetcher([$"http://localhost:{port}"], ApiKey);
|
||||
var result = await fetcher.FetchAsync(deploymentId, revisionHash, TestContext.Current.CancellationToken);
|
||||
|
||||
result.ShouldBe(blob);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WrongKey_ReturnsNull_WhileTheRightKeySucceedsOnTheSameServer()
|
||||
{
|
||||
var blob = RandomBlob(4096);
|
||||
var revisionHash = Sha256Hex(blob);
|
||||
|
||||
var (server, deploymentId, port) = await StartServerAsync(revisionHash, blob);
|
||||
await using var _ = server;
|
||||
|
||||
// Falsifiability control: the interceptor rejects a wrong key -> the fetcher gets an RpcException
|
||||
// -> null. Without this, a green right-key fetch cannot prove the boundary is what carries bytes.
|
||||
var wrong = await Fetcher([$"http://localhost:{port}"], "not-the-key")
|
||||
.FetchAsync(deploymentId, revisionHash, TestContext.Current.CancellationToken);
|
||||
wrong.ShouldBeNull();
|
||||
|
||||
// Same server, right key: succeeds — proving the null above was the auth gate, not a dead server.
|
||||
var right = await Fetcher([$"http://localhost:{port}"], ApiKey)
|
||||
.FetchAsync(deploymentId, revisionHash, TestContext.Current.CancellationToken);
|
||||
right.ShouldBe(blob);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnUnknownDeploymentId_ReturnsNull()
|
||||
{
|
||||
var blob = RandomBlob(4096);
|
||||
var (server, _, port) = await StartServerAsync(Sha256Hex(blob), blob);
|
||||
await using var __ = server;
|
||||
|
||||
var result = await Fetcher([$"http://localhost:{port}"], ApiKey)
|
||||
.FetchAsync(Guid.NewGuid().ToString(), Sha256Hex(blob), TestContext.Current.CancellationToken);
|
||||
|
||||
result.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_FromADeadEndpoint_ToTheRealServer_StillReturnsTheBytes()
|
||||
{
|
||||
var blob = RandomBlob(4096);
|
||||
var revisionHash = Sha256Hex(blob);
|
||||
|
||||
var (server, deploymentId, port) = await StartServerAsync(revisionHash, blob);
|
||||
await using var _ = server;
|
||||
var deadPort = GetFreePort(); // nothing listening
|
||||
|
||||
var fetcher = Fetcher([$"http://localhost:{deadPort}", $"http://localhost:{port}"], ApiKey);
|
||||
var result = await fetcher.FetchAsync(deploymentId, revisionHash, TestContext.Current.CancellationToken);
|
||||
|
||||
result.ShouldBe(blob);
|
||||
}
|
||||
|
||||
// ---- harness --------------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Stands a minimal real Host: an h2c-only Kestrel listener serving the real
|
||||
/// <see cref="DeploymentArtifactGrpcService"/> behind the real <see cref="ConfigServeAuthInterceptor"/>,
|
||||
/// backed by an in-memory config DB seeded with one sealed deployment.
|
||||
/// </summary>
|
||||
private static async Task<(WebApplication Server, string DeploymentId, int Port)> StartServerAsync(
|
||||
string revisionHash, byte[] blob)
|
||||
{
|
||||
var port = GetFreePort();
|
||||
var id = Guid.NewGuid();
|
||||
var dbName = $"artboundary-{id:N}";
|
||||
|
||||
var builder = WebApplication.CreateBuilder();
|
||||
builder.Environment.ApplicationName = typeof(DeploymentArtifactFetchBoundaryTests).Assembly.GetName().Name!;
|
||||
builder.Logging.ClearProviders();
|
||||
|
||||
builder.Services.Configure<ConfigServeOptions>(o => o.ApiKey = ApiKey);
|
||||
builder.Services.AddGrpc(o => o.Interceptors.Add<ConfigServeAuthInterceptor>());
|
||||
builder.Services.AddDbContextFactory<OtOpcUaConfigDbContext>(o => o.UseInMemoryDatabase(dbName));
|
||||
|
||||
builder.WebHost.ConfigureKestrel(k =>
|
||||
k.ListenAnyIP(port, o => o.Protocols = HttpProtocols.Http2));
|
||||
|
||||
var app = builder.Build();
|
||||
app.MapGrpcService<DeploymentArtifactGrpcService>();
|
||||
|
||||
// Seed the sealed deployment the service will serve.
|
||||
await using (var scope = app.Services.CreateAsyncScope())
|
||||
{
|
||||
var factory = scope.ServiceProvider.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
||||
await using var db = await factory.CreateDbContextAsync(TestContext.Current.CancellationToken);
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id,
|
||||
RevisionHash = revisionHash,
|
||||
Status = DeploymentStatus.Sealed,
|
||||
CreatedBy = "test",
|
||||
ArtifactBlob = blob,
|
||||
});
|
||||
await db.SaveChangesAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
await app.StartAsync(TestContext.Current.CancellationToken);
|
||||
return (app, id.ToString(), port);
|
||||
}
|
||||
|
||||
private static GrpcDeploymentArtifactFetcher Fetcher(string[] endpoints, string apiKey)
|
||||
=> new(
|
||||
Options.Create(new ConfigSourceOptions
|
||||
{
|
||||
Mode = ConfigSourceOptions.ModeFetchAndCache,
|
||||
CentralFetchEndpoints = endpoints,
|
||||
ApiKey = apiKey,
|
||||
FetchTimeoutSeconds = 15,
|
||||
}),
|
||||
NullLogger<GrpcDeploymentArtifactFetcher>.Instance);
|
||||
|
||||
private static byte[] RandomBlob(int size)
|
||||
{
|
||||
var blob = new byte[size];
|
||||
for (var i = 0; i < size; i++) blob[i] = (byte)(i * 31 + 7);
|
||||
return blob;
|
||||
}
|
||||
|
||||
private static string Sha256Hex(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static int GetFreePort()
|
||||
{
|
||||
using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
return port;
|
||||
}
|
||||
}
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Configuration;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
using AkkaCluster = Akka.Cluster.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Drives a real <c>ClusterClient</c> across two genuinely separate <see cref="ActorSystem"/>s —
|
||||
/// the only test in the repo where the mesh boundary carries a message over the wire.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The sister project substitutes an in-process relay that unwraps
|
||||
/// <see cref="ClusterClient.Send"/> and forwards. That tests actor <i>logic</i> and cannot
|
||||
/// fail for any transport reason: not a missing receptionist extension, not a service that
|
||||
/// was never registered, not a malformed contact path, not a serialization break. Those are
|
||||
/// precisely the Phase 2 failure modes, so they are what this test exercises.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Two clusters, one ActorSystem name, joined ONLY by ClusterClient.</b> The shared name
|
||||
/// is mandatory — Akka.Remote matches on the full address, so a contact path naming
|
||||
/// <c>otopcua</c> cannot reach a system called anything else. Each system seeds itself and
|
||||
/// neither lists the other, and
|
||||
/// <see cref="The_two_systems_are_separate_clusters_joined_only_by_the_client"/> asserts the
|
||||
/// separation outright: if they ever gossiped into one mesh, every delivery below could be
|
||||
/// plain local messaging and the whole class would be testing nothing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Scope limit.</b> With exactly one node per side, "registered per node" and "registered
|
||||
/// as a cluster singleton" are indistinguishable by construction, so this class does NOT
|
||||
/// cover the per-node property whose structural assertions <c>MeshCommActorPathTests</c>
|
||||
/// dropped. That property's discriminating coverage is live-gate step 8: stop the OLDEST
|
||||
/// central and the survivor's comm actor must still receive a node ack, which a singleton
|
||||
/// hosted on the stopped node cannot.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class MeshClusterClientBoundaryTests : IAsyncLifetime
|
||||
{
|
||||
private const string SharedSystemName = "otopcua";
|
||||
private static readonly TimeSpan Arrival = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// How long the falsifiability control waits before concluding nothing arrived. Deliberately
|
||||
/// shorter than <see cref="Arrival"/> but far longer than a successful delivery takes in
|
||||
/// practice, so a pass means "the boundary is shut", not "the clock ran out first".
|
||||
/// </summary>
|
||||
private static readonly TimeSpan Silence = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// One factory for every client this class creates, because the generation counter that keeps
|
||||
/// client actor names unique is <b>per factory instance</b>. A second factory on the same
|
||||
/// system restarts at generation 1 and collides with the first factory's client — which is
|
||||
/// exactly how this test first failed. Production has one instance per
|
||||
/// <c>CentralCommunicationActor</c>, so the real code path is safe.
|
||||
/// </summary>
|
||||
private readonly DefaultMeshClusterClientFactory _clientFactory = new();
|
||||
|
||||
private ActorSystem _central = null!;
|
||||
private ActorSystem _node = null!;
|
||||
private IActorRef _nodeComm = null!;
|
||||
private IActorRef _centralClient = null!;
|
||||
private TaskCompletionSource<object> _centralInbox = null!;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
_central = StartSelfSeededSystem(withReceptionist: true);
|
||||
_node = StartSelfSeededSystem(withReceptionist: true);
|
||||
await Task.WhenAll(AwaitSingleMemberUpAsync(_central), AwaitSingleMemberUpAsync(_node));
|
||||
|
||||
// Central: the ack landing pad, at the path the node's comm actor sends to.
|
||||
_centralInbox = NewInbox();
|
||||
var centralComm = _central.ActorOf(
|
||||
CaptureActor.Props(_centralInbox), MeshPaths.CentralCommunicationName);
|
||||
ClusterClientReceptionist.Get(_central).RegisterService(centralComm);
|
||||
|
||||
// Node: the REAL comm actor, dialling central through the REAL production client factory.
|
||||
var nodeToCentral = _clientFactory.Create(_node, ReceptionistOf(_central));
|
||||
_nodeComm = _node.ActorOf(
|
||||
NodeCommunicationActor.Props(nodeToCentral), MeshPaths.NodeCommunicationName);
|
||||
ClusterClientReceptionist.Get(_node).RegisterService(_nodeComm);
|
||||
|
||||
_centralClient = _clientFactory.Create(_central, ReceptionistOf(_node));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await ShutdownAsync(_central);
|
||||
await ShutdownAsync(_node);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_two_systems_are_separate_clusters_joined_only_by_the_client()
|
||||
{
|
||||
// The load-bearing precondition of every other test here. Two systems sharing a name in one
|
||||
// process are one gossip accident away from being one cluster, at which point ClusterClient
|
||||
// is no longer the thing under test.
|
||||
AkkaCluster.Get(_central).State.Members.Count.ShouldBe(1,
|
||||
"central must be its own single-member cluster, or the boundary is not being crossed");
|
||||
AkkaCluster.Get(_node).State.Members.Count.ShouldBe(1,
|
||||
"the node must be its own single-member cluster, or the boundary is not being crossed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Central_send_to_all_reaches_the_node_comm_actor_and_lands_on_its_event_stream()
|
||||
{
|
||||
var inbox = NewInbox();
|
||||
_node.EventStream.Subscribe(_node.ActorOf(CaptureActor.Props(inbox)), typeof(RestartDriver));
|
||||
|
||||
var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
|
||||
var delivered = await SendUntilDeliveredAsync(
|
||||
_centralClient,
|
||||
() => new ClusterClient.SendToAll(MeshPaths.NodeCommunication, command),
|
||||
inbox,
|
||||
Arrival);
|
||||
|
||||
delivered.ShouldNotBeNull(
|
||||
"SendToAll must reach the node's registered comm actor and be re-emitted node-locally; "
|
||||
+ "this is the whole central→node leg of the mesh transport");
|
||||
delivered.ShouldBeOfType<RestartDriver>().DriverInstanceId.ShouldBe("driver-1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_node_ack_reaches_central_through_the_node_comm_actors_own_client()
|
||||
{
|
||||
// Told directly rather than routed from central, so a failure here can only be the outbound
|
||||
// leg. The inbound leg has its own test above.
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("node-a"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
CorrelationId.NewId());
|
||||
|
||||
var delivered = await SendUntilDeliveredAsync(
|
||||
_nodeComm, () => ack, _centralInbox, Arrival);
|
||||
|
||||
delivered.ShouldNotBeNull(
|
||||
"the node's ApplyAck must reach central's comm actor, or every deployment times out at "
|
||||
+ "the apply deadline naming nodes that applied successfully");
|
||||
delivered.ShouldBeOfType<ApplyAck>().NodeId.Value.ShouldBe("node-a");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Falsifiability_control_a_node_without_the_receptionist_extension_receives_nothing()
|
||||
{
|
||||
// Without this, the two tests above cannot distinguish "the ClusterClient boundary works"
|
||||
// from "something else in the process delivered the message". Same message, same client
|
||||
// factory, same paths, same comm actor — and it must NOT arrive.
|
||||
//
|
||||
// HONEST SCOPE: this removes the receptionist extension AND, unavoidably, the RegisterService
|
||||
// call — resolving ClusterClientReceptionist.Get to register would materialise the very
|
||||
// extension whose absence is the control. So it falsifies "delivery happens without a
|
||||
// receptionist boundary", not the extension line in isolation. Verified falsifiable: giving
|
||||
// this node the extension and the registration turns the test red, which also proves the send
|
||||
// below is live rather than silently misaddressed.
|
||||
var deaf = StartSelfSeededSystem(withReceptionist: false);
|
||||
try
|
||||
{
|
||||
await AwaitSingleMemberUpAsync(deaf);
|
||||
|
||||
var inbox = NewInbox();
|
||||
deaf.EventStream.Subscribe(deaf.ActorOf(CaptureActor.Props(inbox)), typeof(RestartDriver));
|
||||
|
||||
// Deliberately NOT ClusterClientReceptionist.Get(deaf) — resolving the extension would
|
||||
// materialise the very receptionist whose absence is the control.
|
||||
var comm = deaf.ActorOf(NodeCommunicationActor.Props(null), MeshPaths.NodeCommunicationName);
|
||||
comm.ShouldNotBeNull();
|
||||
|
||||
var client = _clientFactory.Create(_central, ReceptionistOf(deaf));
|
||||
var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
|
||||
|
||||
var deadline = DateTime.UtcNow + Silence;
|
||||
while (DateTime.UtcNow < deadline && !inbox.Task.IsCompleted)
|
||||
{
|
||||
client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, command));
|
||||
await Task.Delay(500, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
inbox.Task.IsCompleted.ShouldBeFalse(
|
||||
"delivery survived removing the receptionist extension, so the two positive tests "
|
||||
+ "above are not measuring the ClusterClient boundary at all");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await ShutdownAsync(deaf);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a single-member cluster on a dynamic port, optionally without the receptionist.
|
||||
/// </summary>
|
||||
/// <param name="withReceptionist">
|
||||
/// <see langword="false"/> drops the receptionist provider from <c>akka.extensions</c>,
|
||||
/// leaving DistributedPubSub in place so only the one variable changes.
|
||||
/// </param>
|
||||
/// <returns>The started system.</returns>
|
||||
private static ActorSystem StartSelfSeededSystem(bool withReceptionist)
|
||||
{
|
||||
var extensions = withReceptionist
|
||||
? string.Empty
|
||||
: "akka.extensions = [\"Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, "
|
||||
+ "Akka.Cluster.Tools\"]\n";
|
||||
|
||||
// Port 0 + a post-start self-Join, rather than a seed-nodes list: the address is not known
|
||||
// until the transport binds, and hard-coding ports makes the test collide with itself under
|
||||
// parallel execution.
|
||||
var config = ConfigurationFactory.ParseString(
|
||||
"akka.remote.dot-netty.tcp.hostname = \"127.0.0.1\"\n"
|
||||
+ "akka.remote.dot-netty.tcp.public-hostname = \"127.0.0.1\"\n"
|
||||
+ "akka.remote.dot-netty.tcp.port = 0\n"
|
||||
+ "akka.cluster.seed-nodes = []\n"
|
||||
+ "akka.loglevel = \"WARNING\"\n"
|
||||
+ extensions)
|
||||
.WithFallback(ZB.MOM.WW.OtOpcUa.Cluster.HoconLoader.LoadBaseConfig());
|
||||
|
||||
var system = ActorSystem.Create(SharedSystemName, config);
|
||||
var cluster = AkkaCluster.Get(system);
|
||||
cluster.Join(cluster.SelfAddress);
|
||||
return system;
|
||||
}
|
||||
|
||||
private static async Task AwaitSingleMemberUpAsync(ActorSystem system)
|
||||
{
|
||||
var cluster = AkkaCluster.Get(system);
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return;
|
||||
await Task.Delay(100, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
cluster.State.Members.Any(m => m.Status == MemberStatus.Up).ShouldBeTrue(
|
||||
$"{cluster.SelfAddress} never reached Up as its own seed");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-sends until the inbox completes or the deadline passes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A ClusterClient establishes contact asynchronously and <c>buffer-size = 0</c> means anything
|
||||
/// sent before that lands is dropped on the floor — by design. A single send would therefore
|
||||
/// race the handshake and fail intermittently for a reason that is not a defect. Re-sending
|
||||
/// weakens nothing: the control test re-sends on exactly the same schedule and must still
|
||||
/// receive nothing.
|
||||
/// </remarks>
|
||||
/// <param name="target">Actor to send to.</param>
|
||||
/// <param name="message">Builds the message for each attempt.</param>
|
||||
/// <param name="inbox">Completion source the receiving capture actor resolves.</param>
|
||||
/// <param name="timeout">How long to keep trying.</param>
|
||||
/// <returns>The delivered message, or <see langword="null"/> if none arrived.</returns>
|
||||
private static async Task<object?> SendUntilDeliveredAsync(
|
||||
IActorRef target,
|
||||
Func<object> message,
|
||||
TaskCompletionSource<object> inbox,
|
||||
TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (inbox.Task.IsCompleted) return await inbox.Task;
|
||||
target.Tell(message());
|
||||
await Task.Delay(250, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
return inbox.Task.IsCompleted ? await inbox.Task : null;
|
||||
}
|
||||
|
||||
private static TaskCompletionSource<object> NewInbox() =>
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
private static ImmutableHashSet<ActorPath> ReceptionistOf(ActorSystem system) =>
|
||||
ImmutableHashSet.Create<ActorPath>(
|
||||
new RootActorPath(AkkaCluster.Get(system).SelfAddress) / "system" / "receptionist");
|
||||
|
||||
private static async Task ShutdownAsync(ActorSystem? system)
|
||||
{
|
||||
if (system is null) return;
|
||||
await system.Terminate().WaitAsync(TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
/// <summary>Completes an inbox with the first message it receives.</summary>
|
||||
private sealed class CaptureActor : ReceiveActor
|
||||
{
|
||||
public static Props Props(TaskCompletionSource<object> sink) =>
|
||||
Akka.Actor.Props.Create(() => new CaptureActor(sink));
|
||||
|
||||
public CaptureActor(TaskCompletionSource<object> sink) => ReceiveAny(m => sink.TrySetResult(m));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the two mesh comm-actor paths against a <b>really booted</b> host.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <c>/user/central-communication</c> and <c>/user/node-communication</c> are the wire
|
||||
/// contract of the central↔node boundary: central sends to a path string, and the node
|
||||
/// registers under one. Nothing else in the codebase asserts they agree. Renaming either
|
||||
/// actor would compile, pass every unit test, deploy — and deliver nothing, with no error
|
||||
/// on either side, because a ClusterClient send to an unregistered path is simply dropped.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Booting the real host is the point. A test that spawned the actors itself would pin the
|
||||
/// constant against itself and prove nothing about the wiring in
|
||||
/// <c>WithOtOpcUaControlPlaneSingletons</c> / <c>WithOtOpcUaRuntimeActors</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class MeshCommActorPathTests
|
||||
{
|
||||
private static async Task AssertActorExistsAsync(ActorSystem system, string path)
|
||||
{
|
||||
var identity = await system.ActorSelection(path)
|
||||
.Ask<ActorIdentity>(new Identify(path), TimeSpan.FromSeconds(10));
|
||||
|
||||
identity.Subject.ShouldNotBeNull(
|
||||
$"{path} is the ClusterClient wire contract for the central↔node boundary; nothing else "
|
||||
+ "pins it, and a mismatch delivers nothing without erroring on either side");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Both_mesh_comm_actors_exist_at_their_contract_paths()
|
||||
{
|
||||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||||
|
||||
// Harness nodes carry admin+driver, so BOTH comm actors live on each of them.
|
||||
await AssertActorExistsAsync(harness.NodeASystem, MeshPaths.CentralCommunication);
|
||||
await AssertActorExistsAsync(harness.NodeASystem, MeshPaths.NodeCommunication);
|
||||
await AssertActorExistsAsync(harness.NodeBSystem, MeshPaths.CentralCommunication);
|
||||
await AssertActorExistsAsync(harness.NodeBSystem, MeshPaths.NodeCommunication);
|
||||
}
|
||||
|
||||
// NOT TESTED HERE: that the comm actors are registered per node rather than as cluster
|
||||
// singletons. Two candidate discriminators were tried and BOTH were invalid:
|
||||
//
|
||||
// * "the path resolves on both nodes" — a ClusterSingletonManager is also created on every
|
||||
// node in the role, so this passes under either shape.
|
||||
// * "no /singleton child exists" — measured against the KNOWN singleton /user/config-publish,
|
||||
// that child is null there too, so Akka.Hosting does not lay singletons out this way and
|
||||
// the assertion proves nothing.
|
||||
//
|
||||
// A sabotage that re-registered CentralCommunicationActor via WithSingleton left both green,
|
||||
// which is what exposed them. Rather than ship an assertion I could not make fail, the property
|
||||
// is left to live-gate step 8 (stop the OLDEST central; the survivor's comm actor must still
|
||||
// receive a node ack, which a singleton hosted on the stopped node cannot).
|
||||
//
|
||||
// NOT MeshClusterClientBoundaryTests: that class proves a send only arrives if the target is
|
||||
// registered with the receptionist, which is necessary but not discriminating — with one node
|
||||
// per side, per-node and singleton registration look identical.
|
||||
}
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using Serilog;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Proves the Phase 2 node-side leg joins up: a command handed to
|
||||
/// <see cref="NodeCommunicationActor"/> reaches the <b>real</b> <see cref="DriverHostActor"/>,
|
||||
/// not just a test probe.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The unit tests either side of this seam both pass with the wiring broken.
|
||||
/// <c>NodeCommunicationActorTests</c> asserts a probe subscribed to the EventStream receives
|
||||
/// the message; the driver-host tests drive the actor by direct <c>Tell</c>. Neither notices
|
||||
/// if <c>DriverHostActor.PreStart</c> never subscribes. That gap is why this file exists.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ordering matters, and it bit this test first time out.</b> <c>ActorOf</c> returns
|
||||
/// before <c>PreStart</c> has run, and an <c>EventStream.Publish</c> with no subscriber is
|
||||
/// dropped silently — there is no buffering and no dead-letter. Relaying immediately after
|
||||
/// spawning therefore fails intermittently. The warm-up below is not ceremony: it is the
|
||||
/// only available proof that <c>PreStart</c> has completed. (In production the driver host
|
||||
/// is spawned at startup, long before any deployment, so the same race does not arise.)
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class EventStreamCommandDeliveryTests : RuntimeActorTestBase
|
||||
{
|
||||
private static DispatchDeployment NewDispatch() => new(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new RevisionHash("rev-does-not-exist"),
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
[Fact]
|
||||
public void A_dispatch_relayed_by_the_comm_actor_reaches_the_real_driver_host()
|
||||
{
|
||||
var ackRouter = CreateTestProbe();
|
||||
var driverHost = Sys.ActorOf(
|
||||
DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref),
|
||||
"driver-host");
|
||||
var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication");
|
||||
|
||||
// Warm-up: a direct Tell whose ack proves PreStart has completed, and with it the
|
||||
// EventStream subscription. Without this the relay below races the subscription.
|
||||
var warmup = NewDispatch();
|
||||
driverHost.Tell(warmup);
|
||||
ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(warmup.DeploymentId);
|
||||
|
||||
// Now the real assertion — relayed exactly as central would send it. The comm actor only
|
||||
// publishes node-locally, so the host must be subscribed for this to arrive at all.
|
||||
var relayed = NewDispatch();
|
||||
comm.Tell(relayed);
|
||||
|
||||
// The deployment id does not exist, so the host answers Failed. That is the stronger
|
||||
// assertion, not a weaker one: an ack at all proves the command was received and handled,
|
||||
// and only PreStart's EventStream subscription can have delivered it.
|
||||
ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(relayed.DeploymentId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CONTROL_the_driver_host_acks_a_direct_tell()
|
||||
{
|
||||
// Falsifiability control. If the test above fails, this one says which half broke: green
|
||||
// here means the host is healthy and the EventStream relay is at fault; red here means the
|
||||
// host would not have acked either way and the relay is not implicated.
|
||||
var ackRouter = CreateTestProbe();
|
||||
var driverHost = Sys.ActorOf(
|
||||
DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref),
|
||||
"driver-host-control");
|
||||
|
||||
var dispatch = NewDispatch();
|
||||
driverHost.Tell(dispatch);
|
||||
|
||||
ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(dispatch.DeploymentId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The alarm leg of the same wiring proof, in its own ActorSystem because the shared
|
||||
/// <see cref="RuntimeActorTestBase"/> harness pins <c>loglevel = WARNING</c> and the only
|
||||
/// observable signal <c>ScriptedAlarmHostActor</c> gives for an unowned alarm is a Debug line.
|
||||
/// </summary>
|
||||
public class AlarmCommandEventStreamDeliveryTests : TestKit
|
||||
{
|
||||
private static string DebugLevelHocon => @"
|
||||
akka {
|
||||
loglevel = ""DEBUG""
|
||||
extensions = [
|
||||
""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools""
|
||||
]
|
||||
actor {
|
||||
provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
|
||||
}
|
||||
remote.dot-netty.tcp {
|
||||
hostname = ""127.0.0.1""
|
||||
port = 0
|
||||
}
|
||||
cluster {
|
||||
seed-nodes = []
|
||||
roles = [""driver""]
|
||||
min-nr-of-members = 1
|
||||
run-coordinated-shutdown-when-down = off
|
||||
}
|
||||
}";
|
||||
|
||||
public AlarmCommandEventStreamDeliveryTests()
|
||||
: base(DebugLevelHocon)
|
||||
{
|
||||
var cluster = Akka.Cluster.Cluster.Get(Sys);
|
||||
cluster.Join(cluster.SelfAddress);
|
||||
AwaitCondition(
|
||||
() => cluster.State.Members.Any(m => m.Status == Akka.Cluster.MemberStatus.Up),
|
||||
TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_alarm_command_relayed_by_the_comm_actor_reaches_the_real_scripted_alarm_host()
|
||||
{
|
||||
// Needed for the same reason as the deploy leg: the comm actor's unit test only proves a
|
||||
// probe receives it, and the alarm-host tests drive the actor by direct Tell. Neither
|
||||
// notices a missing PreStart subscription.
|
||||
var upstream = new DependencyMuxTagUpstreamSource();
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
var engine = new ScriptedAlarmEngine(
|
||||
upstream, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
||||
|
||||
var host = Sys.ActorOf(
|
||||
ScriptedAlarmHostActor.Props(
|
||||
CreateTestProbe().Ref, CreateTestProbe().Ref, upstream, engine, new NodeId("node-A")),
|
||||
"scripted-alarm-host");
|
||||
var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication");
|
||||
|
||||
var cmd = new AlarmCommand("alarm-unowned", "Acknowledge", "alice", "c", null);
|
||||
|
||||
// Warm-up, OUTSIDE the real assertion: a direct Tell whose debug line proves PreStart has
|
||||
// completed and the EventStream subscription is registered. An earlier draft did the direct
|
||||
// Tell INSIDE the assertion window alongside the relay, which made the test vacuous — the
|
||||
// direct Tell alone produced the log the assertion was waiting for.
|
||||
EventFilter.Debug(contains: "ignoring AlarmCommand")
|
||||
.ExpectOne(TimeSpan.FromSeconds(15), () => host.Tell(cmd));
|
||||
|
||||
// The real assertion: relayed ONLY through the comm actor, which publishes node-locally.
|
||||
// The host owns no alarms, so it logs and stops — still a positive delivery signal, because
|
||||
// that line is only reached if the message arrived.
|
||||
EventFilter.Debug(contains: "ignoring AlarmCommand")
|
||||
.ExpectOne(TimeSpan.FromSeconds(15), () => comm.Tell(cmd));
|
||||
}
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication;
|
||||
|
||||
public class NodeCommunicationActorTests : RuntimeActorTestBase
|
||||
{
|
||||
private static DispatchDeployment NewDispatch() => new(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new RevisionHash("rev-1"),
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
[Fact]
|
||||
public void DispatchDeployment_from_central_lands_on_the_node_local_event_stream()
|
||||
{
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(DispatchDeployment));
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
var dispatch = NewDispatch();
|
||||
actor.Tell(dispatch);
|
||||
|
||||
subscriber.ExpectMsg<DispatchDeployment>(TimeSpan.FromSeconds(5)).ShouldBe(dispatch);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_command_is_published_exactly_once()
|
||||
{
|
||||
// The reason inbound commands do NOT go back onto their DPS topic: central SendToAll-s to
|
||||
// every node's comm actor, so a mesh-wide republish would deliver N copies to every
|
||||
// subscriber. One publish per inbound command is the whole point.
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(DispatchDeployment));
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
actor.Tell(NewDispatch());
|
||||
|
||||
subscriber.ExpectMsg<DispatchDeployment>(TimeSpan.FromSeconds(5));
|
||||
subscriber.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RestartDriver_from_central_lands_on_the_event_stream()
|
||||
{
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(RestartDriver));
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
var msg = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
|
||||
actor.Tell(msg);
|
||||
|
||||
subscriber.ExpectMsg<RestartDriver>(TimeSpan.FromSeconds(5)).ShouldBe(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReconnectDriver_from_central_lands_on_the_event_stream()
|
||||
{
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(ReconnectDriver));
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
var msg = new ReconnectDriver("C1", "driver-1", "alice", Guid.NewGuid());
|
||||
actor.Tell(msg);
|
||||
|
||||
subscriber.ExpectMsg<ReconnectDriver>(TimeSpan.FromSeconds(5)).ShouldBe(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmCommand_from_central_lands_on_the_event_stream()
|
||||
{
|
||||
// ScriptedAlarmHostActor is a CHILD of DriverHostActor with no registry key, so there is no
|
||||
// ref to hand this actor at wiring time — the EventStream is what makes the alarm-command
|
||||
// leg possible without a registration handshake.
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(AlarmCommand));
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
var msg = new AlarmCommand("alarm-1", "Acknowledge", "alice", "comment", null);
|
||||
actor.Tell(msg);
|
||||
|
||||
subscriber.ExpectMsg<AlarmCommand>(TimeSpan.FromSeconds(5)).ShouldBe(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplyAck_goes_to_central_as_a_Send_not_a_SendToAll()
|
||||
{
|
||||
// Send, not SendToAll: the deploy coordinator is one singleton behind the central comm
|
||||
// actor's proxy. SendToAll would deliver one ack per central node and the coordinator would
|
||||
// count this node twice.
|
||||
var client = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(client.Ref));
|
||||
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("host-a:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
actor.Tell(ack);
|
||||
|
||||
var sent = client.ExpectMsg<ClusterClient.Send>(TimeSpan.FromSeconds(5));
|
||||
sent.Path.ShouldBe(MeshPaths.CentralCommunication);
|
||||
sent.Message.ShouldBe(ack);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_ack_with_no_central_client_is_dropped_with_a_warning()
|
||||
{
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null));
|
||||
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("host-a:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
EventFilter.Warning(contains: "dropping ApplyAck").ExpectOne(
|
||||
TimeSpan.FromSeconds(5), () => actor.Tell(ack));
|
||||
}
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
using System.Security.Cryptography;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.DeploymentCache;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 4) — the node-side gRPC artifact fetcher's logic: reassembly,
|
||||
/// SHA-256 verification, endpoint failover, and the null-on-any-failure (#485) contract. The real
|
||||
/// gRPC stream across a real Kestrel h2c listener is the Task 8 boundary test; here the client is
|
||||
/// a fake streaming canned chunks, so only the fetcher's own logic is under test.
|
||||
/// </summary>
|
||||
public sealed class GrpcDeploymentArtifactFetcherTests
|
||||
{
|
||||
private const string DeploymentId = "11111111-1111-1111-1111-111111111111";
|
||||
|
||||
private static string Sha256Hex(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static GrpcDeploymentArtifactFetcher Fetcher(
|
||||
string[] endpoints, Func<string, DeploymentArtifactService.DeploymentArtifactServiceClient> factory)
|
||||
=> new(
|
||||
Options.Create(new ConfigSourceOptions
|
||||
{
|
||||
Mode = ConfigSourceOptions.ModeFetchAndCache,
|
||||
CentralFetchEndpoints = endpoints,
|
||||
ApiKey = "k",
|
||||
FetchTimeoutSeconds = 30,
|
||||
}),
|
||||
NullLogger<GrpcDeploymentArtifactFetcher>.Instance,
|
||||
factory);
|
||||
|
||||
[Fact]
|
||||
public async Task Chunks_reassembling_to_the_expected_hash_return_the_exact_bytes()
|
||||
{
|
||||
var blob = new byte[300 * 1024];
|
||||
for (var i = 0; i < blob.Length; i++) blob[i] = (byte)(i % 251);
|
||||
var factory = SingleClient(StreamOf(blob, chunkSize: 128 * 1024));
|
||||
|
||||
var result = await Fetcher(["http://central-1:4055"], factory)
|
||||
.FetchAsync(DeploymentId, Sha256Hex(blob), CancellationToken.None);
|
||||
|
||||
result.ShouldBe(blob);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_hash_mismatch_returns_null()
|
||||
{
|
||||
var blob = new byte[] { 1, 2, 3, 4 };
|
||||
var factory = SingleClient(StreamOf(blob, chunkSize: 2));
|
||||
|
||||
var result = await Fetcher(["http://central-1:4055"], factory)
|
||||
.FetchAsync(DeploymentId, Sha256Hex([9, 9, 9, 9]), CancellationToken.None);
|
||||
|
||||
result.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_stream_returns_null()
|
||||
{
|
||||
var factory = SingleClient(StreamOf([], chunkSize: 128 * 1024));
|
||||
|
||||
var result = await Fetcher(["http://central-1:4055"], factory)
|
||||
.FetchAsync(DeploymentId, Sha256Hex([]), CancellationToken.None);
|
||||
|
||||
result.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task When_the_first_endpoint_is_unavailable_it_fails_over_to_the_second()
|
||||
{
|
||||
var blob = new byte[] { 10, 20, 30, 40, 50 };
|
||||
var consulted = new List<string>();
|
||||
var factory = (string endpoint) =>
|
||||
{
|
||||
consulted.Add(endpoint);
|
||||
return endpoint.Contains("central-1")
|
||||
? (DeploymentArtifactService.DeploymentArtifactServiceClient)
|
||||
new ThrowingClient(new RpcException(new Status(StatusCode.Unavailable, "down")))
|
||||
: new FakeClient(StreamOf(blob, chunkSize: 2));
|
||||
};
|
||||
|
||||
var result = await Fetcher(["http://central-1:4055", "http://central-2:4055"], factory)
|
||||
.FetchAsync(DeploymentId, Sha256Hex(blob), CancellationToken.None);
|
||||
|
||||
result.ShouldBe(blob);
|
||||
// The failover must have actually consulted the second endpoint.
|
||||
consulted.ShouldBe(["http://central-1:4055", "http://central-2:4055"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task When_every_endpoint_throws_it_returns_null()
|
||||
{
|
||||
var factory = (string _) =>
|
||||
(DeploymentArtifactService.DeploymentArtifactServiceClient)
|
||||
new ThrowingClient(new RpcException(new Status(StatusCode.NotFound, "unknown deployment")));
|
||||
|
||||
var result = await Fetcher(["http://central-1:4055", "http://central-2:4055"], factory)
|
||||
.FetchAsync(DeploymentId, Sha256Hex([1]), CancellationToken.None);
|
||||
|
||||
result.ShouldBeNull();
|
||||
}
|
||||
|
||||
// ---- fakes --------------------------------------------------------------------------------
|
||||
|
||||
private static Func<string, DeploymentArtifactService.DeploymentArtifactServiceClient> SingleClient(
|
||||
IReadOnlyList<ArtifactChunk> chunks)
|
||||
=> _ => new FakeClient(chunks);
|
||||
|
||||
private static IReadOnlyList<ArtifactChunk> StreamOf(byte[] blob, int chunkSize)
|
||||
{
|
||||
var chunks = new List<ArtifactChunk>();
|
||||
for (var offset = 0; offset < blob.Length; offset += chunkSize)
|
||||
{
|
||||
var len = Math.Min(chunkSize, blob.Length - offset);
|
||||
chunks.Add(new ArtifactChunk { Data = ByteString.CopyFrom(blob, offset, len) });
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/// <summary>A generated client double whose <c>Fetch</c> streams canned chunks.</summary>
|
||||
private sealed class FakeClient(IReadOnlyList<ArtifactChunk> chunks)
|
||||
: DeploymentArtifactService.DeploymentArtifactServiceClient
|
||||
{
|
||||
public override AsyncServerStreamingCall<ArtifactChunk> Fetch(FetchRequest request, CallOptions options)
|
||||
=> new(
|
||||
new ListStreamReader(chunks),
|
||||
Task.FromResult(new Metadata()),
|
||||
() => Status.DefaultSuccess,
|
||||
() => [],
|
||||
() => { });
|
||||
}
|
||||
|
||||
/// <summary>A generated client double whose <c>Fetch</c> throws (endpoint down / NotFound).</summary>
|
||||
private sealed class ThrowingClient(RpcException toThrow)
|
||||
: DeploymentArtifactService.DeploymentArtifactServiceClient
|
||||
{
|
||||
public override AsyncServerStreamingCall<ArtifactChunk> Fetch(FetchRequest request, CallOptions options)
|
||||
=> throw toThrow;
|
||||
}
|
||||
|
||||
private sealed class ListStreamReader(IReadOnlyList<ArtifactChunk> chunks) : IAsyncStreamReader<ArtifactChunk>
|
||||
{
|
||||
private int _index = -1;
|
||||
|
||||
public ArtifactChunk Current => chunks[_index];
|
||||
|
||||
public Task<bool> MoveNext(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
_index++;
|
||||
return Task.FromResult(_index < chunks.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
+8
-8
@@ -41,7 +41,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
var cache = new StubArtifactCache(cached);
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator: null,
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
@@ -70,7 +70,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
var publishProbe = CreateTestProbe();
|
||||
|
||||
Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator: null,
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publishProbe.Ref,
|
||||
deploymentArtifactCache: new StubArtifactCache(cached)));
|
||||
@@ -85,7 +85,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
{
|
||||
// The pre-cache behaviour, unchanged. A cache miss must not invent a configuration.
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator: null,
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: new StubArtifactCache(current: null)));
|
||||
|
||||
@@ -100,7 +100,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
{
|
||||
// Admin-only graphs and older harnesses pass null; behaviour is identical to a cache miss.
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator: null,
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" }));
|
||||
|
||||
var snapshot = AskDiagnostics(actor);
|
||||
@@ -118,7 +118,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
NewInMemoryDbFactory(), TestNode, coordinator: null,
|
||||
NewInMemoryDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
@@ -140,7 +140,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
deploymentId.ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator: null,
|
||||
db, TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
@@ -156,7 +156,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
{
|
||||
// A cache fault must leave the node exactly where it would have been without a cache.
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator: null,
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: new ThrowingReadCache()));
|
||||
|
||||
@@ -183,7 +183,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
var cache = new StubArtifactCache(current: null);
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator: null,
|
||||
db, TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 6) — <c>FetchAndCache</c> bootstrap. A driver node in this mode
|
||||
/// boots its served state from the LocalDb pointer, reading NO config from central SQL; an empty
|
||||
/// or unreadable cache lands Steady-with-no-revision (the first dispatch fetches), NOT Stale.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every test injects a <see cref="ThrowingDbFactory"/> for central SQL — reaching Steady (and, in
|
||||
/// two of them, applying a dispatch) proves the boot never touched it. This is the FetchAndCache
|
||||
/// counterpart to <see cref="DriverHostActorBootFromCacheTests"/> (which covers the Direct-mode
|
||||
/// SQL-unreachable fallback, where <c>RunningFromCache</c> is TRUE — here it is normal operation,
|
||||
/// so it stays FALSE).
|
||||
/// </remarks>
|
||||
public sealed class DriverHostActorFetchAndCacheBootstrapTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash CachedRev = RevisionHash.Parse(new string('c', 64));
|
||||
|
||||
private static byte[] Artifact() =>
|
||||
JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
|
||||
|
||||
[Fact]
|
||||
public void WithCachedArtifact_RestoresServedState_WithoutReadingCentralSql()
|
||||
{
|
||||
var blob = Artifact();
|
||||
var cached = new CachedDeploymentArtifact(
|
||||
DeploymentId.NewId().ToString(), CachedRev.Value, blob, DateTimeOffset.UtcNow.AddHours(-1));
|
||||
var cache = new StubArtifactCache(cached);
|
||||
var publishProbe = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publishProbe.Ref,
|
||||
deploymentArtifactCache: cache,
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: new RecordingFetcher(Artifact())));
|
||||
|
||||
// Served state rebuilt from the CACHED bytes (no ConfigDb read — the factory throws).
|
||||
var rebuild = publishProbe.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5));
|
||||
rebuild.Artifact.ShouldBe(blob);
|
||||
|
||||
var snapshot = AskDiagnostics(actor);
|
||||
snapshot.CurrentRevision.ShouldBe(CachedRev);
|
||||
// NORMAL operation, not the degraded Direct-mode fallback — a FetchAndCache node can still fetch.
|
||||
snapshot.RunningFromCache.ShouldBeFalse();
|
||||
cache.UnkeyedReads.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithEmptyCache_EntersSteadyNoRevision_AndTheNextDispatchFetches()
|
||||
{
|
||||
var cache = new StubArtifactCache(current: null);
|
||||
var fetcher = new RecordingFetcher(Artifact());
|
||||
var coordinator = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache,
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
var snapshot = AskDiagnostics(actor);
|
||||
snapshot.CurrentRevision.ShouldBeNull();
|
||||
snapshot.RunningFromCache.ShouldBeFalse();
|
||||
|
||||
// Steady, not Stale: a dispatch is serviced (Stale would ignore it, never acking) and fetches.
|
||||
actor.Tell(new DispatchDeployment(DeploymentId.NewId(), CachedRev, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
fetcher.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithAnUnreadableCache_EntersSteadyNoRevision_NotStale()
|
||||
{
|
||||
var fetcher = new RecordingFetcher(Artifact());
|
||||
var coordinator = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: new ThrowingReadCache(),
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
var snapshot = AskDiagnostics(actor);
|
||||
snapshot.CurrentRevision.ShouldBeNull();
|
||||
snapshot.RunningFromCache.ShouldBeFalse();
|
||||
|
||||
// A cache-read fault is NOT Stale (Stale means "central SQL down, retry it" — there is no
|
||||
// config SQL read to recover here). The node is Steady and the next dispatch fetches.
|
||||
actor.Tell(new DispatchDeployment(DeploymentId.NewId(), CachedRev, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
fetcher.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
||||
probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(2));
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
|
||||
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
||||
return probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
private sealed class RecordingFetcher(byte[]? bytes) : IDeploymentArtifactFetcher
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
return Task.FromResult(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Stands in for an unreachable central SQL Server.</summary>
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
=> throw new InvalidOperationException("ConfigDb unreachable");
|
||||
}
|
||||
|
||||
private sealed class StubArtifactCache(CachedDeploymentArtifact? current) : IDeploymentArtifactCache
|
||||
{
|
||||
private int _unkeyedReads;
|
||||
public int UnkeyedReads => Volatile.Read(ref _unkeyedReads);
|
||||
|
||||
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> Task.FromResult(current);
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _unkeyedReads);
|
||||
return Task.FromResult(current);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingReadCache : IDeploymentArtifactCache
|
||||
{
|
||||
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> throw new InvalidOperationException("cache unreadable");
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
=> throw new InvalidOperationException("cache unreadable");
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 5) — the <c>FetchAndCache</c> apply path: on dispatch the node
|
||||
/// fetches the artifact bytes over gRPC, applies them from bytes in hand, and caches them —
|
||||
/// reading NO config from central SQL. A null fetch is an apply failure that keeps last-known-good
|
||||
/// and does not advance the revision (#485).
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorFetchAndCacheTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
|
||||
/// <summary>Builds a minimal but parseable artifact (no drivers) — the fetched-bytes stand-in.</summary>
|
||||
private static byte[] ArtifactBytes() =>
|
||||
JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
|
||||
|
||||
[Fact]
|
||||
public void FetchReturningGoodBytes_AppliesAndCaches_WithoutReadingCentralSql()
|
||||
{
|
||||
// No Deployment row is seeded: if the apply path read ArtifactBlob from SQL it would get empty
|
||||
// bytes and FAIL (#485). Reaching Applied proves the bytes came from the fetcher, not SQL.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var cache = new RecordingArtifactCache();
|
||||
var fetcher = new RecordingFetcher(ArtifactBytes());
|
||||
var deploymentId = DeploymentId.NewId();
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache,
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
AwaitAssert(() => cache.Stores.Count.ShouldBe(1), duration: TimeSpan.FromSeconds(3));
|
||||
cache.Stores[0].RevisionHash.ShouldBe(RevA.Value);
|
||||
fetcher.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FetchReturningNull_FailsTheApply_KeepsRevisionUnchanged_AndCachesNothing()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var cache = new RecordingArtifactCache();
|
||||
var fetcher = new RecordingFetcher(bytes: null); // all endpoints down / NotFound / SHA mismatch
|
||||
var deploymentId = DeploymentId.NewId();
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache,
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
||||
|
||||
Thread.Sleep(300);
|
||||
cache.Stores.ShouldBeEmpty();
|
||||
|
||||
// The revision was NOT advanced: a re-dispatch of the SAME revision fetches AGAIN rather than
|
||||
// short-circuiting as "already current". (Two fetch attempts prove #485's keep-last-known-good.)
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
||||
fetcher.Calls.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RedispatchOfTheAppliedRevision_DoesNotFetchAgain()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var cache = new RecordingArtifactCache();
|
||||
var fetcher = new RecordingFetcher(ArtifactBytes());
|
||||
var deploymentId = DeploymentId.NewId();
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache,
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
// Re-dispatch the now-current revision: the _currentRevision guard short-circuits to an
|
||||
// immediate ACK without a second fetch.
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
Thread.Sleep(200);
|
||||
fetcher.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>Records fetch calls and returns canned bytes (or null for "no answer").</summary>
|
||||
private sealed class RecordingFetcher(byte[]? bytes) : IDeploymentArtifactFetcher
|
||||
{
|
||||
private int _calls;
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
return Task.FromResult(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records every store; GetCurrent* return null (no idempotency shortcut in these tests).</summary>
|
||||
private sealed class RecordingArtifactCache : IDeploymentArtifactCache
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<StoreCall> _stores = [];
|
||||
|
||||
public IReadOnlyList<StoreCall> Stores
|
||||
{
|
||||
get { lock (_gate) { return _stores.ToArray(); } }
|
||||
}
|
||||
|
||||
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
{
|
||||
lock (_gate) { _stores.Add(new StoreCall(clusterId, deploymentId, revisionHash, artifact)); }
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
|
||||
internal sealed record StoreCall(
|
||||
string ClusterId, string DeploymentId, string RevisionHash, byte[] Artifact);
|
||||
}
|
||||
}
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Per-cluster mesh Phase 3 (Task 7) — the #485 "empty/unreadable bytes = no answer, keep
|
||||
/// last-known-good, never tear down to an empty address space" contract, proven on every
|
||||
/// <c>FetchAndCache</c> seam. This adds no new mechanism; it is the regression net that pins the
|
||||
/// NEGATIVE — nothing rebuilds, nothing is torn down — on the failure paths.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorFetchAndCacheUnreadableTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
|
||||
private static byte[] Artifact() =>
|
||||
JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
|
||||
|
||||
[Fact]
|
||||
public void ANullFetchMidSteadyState_KeepsTheServedState_AndFiresNoRebuild()
|
||||
{
|
||||
// revA fetches good bytes and is served; a later revB fetch returns null (all endpoints down /
|
||||
// NotFound / SHA mismatch — all indistinguishable to the actor). The served address space must
|
||||
// NOT be rebuilt for revB and the revision must stay revA.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var fetcher = new MapFetcher(new Dictionary<string, byte[]?>
|
||||
{
|
||||
[RevA.Value] = Artifact(),
|
||||
[RevB.Value] = null,
|
||||
});
|
||||
var publishProbe = CreateTestProbe();
|
||||
var coordinator = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publishProbe.Ref,
|
||||
deploymentArtifactCache: new NullReadCache(),
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
actor.Tell(new DispatchDeployment(DeploymentId.NewId(), RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
publishProbe.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5)); // revA rebuild
|
||||
|
||||
actor.Tell(new DispatchDeployment(DeploymentId.NewId(), RevB, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
||||
|
||||
// The #485 negative: no rebuild for the failed revB — the served address space is untouched.
|
||||
publishProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AZeroLengthFetch_FailsTheApply_ViaTheActorsOwn485Guard()
|
||||
{
|
||||
// The fetcher contract already turns zero bytes into null, but prove the actor ALSO refuses an
|
||||
// empty blob handed to it directly: ReconcileDriversFromBlob's #485 guard fails the apply.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var fetcher = new MapFetcher(new Dictionary<string, byte[]?> { [RevA.Value] = Array.Empty<byte>() });
|
||||
var publishProbe = CreateTestProbe();
|
||||
var coordinator = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publishProbe.Ref,
|
||||
deploymentArtifactCache: new NullReadCache(),
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: fetcher));
|
||||
|
||||
actor.Tell(new DispatchDeployment(DeploymentId.NewId(), RevA, CorrelationId.NewId()));
|
||||
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
||||
publishProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); // nothing applied, nothing rebuilt
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ACorruptCacheAtBoot_DoesNotApplyAPartialConfig()
|
||||
{
|
||||
// A truncated/corrupt cached artifact surfaces as GetCurrentUnkeyedAsync == null (the cache's
|
||||
// integrity check is deliberately indistinguishable from a miss). Boot must land
|
||||
// Steady-no-revision and apply NOTHING — never a partial address space.
|
||||
var publishProbe = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: null,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publishProbe.Ref,
|
||||
deploymentArtifactCache: new NullReadCache(), // corrupt → null, per the cache contract
|
||||
fetchAndCacheMode: true,
|
||||
artifactFetcher: new MapFetcher(new Dictionary<string, byte[]?>())));
|
||||
|
||||
// No rebuild at boot — a null (corrupt/empty) cache is not a configuration to apply.
|
||||
publishProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(750));
|
||||
AskDiagnostics(actor).CurrentRevision.ShouldBeNull();
|
||||
}
|
||||
|
||||
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
||||
probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(2));
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
|
||||
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
||||
return probe.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
private sealed class MapFetcher(Dictionary<string, byte[]?> map) : IDeploymentArtifactFetcher
|
||||
{
|
||||
public Task<byte[]?> FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct)
|
||||
{
|
||||
map.TryGetValue(expectedRevisionHash, out var bytes);
|
||||
return Task.FromResult(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cache that reads empty (a miss, or the indistinguishable corrupt-artifact case).</summary>
|
||||
private sealed class NullReadCache : IDeploymentArtifactCache
|
||||
{
|
||||
public Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
}
|
||||
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
=> throw new InvalidOperationException("ConfigDb unreachable");
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -183,7 +183,7 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
|
||||
Sys.ActorOf(DriverHostActor.Props(
|
||||
NewInMemoryDbFactory(),
|
||||
TestNode,
|
||||
coordinator: null,
|
||||
ackRouter: null,
|
||||
driverMemberCountProvider: () => driverMemberCount,
|
||||
redundancyRoleView: view,
|
||||
replicationPeerHost: PeerHost));
|
||||
|
||||
@@ -139,7 +139,7 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
|
||||
var hostActor = Sys.ActorOf(DriverHostActor.Props(
|
||||
dbFactory: NewInMemoryDbFactory(),
|
||||
localNode: ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId.Parse("host-1"),
|
||||
coordinator: hostProbe.Ref,
|
||||
ackRouter: hostProbe.Ref,
|
||||
dependencyMux: mux));
|
||||
|
||||
// Tell the host an AttributeValuePublished — it should fan out to the mux + subscriber.
|
||||
|
||||
Reference in New Issue
Block a user