diff --git a/CLAUDE.md b/CLAUDE.md
index 3a2f7ece..9073af3f 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -315,6 +315,57 @@ Flip only the site nodes with `OTOPCUA_CONFIG_MODE=FetchAndCache` at `docker com
`docs/Configuration.md` §`ConfigSource` / `ConfigServe`, `docs/Redundancy.md` §"Command transport", and
`docs/plans/2026-07-22-mesh-phase3-config-fetch-and-cache.md`.
+## Live telemetry transport (`Telemetry` / `TelemetryDial`)
+
+Per-cluster mesh **Phase 5** (code-complete on `feat/mesh-phase5`, live gate pending) added a second
+transport for the four live-observability channels the AdminUI's `/alerts`, `/script-log`, and
+`/hosts` panels feed on, selected by `Telemetry:Mode` (node/serve side) and `TelemetryDial:Mode`
+(central/dial side): `Dps` — the four existing DPS SignalR bridges, **the default**, today's
+behaviour — or `Grpc`, one gRPC server-streaming contract carrying all four channels as `oneof`
+event kinds. **The direction is inverted from `ConfigSource`/`MeshTransport`: the driver node hosts
+the gRPC server (dedicated Kestrel h2c port), and central is the client that dials in** — mirroring
+ScadaBridge's `SiteStreamService` shape, and required so telemetry survives once the meshes split
+(Phase 6) and central no longer shares a gossip ring with a site node. Things worth knowing before
+touching it:
+
+- **Migrated (4 channels):** `alerts` (`AlarmTransitionEvent`), `script-logs` (`ScriptLogEntry`),
+ `driver-health` (`DriverHealthChanged`), `driver-resilience-status`
+ (`DriverResilienceStatusChanged`) — feeding the exact same central sinks
+ (`IInProcessBroadcaster<...>`, `IDriverStatusSnapshotStore`, `IDriverResilienceStatusStore`) in
+ either mode.
+- **Deferred, with reasons (do not read "seven observability topics" from the program sketch as the
+ delivered scope):** `redundancy-state` stays on DPS in every mode (pair-local, built from
+ `Cluster.State`, drives ServiceLevel + the Primary gate — genuinely mesh-bound by design, not an
+ observability panel); `fleet-status` is central-internal (`FleetStatusBroadcaster` builds it from
+ the admin node's own membership events, `Fleet.razor` polls the Config DB and ignores the feed —
+ never a node→central stream); `deployment-acks` already rides the Phase 2 `ClusterClient`
+ transport as a command-plane reply, not telemetry.
+- **The node ALWAYS hosts and ALWAYS double-publishes, in both modes.** Every publish seam emits
+ into a node-local `ITelemetryLocalHub` **and** still publishes DPS, unconditionally; the gRPC
+ server binds whenever `Telemetry:GrpcListenPort > 0`, independent of `Telemetry:Mode`. Only
+ `TelemetryDial:Mode` on central actually switches anything: which of the two always-available
+ sources (DPS bridges vs. the `TelemetryDialSupervisor` dialer) feeds the sinks. This is what
+ makes the flip a config change, not a coordinated redeploy — same dark-switch discipline as
+ [MeshTransport](#mesh-command-transport-meshtransport) and [ConfigSource](#config-source-configsource--configserve).
+- **Authenticated from day one, fail-closed** — a shared node bearer key (`Telemetry:ApiKey` ==
+ `TelemetryDial:ApiKey`), gated by `TelemetryStreamAuthInterceptor`
+ (`CryptographicOperations.FixedTimeEquals`, `PermissionDenied` on mismatch, an unset key rejects
+ every call). This **supersedes** design doc §6.3's "match ScadaBridge's unauthenticated posture
+ for now" — ScadaBridge itself has since closed that gap with this identical interceptor pattern.
+- **Reconnect story:** central runs one reconnecting dialer per driver node (discovered from
+ `ClusterNode.Host`/`GrpcPort`, refreshed every `TelemetryDial:ContactRefreshSeconds`), immediate
+ first retry then ~5s fixed backoff, generation-stamped so a superseded stream's late error/event
+ is ignored, and it never permanently gives up (observability, not data plane). The node-local hub
+ replays last-value snapshots for `driver-health`/`driver-resilience-status` on every (re)subscribe
+ so those stores re-prime immediately after a reconnect; `alerts`/`script-logs` are append logs and
+ tolerate the gap. The `/alerts`/`/script-log` pill reflects aggregate stream health (live when ≥1
+ node stream is up) — the same semantics the DPS pill has always had.
+
+See `docs/Telemetry.md` for the full architecture, `docs/Configuration.md`
+§`Telemetry`/`TelemetryDial` for the config keys, and
+`docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md` for the implementation plan and live
+gate procedure.
+
## LocalDb pair-local store (Phases 1 + 2)
Every **driver-role** node keeps a consolidated `ZB.MOM.WW.LocalDb` SQLite database (the retired
diff --git a/ZB.MOM.WW.OtOpcUa.slnx b/ZB.MOM.WW.OtOpcUa.slnx
index a1090eab..b6647bd2 100644
--- a/ZB.MOM.WW.OtOpcUa.slnx
+++ b/ZB.MOM.WW.OtOpcUa.slnx
@@ -76,6 +76,7 @@
+
diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml
index 9db7aac3..5330a628 100644
--- a/docker-dev/docker-compose.yml
+++ b/docker-dev/docker-compose.yml
@@ -40,6 +40,23 @@
# + seeded clusters across `docker compose up` cycles; without it a recreate
# silently drops the OtOpcUa database.
#
+# Per-cluster mesh Phase 5 (gRPC live-telemetry stream): every driver-role node (all six,
+# including the fused central pair) carries Telemetry__GrpcListenPort=4056 (dedicated h2c
+# listener, verified free against every other per-node port: Akka 4053, ConfigServe 4055 on
+# central, LocalDb sync 9001 on site-a, OPC UA 4840, HTTP 9000) + Telemetry__ApiKey
+# (committed dev key, matches the SQL password / secrets KEK / ConfigServe key exception).
+# central-1/central-2 also carry the matching TelemetryDial__ApiKey. Both Telemetry__Mode and
+# TelemetryDial__Mode are left UNSET here, so every node defaults to "Dps" (dark — telemetry
+# keeps riding the existing DistributedPubSub topic). The ClusterNode.GrpcPort column (seeded
+# by docker-dev/seed/seed-clusters.sql) points central's dial supervisor at each node's :4056.
+# Unlike MeshTransport/ConfigSource, no `${VAR:-Dps}` interpolation is wired for either Mode key
+# (a bare shell export does nothing here — these keys aren't referenced anywhere in this file, so
+# there is nothing for it to substitute into). To run the Phase 5 live gate, manually add
+# `Telemetry__Mode: "Grpc"` to the `environment:` block of all six driver nodes (central-1,
+# central-2, site-a-1, site-a-2, site-b-1, site-b-2) and `TelemetryDial__Mode: "Grpc"` to
+# central-1's and central-2's blocks — or supply both via a docker-compose.override.yml carrying
+# the same keys — then `docker compose -f docker-dev/docker-compose.yml up -d` to recreate.
+#
# Usage:
# docker compose -f docker-dev/docker-compose.yml up -d --build
# open http://localhost:9200 # central Blazor admin UI
@@ -186,6 +203,16 @@ services:
ConfigServe__GrpcListenPort: "4055"
ConfigServe__ApiKey: "configserve-docker-dev-key"
ConfigSource__Mode: "Direct"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream). DARK SWITCH — Telemetry:Mode is left
+ # unset here (defaults to "Dps"), so the node keeps publishing telemetry on the existing
+ # DistributedPubSub topic; the dedicated gRPC listener below binds regardless (the node "always
+ # hosts" per docs/Telemetry.md) so flipping the mode later is a config change, not a redeploy.
+ # Port 4056 checked free against every other port this node binds: Akka 4053, ConfigServe 4055,
+ # OPC UA 4840 (container-internal), HTTP 9000. TelemetryDial__ApiKey (below) is central's
+ # dial-side key and must equal every node's Telemetry:ApiKey (fail-closed interceptor).
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
+ TelemetryDial__ApiKey: "telemetry-docker-dev-key"
# Both redundancy peers are seeds (#459 / ScadaBridge parity) so a restarted node can
# re-join the mesh via EITHER peer, not only central-1.
#
@@ -282,6 +309,11 @@ services:
ConfigServe__GrpcListenPort: "4055"
ConfigServe__ApiKey: "configserve-docker-dev-key"
ConfigSource__Mode: "Direct"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream) — see central-1. DARK SWITCH: Mode
+ # stays unset (⇒ Dps); the dedicated :4056 listener binds regardless.
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
+ TelemetryDial__ApiKey: "telemetry-docker-dev-key"
# Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. SELF FIRST:
# central-2 lists itself as seed-nodes[0], which is what lets it cold-start while central-1
# is down (it previously listed central-1 first and simply never came Up in that case).
@@ -374,6 +406,13 @@ services:
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
ConfigSource__ApiKey: "configserve-docker-dev-key"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream) — see central-1/central-2. DARK
+ # SWITCH: Telemetry:Mode stays unset (⇒ Dps); the dedicated :4056 listener binds regardless
+ # (the node "always hosts" per docs/Telemetry.md). Port checked free against this node's
+ # other ports: Akka 4053, OPC UA 4840, HTTP 9000, LocalDb sync 9001 (below). ApiKey must
+ # equal central's TelemetryDial:ApiKey (fail-closed interceptor).
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
# Site nodes are deliberately NOT seeds — they join the central pair's mesh. The self-first
# seed rule is conditional for exactly this reason (it binds only when a node's own address
# is in its own seed list), so these configs are exempt rather than broken.
@@ -462,6 +501,10 @@ services:
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
ConfigSource__ApiKey: "configserve-docker-dev-key"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream) — see central-1/site-a-1. DARK
+ # SWITCH: Telemetry:Mode stays unset (⇒ Dps); the dedicated :4056 listener binds regardless.
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
Cluster__Roles__0: "driver"
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
@@ -539,6 +582,10 @@ services:
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
ConfigSource__ApiKey: "configserve-docker-dev-key"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream) — see central-1/site-a-1. DARK
+ # SWITCH: Telemetry:Mode stays unset (⇒ Dps); the dedicated :4056 listener binds regardless.
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
Cluster__Roles__0: "driver"
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
@@ -599,6 +646,10 @@ services:
ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055"
ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055"
ConfigSource__ApiKey: "configserve-docker-dev-key"
+ # Per-cluster mesh Phase 5 (gRPC live-telemetry stream) — see central-1/site-a-1. DARK
+ # SWITCH: Telemetry:Mode stays unset (⇒ Dps); the dedicated :4056 listener binds regardless.
+ Telemetry__GrpcListenPort: "4056"
+ Telemetry__ApiKey: "telemetry-docker-dev-key"
Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
Cluster__Roles__0: "driver"
# Mesh command transport (per-cluster mesh Phase 2). DARK SWITCH: the rig comes up on "Dps",
diff --git a/docker-dev/seed/seed-clusters.sql b/docker-dev/seed/seed-clusters.sql
index a2edf468..62bfa596 100644
--- a/docker-dev/seed/seed-clusters.sql
+++ b/docker-dev/seed/seed-clusters.sql
@@ -16,6 +16,12 @@
-- Host = Compose service name (resolves inside the otopcua-dev network).
-- OpcUaPort stays at the container-internal 4840; the host-side port mapping is in
-- docker-compose.yml ports: blocks and is irrelevant to ClusterNode rows.
+--
+-- GrpcPort (per-cluster mesh Phase 5): every node's dedicated live-telemetry gRPC listener,
+-- matching Telemetry__GrpcListenPort=4056 set on all six host nodes in docker-compose.yml.
+-- TelemetryNodeSource (central) reads Host + GrpcPort to build its dial-target set
+-- (http://{Host}:{GrpcPort}); a NULL GrpcPort is skipped with a Warning. Mirrors how AkkaPort
+-- was seeded for Phase 1's ClusterNodeAddressReconcilerActor.
SET NOCOUNT ON;
SET XACT_ABORT ON;
@@ -64,13 +70,13 @@ IF NOT EXISTS (SELECT 1 FROM dbo.ServerCluster WHERE ClusterId = 'SITE-B')
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'central-1:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('central-1:4053', 'MAIN', 'central-1', 4840, 8081, 4053, 'urn:OtOpcUa:central-1', 200, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('central-1:4053', 'MAIN', 'central-1', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:central-1', 200, 1, 'docker-dev-seed');
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'central-2:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('central-2:4053', 'MAIN', 'central-2', 4840, 8081, 4053, 'urn:OtOpcUa:central-2', 150, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('central-2:4053', 'MAIN', 'central-2', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:central-2', 150, 1, 'docker-dev-seed');
------------------------------------------------------------------------------
-- ClusterNode — site A
@@ -78,13 +84,13 @@ IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'central-2:4053')
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'site-a-1:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('site-a-1:4053', 'SITE-A', 'site-a-1', 4840, 8081, 4053, 'urn:OtOpcUa:site-a-1', 200, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('site-a-1:4053', 'SITE-A', 'site-a-1', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:site-a-1', 200, 1, 'docker-dev-seed');
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'site-a-2:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('site-a-2:4053', 'SITE-A', 'site-a-2', 4840, 8081, 4053, 'urn:OtOpcUa:site-a-2', 150, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('site-a-2:4053', 'SITE-A', 'site-a-2', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:site-a-2', 150, 1, 'docker-dev-seed');
------------------------------------------------------------------------------
-- ClusterNode — site B
@@ -92,13 +98,13 @@ IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'site-a-2:4053')
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'site-b-1:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('site-b-1:4053', 'SITE-B', 'site-b-1', 4840, 8081, 4053, 'urn:OtOpcUa:site-b-1', 200, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('site-b-1:4053', 'SITE-B', 'site-b-1', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:site-b-1', 200, 1, 'docker-dev-seed');
IF NOT EXISTS (SELECT 1 FROM dbo.ClusterNode WHERE NodeId = 'site-b-2:4053')
INSERT INTO dbo.ClusterNode
- (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
- VALUES ('site-b-2:4053', 'SITE-B', 'site-b-2', 4840, 8081, 4053, 'urn:OtOpcUa:site-b-2', 150, 1, 'docker-dev-seed');
+ (NodeId, ClusterId, Host, OpcUaPort, DashboardPort, AkkaPort, GrpcPort, ApplicationUri, ServiceLevelBase, Enabled, CreatedBy)
+ VALUES ('site-b-2:4053', 'SITE-B', 'site-b-2', 4840, 8081, 4053, 4056, 'urn:OtOpcUa:site-b-2', 150, 1, 'docker-dev-seed');
------------------------------------------------------------------------------
-- Galaxy MxAccess gateway — INTENTIONALLY NOT SEEDED (retired 2026-06-15)
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 734867d8..0b3b76dd 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -177,6 +177,60 @@ PersistNodeAck`) and LDAP group→role DB grants (a driver-only node maps LDAP g
the `Security:Ldap:GroupToRole` appsettings baseline only — see `docs/Redundancy.md` and
`CLAUDE.md` §"Phase 4" for the full picture, including the client-visible ServiceLevel change).
+### `Telemetry` / `TelemetryDial` (live telemetry transport)
+
+- **Purpose:** [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) **Phase 5**. Selects how
+ the four live-observability channels (`alerts`, `script-logs`, `driver-health`,
+ `driver-resilience-status`) reach central: `Dps` (DistributedPubSub — the four existing SignalR
+ bridges, today's behaviour) or `Grpc` (a driver node hosts one gRPC server-streaming contract over a
+ dedicated h2c port; central dials in and feeds the same in-process sinks). `Telemetry` is the
+ **node/serve** side; `TelemetryDial` is the **central/dial** side.
+- **Options classes:** `TelemetryOptions` (`SectionName = "Telemetry"`) + `TelemetryDialOptions`
+ (`SectionName = "TelemetryDial"`), both in `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/`. Bound by
+ `AddOtOpcUaCluster(config)`; their validators run at `ValidateOnStart`.
+
+**`Telemetry` (node-side, serve):**
+
+| Key | Type | Default | Meaning |
+|---|---|---|---|
+| `Mode` | string | `Dps` | `Dps` = no change, node just keeps publishing DPS as it always has. `Grpc` = central-ingest hint (see `TelemetryDial:Mode` below — this key does not gate anything on the node itself). Any other value **refuses to start**. |
+| `GrpcListenPort` | int | `0` | Dedicated **h2c-only** Kestrel port for the telemetry stream service. `0` = disabled (nothing bound). The node hosts the server whenever this is `> 0`, **regardless of `Mode`**. |
+| `ApiKey` | string | `""` | Shared bearer key the serve-side interceptor checks (constant-time, **fail-closed**: an unset key rejects every call). **Supply via env `Telemetry__ApiKey` — never commit.** |
+
+**`TelemetryDial` (central-side, dial):**
+
+| Key | Type | Default | Meaning |
+|---|---|---|---|
+| `Mode` | string | `Dps` | `Dps` = the four DPS SignalR bridges (alert, script-log, driver-status, resilience) subscribe as today. `Grpc` = those four bridges are **not spawned**; the `TelemetryDialSupervisor` actor dials every driver node instead and feeds the same sinks. Any other value **refuses to start**. |
+| `ApiKey` | string | `""` | Shared bearer key; must equal the nodes' `Telemetry:ApiKey`. **Supply via env `TelemetryDial__ApiKey` — never commit.** Required under `Grpc`. |
+| `ContactRefreshSeconds` | int | `60` | How often central rebuilds its dial-target set from the `ClusterNode` rows (`Host` + `GrpcPort`). |
+| `CallTimeoutSeconds` | int | `30` | Per-call deadline for the streaming RPC's keepalive/health bookkeeping. |
+
+**It is a dark switch; the node always hosts.** Unlike `ConfigServe`/`MeshTransport`, where the serve
+side is gated on a role, the telemetry server binds whenever `Telemetry:GrpcListenPort > 0` — hosting
+is unconditional on a driver-role node, in both modes, because the node also always emits into its
+local hub and always publishes DPS. Only `TelemetryDial:Mode` on central actually changes behaviour:
+which of the two already-always-available sources (DPS bridges vs. the gRPC dial supervisor) feeds the
+AdminUI sinks. Flipping either side is a config change plus restart, not a redeploy — the same
+discipline as `MeshTransport:Mode` and `ConfigSource:Mode`.
+
+**A driver-role node with `Telemetry:Mode=Grpc` must set `GrpcListenPort > 0`** (nothing to serve
+otherwise) — enforced by `TelemetryOptionsValidator`. **`Grpc` mode on either side requires a
+non-empty `ApiKey`** — fail-closed, same as `ConfigServe`/`LocalDb:Replication`.
+
+**Three DPS observability topics stay out of scope for this transport, on purpose:**
+`redundancy-state` (pair-local, built from `Cluster.State`, stays on DPS in both modes — see
+[`Redundancy.md`](Redundancy.md#command-transport-centralnode)), `fleet-status` (central-internal;
+`Fleet.razor` polls the Config DB, not any stream), and `deployment-acks` (already rides the Phase 2
+`ClusterClient` transport as a command-plane reply). See
+[`docs/Telemetry.md`](Telemetry.md) for the full architecture, the reconnect/generation-stamped
+dialer story, and the rationale for the node-hosts/central-dials inversion.
+
+On the docker-dev rig every driver-role node carries `Telemetry:GrpcListenPort` (`4056`) + a
+committed dev key, and central carries the matching `TelemetryDial:ApiKey`; both `Mode` keys stay
+unset (⇒ `Dps`) until the rig is flipped with `Telemetry__Mode=Grpc` / `TelemetryDial__Mode=Grpc` at
+`docker compose up`.
+
### `ConnectionStrings` → `ConfigDb`
- **Purpose:** the central Config DB connection string. **Required on admin-role nodes only** — `Program.cs` calls `AddOtOpcUaConfigDb` when `hasAdmin` (per-cluster mesh Phase 4). A driver-only node holds no `ConfigDb` connection string; it fetches its configuration via `ConfigSource:Mode=FetchAndCache` instead (see above). A fused `admin,driver` node still requires it.
@@ -283,6 +337,7 @@ These are consumed by the driver **integration-test fixtures** (under `tests/Dri
- [`security.md`](security.md) — transport security, OPC UA authentication, LDAP (`Security:Ldap`), data-plane ACLs, control-plane roles.
- [`Redundancy.md`](Redundancy.md) — the `Cluster` section in the context of warm/hot redundancy, ServiceLevel, peer discovery.
+- [`Telemetry.md`](Telemetry.md) — the `Telemetry`/`TelemetryDial` gRPC live-telemetry stream: direction, dark switch, the four migrated channels + three deferrals, auth, reconnect story.
- [`ServiceHosting.md`](ServiceHosting.md) — role-based host wiring and `OTOPCUA_ROLES`.
- [`docs/drivers/Galaxy.md`](drivers/Galaxy.md) — Galaxy/MxAccess driver overview.
- [`docs/v2/config-db-schema.md`](v2/config-db-schema.md) — the full Config DB schema.
diff --git a/docs/Redundancy.md b/docs/Redundancy.md
index 2733b966..d08756c8 100644
--- a/docs/Redundancy.md
+++ b/docs/Redundancy.md
@@ -465,6 +465,15 @@ are untouched, and `redundancy-state` itself stays on DPS (it is bidirectional a
`Cluster.State`, which makes it genuinely mesh-bound — the hardest remaining dependency before the
meshes can split).
+**Observability telemetry took the opposite path (Phase 5).** The four live-panel channels
+(`alerts`, `script-logs`, `driver-health`, `driver-resilience-status`) moved off DPS onto a
+dedicated gRPC server-streaming contract — each driver node hosts, central dials in — selected by
+`Telemetry:Mode`/`TelemetryDial:Mode` (default `Dps`). `redundancy-state` is deliberately **not**
+part of that migration: it stays on DPS in every mode because it is pair-local and Cluster.State-built,
+not a fleet-wide observability broadcast. See [`docs/Telemetry.md`](Telemetry.md) for the full
+architecture and [`Configuration.md` § `Telemetry`/`TelemetryDial`](Configuration.md#telemetry--telemetrydial-live-telemetry-transport)
+for the config keys.
+
**Config bytes travel out-of-band (Phase 3).** Commands stay on one of the two transports above, but
the *configuration artifact itself* does not ride any Akka message — `DispatchDeployment` is
payload-free (`DeploymentId` + `RevisionHash`). Under `ConfigSource:Mode = FetchAndCache` (see
diff --git a/docs/Telemetry.md b/docs/Telemetry.md
new file mode 100644
index 00000000..a24570f1
--- /dev/null
+++ b/docs/Telemetry.md
@@ -0,0 +1,190 @@
+# Live Telemetry Transport (v2)
+
+## Overview
+
+The AdminUI's live observability panels (`/alerts`, `/script-log`, `/hosts` driver table, driver
+resilience status) are fed by four node→central event channels. Historically all four rode
+DistributedPubSub (DPS) on the shared Akka mesh — the same gossip ring that carries redundancy
+state and the command-plane topics. Per-cluster mesh **Phase 5** adds a second transport for these
+four channels only: one gRPC server-streaming contract, selected per node/central pair by
+`Telemetry:Mode` / `TelemetryDial:Mode` (`Dps` default | `Grpc`).
+
+This is the same motivation as [Phase 2's `MeshTransport`](Configuration.md#meshtransport-centralnode-command-transport)
+and [Phase 3's `ConfigSource`/`ConfigServe`](Configuration.md#configsource--configserve-config-fetch-and-cache):
+DPS only works when central and the node share a gossip ring, and [Phase 6](plans/2026-07-21-per-cluster-mesh-design.md)
+splits the fleet into one Akka mesh per application `Cluster`. Once that split lands, a driver node
+in a site's mesh is no longer a cluster member of central's mesh, so DPS can no longer reach it —
+telemetry needs its own out-of-band transport, same as commands (Phase 2) and config bytes (Phase 3).
+
+## Direction: the node hosts, central dials
+
+**Load-bearing and easy to get backwards:** each **driver** node hosts the telemetry gRPC server
+(a dedicated Kestrel h2c listener); **central (admin role) is the client** and dials in. This
+mirrors the inversion ScadaBridge already uses for the same problem (its `SiteStreamService`).
+
+```
+ driver node central (admin)
+┌─────────────────────┐ ┌──────────────────────────┐
+│ 4 publish seams │ │ TelemetryDialSupervisor │
+│ -> node-local hub │ Subscribe │ - discovers ClusterNode │
+│ -> DPS (unchanged) │◄─────────────│ rows (Host+GrpcPort) │
+│ │ (gRPC │ - one reconnecting │
+│ TelemetryStreamGrpc- │ stream) │ dialer per node │
+│ Service (h2c server) │─────────────►│ - feeds the SAME sinks │
+│ Telemetry:GrpcListen- │ TelemetryEvent │ the DPS bridges feed │
+│ Port │ (oneof, 4 kinds)│ today │
+└─────────────────────┘ └──────────────────────────┘
+```
+
+A fused `admin,driver` node both hosts (as driver) and dials (as admin) — it dials itself plus its
+redundant pair peer, the same way central dials site nodes.
+
+Central discovers driver-node telemetry endpoints from the `ClusterNode` table (`Host` +
+`GrpcPort`, the latter added in Phase 1 for exactly this purpose) — the same DB-sourced discovery
+Phase 1's ack set and Phase 2's `ClusterClient` contact set already use. No shared gossip
+membership is required to find or dial a node.
+
+## The dark switch
+
+`Telemetry:Mode` (node/serve side) and `TelemetryDial:Mode` (central/dial side) each default to
+`Dps` and can be independently set to `Grpc`. Both code paths are compiled into every binary —
+flipping either flag is an appsettings/env change plus a restart, **not a redeploy or rebuild** —
+the same discipline as `MeshTransport:Mode` and `ConfigSource:Mode`.
+
+What actually changes per mode, precisely:
+
+- **The node ALWAYS does both things, regardless of `Telemetry:Mode`.** Every one of the four
+ publish seams emits into the node-local `ITelemetryLocalHub` **and** publishes to DPS,
+ unconditionally. The node additionally **always** hosts the gRPC server whenever
+ `Telemetry:GrpcListenPort > 0` — hosting is not mode-gated at all. `Telemetry:Mode` only affects
+ central's own bookkeeping about which side it expects to be consulted (it is otherwise inert on
+ the node).
+- **Only central's ingest source switches**, driven by `TelemetryDial:Mode`:
+ - `Dps` (default) — today's four DPS SignalR bridges (alert, script-log, driver-status,
+ resilience) subscribe and feed the AdminUI sinks. Unchanged behavior.
+ - `Grpc` — those four DPS bridges are **not spawned**. Instead the `TelemetryDialSupervisor`
+ actor dials every driver node's gRPC stream and feeds the **identical** in-process sinks
+ (`IInProcessBroadcaster`, `IInProcessBroadcaster`,
+ `IDriverStatusSnapshotStore`, `IDriverResilienceStatusStore`).
+- **The AdminUI panels themselves are untouched.** `/alerts`, `/script-log`, and the `/hosts`
+ driver table all read from the same sinks in both modes — only the sinks' upstream feed swaps.
+ The `fleet-status` bridge (out of Phase 5 scope — see below) stays on DPS in both modes.
+
+Because hosting is unconditional and both ingest paths are always compiled in, a fleet can be
+flipped node-by-node and central-by-central with no coordination window where telemetry is lost —
+the node is always serving, so central can switch to `Grpc` whenever it likes.
+
+## The four migrated channels
+
+| DPS topic (today) | Domain record | Central sink fed |
+|---|---|---|
+| `alerts` | `AlarmTransitionEvent` | `IInProcessBroadcaster` (+ `AlertHub`); `/alerts` page; live/disconnected pill |
+| `script-logs` | `ScriptLogEntry` | `IInProcessBroadcaster` (+ `ScriptLogHub`); `/script-log` page; live pill |
+| `driver-health` | `DriverHealthChanged` | `IDriverStatusSnapshotStore`; `/hosts` driver table |
+| `driver-resilience-status` | `DriverResilienceStatusChanged` | `IDriverResilienceStatusStore` |
+
+These four are carried as `oneof` event kinds on one `TelemetryStreamService.Subscribe` RPC — a
+single stream per (central, driver-node) pair carries all four, rather than one stream per topic.
+Proto field evolution is additive-only (never renumber/reuse a tag), locked by a contract test that
+reflects over the `oneof` cases.
+
+## The three deferred channels (NOT migrated in Phase 5 — do not read this as "seven done")
+
+The program sketch originally named seven observability topics for Phase 5. Three were scoped out,
+each for a distinct, settled reason:
+
+- **`redundancy-state`** — bidirectional, built directly from `Cluster.State`, and **pair-local**:
+ it drives ServiceLevel and the Primary data-plane gate, consumed in-process by
+ `OpcUaPublishActor`, `ScriptedAlarmHostActor`, `DriverHostActor`, and `HistorianAdapterActor`.
+ It stays on DPS in **both** `MeshTransport` modes today, and it is genuinely mesh-bound by
+ design — under Phase 6 each pair keeps sharing its own small mesh, so DPS keeps working for it
+ in-mesh. Central's *display* of each pair's redundancy state is a Phase 6 cross-mesh concern
+ (a future observability channel, once central no longer shares gossip with any site pair), not a
+ Phase 5 telemetry-panel migration.
+- **`fleet-status`** — **central-internal**, not a node→central stream at all. The admin singleton
+ `FleetStatusBroadcaster` builds it from the admin node's own cluster membership/reachability/
+ leader events, and `Fleet.razor` **polls the Config DB** and ignores the feed entirely. There is
+ nothing here for a per-node gRPC stream to carry. Revisit in Phase 6, once central genuinely
+ loses gossip visibility of site nodes and needs another way to know a pair's membership.
+- **`deployment-acks`** — already migrated, but onto a different transport: it rides the Phase 2
+ `ClusterClient` transport (`MeshTransport:Mode=ClusterClient`) as a command-plane reply, not an
+ observability broadcast. It was never a Phase 5 candidate.
+
+See [`docs/Redundancy.md`](Redundancy.md#command-transport-centralnode) for how `redundancy-state`
+and the command transports fit together.
+
+## Authentication — fail-closed from day one
+
+A shared node bearer key gates the stream: `Telemetry:ApiKey` (node/serve side) must equal
+`TelemetryDial:ApiKey` (central/dial side). `TelemetryStreamAuthInterceptor` enforces it — path-scoped
+to `/telemetry.v1.TelemetryStreamService/`, comparing the `Authorization: Bearer` token with
+`CryptographicOperations.FixedTimeEquals`, and rejecting with `PermissionDenied`. An **unset** key
+rejects every call rather than allowing an open stream — the same fail-closed posture as
+`ConfigServeAuthInterceptor` and `LocalDbSyncAuthInterceptor`.
+
+This **supersedes** [the design doc's §6.3](plans/2026-07-21-per-cluster-mesh-design.md), which had
+provisionally decided to "match ScadaBridge's unauthenticated posture for now" for inter-cluster
+transports. ScadaBridge itself has since closed that gap with this identical
+interceptor pattern, so Phase 5 ships authenticated from the start rather than deferring auth to a
+later hardening pass. See the design doc's superseded note for the full history.
+
+## Reconnect and reliability
+
+Central's `TelemetryDialSupervisor` (an admin-role actor) runs **one reconnecting dialer per
+driver node**:
+
+- **Discovery** — enabled, non-maintenance `ClusterNode` rows are read on start and refreshed every
+ `TelemetryDial:ContactRefreshSeconds` (default 60), plus on an admin-side topology change. A row
+ with a null `GrpcPort` is skipped (that node exposes no telemetry endpoint — logged, not an
+ error). Dial targets are added/removed as the row set changes.
+- **Reconnect backoff** — immediate first retry, then a fixed ~5 second backoff, indefinitely. The
+ dialer never gives up permanently: this is an observability channel, not the data plane, so a
+ persistently-unreachable node just means a persistently-stale panel, not a fleet fault.
+- **Generation stamping** — each dialer carries a monotonically-increasing generation counter, so
+ a late error or late event from a superseded stream (e.g. one that raced a reconnect) is
+ recognized and ignored rather than corrupting the current stream's state.
+- **Snapshot replay on reconnect** — the node-local hub keeps a last-value cache for
+ `driver-health` and `driver-resilience-status` (there is nothing meaningful to replay for
+ `alerts`/`script-logs`, which are append logs). On every new `Subscribe` — including a
+ reconnect — the hub drains the cached snapshots to the new stream before live deltas, so
+ `IDriverStatusSnapshotStore` / `IDriverResilienceStatusStore` re-prime immediately rather than
+ sitting stale until the next natural event. `alerts`/`script-logs` simply tolerate the gap — a
+ reconnect loses whatever transitions happened while disconnected, same as a DPS resubscribe would.
+- **Connection indicator** — the `/alerts` and `/script-log` live/disconnected pill is driven by
+ **aggregate** stream health across all dialers: connected when **at least one** node stream is
+ up, disconnected when all are down. This matches today's DPS `SubscribeAck`/`PostStop` pill
+ semantics — the pill has never meant "every node is live," only "the panel has a source." The
+ two store-backed panels (`driver-health`, `driver-resilience-status`) carry no fleet-wide
+ connection flag today, on gRPC or on DPS; per-row staleness is unchanged.
+
+## Configuration reference
+
+See [`docs/Configuration.md` § `Telemetry` / `TelemetryDial`](Configuration.md#telemetry--telemetrydial-live-telemetry-transport)
+for the full keys table. In brief:
+
+- **`Telemetry`** (node/serve side): `Mode` (`Dps`|`Grpc`, default `Dps`), `GrpcListenPort` (`0` =
+ disabled — the driver node's dedicated telemetry h2c port), `ApiKey` (the shared node key; supply
+ via `${secret:}`/env, never commit).
+- **`TelemetryDial`** (central side): `Mode` (`Dps`|`Grpc`), `ApiKey` (must equal the nodes'
+ `Telemetry:ApiKey`), `ContactRefreshSeconds` (default `60`), `CallTimeoutSeconds` (default `30`).
+
+A driver-role node with `Telemetry:Mode=Grpc` must set `GrpcListenPort > 0` (nothing to serve
+otherwise), and `Grpc` mode on either side requires a non-empty `ApiKey` — both are enforced by
+`ValidateOnStart` validators, fail-fast at boot.
+
+## Relationship to the rest of the mesh program
+
+- **Same dark-switch discipline as Phases 2 and 3.** `MeshTransport:Mode`, `ConfigSource:Mode`, and
+ `Telemetry:Mode`/`TelemetryDial:Mode` are all independently-flippable, restart-only config
+ changes, and in every case the "new" side is always wired so the flip needs no coordinated
+ redeploy.
+- **Why the inversion matters for Phase 6.** Every other cross-boundary transport in this program
+ (`MeshTransport`, `ConfigSource`/`ConfigServe`) already has central as the addressable, discoverable
+ side and the node as the caller or the callee-by-address. Telemetry is the one channel that is
+ naturally node-sourced and fleet-wide-fanned, so putting central in the client role — dialing out
+ to each node by its `ClusterNode`-recorded address — is what lets telemetry survive the mesh split
+ cleanly: central never needs cluster membership with a site's mesh to keep watching it.
+- **Status:** code-complete on `feat/mesh-phase5`; the live gate (flip the docker-dev rig to `Grpc`,
+ confirm all panels stay green with the DPS telemetry bridges unspawned, kill-and-reconnect a site
+ node) has not yet run. See `docs/plans/2026-07-22-per-cluster-mesh-program.md` § Phase 5 and
+ `docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md` for the implementation plan.
diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md
index 8ccd3ed6..8a39d63b 100644
--- a/docs/plans/2026-07-21-per-cluster-mesh-design.md
+++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md
@@ -285,6 +285,20 @@ trusted network. Recorded as an accepted risk rather than an oversight, with two
it is revisited: `LocalDbSyncAuthInterceptor` is already in our tree as a fail-closed template, and
the surface most worth gating first is any Pull-style RPC that returns historical rows.
+**SUPERSEDED 2026-07-23.** The Phase 5 telemetry stream (`TelemetryStreamService` — the four
+node→central observability channels) shipped **authenticated from day one**: a shared node bearer
+key (`Telemetry:ApiKey` == `TelemetryDial:ApiKey`), enforced by a fail-closed
+`TelemetryStreamAuthInterceptor` (`CryptographicOperations.FixedTimeEquals`, `PermissionDenied` on
+mismatch, and — load-bearing — an **unset** key rejects every call rather than allowing an open
+stream). This is exactly the `LocalDbSyncAuthInterceptor`/`ConfigServeAuthInterceptor` pattern this
+section flagged as the fail-closed template to reach for, applied immediately rather than deferred.
+ScadaBridge itself has since closed the same "unauthenticated inter-cluster transport" gap with this
+identical interceptor pattern, so this decision no longer matches ScadaBridge's current posture
+either — it matches where ScadaBridge moved to. See [`docs/Telemetry.md`](../Telemetry.md) §
+Authentication for the full detail. The "return historical rows" surface this section called out as
+worth gating first (Pull-style RPCs) remains open for `ConfigServe`/`ConfigSource` and any future
+such surface — only the telemetry stream is closed by this note.
+
## 7. Sequencing sketch
Deliberately not a task plan — per-phase plans follow, one at a time.
diff --git a/docs/plans/2026-07-22-per-cluster-mesh-program.md b/docs/plans/2026-07-22-per-cluster-mesh-program.md
index 301c5f40..15d5e7f6 100644
--- a/docs/plans/2026-07-22-per-cluster-mesh-program.md
+++ b/docs/plans/2026-07-22-per-cluster-mesh-program.md
@@ -202,16 +202,66 @@ Phase 4 as code-complete, not verified, until this lands.**
**no ConfigDb connection string configured at all**; grep-level proof no driver-branch service can
resolve the ConfigDb context.
-### Phase 5 — gRPC stream contract for live telemetry
-**Scope:** one server-streaming contract carrying a `oneof` event, **cluster nodes host the gRPC
-server, central dials in** (the inverted direction is the load-bearing ScadaBridge finding —
-design §2); migrate the seven observability topics (`alerts`, `driver-health`,
-`driver-resilience-status`, `fleet-status`, `script-logs`, plus redundancy-state distribution and
-deployment-acks if Phase 2 left them on DPS); additive-only field evolution, contract locked by
-test; per-panel reconnect story for the AdminUI (design §8 — losing gossip loses free fleet
-observability).
-**Exit gate:** all AdminUI live panels green against a pair with DPS telemetry topics deleted;
-kill-and-reconnect of the central dialer recovers every stream.
+### Phase 5 — gRPC stream contract for live telemetry — **DONE 2026-07-23 (live gate PASSED)**
+**Scope as shipped (narrowed from the program sketch's "seven observability topics," settled with the
+user 2026-07-23 — mirrors how Phases 1 and 3 also honestly scoped down):** one server-streaming gRPC
+contract carrying a `oneof` event with **four** kinds, **driver nodes host the gRPC server (dedicated
+Kestrel h2c port), central dials in** — the inverted direction is the load-bearing ScadaBridge finding
+(design §2), required so telemetry survives the Phase 6 mesh split without shared gossip membership.
+
+**The four migrated channels:**
+
+| DPS topic | Domain record | Central sink |
+|---|---|---|
+| `alerts` | `AlarmTransitionEvent` | `IInProcessBroadcaster` (+ `AlertHub`) |
+| `script-logs` | `ScriptLogEntry` | `IInProcessBroadcaster` (+ `ScriptLogHub`) |
+| `driver-health` | `DriverHealthChanged` | `IDriverStatusSnapshotStore` |
+| `driver-resilience-status` | `DriverResilienceStatusChanged` | `IDriverResilienceStatusStore` |
+
+**Three deferred, each with a settled reason (NOT migrated — do not read the program sketch's original
+"seven" as the delivered scope):**
+- `redundancy-state` — bidirectional, built from `Cluster.State`, pair-local control plane driving
+ ServiceLevel + the Primary gate. Stays on DPS in every mode; under Phase 6 it works in-mesh per pair.
+ Central's cross-pair *display* of redundancy is a Phase 6 concern, not a Phase 5 panel.
+- `fleet-status` — central-internal; `FleetStatusBroadcaster` builds it from the admin node's own
+ cluster membership events and `Fleet.razor` polls the Config DB, ignoring the feed. Not a
+ node→central stream at all. Revisit in Phase 6 once central loses site-node gossip visibility.
+- `deployment-acks` — already migrated, but onto the Phase 2 `ClusterClient` transport, as a
+ command-plane reply, not an observability broadcast.
+
+**Dark switch:** `Telemetry:Mode` (node/serve, default `Dps`) and `TelemetryDial:Mode` (central/dial,
+default `Dps`) flip independently to `Grpc`; both code paths are compiled into every binary, so
+flipping is a config + restart, not a redeploy. The node **always** hosts the gRPC server whenever
+`Telemetry:GrpcListenPort > 0` and **always** emits into its node-local hub + publishes DPS, in both
+modes — only central's ingest source (`TelemetryDial:Mode`) actually switches which of the two
+always-available sources feeds the AdminUI sinks. The AdminUI panels/components are unchanged; only
+the sinks' upstream feed swaps.
+
+**Auth from day one, fail-closed:** a shared node bearer key (`Telemetry:ApiKey` ==
+`TelemetryDial:ApiKey`), gated by `TelemetryStreamAuthInterceptor`
+(`CryptographicOperations.FixedTimeEquals`, `PermissionDenied` on mismatch, unset key rejects every
+call). This supersedes design §6.3's "match ScadaBridge's unauthenticated posture for now" — ScadaBridge
+itself has since closed that gap with this identical interceptor pattern (see the supersede note added
+to design §6.3, and `docs/Telemetry.md` § Authentication).
+
+**Reconnect story:** central's `TelemetryDialSupervisor` runs one reconnecting dialer per driver node,
+discovered from `ClusterNode` rows (`Host`+`GrpcPort`, refreshed every
+`TelemetryDial:ContactRefreshSeconds`); immediate first retry then ~5s fixed backoff, generation-stamped
+so a superseded stream's late error/event is ignored; the dialer never permanently gives up
+(observability, not data plane). The node-local hub replays last-value snapshots for
+`driver-health`/`driver-resilience-status` on every (re)subscribe so those stores re-prime immediately;
+`alerts`/`script-logs` are append logs and tolerate the gap. The `/alerts`/`/script-log` pill reflects
+aggregate stream health (live when ≥1 node stream is up), matching today's DPS pill semantics.
+
+**Docs shipped:** `docs/Telemetry.md` (new, canonical guide), `docs/Configuration.md` §
+`Telemetry`/`TelemetryDial`, `docs/Redundancy.md` cross-reference, this section, and the design §6.3
+supersede note.
+
+**Exit gate (not yet run):** all AdminUI live panels green against the docker-dev rig with
+`Telemetry:Mode=Grpc`/`TelemetryDial:Mode=Grpc` and the four DPS telemetry bridges NOT spawned;
+kill-and-reconnect of a site node's dialer recovers every stream (driver-health/resilience re-prime
+from the node hub's snapshot replay). Treat Phase 5 as **code-complete, not verified**, until this
+lands — see `docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md` Task 12.
### Phase 6 — Mesh partition + co-location topology
**Scope:** per-cluster seed nodes — ~~adopt ScadaBridge's self-first ordering and RETIRE the
@@ -267,6 +317,6 @@ resource sizing on the site VMs should be checked once both products run the ful
| 2 ClusterClient transport | **DONE 2026-07-22** (live gate PASSED, shipped dark) — shipped dark (`MeshTransport:Mode=Dps` default), both comm actors registered in both modes. Corrections: `SendToAll` not `Send`, one fleet-wide client not one per cluster, `/user/node-communication` not `/user/cluster-communication`. The gate's "Ask timing out" item does not exist to test — no cross-boundary Ask in this phase; see the phase section. |
| 3 fetch-and-cache | **DONE 2026-07-23**, merged `d01b0695`, live gate PASSED, pushed to origin (scadaproj umbrella index updated + pushed `b5e7bc8`). gRPC fetch RPC + shared node key; dark switch `ConfigSource:Mode` (Direct default). Rig flip: `OTOPCUA_CONFIG_MODE=FetchAndCache` on the site nodes. See `2026-07-22-mesh-phase3-config-fetch-and-cache.md`. |
| 4 cut driver ConfigDb | **DONE 2026-07-23 — live gate PASSED** on `feat/mesh-phase4` (Tasks 0–8 + 1b + 10–11; Task 9 table-drop deferred). ConfigDb admin-only; driver-only ⇒ FetchAndCache (validator); `DbHealthProbeActor` not spawned driver-only (client-visible ServiceLevel 240/250 with central SQL down — proven live); alarm condition state in replicated LocalDb `alarm_condition_state`; central persists acks; `OpcUaPublish` guard split fixes the driver-only address-space wipe; driver-only LDAP maps from appsettings only (6th consumer found mid-phase). Gate: deploy sealed green w/ 4 DB-less site nodes acking, ServiceLevel held 240 w/ SQL down, restart booted last-known-good from the LocalDb pointer. See `2026-07-23-mesh-phase4-cut-driver-configdb.md` + `2026-07-23-mesh-phase4-live-gate.md`. |
-| 5 gRPC telemetry | not started |
+| 5 gRPC telemetry | **DONE 2026-07-23 — live gate PASSED** on `feat/mesh-phase5`. 4-channel scope (`alerts`/`script-logs`/`driver-health`/`driver-resilience-status`), 3 deferred with rationale (`redundancy-state`, `fleet-status`, `deployment-acks`); dark switch `Telemetry:Mode`/`TelemetryDial:Mode` (Dps default); node hosts server / central dials; auth-from-day-one fail-closed bearer key, superseding design §6.3. Gate: full 12-stream mesh formed in Grpc mode (2 inbound per driver node, 6 outbound per central), AdminUI pill live (data path proven), kill-and-reconnect of a node recovered its stream in ~5s; Dps baseline dials nothing. Surfaced an upgrade gotcha — `ClusterNode.GrpcPort` must be populated on existing deployments (fresh installs seed it; nullable column doesn't backfill) — which live-validated the graceful null-`GrpcPort` skip. See `docs/Telemetry.md`, `2026-07-23-mesh-phase5-grpc-telemetry-stream.md` + `2026-07-23-mesh-phase5-live-gate.md`. |
| 6 mesh partition + co-location | not started |
| 7 drill + live gates | not started |
diff --git a/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md b/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md
new file mode 100644
index 00000000..0390a1f7
--- /dev/null
+++ b/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md
@@ -0,0 +1,650 @@
+# Per-Cluster Mesh Phase 5 — gRPC live-telemetry stream Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: execute this with superpowers-extended-cc:subagent-driven-development
+> (chosen for the program's prior phases). Fresh subagent per task; classification-driven review chain.
+
+**Goal:** Replace the DPS fan-out of the live-telemetry observability channels with one gRPC
+server-streaming contract — **each driver node hosts the telemetry gRPC server (Kestrel h2c),
+central dials in** — so AdminUI live panels keep working once the meshes split (Phase 6) and no
+longer depend on a shared Akka gossip ring for fleet observability. Ships as a per-node **dark
+switch** (`Telemetry:Mode` = `Dps` default | `Grpc`), authenticated fail-closed from day one.
+
+**Architecture:** Mirror ScadaBridge's `SiteStreamService` shape (recon'd 2026-07-23), adapted to
+OtOpcUa's substrate. Node side: the four telemetry publish seams also emit into a **node-local
+in-process hub** (`ITelemetryLocalHub`) — DPS publishing is left untouched, so the switch is pure —
+and a streaming gRPC service fans the hub to connected clients through per-subscriber bounded
+`DropOldest` channels. Central side: a supervisor actor discovers driver-node gRPC endpoints from
+`ClusterNode` rows (`Host` + `GrpcPort`, added in Phase 1), keeps one reconnecting dialer per node,
+converts streamed events back to the domain records, and feeds **the exact same in-process sinks the
+DPS SignalR bridges feed today** (`IInProcessBroadcaster`,
+`IInProcessBroadcaster`, `IDriverStatusSnapshotStore`,
+`IDriverResilienceStatusStore`). The AdminUI components are untouched: only the bridge's *upstream*
+swaps.
+
+**Tech Stack:** .NET 10, `Grpc.AspNetCore` / `Grpc.Net.Client` (already in-repo via Phase 3),
+`Grpc.Tools` codegen, Akka.NET, `System.Threading.Channels`, xUnit + Shouldly.
+
+---
+
+## Scope (settled with the user 2026-07-23)
+
+**In scope — migrate these four node→central observability channels** to the stream as four `oneof`
+event kinds:
+
+| DPS topic today | Message record | Central sink fed today |
+|---|---|---|
+| `alerts` | `AlarmTransitionEvent` | `IInProcessBroadcaster` (+ `AlertHub`) |
+| `script-logs` | `ScriptLogEntry` | `IInProcessBroadcaster` (+ `ScriptLogHub`) |
+| `driver-health` | `DriverHealthChanged` | `IDriverStatusSnapshotStore` (+ `DriverStatusHub`) |
+| `driver-resilience-status` | `DriverResilienceStatusChanged` | `IDriverResilienceStatusStore` (no hub) |
+
+**Explicitly deferred, with rationale (do NOT migrate in Phase 5):**
+
+- **`redundancy-state`** — bidirectional, built from `Cluster.State`, **pair-local control-plane**
+ that drives ServiceLevel + the Primary gate (consumed by `OpcUaPublishActor`,
+ `ScriptedAlarmHostActor`, `DriverHostActor`, `HistorianAdapterActor`). It stays on DPS in both
+ MeshTransport modes today, and under Phase 6 it is pair-local and works in-mesh. Central's *display*
+ of each pair's redundancy is a Phase 6 cross-mesh concern (possibly a later added event kind), not a
+ Phase 5 observability panel.
+- **`fleet-status`** — **central-internal**: `FleetStatusBroadcaster` (admin singleton) builds it
+ from the admin node's own cluster membership/reachability/leader events; `Fleet.razor` **polls the
+ ConfigDB** and ignores the feed entirely. It is not a node→central stream, and its live UI path is
+ already DB-polled. Revisit in Phase 6 when central loses gossip visibility of site nodes.
+- **`deployment-acks`** — already rides the Phase 2 ClusterClient transport when
+ `MeshTransport:Mode=ClusterClient`; it is a command-plane reply, not telemetry.
+
+This narrowing mirrors how Phases 1 and 3 honestly scoped down from the program sketch. The program
+doc's Phase 5 line lists all seven; this plan records the four that are genuinely live node→central
+observability and defers the rest with reasons above. **Update the program doc + design §6.3 in
+Task 10.**
+
+## Direction & the dark switch (read before any task)
+
+- **Node = server, central = client** (the load-bearing ScadaBridge inversion). Telemetry
+ originates on driver nodes; central/admin consumes it. A fused `admin,driver` node both hosts (as
+ driver) and dials (as admin) — it dials itself plus its pair peer, same as central dials site nodes.
+- **`Telemetry:Mode` is read at startup; both code paths are compiled into every binary.** Flipping
+ the flag is an appsettings/env change + restart — NOT a rebuild — exactly the Phase 2/3 dark-switch
+ discipline (`OTOPCUA_CONFIG_MODE` on the rig). The node **always** hosts the gRPC server when
+ `Telemetry:GrpcListenPort > 0` and **always** emits into the local hub AND publishes DPS, in both
+ modes — so central can ingest from either side without a node redeploy. Only **central's ingest
+ source** switches: `Dps` → today's four DPS SignalR bridges subscribe and feed the sinks; `Grpc` →
+ those four bridges are NOT spawned and the dial supervisor feeds the identical sinks instead.
+- **Auth from day one.** Reuse the fail-closed `FixedTimeEquals` bearer interceptor pattern
+ (`ConfigServeAuthInterceptor` / `LocalDbSyncAuthInterceptor`). Shared node key
+ `Telemetry:ApiKey` (serve side) == `TelemetryDial:ApiKey` (central side). This supersedes design
+ §6.3's "unauthenticated for now" — ScadaBridge itself closed that gap with this same pattern.
+
+## Reuse map (from recon 2026-07-23 — exact sites)
+
+- **Proto + codegen:** add `telemetry.proto` beside `src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/deployment_artifact.proto`; register a `` in `ZB.MOM.WW.OtOpcUa.Commons.csproj:29` (same block as the existing item). Generated types are shared by node (Runtime/Host) and central (AdminUI) from the one Commons reference.
+- **Server hosting + Kestrel h2c:** the dedicated-listener block `Program.cs:434-533` and the `MapGrpcService` gate `Program.cs:577-580`. Add `telemetryListenPort` alongside `syncListenPort`/`configServeGrpcPort`.
+- **Auth interceptor:** copy `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ConfigServeAuthInterceptor.cs`; add to the shared `AddGrpc` pipeline at `Program.cs:425-431`.
+- **Client dialing:** `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Deployment/GrpcDeploymentArtifactFetcher.cs` (channel-cache, Bearer metadata, h2c `GrpcChannel.ForAddress`, linked-CTS deadline).
+- **Options + validator:** `ConfigSourceOptions`/`ConfigSourceOptionsValidator` in `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/`; registered via `AddValidatedOptions` in `ServiceCollectionExtensions.AddOtOpcUaCluster` (`:32`).
+- **Node discovery:** `CentralCommunicationActor.cs:198-219` already reads `ClusterNodes` (enabled, non-maintenance) selecting `NodeId, Host, AkkaPort` — extend the same query shape to `GrpcPort` for telemetry dial targets. `ClusterNode.GrpcPort` (nullable) already exists (`Entities/ClusterNode.cs:46`) explicitly for "the Phase 5 telemetry stream."
+- **Central sinks (the untouched seam):** registered in `HubServiceCollectionExtensions.AddOtOpcUaDriverStatusServices` (`:29-35`); DPS bridges spawned in `WithOtOpcUaSignalRBridges` (`:53-83`).
+- **ScadaBridge reference to mirror:** `SiteStreamGrpcServer.cs` (relay-actor + bounded `DropOldest` channel + lifecycle/cleanup + concurrency cap + max-stream-lifetime), `SiteStreamGrpcClient.cs` / `SiteStreamGrpcClientFactory.cs` (channel cache, keepalive), `SiteAlarmAggregatorActor.cs` (generation-stamped, budget-limited, self-healing reconnect), `ProtoContractTests.cs` (reflection-over-oneof contract lock).
+
+---
+
+## Tasks
+
+### Task 0: Telemetry proto contract + codegen + contract-lock test
+
+**Classification:** standard
+**Estimated implement time:** ~5 min
+**Parallelizable with:** Task 1, Task 2
+
+**Files:**
+- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/telemetry.proto`
+- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj` (add the `` item next to line 29)
+- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests/Protos/TelemetryProtoContractTests.cs` (create; match the existing Commons.Tests project layout — if there is no Commons.Tests project, add the test to the nearest existing Core test project that already references generated Commons types and note it)
+
+**Step 1: Write `telemetry.proto`.** Package `telemetry.v1`, `csharp_namespace = ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1`. One server-streaming RPC + a `oneof` envelope with the four event kinds. Every enum carries a `*_UNSPECIFIED = 0` zero value; every field comment marks additive intent.
+
+```proto
+syntax = "proto3";
+package telemetry.v1;
+option csharp_namespace = "ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1";
+
+import "google/protobuf/timestamp.proto";
+
+// Central dials each driver node and opens Subscribe; the node streams its own live telemetry.
+// Additive-only field evolution: never renumber/reuse a tag; a pre-field peer must read a new
+// field's proto3 default correctly. Locked by TelemetryProtoContractTests.
+service TelemetryStreamService {
+ rpc Subscribe(TelemetryStreamRequest) returns (stream TelemetryEvent);
+}
+
+message TelemetryStreamRequest {
+ string correlation_id = 1; // safe Akka path element; becomes the relay actor name on the node
+}
+
+message TelemetryEvent {
+ string correlation_id = 1;
+ oneof event {
+ AlarmTransition alarm_transition = 2; // <- DPS topic "alerts"
+ ScriptLog script_log = 3; // <- DPS topic "script-logs"
+ DriverHealth driver_health = 4; // <- DPS topic "driver-health"
+ DriverResilienceStatus driver_resilience = 5; // <- DPS topic "driver-resilience-status"
+ }
+}
+
+// The four event bodies below mirror the C# records field-for-field. Encode the domain records'
+// enums as proto enums (UNSPECIFIED=0), and DateTimes as google.protobuf.Timestamp.
+message AlarmTransition { /* fields mirroring Commons/Messages/Alerts/AlarmTransitionEvent.cs */ }
+message ScriptLog { /* fields mirroring Commons/Messages/Logging/ScriptLogEntry.cs */ }
+message DriverHealth { /* fields mirroring Commons/Messages/Drivers/DriverHealthChanged.cs */ }
+message DriverResilienceStatus{ /* fields mirroring Commons/Messages/Drivers/DriverResilienceStatusChanged.cs */ }
+```
+
+The implementer MUST open each of the four domain records (paths in the scope table above) and
+transcribe every field into the corresponding message with a stable tag number, choosing
+`Timestamp`/`Int32`/`string`/`bool`/enum as the type dictates. Where a record enum exists, define a
+matching proto enum with `_UNSPECIFIED = 0`. This is the contract — get it complete, because the
+mapping tasks (7) and the lock test below depend on it.
+
+**Step 2: Register codegen.** In the `.csproj`, next to the existing `deployment_artifact.proto` item:
+```xml
+
+```
+
+**Step 3: Write the contract-lock test (fails first).** Mirror ScadaBridge `ProtoContractTests.AllOneofVariants_HaveConversionHandlers`: reflect over `Enum.GetValues()` minus `None`, assert count- and membership-equality against a hand-maintained `HandledCases` array that Task 7's converter will also key off. This fails until the proto compiles and the array is filled.
+
+```csharp
+[Fact]
+public void EveryOneofVariant_IsAccountedFor()
+{
+ var variants = Enum.GetValues()
+ .Where(c => c != TelemetryEvent.EventOneofCase.None).ToArray();
+ variants.ShouldBe(TelemetryProtoContract.HandledCases, ignoreOrder: true);
+}
+```
+Add a tiny `TelemetryProtoContract.HandledCases` constant array in Commons (the single source both
+this test and the Task-7 converter reference) listing the four cases.
+
+**Step 4: `dotnet build ZB.MOM.WW.OtOpcUa.slnx`** — expect the generated types to appear; test goes green.
+
+**Step 5: Commit.** `feat(mesh-phase5): telemetry.proto contract + oneof + contract-lock test`
+
+---
+
+### Task 1: Telemetry options (serve + dial) + validator + registration
+
+**Classification:** small
+**Estimated implement time:** ~4 min
+**Parallelizable with:** Task 0, Task 2
+
+**Files:**
+- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/TelemetryOptions.cs`
+- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs` (register in `AddOtOpcUaCluster`, ~lines 40-49 block)
+- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/TelemetryOptionsValidatorTests.cs`
+
+**Step 1:** Define two options classes and one validator, mirroring `ConfigSourceOptions`/`ConfigServeOptions` + `ConfigSourceOptionsValidator` (which reads roles from `IConfiguration`, NOT `IClusterRoleInfo`).
+
+```csharp
+public sealed class TelemetryOptions // section "Telemetry" (serve side, node)
+{
+ public const string SectionName = "Telemetry";
+ public string Mode { get; set; } = "Dps"; // Dps | Grpc (central-ingest selector; harmless on node)
+ public int GrpcListenPort { get; set; } // 0 = disabled; driver node's telemetry h2c port
+ public string ApiKey { get; set; } = ""; // shared node key; supply via ${secret:}/env
+}
+
+public sealed class TelemetryDialOptions // section "TelemetryDial" (central)
+{
+ public const string SectionName = "TelemetryDial";
+ public string Mode { get; set; } = "Dps"; // Dps | Grpc
+ public string ApiKey { get; set; } = ""; // must equal the nodes' Telemetry:ApiKey
+ public int ContactRefreshSeconds { get; set; } = 60;
+ public int CallTimeoutSeconds { get; set; } = 30;
+}
+```
+
+**Step 2: Validator rules** (`TelemetryOptionsValidator : IValidateOptions`, ctor-inject `IConfiguration`, read `Cluster:Roles` like `ConfigSourceOptionsValidator:98`):
+- `Mode` must be `Dps` or `Grpc` (case-insensitive) — else Fail.
+- A **driver-role** node with `Mode=Grpc` must have `GrpcListenPort > 0` — else Fail ("nothing to serve").
+- `Mode=Grpc` with a non-empty role set must have a non-empty `ApiKey` (fail-closed: refuse to host an un-keyed telemetry surface) — else Fail.
+- Add a sibling `TelemetryDialOptionsValidator`: `Mode` in {Dps,Grpc}; `Mode=Grpc` ⇒ `ApiKey` non-empty.
+
+**Step 3: Register** via `AddValidatedOptions(configuration, TelemetryOptions.SectionName)` and the dial equivalent, in `AddOtOpcUaCluster`.
+
+**Step 4:** Tests — Grpc+driver+port0 fails; Grpc+empty-key fails; Dps passes with defaults; unknown Mode fails. Run `dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests`.
+
+**Step 5: Commit.** `feat(mesh-phase5): Telemetry/TelemetryDial options + fail-closed validators`
+
+---
+
+### Task 2: Node-local in-process telemetry hub
+
+**Classification:** standard
+**Estimated implement time:** ~5 min
+**Parallelizable with:** Task 0, Task 1
+
+**Files:**
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/ITelemetryLocalHub.cs`
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs`
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Telemetry/TelemetryLocalHubTests.cs`
+
+**Design:** a process-wide singleton carrying **this node's own** telemetry (never cluster-wide — that
+is what keeps it correct on both the current single mesh and the Phase-6 pair mesh; DPS would leak
+peers' events). It holds a **domain-typed** union (not proto) so the publish seams stay decoupled from
+generated types. Two responsibilities:
+
+1. `Emit(TelemetryItem)` — fan the item to every currently-subscribed writer through **bounded
+ per-subscriber channels with `FullMode = DropOldest`** (live view is lossy under backpressure by
+ design; a slow central never blocks the node).
+2. **Snapshot replay for last-value channels.** Keep a last-value cache for `DriverHealth` (keyed by
+ driver instance) and `DriverResilienceStatus` (keyed by `(instance, host)`); `alerts`/`script-logs`
+ are append logs and are NOT cached. On `Subscribe`, first drain the cached snapshots to the new
+ writer, then attach it for live deltas — the simplified equivalent of ScadaBridge's seed-then-stream,
+ so a central reconnect immediately re-primes the driver-health/resilience stores.
+
+```csharp
+public abstract record TelemetryItem
+{
+ public sealed record Alarm(AlarmTransitionEvent E) : TelemetryItem;
+ public sealed record Script(ScriptLogEntry E) : TelemetryItem;
+ public sealed record Health(DriverHealthChanged E) : TelemetryItem;
+ public sealed record Resilience(DriverResilienceStatusChanged E) : TelemetryItem;
+}
+
+public interface ITelemetryLocalHub
+{
+ void Emit(TelemetryItem item);
+ // Returns a reader that first yields cached snapshots, then live deltas; disposing the
+ // subscription detaches and completes the channel.
+ ITelemetrySubscription Subscribe(int boundedCapacity);
+}
+```
+
+`TelemetryLocalHub` uses a `ConcurrentDictionary>` of subscribers +
+two `ConcurrentDictionary` snapshot caches. `Emit` updates the snapshot cache (for Health/Resilience)
+then `TryWrite`s to each subscriber (drop-oldest handled by the bounded channel). `Subscribe` snapshots
+the caches into the new channel under a brief lock ordering guarantee (cache-then-attach) so no delta
+is lost across the attach boundary.
+
+**Register** as a singleton in the driver-role DI branch (Task 3 wires the producers; do the
+`AddSingleton()` here in the Runtime `ServiceCollectionExtensions`
+`hasDriver` path, or wherever driver-role Runtime services register — locate and match).
+
+**Tests:** Emit-before-Subscribe is not seen except via snapshot (Health/Resilience last-value IS seen;
+Alarm/Script are not); two subscribers each get their own copy; a full channel drops oldest not newest;
+dispose detaches. Run the Runtime.Tests subset.
+
+**Step 5: Commit.** `feat(mesh-phase5): node-local telemetry hub (snapshot-replay + drop-oldest fan-out)`
+
+---
+
+### Task 3: Tap the four publish seams into the hub
+
+**Classification:** standard
+**Estimated implement time:** ~5 min
+**Parallelizable with:** none (needs Task 2)
+
+**Files (all Modify) — add a hub `Emit` beside the existing DPS `Publish`, leaving DPS intact:**
+- `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/AkkaDriverHealthPublisher.cs:36` (driver-health)
+- `src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs:80` (resilience)
+- `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs:379` + `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs:1405` (alerts — two producers)
+- `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs:261` + `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Scripting/DpsScriptLogPublisher.cs:46` (script-logs — two producers)
+- Test: extend the nearest existing publisher tests, or add `tests/.../Telemetry/PublishSeamEmitsToHubTests.cs`
+
+**Approach:** inject `ITelemetryLocalHub` into each producer (constructor for the services/publishers;
+for actors, pass via `Props` — match how each already receives dependencies). At each seam, immediately
+after the existing `Mediator.Tell(new Publish(...))`, add `_telemetryHub.Emit(new TelemetryItem.Xxx(msg))`.
+**Do NOT remove or gate the DPS publish** — it is the `Dps`-mode path and must remain unconditional so
+the switch stays pure. The hub is a no-op sink until a client subscribes, so this is safe on every node
+regardless of mode.
+
+For actors constructed where a hub isn't readily resolvable, resolve the singleton once at spawn and
+thread it through `Props` (do not `DependencyResolver` inside the actor per-message). Where a producer
+is only present on driver-role nodes, the hub singleton is guaranteed registered (Task 2).
+
+**Tests:** each producer, when driven, results in exactly one `hub.Emit` of the correct `TelemetryItem`
+subtype carrying the same payload it published to DPS. Use a fake `ITelemetryLocalHub` capturing emits.
+
+**Step 5: Commit.** `feat(mesh-phase5): tap the 4 telemetry publish seams into the local hub (DPS intact)`
+
+---
+
+### Task 4: Node-side gRPC streaming service
+
+**Classification:** high-risk
+**Estimated implement time:** ~5 min (mirror ScadaBridge closely)
+**Parallelizable with:** none (needs Task 0, Task 2)
+
+**Files:**
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryStreamGrpcService.cs` (place beside `DeploymentArtifactService.cs`; alias the generated base to avoid the name collision, as that file does at its lines 9-10)
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryProtoMap.Node.cs` (domain→proto mapping; the reverse of Task 7)
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/TelemetryStreamGrpcServiceTests.cs`
+
+**Design — port `SiteStreamGrpcServer.RunSubscriptionStreamAsync` (`.cs:253-385`) shape:**
+
+```csharp
+public sealed class TelemetryStreamGrpcService : GeneratedTelemetryBase
+{
+ // ctor: ITelemetryLocalHub hub, ILogger, IOptions (for a GrpcMaxConcurrentStreams knob)
+ public override async Task Subscribe(TelemetryStreamRequest request,
+ IServerStreamWriter responseStream, ServerCallContext context)
+ {
+ // 1. Validate correlation_id is a safe id (it labels the subscription; reject empty/oversized).
+ // 2. Concurrency cap (default 100) -> throw RpcException(ResourceExhausted) when exceeded.
+ // 3. sub = _hub.Subscribe(boundedCapacity: 1000); // snapshot-then-live, DropOldest inside the hub
+ // 4. MaxStreamLifetime linked-CTS CancelAfter (default 4h) ORed with context.CancellationToken,
+ // so a zombie stream terminates even if h2c keepalive misses it.
+ // 5. await foreach (item in sub.Reader.ReadAllAsync(ct))
+ // await responseStream.WriteAsync(TelemetryProtoMap.ToProto(item, request.CorrelationId), ct);
+ // 6. finally: sub.Dispose(); balance any opened/closed gauge; swallow OperationCanceled as normal.
+ }
+}
+```
+
+Key correctness points (from the ScadaBridge recon — do not skip):
+- The **bounded DropOldest channel lives in the hub** (Task 2), so the service just pumps its reader —
+ a slow/blocked central cannot back-pressure the node's actor threads.
+- Wrap the whole body so a client disconnect (`RpcException`/`OperationCanceledException`) exits cleanly
+ and disposes the subscription; never let it bubble as a fault.
+- No readiness race: the hub is a plain singleton available at host build, so unlike ScadaBridge's
+ `SetReady(ActorSystem)` gate there is nothing to defer — but if the driver actor system isn't up yet
+ the hub simply has no snapshots and no deltas, which is fine.
+
+**`TelemetryProtoMap.ToProto`**: switch on `TelemetryItem` subtype → build the matching proto message,
+wrap in `TelemetryEvent { CorrelationId, = ... }`. Transcribe every field (the reverse of the
+`.proto` transcription in Task 0). DateTimes → `Timestamp.FromDateTime(utc)`.
+
+**Tests:** in-memory — subscribe, emit each of the four item kinds into the hub, assert the service
+writes the correct proto event with fields intact; a cancelled `ServerCallContext` ends the stream and
+disposes; the concurrency cap throws `ResourceExhausted` past the limit. (Full two-host wire test is
+covered by the Task-12 live gate + optionally a Task-8 boundary test.)
+
+**Step 5: Commit.** `feat(mesh-phase5): node-side TelemetryStreamService (hub -> server-streaming)`
+
+---
+
+### Task 5: Telemetry stream auth interceptor
+
+**Classification:** small
+**Estimated implement time:** ~3 min
+**Parallelizable with:** none (needs Task 1)
+
+**Files:**
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/TelemetryStreamAuthInterceptor.cs` (copy `ConfigServeAuthInterceptor.cs` verbatim, change the gated prefix + options source)
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/TelemetryStreamAuthInterceptorTests.cs`
+
+**Steps:**
+- `ServicePrefix = "/telemetry.v1.TelemetryStreamService/"`; read `IOptions.ApiKey`.
+- Override all four handler kinds (especially **ServerStreaming** — that is the RPC shape here) so the
+ gate holds; non-matching paths pass through; empty key ⇒ throw (fail-closed); `FixedTimeEquals` on the
+ Bearer token; reject with `PermissionDenied`.
+- **Keep exactly one public constructor** (the ScadaBridge recon flagged that `Grpc.AspNetCore` silently
+ stops authorizing an interceptor with >1 ctor — pin it with a reflection test).
+
+**Tests:** right key passes; wrong key `PermissionDenied`; empty configured key rejects all; a call to a
+different service path passes through.
+
+**Step 5: Commit.** `feat(mesh-phase5): fail-closed bearer interceptor for the telemetry stream`
+
+---
+
+### Task 6: Kestrel wiring + map the node telemetry server
+
+**Classification:** standard
+**Estimated implement time:** ~4 min
+**Parallelizable with:** none (needs Task 1, Task 4, Task 5)
+
+**Files:**
+- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs` (the `AddGrpc` block 425-431, the dedicated-listener block 434-533, and the map block 577-580)
+- Test: none new (covered by Task 12 live gate); build must stay green
+
+**Steps — extend, do not rewrite, the existing block:**
+
+1. **Interceptor registration** (`Program.cs:425-431`): inside the `AddGrpc(o => …)`, add
+ `if (hasDriver) o.Interceptors.Add();` (alongside the LocalDbSync one —
+ both are driver-side, both path-scoped, harmless when their service is unmapped).
+
+2. **Port resolution** (near `:453-454`):
+ ```csharp
+ var telemetryListenPort = hasDriver ? builder.Configuration.GetValue("Telemetry:GrpcListenPort") : 0;
+ ```
+ Add `|| telemetryListenPort > 0` to the block guard at `:455`.
+
+3. **Bind it** inside the `ConfigureKestrel` closure (`:517-526`): capture `var telemetryPortToBind = telemetryListenPort;` beside the other two, and add
+ `if (telemetryPortToBind > 0) kestrel.ListenAnyIP(telemetryPortToBind, o => o.Protocols = HttpProtocols.Http2);`.
+
+4. **HTTPS-refuse branch** (`:500-511`): set `telemetryListenPort = 0;` too, and add it to the log
+ message (all dedicated listeners disable together when the host serves HTTPS).
+
+5. **Map the service** (after `:580`):
+ ```csharp
+ if (hasDriver && telemetryListenPort > 0)
+ app.MapGrpcService();
+ ```
+
+6. Update the block's header comment (434-452) to mention the third dedicated port.
+
+**Verify:** `dotnet build`; a fused node with all three ports set binds all three exactly once (the
+"re-bind existing surface once" invariant already holds — the new port is just one more
+`ListenAnyIP`). A driver-only node with only `Telemetry:GrpcListenPort` set binds only that.
+
+**Step 5: Commit.** `feat(mesh-phase5): host the telemetry gRPC server on driver nodes (dedicated h2c port)`
+
+---
+
+### Task 7: Central per-node dialer client + proto→domain mapping
+
+**Classification:** high-risk
+**Estimated implement time:** ~5 min
+**Parallelizable with:** none (needs Task 0)
+
+**Files:**
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Telemetry/TelemetryStreamClient.cs`
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Telemetry/TelemetryProtoMap.Central.cs` (proto→domain; reverse of Task 4; keyed off `TelemetryProtoContract.HandledCases`)
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Telemetry/TelemetryProtoMapTests.cs`
+
+**Design — mirror `SiteStreamGrpcClient.cs` + `GrpcDeploymentArtifactFetcher` channel handling:**
+
+```csharp
+public sealed class TelemetryStreamClient : IDisposable
+{
+ // ctor: string endpoint (http://host:port), string apiKey, IClientFactory seam for tests
+ // - GrpcChannel.ForAddress(endpoint) over http:// => prior-knowledge h2c; cache one channel per endpoint.
+ // - HTTP/2 keepalive: PingDelay 15s / Timeout 10s / Always (SocketsHttpHandler on the channel).
+ public async Task RunAsync(string correlationId, Action onEvent,
+ Action onError, CancellationToken ct)
+ {
+ // headers: authorization: Bearer {apiKey}
+ // using call = client.Subscribe(new TelemetryStreamRequest{CorrelationId=correlationId}, headers, ct)
+ // await foreach (evt in call.ResponseStream.ReadAllAsync(ct)) onEvent(TelemetryProtoMap.ToDomain(evt));
+ // RpcException(Cancelled) on shutdown => normal; anything else => onError (the reconnect trigger).
+ }
+}
+```
+
+`TelemetryProtoMap.ToDomain(TelemetryEvent)` switches on `evt.EventCase` and reconstructs the domain
+record (`AlarmTransitionEvent` / `ScriptLogEntry` / `DriverHealthChanged` / `DriverResilienceStatusChanged`).
+It MUST cover exactly `TelemetryProtoContract.HandledCases` — a `default: throw` on an unknown case makes
+"new event added but not mapped" a loud runtime failure, and the Task-0 lock test makes it a compile-time-ish
+guard.
+
+**Tests:** round-trip each domain record → `TelemetryProtoMap.ToProto` (Task 4) → `ToDomain` → assert
+field-equality across all four kinds and all enum values (wire-fidelity, like ScadaBridge
+`ProtoRoundtripTests`); an unmapped `EventCase` throws.
+
+**Step 5: Commit.** `feat(mesh-phase5): central telemetry dialer client + proto<->domain round-trip`
+
+---
+
+### Task 8: Central dial supervisor actor (discovery + reconnect + feed sinks)
+
+**Classification:** high-risk
+**Estimated implement time:** ~5 min
+**Parallelizable with:** none (needs Task 1, Task 7)
+
+**Files:**
+- Create: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Telemetry/TelemetryDialSupervisor.cs`
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Telemetry/TelemetryDialSupervisorTests.cs`
+
+**Design — an admin-role actor holding one dialer per driver node, feeding the existing central sinks:**
+
+- **Discovery:** on `PreStart` and every `TelemetryDialOptions.ContactRefreshSeconds` (+ on an
+ admin-change signal if one is already wired for `CentralCommunicationActor`), read enabled,
+ non-maintenance `ClusterNode` rows selecting `NodeId, Host, GrpcPort` (extend the
+ `CentralCommunicationActor.cs:198-219` query). A row with `GrpcPort == null` is skipped with a Warning
+ (the node exposes no telemetry port — honest, not an error). Build `http://{Host}:{GrpcPort}` dial
+ targets; add dialers for new nodes, stop dialers for removed ones.
+- **Per-node dialer loop** (mirror `SiteAlarmAggregatorActor.OpenGrpcStream`/`HandleGrpcError`,
+ simplified — one node per dialer, no NodeA/NodeB flip since each `ClusterNode` row is its own node):
+ - Open `TelemetryStreamClient.RunAsync`; `onEvent` posts each `TelemetryItem` to the supervisor which
+ routes to the sink (below); `onError` schedules a reconnect.
+ - **Reconnect:** immediate first retry, then fixed `_reconnectDelay` (5s) backoff. A **monotonic
+ generation stamp** per node makes late errors from a superseded stream ignorable. The dialer does
+ **not** die on repeated failure — it keeps retrying (observability, not data plane); log the first
+ failure + every Nth.
+ - A `correlationId` per (node, generation) — a safe id string, e.g. `central-{NodeId}-{gen}`.
+- **Feed the sinks (the untouched seam):** inject the four singletons and route by item type:
+ - `Alarm` → `IInProcessBroadcaster.Publish(e)`
+ - `Script` → `IInProcessBroadcaster.Publish(e)`
+ - `Health` → `IDriverStatusSnapshotStore.Upsert(e)`
+ - `Resilience` → `IDriverResilienceStatusStore.Upsert(e)`
+- **Connection indicator:** drive the two broadcasters' `SetConnected(...)` off aggregate stream health
+ — connected when **≥1** node stream is up, disconnected when all are down — matching today's
+ DPS `SubscribeAck`/`PostStop` pill semantics on `/alerts` and `/script-log`. (The two store-backed
+ panels have no connection flag today; leave their per-row staleness as-is — noted for Task 10 docs.)
+
+**Registration** is done in Task 9 (mode-gated). This task only builds + unit-tests the actor with a
+fake `TelemetryStreamClient` factory and fake sinks.
+
+**Tests:** discovery adds/removes dialers as the (fake) `ClusterNode` set changes; a null `GrpcPort`
+row is skipped; an emitted item of each kind lands on the correct sink; a stream error triggers a
+scheduled reconnect with an incremented generation; a late event from a superseded generation is
+dropped; broadcaster `IsConnected` flips true on first up / false on all-down.
+
+**Step 5: Commit.** `feat(mesh-phase5): central telemetry dial supervisor (discover + reconnect + feed sinks)`
+
+---
+
+### Task 9: Wire the dark switch (central ingest source)
+
+**Classification:** standard
+**Estimated implement time:** ~4 min
+**Parallelizable with:** none (needs Task 8)
+
+**Files:**
+- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs` (`WithOtOpcUaSignalRBridges`, lines 53-83)
+- Modify: the AdminUI Akka configurator call site that invokes `WithOtOpcUaSignalRBridges` (locate; it's inside the `hasAdmin` branch per the method's own doc example) — pass or resolve the mode
+- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Hubs/TelemetryModeWiringTests.cs`
+
+**Steps:**
+- Inside `WithActors`, resolve `IOptions` and read `Mode`.
+- **`Mode == Dps` (default):** spawn all five bridges exactly as today (unchanged behaviour).
+- **`Mode == Grpc`:** spawn the **fleet-status bridge only** (out of Phase 5 scope, stays on DPS), and
+ do **NOT** spawn the four telemetry DPS bridges (alert, script-log, driver-status, resilience).
+ Instead spawn `TelemetryDialSupervisor` (Task 8), resolving the four sink singletons + the
+ `IDbContextFactory` (admin nodes have ConfigDb) + `IOptions`.
+- The sinks (`AddOtOpcUaDriverStatusServices`, `:29-35`) are registered identically in both modes — no
+ change there; that is the whole point of the swap.
+
+**Guard rails:**
+- Keep the fleet-status bridge on DPS in both modes (deferred).
+- A `Grpc`-mode admin node still needs the sinks registered (it does — same call), so the AdminUI
+ components resolve them regardless.
+
+**Tests:** with `Mode=Dps`, the registry has the four telemetry bridge keys and no supervisor; with
+`Mode=Grpc`, it has the supervisor and the fleet-status bridge but none of the four telemetry bridge
+keys. (Use the Akka.Hosting `ActorRegistry` + a TestKit or the existing bridge-registration test
+harness if one exists.)
+
+**Step 5: Commit.** `feat(mesh-phase5): dark-switch central telemetry ingest (Dps bridges | Grpc dialer)`
+
+---
+
+### Task 10: Docs — Telemetry section, supersede §6.3, program status
+
+**Classification:** small
+**Estimated implement time:** ~4 min
+**Parallelizable with:** Task 11 (disjoint files)
+
+**Files (Modify/Create):**
+- Create: `docs/Telemetry.md` — the stream architecture (direction, dark switch, auth, the four channels, deferred three + why, reconnect story, per-panel connection indicator status).
+- Modify: `docs/Configuration.md` — add `Telemetry` + `TelemetryDial` sections (keys table, the `Mode` dark switch, port + shared-key `${secret:}` guidance).
+- Modify: `docs/Redundancy.md` — a short "Command/observability transport" cross-ref noting redundancy-state stays DPS (pair-local) and telemetry moved to gRPC.
+- Modify: `docs/plans/2026-07-21-per-cluster-mesh-design.md` §6.3 — supersede note: "telemetry stream authenticated from day one (fail-closed bearer), superseding the earlier 'unauthenticated for now'; ScadaBridge closed the same gap identically."
+- Modify: `docs/plans/2026-07-22-per-cluster-mesh-program.md` — Phase 5 section + Tracking row: record the four-channel scope, the three deferrals + rationale, the dark switch, auth-from-day-one.
+- Modify: `CLAUDE.md` — a "Live telemetry transport (`Telemetry`/`TelemetryDial`)" section mirroring the MeshTransport/ConfigSource sections (default `Dps`, node hosts server, central dials, four channels, deferred three, fail-closed key).
+
+**Step 5: Commit.** `docs(mesh-phase5): telemetry stream — Telemetry.md, config, supersede §6.3, program status`
+
+---
+
+### Task 11: Rig config — telemetry ports + keys + flip env
+
+**Classification:** small
+**Estimated implement time:** ~4 min
+**Parallelizable with:** Task 10 (disjoint files); needs Task 6 + Task 9 landed for the flip to mean anything
+
+**Files:**
+- Modify: `docker-dev/docker-compose.yml`
+- Modify: any committed `appsettings.json` defaults that must carry the (default-OFF) keys — verify none are needed since `Mode` defaults to `Dps` and port defaults to 0.
+
+**Steps:**
+- Give every **driver-role** node (central-1/2 fused + the four site nodes) `Telemetry__GrpcListenPort`
+ (pick a non-colliding port — verify against 4053 Akka / 4055 ConfigServe / the LocalDb sync port; use
+ e.g. **4056**) and `Telemetry__ApiKey: "telemetry-docker-dev-key"` (committed dev-secret exception, like
+ `configserve-docker-dev-key`).
+- Give the **central (admin)** nodes `TelemetryDial__ApiKey: "telemetry-docker-dev-key"` (matching).
+- Add each driver node's `GrpcPort` to its `ClusterNode` seed row (the rig's SQL seed / migration seed) so
+ central discovers the telemetry endpoint — mirror how `AkkaPort` was seeded in Phase 1.
+- Leave `Telemetry__Mode`/`TelemetryDial__Mode` unset (⇒ `Dps` default). Document the flip in the compose
+ header: set `Telemetry__Mode=Grpc` on the driver nodes **and** `TelemetryDial__Mode=Grpc` on the central
+ nodes at `docker compose up`, then recreate.
+- `lmxopcua-fix`/rig note: this is the local `docker-dev` rig, not the shared driver-fixture host.
+
+**Step 5: Commit.** `chore(mesh-phase5): rig telemetry ports + shared key + ClusterNode.GrpcPort seed`
+
+---
+
+### Task 12: Live gate
+
+**Classification:** high-risk (gate, not a code change — but the phase is not done until it passes)
+**Estimated implement time:** live rig run (not subagent wall-time)
+**Parallelizable with:** none (needs everything)
+
+**Files:**
+- Create: `docs/plans/2026-07-23-mesh-phase5-live-gate.md` (record)
+
+**Procedure (on the local `docker-dev` rig, matching prior phases' gates):**
+1. Build the Phase-5 image (`--platform linux/amd64` build stage; `--progress=plain`) and recreate the rig in **default `Dps` mode** — confirm all six nodes up, `/alerts`, `/script-log`, `/hosts` panels live exactly as before (regression check: DPS path unchanged).
+2. Flip: set `Telemetry__Mode=Grpc` on all driver nodes and `TelemetryDial__Mode=Grpc` on the central nodes; `docker compose up` (recreate).
+3. **Exit gate leg A — panels green over gRPC:** drive a scripted-alarm transition and a script-log emission and a driver health/resilience change; confirm `/alerts`, `/script-log`, and the `/hosts` driver table update live — **with the four DPS telemetry bridges NOT spawned** (grep the central logs to confirm the Grpc branch ran and the four bridge keys are absent). Confirm the `/alerts` + `/script-log` connection pill reads "live".
+4. **Exit gate leg B — kill-and-reconnect:** `docker kill` a site driver node (or drop its telemetry port), watch the central dialer log the error + retries and the pill flip to disconnected; restart the node and confirm the dialer reconnects, the driver-health/resilience stores re-prime from the node hub's snapshot replay, and the pill returns to live — **every stream recovers**.
+5. **Optionally** confirm the auth gate: a dial with a wrong `TelemetryDial:ApiKey` gets `PermissionDenied` and no panel data (a deliberate misconfig probe).
+6. Record all legs + evidence in the gate doc; note any deviations (as Phases 1-4 gates did).
+
+**Exit gate (program):** all AdminUI live panels green against the rig with the telemetry DPS bridges
+off; kill-and-reconnect of the central dialer recovers every stream.
+
+**Step: Commit** the gate record; then finish per superpowers-extended-cc:finishing-a-development-branch
+(merge `feat/mesh-phase5` to master locally, push, update the scadaproj umbrella index, update memory).
+
+---
+
+## Risks / watch-items carried into execution
+
+- **Single-mesh correctness of the node-local hub.** The hub must carry only *this node's* events
+ (Task 2) — never subscribe it to cluster DPS, or on the current single mesh every node would stream
+ every other node's events and central would double-count. Snapshot replay + drop-oldest come from the
+ hub, not the service.
+- **Do not gate/remove the DPS publish** at the four seams (Task 3) — that is the `Dps`-mode path and
+ the pure-dark-switch guarantee. The hub is a no-op until a client connects.
+- **Interceptor single-ctor pin** (Task 5) — `Grpc.AspNetCore` silently disables an interceptor with
+ >1 public ctor.
+- **Kestrel "re-bind once" invariant** (Task 6) — the third dedicated port is just one more
+ `ListenAnyIP`; the existing-surface re-bind still happens exactly once. Don't duplicate it.
+- **Frame size is a non-issue** — server-streaming telemetry events are small and never cross Akka
+ remoting; the 128 KB ClusterClient frame trap does not apply to this path.
+- **Additive-only proto evolution** — new event kinds/fields only; the Task-0 lock test + the
+ Task-7 `default: throw` are the guard. Never renumber a tag.
+- **Deferred-channel honesty** — redundancy-state / fleet-status / deployment-acks are out by design,
+ documented in Task 10; a future reviewer must not read "seven topics" in the program doc and think
+ Phase 5 missed three.
diff --git a/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md.tasks.json b/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md.tasks.json
new file mode 100644
index 00000000..e5a5a79e
--- /dev/null
+++ b/docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md.tasks.json
@@ -0,0 +1,22 @@
+{
+ "planPath": "docs/plans/2026-07-23-mesh-phase5-grpc-telemetry-stream.md",
+ "program": "per-cluster-mesh",
+ "phase": 5,
+ "branch": "feat/mesh-phase5",
+ "tasks": [
+ {"id": 0, "subject": "Task 0: telemetry.proto contract + codegen + contract-lock test", "classification": "standard", "status": "completed", "commit": "a845a6d2", "parallelizableWith": [1, 2]},
+ {"id": 1, "subject": "Task 1: Telemetry/TelemetryDial options + fail-closed validators", "classification": "small", "status": "completed", "commit": "53ae0100", "parallelizableWith": [0, 2]},
+ {"id": 2, "subject": "Task 2: node-local telemetry hub (snapshot-replay + drop-oldest)", "classification": "standard", "status": "completed", "commit": "9882b1d2+7bdb6d00", "parallelizableWith": [0, 1]},
+ {"id": 3, "subject": "Task 3: tap the 4 publish seams into the hub (DPS intact)", "classification": "standard", "status": "completed", "commit": "8e5090ed+909d7535", "blockedBy": [2]},
+ {"id": 4, "subject": "Task 4: node-side TelemetryStreamService (hub -> server-streaming)","classification": "high-risk", "status": "completed", "commit": "e742fee4+2e70ef88", "blockedBy": [0, 2]},
+ {"id": 5, "subject": "Task 5: fail-closed bearer interceptor for the telemetry stream", "classification": "small", "status": "completed", "commit": "50f1620d", "blockedBy": [1]},
+ {"id": 6, "subject": "Task 6: host the telemetry gRPC server on driver nodes (Kestrel)", "classification": "standard", "status": "completed", "commit": "f131c1cc+af9c9d78", "blockedBy": [1, 4, 5]},
+ {"id": 7, "subject": "Task 7: central telemetry dialer client + proto<->domain round-trip","classification": "high-risk","status": "completed", "commit": "c78034f0+a279a43e", "blockedBy": [0]},
+ {"id": 8, "subject": "Task 8: central telemetry dial supervisor (discover+reconnect+sinks)","classification": "high-risk","status": "completed", "commit": "1104785c+84fa2e1e", "blockedBy": [1, 7]},
+ {"id": 9, "subject": "Task 9: dark-switch central telemetry ingest (Dps bridges | Grpc)", "classification": "standard", "status": "completed", "commit": "d15c613e", "blockedBy": [8]},
+ {"id": 10, "subject": "Task 10: docs — Telemetry.md, config, supersede §6.3, program status","classification": "small", "status": "completed", "commit": "e99ea40e", "parallelizableWith": [11], "blockedBy": [9]},
+ {"id": 11, "subject": "Task 11: rig telemetry ports + shared key + ClusterNode.GrpcPort seed","classification": "small", "status": "completed", "commit": "ffb75b78", "note": "port 4056, docker compose config validated", "parallelizableWith": [10], "blockedBy": [6, 9]},
+ {"id": 12, "subject": "Task 12: live gate (panels green over gRPC + kill-and-reconnect)", "classification": "high-risk","status": "completed", "note": "PASSED — 12-stream mesh, pill live, kill/reconnect recovered; found GrpcPort upgrade gotcha. Record: 2026-07-23-mesh-phase5-live-gate.md", "blockedBy": [10, 11]}
+ ],
+ "lastUpdated": "2026-07-23T00:00:00Z"
+}
diff --git a/docs/plans/2026-07-23-mesh-phase5-live-gate.md b/docs/plans/2026-07-23-mesh-phase5-live-gate.md
new file mode 100644
index 00000000..63c8f190
--- /dev/null
+++ b/docs/plans/2026-07-23-mesh-phase5-live-gate.md
@@ -0,0 +1,102 @@
+# Per-Cluster Mesh Phase 5 — Live Gate Record
+
+**Date:** 2026-07-23
+**Branch:** `feat/mesh-phase5` (Phase-5 image built + rig recreated with Phase-5 code)
+**Rig:** local `docker-dev` — central-1/2 (`admin,driver`, fused), site-a-1/2, site-b-1/2 (`driver`).
+Telemetry served on **:4056** (h2c), shared key `telemetry-docker-dev-key`.
+
+**Result: PASSED.** Every load-bearing exit-gate leg proven live. In `Telemetry:Mode=Grpc`, central
+dials each driver node's telemetry gRPC server, the full stream mesh forms, real telemetry traverses
+node→hub→gRPC→central→sink (AdminUI pill goes live), and kill-and-reconnect of a node recovers its
+stream automatically. In the default `Dps` mode the node hosts the server but central does not dial.
+
+Evidence was gathered primarily at the **TCP-socket level** (`/proc/net/tcp6`, port 4056 = hex
+`0FD8`) because it is decisive and log-independent: an established stream is an established stream.
+
+## What was verified
+
+### Leg 0 — Dps baseline (headline: node always hosts, central does NOT dial)
+Fresh rig in the committed default (`Telemetry:Mode` unset ⇒ `Dps`). site-a-1 log:
+`Now listening on: http://[::]:4056` — the dedicated h2c telemetry listener binds in Dps mode too
+(the node **always** hosts the server). Socket check: site-a-1 has a **LISTEN** on `[::]:4056` and
+**zero** inbound ESTABLISHED — central does not dial in Dps mode. Cluster formed clean (central-1 sees
+central-2 + all four site nodes at `Up`), OPC UA up, drivers subscribed. Baseline correct.
+
+### Leg 1 — Grpc flip → full stream mesh forms ✅
+Flipped via a `docker-compose.override.yml` (gate artifact, since removed) setting
+`TelemetryDial__Mode=Grpc` on the central pair and `Telemetry__Mode=Grpc` on all six driver nodes,
+then recreated. After the central pair discovered the (populated) `ClusterNode.GrpcPort` rows and
+dialed:
+
+| Node | inbound telemetry streams on :4056 |
+|---|---|
+| central-1 | 2 |
+| central-2 | 2 |
+| site-a-1 | 2 |
+| site-a-2 | 2 |
+| site-b-1 | 2 |
+| site-b-2 | 2 |
+
+Both central nodes each held **6 outbound** streams (one per driver node incl. self). **12 streams
+total** — the exact designed topology (both admins dial every driver node). Zero telemetry failures in
+the steady window. Dial happens **only** in Grpc mode (Leg 0 had zero); discovery is DB-sourced from
+`ClusterNode` (Host + GrpcPort), matching Phase 1 + the rig seed.
+
+### Leg 2 — data path end-to-end (AdminUI pill live) ✅
+`http://localhost:9200/alerts` connection pill read **"live"**. This is the load-bearing data-path
+proof: the dial supervisor marks a node **Connected — and flips both broadcasters' `SetConnected(true)`
+— only on its FIRST received telemetry event**. So "live" means a real telemetry event (the
+5 s-periodic `driver-resilience-status`) traversed the node's publish seam → node-local hub → gRPC
+stream → central dial supervisor → `TelemetryReceived` → the in-process sink the Blazor panel reads.
+Transport *and* payload proven, not just a connected socket. (The panel's static "…from the cluster's
+DPS topic" caption is now stale wording — cosmetic, the broadcaster seam is unchanged, only its
+upstream swapped; noted as a doc-copy follow-up, non-blocking.)
+
+### Leg 3 — kill-and-reconnect recovers every stream ✅
+`docker kill otopcua-dev-site-b-1-1`:
+- central-1 logged `Telemetry stream to node site-b-1:4053 failed (failure streak 1); reconnecting`.
+- central-1 outbound streams dropped 6 → 5; the pill stayed **live** (the other 5 streams carried the
+ panel — a single node loss does not blank the fleet view).
+
+`docker start otopcua-dev-site-b-1-1`: within **5 s** of the node rejoining, site-b-1 was back to **2
+inbound** and central-1 back to **6 outbound**, with **zero** ongoing telemetry failures. The
+generation-stamped reconnect recovered the stream automatically. (The same recovery was independently
+observed as a transient at central startup, when central briefly dialed its own/peer server before it
+finished booting — failure-streak-1 → reconnect → 6 streams.)
+
+### Auth — covered by the Task 6 integration test (not re-probed live)
+`TelemetryListenerTests` (Host.IntegrationTests) already proves the mapped, interceptor-gated endpoint
+with two real gRPC calls: **no bearer ⇒ `PermissionDenied`** (which can only originate in
+`TelemetryStreamAuthInterceptor` on a *mapped* service — an unmapped service returns `Unimplemented`),
+and correct bearer ⇒ a clean stream. The rig ran with matching keys throughout; a key mismatch would
+have surfaced as `PermissionDenied` reconnect-spam (it did not).
+
+## Finding — `ClusterNode.GrpcPort` must be populated on an EXISTING deployment (upgrade gotcha)
+The first Grpc flip produced **no** dial connections. Root cause was **not** a code defect: all six
+`ClusterNode` rows had `GrpcPort = NULL`. The rig's SQL volume persisted from before this branch, and
+the seed is idempotent **INSERT-if-not-exists** — so it never backfilled the new nullable `GrpcPort`
+onto pre-existing rows (`AkkaPort` was non-null only because its Phase-1 column carries a NOT NULL
+default `4053`; `GrpcPort` is nullable with no default). The dial supervisor behaved **correctly**: it
+spawned in Grpc mode, ran discovery, and skipped all six nodes with the throttled
+`ClusterNode has no GrpcPort … skipped … (further skips of this node are silent)` warning (the
+exact throttle added in Task 8's review fix). Populating `GrpcPort = 4056` and bouncing the central
+pair produced the full 12-stream mesh immediately.
+
+**Operational takeaway (record in the upgrade notes):** a **fresh** install seeds `GrpcPort` correctly
+(the Phase-11 seed INSERT includes it), but an **existing** deployment upgrading to Phase 5 must
+populate `ClusterNode.GrpcPort` per node — via the AdminUI node edit (Phase 1 surfaced the field) or a
+data update — or the telemetry dial silently finds no targets (loud-but-throttled warning, last-known
+DPS panels unaffected since the flip is per-node). Optional rig follow-up: make the docker-dev seed
+UPSERT the port columns so a persisted volume backfills on re-run (not blocking; this volume is now
+populated).
+
+## Cleanup
+`docker-compose.override.yml` removed; rig recreated to the `Dps` default. `ClusterNode.GrpcPort`
+left populated at 4056 (correct — matches the fresh-install seed intent).
+
+## Verdict
+**Phase 5 live gate PASSED.** Driver nodes host the telemetry gRPC server; central dials in only in
+Grpc mode; the full stream mesh forms from DB-sourced discovery; real telemetry reaches the AdminUI
+panels; kill-and-reconnect recovers every stream. The one issue surfaced was a rig/upgrade data gap
+(`GrpcPort` backfill), not a Phase-5 code defect — and it validated the graceful null-`GrpcPort`
+handling live.
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs
index 4c9e567e..34e0e3cb 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs
@@ -48,6 +48,14 @@ public static class ServiceCollectionExtensions
configuration, ConfigSourceOptions.SectionName);
services.Configure(configuration.GetSection(ConfigServeOptions.SectionName));
+ // Per-cluster mesh Phase 5: which transport carries a node's live-telemetry stream (node
+ // serve side + central dial side). Validated at startup for the same reason ConfigSource is —
+ // a Grpc-mode node with no listen port or no key produces silence, not an error.
+ services.AddValidatedOptions(
+ configuration, TelemetryOptions.SectionName);
+ services.AddValidatedOptions(
+ configuration, TelemetryDialOptions.SectionName);
+
services.AddSingleton();
return services;
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/TelemetryOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/TelemetryOptions.cs
new file mode 100644
index 00000000..b9fc6d9a
--- /dev/null
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/TelemetryOptions.cs
@@ -0,0 +1,191 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+
+namespace ZB.MOM.WW.OtOpcUa.Cluster;
+
+///
+/// Node-side selection of the transport a node's live-telemetry stream is carried over
+/// (per-cluster mesh Phase 5). The dark switch that lets a node stop publishing telemetry over
+/// the mesh-wide DistributedPubSub topic and instead serve it over a dedicated gRPC stream that
+/// does not require sharing a gossip ring with whoever is watching.
+///
+public sealed class TelemetryOptions
+{
+ /// Configuration section name.
+ public const string SectionName = "Telemetry";
+
+ /// Publish telemetry over the mesh-wide DistributedPubSub topic — the default.
+ public const string ModeDps = "Dps";
+
+ /// Serve telemetry over a dedicated gRPC stream.
+ public const string ModeGrpc = "Grpc";
+
+ ///
+ /// (default) or . Any other value fails host
+ /// start.
+ ///
+ public string Mode { get; set; } = ModeDps;
+
+ ///
+ /// Dedicated gRPC listen port for the telemetry stream. 0 (the default) disables it —
+ /// nothing is bound. Required on a driver-role node under .
+ ///
+ public int GrpcListenPort { get; set; }
+
+ ///
+ /// Shared bearer key the serve-side interceptor checks. Supply via the environment
+ /// (Telemetry__ApiKey) — never commit it. Required under on any
+ /// roled node — fail-closed against hosting an un-keyed telemetry surface.
+ ///
+ public string ApiKey { get; set; } = string.Empty;
+}
+
+///
+/// Central-side selection of how central dials a node for live telemetry (per-cluster mesh
+/// Phase 5). Mirrors 's mode but carries the dial-side knobs
+/// (contact refresh cadence, per-call timeout) instead of a listen port.
+///
+public sealed class TelemetryDialOptions
+{
+ /// Configuration section name.
+ public const string SectionName = "TelemetryDial";
+
+ /// Read telemetry from the mesh-wide DistributedPubSub topic — the default.
+ public const string ModeDps = "Dps";
+
+ /// Dial nodes' dedicated gRPC telemetry streams.
+ public const string ModeGrpc = "Grpc";
+
+ ///
+ /// (default) or . Any other value fails host
+ /// start.
+ ///
+ public string Mode { get; set; } = ModeDps;
+
+ ///
+ /// Shared bearer key; must equal a dialled node's .
+ /// Supply via the environment (TelemetryDial__ApiKey) — never commit it. Required
+ /// under .
+ ///
+ public string ApiKey { get; set; } = string.Empty;
+
+ /// How often central refreshes its set of dialable node contacts, in seconds.
+ public int ContactRefreshSeconds { get; set; } = 60;
+
+ /// Per-call deadline for a gRPC telemetry dial, in seconds.
+ public int CallTimeoutSeconds { get; set; } = 30;
+}
+
+///
+/// Fails the host at startup on a shape that would leave a node
+/// unable to serve its live-telemetry stream.
+///
+///
+/// Like , every fault caught here otherwise surfaces as
+/// an absence — a telemetry consumer that simply never sees data from this node, with no
+/// stack trace and no failing node to point at; refusing to start is cheaper to diagnose.
+///
+public sealed class TelemetryOptionsValidator : IValidateOptions
+{
+ private readonly IConfiguration _configuration;
+
+ ///
+ /// DI-constructed by AddValidatedOptions (a plain AddSingleton), so a
+ /// constructor dependency is safe here. — not
+ /// IClusterRoleInfo — is the source of this node's roles: IClusterRoleInfo's
+ /// implementation needs the ActorSystem, which does not exist yet at
+ /// ValidateOnStart time.
+ ///
+ public TelemetryOptionsValidator(IConfiguration configuration)
+ {
+ _configuration = configuration;
+ }
+
+ ///
+ public ValidateOptionsResult Validate(string? name, TelemetryOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+
+ var errors = new List();
+
+ var isDps = string.Equals(options.Mode, TelemetryOptions.ModeDps, StringComparison.OrdinalIgnoreCase);
+ var isGrpc = string.Equals(options.Mode, TelemetryOptions.ModeGrpc, StringComparison.OrdinalIgnoreCase);
+
+ if (!isDps && !isGrpc)
+ {
+ errors.Add(
+ $"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.Mode)} is '{options.Mode}'. "
+ + $"Expected '{TelemetryOptions.ModeDps}' or '{TelemetryOptions.ModeGrpc}'.");
+ }
+
+ var roles = _configuration.GetSection("Cluster:Roles").Get() ?? Array.Empty();
+ var isDriver = Array.IndexOf(roles, "driver") >= 0;
+
+ if (isGrpc)
+ {
+ if (isDriver && options.GrpcListenPort <= 0)
+ {
+ errors.Add(
+ $"Cluster:Roles has 'driver' and {TelemetryOptions.SectionName}:"
+ + $"{nameof(TelemetryOptions.Mode)} is '{TelemetryOptions.ModeGrpc}', but "
+ + $"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.GrpcListenPort)} is "
+ + $"{options.GrpcListenPort}. A driver node in Grpc telemetry mode must set "
+ + $"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.GrpcListenPort)}.");
+ }
+
+ if (roles.Length > 0 && string.IsNullOrEmpty(options.ApiKey))
+ {
+ errors.Add(
+ $"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.ApiKey)} is empty. Under "
+ + $"{TelemetryOptions.ModeGrpc} the shared bearer key is the whole authentication "
+ + "boundary to this node's telemetry surface; without it every dial is rejected. "
+ + $"Supply it via the environment ({TelemetryOptions.SectionName}__ApiKey).");
+ }
+ }
+
+ return errors.Count == 0
+ ? ValidateOptionsResult.Success
+ : ValidateOptionsResult.Fail(string.Join(" ", errors));
+ }
+}
+
+///
+/// Fails the host at startup on a shape that would leave
+/// central unable to dial a node's live-telemetry stream.
+///
+public sealed class TelemetryDialOptionsValidator : IValidateOptions
+{
+ ///
+ public ValidateOptionsResult Validate(string? name, TelemetryDialOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+
+ var errors = new List();
+
+ var isDps = string.Equals(
+ options.Mode, TelemetryDialOptions.ModeDps, StringComparison.OrdinalIgnoreCase);
+ var isGrpc = string.Equals(
+ options.Mode, TelemetryDialOptions.ModeGrpc, StringComparison.OrdinalIgnoreCase);
+
+ if (!isDps && !isGrpc)
+ {
+ errors.Add(
+ $"{TelemetryDialOptions.SectionName}:{nameof(TelemetryDialOptions.Mode)} is "
+ + $"'{options.Mode}'. Expected '{TelemetryDialOptions.ModeDps}' or "
+ + $"'{TelemetryDialOptions.ModeGrpc}'.");
+ }
+
+ if (isGrpc && string.IsNullOrEmpty(options.ApiKey))
+ {
+ errors.Add(
+ $"{TelemetryDialOptions.SectionName}:{nameof(TelemetryDialOptions.ApiKey)} is empty. "
+ + $"Under {TelemetryDialOptions.ModeGrpc} the shared bearer key is the whole "
+ + "authentication boundary to a node's telemetry surface; without it every dial is "
+ + $"rejected. Supply it via the environment ({TelemetryDialOptions.SectionName}__ApiKey).");
+ }
+
+ return errors.Count == 0
+ ? ValidateOptionsResult.Success
+ : ValidateOptionsResult.Fail(string.Join(" ", errors));
+ }
+}
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/TelemetryProtoContract.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/TelemetryProtoContract.cs
new file mode 100644
index 00000000..b251a542
--- /dev/null
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/TelemetryProtoContract.cs
@@ -0,0 +1,25 @@
+using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.Commons.Protos;
+
+///
+/// The single source of truth for which variants the
+/// Phase 5 telemetry transport actually handles. Both the contract-lock test (which proves this
+/// list stays in lock-step with the generated oneof) and the later oneof↔domain converter
+/// reference this array, so adding a fifth channel to telemetry.proto forces a matching
+/// entry here or the build/test goes red.
+///
+public static class TelemetryProtoContract
+{
+ ///
+ /// Exactly the four telemetry oneof cases mirrored from the domain records:
+ /// alerts / script-logs / driver-health / driver-resilience-status.
+ ///
+ public static readonly TelemetryEvent.EventOneofCase[] HandledCases =
+ [
+ TelemetryEvent.EventOneofCase.AlarmTransition,
+ TelemetryEvent.EventOneofCase.ScriptLog,
+ TelemetryEvent.EventOneofCase.DriverHealth,
+ TelemetryEvent.EventOneofCase.DriverResilience,
+ ];
+}
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/telemetry.proto b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/telemetry.proto
new file mode 100644
index 00000000..0997fb2e
--- /dev/null
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos/telemetry.proto
@@ -0,0 +1,83 @@
+syntax = "proto3";
+
+package telemetry.v1;
+
+option csharp_namespace = "ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1";
+
+import "google/protobuf/timestamp.proto";
+
+// Per-cluster mesh Phase 5: the four live-telemetry channels that used to fan out over
+// DistributedPubSub (alerts / script-logs / driver-health / driver-resilience-status) now travel over
+// ONE gRPC server-streaming contract. Central dials each driver node and opens Subscribe; the node
+// streams its own live telemetry as a sequence of TelemetryEvent envelopes.
+//
+// Additive-only field evolution: never renumber or reuse a tag; a pre-field peer must read a new
+// field's proto3 default correctly (that is why nullable domain fields are modelled with `optional`
+// for explicit presence). Locked by the contract test in the Commons test project.
+service TelemetryStreamService {
+ rpc Subscribe(TelemetryStreamRequest) returns (stream TelemetryEvent);
+}
+
+message TelemetryStreamRequest {
+ string correlation_id = 1;
+}
+
+message TelemetryEvent {
+ string correlation_id = 1;
+ oneof event {
+ AlarmTransition alarm_transition = 2; // <- "alerts" / AlarmTransitionEvent
+ ScriptLog script_log = 3; // <- "script-logs" / ScriptLogEntry
+ DriverHealth driver_health = 4; // <- "driver-health" / DriverHealthChanged
+ DriverResilienceStatus driver_resilience = 5; // <- "driver-resilience-status" / DriverResilienceStatusChanged
+ }
+}
+
+// Mirrors ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts.AlarmTransitionEvent.
+message AlarmTransition {
+ string alarm_id = 1;
+ string equipment_path = 2;
+ string alarm_name = 3;
+ string transition_kind = 4; // string in the record (Activated/Cleared/...); kept as string
+ int32 severity = 5;
+ string message = 6;
+ string user = 7;
+ google.protobuf.Timestamp timestamp_utc = 8;
+ string alarm_type_name = 9; // record default "AlarmCondition"
+ optional string comment = 10; // nullable in the record — presence distinguishes null from ""
+ optional bool historize_to_aveva = 11; // bool? in the record — three states (null / true / false)
+ repeated string referencing_equipment_paths = 12; // null in the record is treated as empty
+}
+
+// Mirrors ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging.ScriptLogEntry.
+message ScriptLog {
+ string script_id = 1;
+ string level = 2; // string in the record (Trace/Debug/Information/...); kept as string
+ string message = 3;
+ google.protobuf.Timestamp timestamp_utc = 4;
+ optional string virtual_tag_id = 5; // nullable in the record
+ optional string alarm_id = 6; // nullable in the record
+ optional string equipment_id = 7; // nullable in the record
+}
+
+// Mirrors ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers.DriverHealthChanged.
+message DriverHealth {
+ string cluster_id = 1;
+ string driver_instance_id = 2;
+ string state = 3; // DriverState-as-string in the record; kept as string
+ google.protobuf.Timestamp last_successful_read_utc = 4; // DateTime? — absent Timestamp encodes null
+ optional string last_error = 5; // nullable in the record
+ int32 error_count_5min = 6;
+ google.protobuf.Timestamp published_utc = 7;
+}
+
+// Mirrors ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers.DriverResilienceStatusChanged.
+message DriverResilienceStatus {
+ string driver_instance_id = 1;
+ string host_name = 2;
+ bool breaker_open = 3;
+ int32 consecutive_failures = 4;
+ int32 current_in_flight = 5;
+ google.protobuf.Timestamp last_breaker_open_utc = 6; // DateTime? — absent Timestamp encodes null
+ google.protobuf.Timestamp last_sampled_utc = 7;
+ google.protobuf.Timestamp published_utc = 8;
+}
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj
index acad5d43..5e55544f 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/ZB.MOM.WW.OtOpcUa.Commons.csproj
@@ -27,6 +27,7 @@
+
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs
index f3675564..cf664c67 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs
@@ -1,7 +1,14 @@
using Akka.Actor;
using Akka.Hosting;
using Microsoft.AspNetCore.SignalR;
+using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
@@ -12,6 +19,7 @@ public static class HubServiceCollectionExtensions
public const string ScriptLogSignalRBridgeName = "script-log-signalr-bridge";
public const string DriverStatusSignalRBridgeName = "driver-status-signalr-bridge";
public const string DriverResilienceStatusBridgeName = "driver-resilience-status-bridge";
+ public const string TelemetryDialSupervisorName = "telemetry-dial-supervisor";
///
/// Registers the in-process live-push services the AdminUI's Blazor Server panels read
@@ -54,30 +62,69 @@ public static class HubServiceCollectionExtensions
{
builder.WithActors((system, registry, resolver) =>
{
+ // Fleet-status always stays on DPS in both modes (deferred / out of Phase 5 scope) — it is
+ // never gated by the telemetry dark switch.
var fleetHub = resolver.GetService>();
var fleetBridge = system.ActorOf(FleetStatusSignalRBridge.Props(fleetHub), FleetStatusSignalRBridgeName);
registry.Register(fleetBridge);
- var alertHub = resolver.GetService>();
+ // The four telemetry sinks are registered identically in both modes; only the upstream that
+ // feeds them swaps (DPS bridges vs. the gRPC dial supervisor). Resolve them once, above the
+ // branch, so both paths feed the SAME singletons the Blazor panels read.
var alertBroadcaster = resolver.GetService>();
- var alertBridge = system.ActorOf(AlertSignalRBridge.Props(alertHub, alertBroadcaster), AlertSignalRBridgeName);
- registry.Register(alertBridge);
-
- var scriptLogHub = resolver.GetService>();
var scriptLogBroadcaster = resolver.GetService>();
- var scriptLogBridge = system.ActorOf(ScriptLogSignalRBridge.Props(scriptLogHub, scriptLogBroadcaster), ScriptLogSignalRBridgeName);
- registry.Register(scriptLogBridge);
-
- var driverStatusHub = resolver.GetService>();
var driverStatusStore = resolver.GetService();
- var driverStatusBridge = system.ActorOf(DriverStatusSignalRBridge.Props(driverStatusHub, driverStatusStore), DriverStatusSignalRBridgeName);
- registry.Register(driverStatusBridge);
-
- // Resilience-status bridge: DPS topic -> in-process store (no SignalR hub — the panel reads
- // the store directly, and resilience has no browser-JS consumer).
var resilienceStore = resolver.GetService();
- var resilienceBridge = system.ActorOf(DriverResilienceStatusBridge.Props(resilienceStore), DriverResilienceStatusBridgeName);
- registry.Register(resilienceBridge);
+
+ // Phase 5 dark switch. Absent options ⇒ Dps (today's behaviour); case-insensitive.
+ var telemetryMode = resolver.GetService>()?.Value.Mode
+ ?? TelemetryDialOptions.ModeDps;
+
+ if (string.Equals(telemetryMode, TelemetryDialOptions.ModeGrpc, StringComparison.OrdinalIgnoreCase))
+ {
+ // Grpc: central dials each enabled node's dedicated telemetry stream, feeding the SAME
+ // four sinks the DPS bridges feed. No DPS telemetry bridges are spawned.
+ var options = resolver.GetService>()!.Value;
+ var dbFactory = resolver.GetService>();
+ var loggerFactory = resolver.GetService();
+
+ var nodeSource = TelemetryNodeSource.Create(
+ dbFactory!, loggerFactory!.CreateLogger(typeof(TelemetryNodeSource).FullName!));
+ var dialLoop = TelemetryNodeSource.CreateDialLoop(
+ options.ApiKey, loggerFactory.CreateLogger());
+
+ var supervisor = system.ActorOf(
+ TelemetryDialSupervisor.Props(
+ nodeSource,
+ dialLoop,
+ alertBroadcaster,
+ scriptLogBroadcaster,
+ driverStatusStore,
+ resilienceStore,
+ options),
+ TelemetryDialSupervisorName);
+ registry.Register(supervisor);
+ }
+ else
+ {
+ // Dps (default): the four DPS bridges subscribe their mesh-wide topics and feed the sinks.
+ var alertHub = resolver.GetService>();
+ var alertBridge = system.ActorOf(AlertSignalRBridge.Props(alertHub, alertBroadcaster), AlertSignalRBridgeName);
+ registry.Register(alertBridge);
+
+ var scriptLogHub = resolver.GetService>();
+ var scriptLogBridge = system.ActorOf(ScriptLogSignalRBridge.Props(scriptLogHub, scriptLogBroadcaster), ScriptLogSignalRBridgeName);
+ registry.Register(scriptLogBridge);
+
+ var driverStatusHub = resolver.GetService>();
+ var driverStatusBridge = system.ActorOf(DriverStatusSignalRBridge.Props(driverStatusHub, driverStatusStore), DriverStatusSignalRBridgeName);
+ registry.Register(driverStatusBridge);
+
+ // Resilience-status bridge: DPS topic -> in-process store (no SignalR hub — the panel reads
+ // the store directly, and resilience has no browser-JS consumer).
+ var resilienceBridge = system.ActorOf(DriverResilienceStatusBridge.Props(resilienceStore), DriverResilienceStatusBridgeName);
+ registry.Register(resilienceBridge);
+ }
});
return builder;
}
@@ -89,3 +136,6 @@ public sealed class AlertSignalRBridgeKey { }
public sealed class ScriptLogSignalRBridgeKey { }
public sealed class DriverStatusSignalRBridgeKey { }
public sealed class DriverResilienceStatusBridgeKey { }
+
+/// Marker key for lookup of the Grpc-mode telemetry dial supervisor.
+public sealed class TelemetryDialSupervisorKey { }
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Telemetry/TelemetryDialSupervisor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Telemetry/TelemetryDialSupervisor.cs
new file mode 100644
index 00000000..d11b1f9d
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Telemetry/TelemetryDialSupervisor.cs
@@ -0,0 +1,499 @@
+using Akka.Actor;
+using Akka.Event;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
+
+///
+/// A single dialable driver node: its logical id and the prior-knowledge
+/// http://host:port gRPC telemetry endpoint central connects to.
+///
+/// The node's stable logical id (the ClusterNode.NodeId).
+/// The http://{Host}:{GrpcPort} h2c telemetry endpoint.
+public sealed record TelemetryDialTarget(string NodeId, string Endpoint);
+
+///
+/// The dial-loop seam: opens one telemetry stream to and pumps mapped
+/// domain records to until the stream ends or
+/// fires, routing a transport failure to . The production
+/// implementation wraps TelemetryStreamClient + TelemetryProtoMapCentral; tests
+/// substitute a fake that captures the callbacks and drives events/failures by hand.
+///
+/// The node to dial.
+/// Per-generation stream correlation id.
+/// Sink for each mapped domain record (already projected from the wire).
+/// Invoked on a transport failure (the reconnect trigger).
+/// Cancels the stream (a normal, supervisor-initiated shutdown).
+/// A task that completes when the stream ends or is cancelled.
+public delegate Task TelemetryDialLoop(
+ TelemetryDialTarget target,
+ string correlationId,
+ Action
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/TelemetryStreamAuthInterceptor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/TelemetryStreamAuthInterceptor.cs
new file mode 100644
index 00000000..7f331c7b
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/TelemetryStreamAuthInterceptor.cs
@@ -0,0 +1,165 @@
+using System.Security.Cryptography;
+using System.Text;
+using Grpc.Core;
+using Grpc.Core.Interceptors;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+///
+/// Gates the driver-node telemetry stream endpoint (per-cluster mesh Phase 5). The
+/// TelemetryStreamService.Subscribe RPC streams a node's live alerts / script-logs /
+/// driver-health / driver-resilience-status to whoever dials it; anything able to reach a driver
+/// node's telemetry port could otherwise pull that live feed out-of-band. This interceptor is the
+/// ONLY inbound auth on that endpoint.
+///
+///
+///
+/// Scoped by method path. Only calls under
+/// /telemetry.v1.TelemetryStreamService/ are gated; every other method on the shared
+/// gRPC pipeline (config-serve, LocalDb sync) passes through untouched, so this can share one
+/// AddGrpc registration with and
+/// .
+///
+///
+/// Fail-closed. With no Telemetry:ApiKey configured, NO dial is accepted,
+/// authenticated or not. Treating "no key" as "no auth required" would expose the live
+/// telemetry feed on exactly the default shape most nodes ship with. The caller dialing in
+/// must present the same TelemetryDial:ApiKey (the shared node key), so a key typo
+/// stops telemetry delivery outright rather than degrading to unauthenticated.
+///
+///
+/// Comparison is over UTF-8 bytes, so a
+/// wrong key cannot be recovered byte-by-byte from response timing. Length differences are
+/// unavoidably observable and are not sensitive.
+///
+///
+/// Exactly one public constructor.Grpc.AspNetCore silently stops invoking an
+/// interceptor that has more than one public ctor — see
+/// TelemetryStreamAuthInterceptorTests.HasExactlyOnePublicConstructor.
+///
+///
+public sealed class TelemetryStreamAuthInterceptor : Interceptor
+{
+ private const string ServicePrefix = "/telemetry.v1.TelemetryStreamService/";
+ private const string AuthorizationHeader = "authorization";
+ private const string BearerPrefix = "Bearer ";
+
+ private readonly IOptions _options;
+ private readonly ILogger _logger;
+
+ /// Creates the interceptor.
+ /// Telemetry options; ApiKey is the expected bearer token.
+ /// Logger for denial diagnostics.
+ public TelemetryStreamAuthInterceptor(
+ IOptions options,
+ ILogger logger)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(logger);
+
+ _options = options;
+ _logger = logger;
+ }
+
+ ///
+ public override Task UnaryServerHandler(
+ TRequest request,
+ ServerCallContext context,
+ UnaryServerMethod continuation)
+ {
+ Authorize(context);
+ return continuation(request, context);
+ }
+
+ ///
+ public override Task DuplexStreamingServerHandler(
+ IAsyncStreamReader requestStream,
+ IServerStreamWriter responseStream,
+ ServerCallContext context,
+ DuplexStreamingServerMethod continuation)
+ {
+ Authorize(context);
+ return continuation(requestStream, responseStream, context);
+ }
+
+ ///
+ public override Task ClientStreamingServerHandler(
+ IAsyncStreamReader requestStream,
+ ServerCallContext context,
+ ClientStreamingServerMethod continuation)
+ {
+ Authorize(context);
+ return continuation(requestStream, context);
+ }
+
+ ///
+ public override Task ServerStreamingServerHandler(
+ TRequest request,
+ IServerStreamWriter responseStream,
+ ServerCallContext context,
+ ServerStreamingServerMethod continuation)
+ {
+ Authorize(context);
+ return continuation(request, responseStream, context);
+ }
+
+ ///
+ /// Throws with if this is
+ /// a telemetry-stream call that does not carry the configured bearer token. Non-telemetry
+ /// calls return immediately.
+ ///
+ private void Authorize(ServerCallContext context)
+ {
+ if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
+ return;
+
+ var expected = _options.Value.ApiKey;
+ if (string.IsNullOrEmpty(expected))
+ {
+ _logger.LogWarning(
+ "Rejected a telemetry stream call to {Method}: no Telemetry:ApiKey is configured, so " +
+ "the telemetry endpoint is closed. Configure the shared node key on this node and every " +
+ "dialing caller.",
+ context.Method);
+ throw new RpcException(new Status(
+ StatusCode.PermissionDenied,
+ "Telemetry stream is not accepting connections: no API key is configured on this node."));
+ }
+
+ var presented = ExtractBearerToken(context.RequestHeaders);
+ if (presented is null || !FixedTimeEquals(presented, expected))
+ {
+ _logger.LogWarning(
+ "Rejected a telemetry stream call to {Method}: {Reason}.",
+ context.Method,
+ presented is null ? "no bearer token presented" : "bearer token did not match");
+ throw new RpcException(new Status(
+ StatusCode.PermissionDenied,
+ "Telemetry stream authentication failed."));
+ }
+ }
+
+ private static string? ExtractBearerToken(Metadata headers)
+ {
+ // gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
+ // Metadata in a test behaves the same as a real request.
+ foreach (var entry in headers)
+ {
+ if (!string.Equals(entry.Key, AuthorizationHeader, StringComparison.OrdinalIgnoreCase))
+ continue;
+
+ var value = entry.Value;
+ if (value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase))
+ return value[BearerPrefix.Length..];
+ }
+
+ return null;
+ }
+
+ private static bool FixedTimeEquals(string presented, string expected)
+ => CryptographicOperations.FixedTimeEquals(
+ Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
+}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs
index 76d1265d..f7eaac28 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Host.Drivers;
@@ -23,22 +24,26 @@ public sealed class DriverResilienceStatusPublisherService : BackgroundService
private readonly DriverResilienceStatusTracker _tracker;
private readonly Func _actorSystemAccessor;
private readonly ILogger _logger;
+ private readonly ITelemetryLocalHub _hub;
private readonly TimeSpan _interval;
/// Initializes a new instance of .
/// The process-singleton resilience-status tracker to read each tick.
/// Lazy accessor for the Akka whose DPS mediator publishes the snapshots.
/// Logger for publish diagnostics.
+ /// The node-local live-telemetry hub each published snapshot is also emitted into (Phase 5).
/// Publish cadence; defaults to 5 s when null.
public DriverResilienceStatusPublisherService(
DriverResilienceStatusTracker tracker,
Func actorSystemAccessor,
ILogger logger,
+ ITelemetryLocalHub hub,
TimeSpan? interval = null)
{
_tracker = tracker;
_actorSystemAccessor = actorSystemAccessor;
_logger = logger;
+ _hub = hub;
_interval = interval ?? DefaultInterval;
}
@@ -77,7 +82,12 @@ public sealed class DriverResilienceStatusPublisherService : BackgroundService
var mediator = DistributedPubSub.Get(_actorSystemAccessor()).Mediator;
foreach (var message in messages)
+ {
mediator.Tell(new Publish(DriverResilienceStatusChanged.TopicName, message));
+ // Phase 5: fan the same snapshot into the node-local live-telemetry hub (no-op until a
+ // gRPC client subscribes). The DPS publish above is unchanged — a strictly additive tap.
+ _hub.Emit(new TelemetryItem.Resilience(message));
+ }
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryProtoMapNode.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryProtoMapNode.cs
new file mode 100644
index 00000000..8682a16f
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryProtoMapNode.cs
@@ -0,0 +1,171 @@
+using Google.Protobuf.WellKnownTypes;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
+using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Grpc;
+
+///
+/// Node-side projection of a domain onto its wire
+/// envelope (per-cluster mesh Phase 5). The exact reverse of
+/// telemetry.proto: it transcribes every domain field onto the matching proto field and
+/// selects the envelope's oneof arm from the subtype.
+///
+///
+///
+/// DateTime → Timestamp is Kind-defensive. throws
+/// unless the source is . Producers
+/// use today, but a stray or
+/// value must never crash the stream — so a Local value
+/// is converted and an Unspecified value is assumed already-UTC (matching the producer intent
+/// and preserving the instant on any machine).
+///
+///
+/// Nullable presence is preserved. A null nullable-DateTime leaves the proto
+/// Timestamp unset (absent == null per the contract); a null optional string leaves the
+/// proto field unset so the generated Has… presence distinguishes null from "".
+///
+///
+public static class TelemetryProtoMapNode
+{
+ /// Projects onto a wire .
+ /// The domain telemetry item to project.
+ /// Stream correlation id echoed on every envelope.
+ /// The wire envelope with the matching oneof arm populated.
+ public static TelemetryEvent ToProto(TelemetryItem item, string correlationId)
+ {
+ ArgumentNullException.ThrowIfNull(item);
+
+ return item switch
+ {
+ TelemetryItem.Alarm a => new TelemetryEvent
+ {
+ CorrelationId = correlationId,
+ AlarmTransition = MapAlarm(a.E),
+ },
+ TelemetryItem.Script s => new TelemetryEvent
+ {
+ CorrelationId = correlationId,
+ ScriptLog = MapScript(s.E),
+ },
+ TelemetryItem.Health h => new TelemetryEvent
+ {
+ CorrelationId = correlationId,
+ DriverHealth = MapHealth(h.E),
+ },
+ TelemetryItem.Resilience r => new TelemetryEvent
+ {
+ CorrelationId = correlationId,
+ DriverResilience = MapResilience(r.E),
+ },
+ _ => throw new ArgumentOutOfRangeException(
+ nameof(item), item.GetType().Name, "Unknown TelemetryItem subtype"),
+ };
+ }
+
+ private static AlarmTransition MapAlarm(AlarmTransitionEvent e)
+ {
+ // Required (non-optional) proto string setters throw ArgumentNullException on null, and the
+ // domain records carry no runtime null guard (nullable-ref annotations are compile-time only).
+ // Coalesce every required string to "" so a stray null can never throw out of the pump and kill
+ // the stream. The `optional` fields below keep their null-vs-"" presence guards.
+ var msg = new AlarmTransition
+ {
+ AlarmId = e.AlarmId ?? "",
+ EquipmentPath = e.EquipmentPath ?? "",
+ AlarmName = e.AlarmName ?? "",
+ TransitionKind = e.TransitionKind ?? "",
+ Severity = e.Severity,
+ Message = e.Message ?? "",
+ User = e.User ?? "",
+ TimestampUtc = ToUtcTimestamp(e.TimestampUtc),
+ AlarmTypeName = e.AlarmTypeName ?? "",
+ };
+
+ if (e.Comment is not null)
+ msg.Comment = e.Comment;
+ if (e.HistorizeToAveva is not null)
+ msg.HistorizeToAveva = e.HistorizeToAveva.Value;
+ msg.ReferencingEquipmentPaths.AddRange(e.ReferencingEquipmentPaths ?? Enumerable.Empty());
+
+ return msg;
+ }
+
+ private static ScriptLog MapScript(ScriptLogEntry e)
+ {
+ var msg = new ScriptLog
+ {
+ ScriptId = e.ScriptId ?? "",
+ Level = e.Level ?? "",
+ Message = e.Message ?? "",
+ TimestampUtc = ToUtcTimestamp(e.TimestampUtc),
+ };
+
+ if (e.VirtualTagId is not null)
+ msg.VirtualTagId = e.VirtualTagId;
+ if (e.AlarmId is not null)
+ msg.AlarmId = e.AlarmId;
+ if (e.EquipmentId is not null)
+ msg.EquipmentId = e.EquipmentId;
+
+ return msg;
+ }
+
+ private static DriverHealth MapHealth(DriverHealthChanged e)
+ {
+ var msg = new DriverHealth
+ {
+ ClusterId = e.ClusterId ?? "",
+ DriverInstanceId = e.DriverInstanceId ?? "",
+ State = e.State ?? "",
+ ErrorCount5Min = e.ErrorCount5Min,
+ PublishedUtc = ToUtcTimestamp(e.PublishedUtc),
+ };
+
+ if (e.LastSuccessfulReadUtc is not null)
+ msg.LastSuccessfulReadUtc = ToUtcTimestamp(e.LastSuccessfulReadUtc.Value);
+ if (e.LastError is not null)
+ msg.LastError = e.LastError;
+
+ return msg;
+ }
+
+ private static DriverResilienceStatus MapResilience(DriverResilienceStatusChanged e)
+ {
+ var msg = new DriverResilienceStatus
+ {
+ DriverInstanceId = e.DriverInstanceId ?? "",
+ HostName = e.HostName ?? "",
+ BreakerOpen = e.BreakerOpen,
+ ConsecutiveFailures = e.ConsecutiveFailures,
+ CurrentInFlight = e.CurrentInFlight,
+ LastSampledUtc = ToUtcTimestamp(e.LastSampledUtc),
+ PublishedUtc = ToUtcTimestamp(e.PublishedUtc),
+ };
+
+ if (e.LastBreakerOpenUtc is not null)
+ msg.LastBreakerOpenUtc = ToUtcTimestamp(e.LastBreakerOpenUtc.Value);
+
+ return msg;
+ }
+
+ ///
+ /// Converts a domain to a proto without ever
+ /// throwing on Kind: Utc is used as-is, Local is converted, and Unspecified is assumed
+ /// already-UTC (the producer contract — — with the wall-clock
+ /// ticks preserved so the instant round-trips).
+ ///
+ private static Timestamp ToUtcTimestamp(DateTime dt)
+ {
+ var utc = dt.Kind switch
+ {
+ DateTimeKind.Utc => dt,
+ DateTimeKind.Local => dt.ToUniversalTime(),
+ _ => DateTime.SpecifyKind(dt, DateTimeKind.Utc),
+ };
+
+ return Timestamp.FromDateTime(utc);
+ }
+}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryStreamGrpcService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryStreamGrpcService.cs
new file mode 100644
index 00000000..b2e7c634
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/TelemetryStreamGrpcService.cs
@@ -0,0 +1,123 @@
+using Grpc.Core;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
+// The generated service container is itself named TelemetryStreamService; alias its nested server
+// base so this impl can keep the natural name without colliding with the generated type.
+using GeneratedServiceBase =
+ ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1.TelemetryStreamService.TelemetryStreamServiceBase;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Grpc;
+
+///
+/// Node-side live-telemetry streaming server (per-cluster mesh Phase 5). Central dials a driver
+/// node and opens ; the node attaches a fresh reader on its
+/// and pumps this node's own telemetry
+/// (alarm-transitions / script-logs / driver-health / driver-resilience) to the caller as a
+/// sequence of envelopes until the client cancels or the
+/// max-lifetime cap fires.
+///
+///
+/// The hub carries ONLY this node's telemetry, so a per-node stream never double-counts a peer's
+/// events. Fan-out is lossy under backpressure by design (the per-subscriber channel is
+/// bounded/DropOldest inside the hub). A simple process-wide concurrency cap sheds excess dials
+/// with , and every accepted stream is bounded by a
+/// linked max-lifetime CTS.
+///
+public sealed class TelemetryStreamGrpcService : GeneratedServiceBase
+{
+ /// Process-wide cap on concurrently-open telemetry streams.
+ private const int MaxConcurrentStreams = 100;
+
+ /// Upper bound on an accepted correlation id (defends against absurd inputs).
+ private const int MaxCorrelationIdLength = 256;
+
+ /// Per-subscriber bounded-channel capacity requested from the hub.
+ private const int SubscriptionCapacity = 1000;
+
+ /// Hard ceiling on a single stream's lifetime; the linked CTS cancels the pump when it fires.
+ private static readonly TimeSpan MaxStreamLifetime = TimeSpan.FromHours(4);
+
+ /// Process-wide count of currently-open streams; mutated only via .
+ private static int _activeStreams;
+
+ private readonly ITelemetryLocalHub _hub;
+ private readonly ILogger _log;
+
+ /// Initializes a new instance of the class.
+ /// The node-local telemetry fan-out hub this stream drains.
+ /// Logger.
+ public TelemetryStreamGrpcService(ITelemetryLocalHub hub, ILogger log)
+ {
+ _hub = hub;
+ _log = log;
+ }
+
+ ///
+ public override async Task Subscribe(
+ TelemetryStreamRequest request,
+ IServerStreamWriter responseStream,
+ ServerCallContext context)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ ArgumentNullException.ThrowIfNull(responseStream);
+ ArgumentNullException.ThrowIfNull(context);
+
+ var correlationId = request.CorrelationId;
+ if (string.IsNullOrWhiteSpace(correlationId) || correlationId.Length > MaxCorrelationIdLength)
+ throw new RpcException(new Status(
+ StatusCode.InvalidArgument,
+ "correlation_id must be non-empty and at most 256 characters"));
+
+ // Concurrency cap. The increment is ALWAYS paired: on rejection we decrement here and throw
+ // before entering the try; on acceptance the finally decrements exactly once. No path
+ // decrements without a matching successful increment, and no path double-decrements.
+ var active = Interlocked.Increment(ref _activeStreams);
+ if (active > MaxConcurrentStreams)
+ {
+ Interlocked.Decrement(ref _activeStreams);
+ _log.LogWarning(
+ "Telemetry stream rejected (correlationId={CorrelationId}): {Active} open streams exceeds cap {Cap}",
+ correlationId, active - 1, MaxConcurrentStreams);
+ throw new RpcException(new Status(
+ StatusCode.ResourceExhausted, "too many concurrent telemetry streams"));
+ }
+
+ try
+ {
+ using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
+ lifetime.CancelAfter(MaxStreamLifetime);
+ using var sub = _hub.Subscribe(SubscriptionCapacity);
+
+ _log.LogDebug("Telemetry stream opened (correlationId={CorrelationId})", correlationId);
+
+ try
+ {
+ await foreach (var item in sub.Reader.ReadAllAsync(lifetime.Token).ConfigureAwait(false))
+ await responseStream
+ .WriteAsync(TelemetryProtoMapNode.ToProto(item, correlationId), lifetime.Token)
+ .ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ // Normal termination: the client disconnected (context token) or the max-lifetime cap
+ // fired (linked token). Neither is an error — swallow and let the stream close cleanly.
+ }
+ catch (Exception ex) when (ex is IOException or RpcException)
+ {
+ // Routine mid-stream disconnect: central's TelemetryStreamClient reconnects on ANY
+ // non-Cancelled error, so a broken pipe / connection-reset here is expected churn, not a
+ // fault. Swallow at Debug so grpc-dotnet doesn't log it at Error on every reconnect.
+ // (A genuinely unexpected exception still escapes to surface as an error.)
+ _log.LogDebug(
+ ex, "Telemetry client disconnected mid-stream (correlationId={CorrelationId})", correlationId);
+ }
+
+ _log.LogDebug("Telemetry stream closed (correlationId={CorrelationId})", correlationId);
+ }
+ finally
+ {
+ Interlocked.Decrement(ref _activeStreams);
+ }
+ }
+}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs
index 3204646c..8853dca4 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs
@@ -22,6 +22,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Host;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Drivers;
+using ZB.MOM.WW.OtOpcUa.Host.Grpc;
using ZB.MOM.WW.OtOpcUa.Host.Engines;
using ZB.MOM.WW.OtOpcUa.Host.Health;
using ZB.MOM.WW.OtOpcUa.Host.Logging;
@@ -34,6 +35,7 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using ZB.MOM.WW.OtOpcUa.Runtime.Scripting;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
using ZB.MOM.WW.OtOpcUa.Runtime;
using ZB.MOM.WW.Auth.Abstractions.Roles;
@@ -310,7 +312,9 @@ if (hasDriver)
? parsedLevel
: LogEventLevel.Information;
builder.Services.AddSingleton(sp =>
- new DpsScriptLogPublisher(() => sp.GetRequiredService()));
+ new DpsScriptLogPublisher(
+ () => sp.GetRequiredService(),
+ sp.GetRequiredService()));
builder.Services.AddSingleton(sp => new ScriptRootLogger(
ScriptRootLoggerFactory.Build(
sp.GetRequiredService(), scriptLogFilePath, scriptLogTopicMinLevel, Serilog.Log.Logger)));
@@ -413,46 +417,54 @@ builder.Services.AddOtOpcUaSecrets(builder.Configuration);
builder.Services.AddOtOpcUaHealth(hasAdmin);
builder.Services.AddOtOpcUaObservability(builder.Configuration);
-// gRPC server plumbing shared by two endpoints: the LocalDb passive sync endpoint (driver-role,
-// mapped below on hasDriver && syncListenPort > 0) and the ConfigServe artifact endpoint
-// (admin-role, mapped on hasAdmin && configServeGrpcPort > 0). Each interceptor is scoped strictly
-// to its own service path, so both can share one pipeline and each is a harmless pass-through when
-// its service is unmapped. Both fail-closed with no ApiKey configured — the interceptor is the ONLY
-// inbound auth on either endpoint (the libraries verify nothing). Registered whenever either role is
-// present so an admin-only node still serves config and a driver-only node still gates sync.
+// gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,
+// mapped below on hasDriver && syncListenPort > 0), the ConfigServe artifact endpoint (admin-role,
+// mapped on hasAdmin && configServeGrpcPort > 0), and the telemetry-stream endpoint (driver-role,
+// mapped on hasDriver && telemetryListenPort > 0, per-cluster mesh Phase 5). Each interceptor is
+// scoped strictly to its own service path, so all three can share one pipeline and each is a
+// harmless pass-through when its service is unmapped. All three fail-closed with no ApiKey
+// configured (LocalDbSyncAuthInterceptor, ConfigServeAuthInterceptor, TelemetryStreamAuthInterceptor)
+// — the interceptor is the ONLY inbound auth on any endpoint (the libraries verify nothing).
+// Registered whenever either role is present so an admin-only node still serves config and a
+// driver-only node still gates sync + telemetry.
if (hasDriver || hasAdmin)
{
builder.Services.AddGrpc(o =>
{
if (hasDriver)
+ {
o.Interceptors.Add();
+ o.Interceptors.Add();
+ }
if (hasAdmin)
o.Interceptors.Add();
});
}
// ---------------------------------------------------------------------------------------------
-// Dedicated h2c listeners (both default-OFF): the LocalDb sync endpoint (driver) and the
-// ConfigServe artifact endpoint (admin, per-cluster mesh Phase 3).
+// Dedicated h2c listeners (all default-OFF): the LocalDb sync endpoint (driver), the ConfigServe
+// artifact endpoint (admin, per-cluster mesh Phase 3), and the telemetry-stream endpoint (driver,
+// per-cluster mesh Phase 5).
//
// DANGER: any explicit Kestrel Listen* call makes Kestrel IGNORE ASPNETCORE_URLS/urls ENTIRELY
// (it logs "Overriding address(es)"). The host has no ConfigureKestrel today and binds solely via
// that configuration, so adding a listener naively would silently unbind the AdminUI + deploy API
// behind Traefik. Everything the host was already asked to serve is therefore re-bound explicitly
-// in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have BOTH
+// in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have ALL THREE
// dedicated ports set, and re-applying the existing surface per-port would double-bind it and throw
// "address already in use". So the existing surface is computed and applied once, then whichever of
-// the two dedicated ports are configured are added on top.
+// the three dedicated ports are configured are added on top.
//
-// Each listener is HTTP/2-ONLY on purpose: both clients speak prior-knowledge h2c, which a cleartext
+// Each listener is HTTP/2-ONLY on purpose: every client speaks prior-knowledge h2c, which a cleartext
// Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence dedicated ports
// rather than multiplexing onto the main one.
//
-// When both ports are 0 (the default) none of this runs and URL binding is untouched.
+// When all three ports are 0 (the default) none of this runs and URL binding is untouched.
// ---------------------------------------------------------------------------------------------
var syncListenPort = hasDriver ? LocalDbRegistration.SyncListenPort(builder.Configuration) : 0;
var configServeGrpcPort = hasAdmin ? builder.Configuration.GetValue("ConfigServe:GrpcListenPort") : 0;
-if (syncListenPort > 0 || configServeGrpcPort > 0)
+var telemetryListenPort = hasDriver ? builder.Configuration.GetValue("Telemetry:GrpcListenPort") : 0;
+if (syncListenPort > 0 || configServeGrpcPort > 0 || telemetryListenPort > 0)
{
// Mirror Kestrel's own source precedence: URLS wins; else the HTTP_PORTS/HTTPS_PORTS bare-port
// vars; else Kestrel's localhost:5000 default. Missing the HTTP_PORTS leg is not academic — the
@@ -501,19 +513,22 @@ if (syncListenPort > 0 || configServeGrpcPort > 0)
{
Log.Error(
"A dedicated h2c listener is requested (LocalDb:SyncListenPort={SyncPort}, " +
- "ConfigServe:GrpcListenPort={ConfigServePort}) but this host serves HTTPS endpoint(s) ({Urls}). " +
+ "ConfigServe:GrpcListenPort={ConfigServePort}, Telemetry:GrpcListenPort={TelemetryPort}) but this " +
+ "host serves HTTPS endpoint(s) ({Urls}). " +
"Binding a dedicated listener requires re-binding every existing endpoint explicitly, and the " +
- "HTTPS certificate configuration cannot be replayed safely. BOTH dedicated listeners are DISABLED; " +
+ "HTTPS certificate configuration cannot be replayed safely. ALL dedicated listeners are DISABLED; " +
"terminate TLS upstream (as the docker-dev rig does) or leave these features off on this node.",
- syncListenPort, configServeGrpcPort, configuredUrls);
+ syncListenPort, configServeGrpcPort, telemetryListenPort, configuredUrls);
syncListenPort = 0;
configServeGrpcPort = 0;
+ telemetryListenPort = 0;
}
else
{
// Capture locals so the ConfigureKestrel closure does not observe later reassignment.
var syncPortToBind = syncListenPort;
var configServePortToBind = configServeGrpcPort;
+ var telemetryPortToBind = telemetryListenPort;
builder.WebHost.ConfigureKestrel(kestrel =>
{
foreach (var binding in existingBindings)
@@ -523,12 +538,15 @@ if (syncListenPort > 0 || configServeGrpcPort > 0)
kestrel.ListenAnyIP(syncPortToBind, o => o.Protocols = HttpProtocols.Http2);
if (configServePortToBind > 0)
kestrel.ListenAnyIP(configServePortToBind, o => o.Protocols = HttpProtocols.Http2);
+ if (telemetryPortToBind > 0)
+ kestrel.ListenAnyIP(telemetryPortToBind, o => o.Protocols = HttpProtocols.Http2);
});
Log.Information(
- "Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}); " +
- "re-bound existing endpoint(s) {Urls}.",
- syncListenPort, configServeGrpcPort, configuredUrls ?? "http://localhost:5000 (Kestrel default)");
+ "Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}, " +
+ "telemetry=:{TelemetryPort}); re-bound existing endpoint(s) {Urls}.",
+ syncListenPort, configServeGrpcPort, telemetryListenPort,
+ configuredUrls ?? "http://localhost:5000 (Kestrel default)");
}
}
@@ -579,6 +597,13 @@ if (hasAdmin && configServeGrpcPort > 0)
app.MapGrpcService();
}
+// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
+// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
+if (hasDriver && telemetryListenPort > 0)
+{
+ app.MapGrpcService();
+}
+
app.MapOtOpcUaHealth();
app.MapOtOpcUaMetrics();
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/AkkaDriverHealthPublisher.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/AkkaDriverHealthPublisher.cs
index 4f8e7a59..6f93d39f 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/AkkaDriverHealthPublisher.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/AkkaDriverHealthPublisher.cs
@@ -2,6 +2,7 @@ using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
@@ -17,10 +18,16 @@ public sealed class AkkaDriverHealthPublisher : IDriverHealthPublisher
public const string TopicName = DriverHealthChanged.TopicName;
private readonly ActorSystem _system;
+ private readonly ITelemetryLocalHub _hub;
/// Initializes a new instance of .
/// The Akka actor system used to resolve the DPS mediator.
- public AkkaDriverHealthPublisher(ActorSystem system) => _system = system;
+ /// The node-local live-telemetry hub each snapshot is also emitted into (Phase 5).
+ public AkkaDriverHealthPublisher(ActorSystem system, ITelemetryLocalHub hub)
+ {
+ _system = system;
+ _hub = hub;
+ }
///
public void Publish(string clusterId, string driverInstanceId, DriverHealth health, int errorCount5Min)
@@ -34,5 +41,8 @@ public sealed class AkkaDriverHealthPublisher : IDriverHealthPublisher
errorCount5Min,
DateTime.UtcNow);
DistributedPubSub.Get(_system).Mediator.Tell(new Publish(TopicName, msg));
+ // Phase 5: fan the same snapshot into the node-local live-telemetry hub (no-op until a gRPC
+ // client subscribes). The DPS publish above is unchanged — the hub is a strictly additive tap.
+ _hub.Emit(new TelemetryItem.Health(msg));
}
}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
index bc48abb2..c1025d56 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
@@ -26,6 +26,7 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
@@ -118,6 +119,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
///
private readonly IAlarmStateStore? _alarmStateStore;
+ /// Optional node-local live-telemetry hub (Phase 5): each Primary-gated native-alarm
+ /// alerts transition is also emitted here, and it is threaded into the spawned VirtualTag +
+ /// ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.
+ private readonly ITelemetryLocalHub? _telemetryHub;
+
private readonly CommonsNodeId _localNode;
private readonly IActorRef? _ackRouter;
private readonly IDriverFactory _driverFactory;
@@ -397,6 +403,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// driver-role node this is the replicated LocalDb store, so condition state persists with no
/// ConfigDb; null skips spawning the alarm host (the ConfigDb-backed EF fallback was retired in
/// Phase 4 Task 9). Defaults to null.
+ /// Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
+ /// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
+ /// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).
/// The Akka.NET used to spawn this actor.
public static Props Props(
IDbContextFactory? dbFactory,
@@ -420,7 +429,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
- IAlarmStateStore? alarmStateStore = null) =>
+ IAlarmStateStore? alarmStateStore = null,
+ ITelemetryLocalHub? telemetryHub = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -431,7 +441,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
- alarmStateStore));
+ alarmStateStore, telemetryHub));
/// Initializes a new DriverHostActor with the specified dependencies.
/// Database context factory for configuration database access.
@@ -469,6 +479,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// Per-cluster mesh Phase 4: scripted-alarm condition-state store. On a
/// driver-role node this is the replicated LocalDb store; null skips spawning the alarm host (the
/// ConfigDb-backed EF fallback was retired in Phase 4 Task 9).
+ /// Per-cluster mesh Phase 5: optional node-local live-telemetry hub each
+ /// Primary-gated native-alarm transition is also emitted into, and which is threaded into the spawned
+ /// VirtualTag + ScriptedAlarm hosts. Null (admin-only nodes / tests) skips the emit and passes null down.
public DriverHostActor(
IDbContextFactory? dbFactory,
CommonsNodeId localNode,
@@ -491,8 +504,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
string? replicationPeerHost = null,
bool fetchAndCacheMode = false,
IDeploymentArtifactFetcher? artifactFetcher = null,
- IAlarmStateStore? alarmStateStore = null)
+ IAlarmStateStore? alarmStateStore = null,
+ ITelemetryLocalHub? telemetryHub = null)
{
+ _telemetryHub = telemetryHub;
_deploymentArtifactCache = deploymentArtifactCache;
_redundancyRoleView = redundancyRoleView;
_replicationPeerHost = replicationPeerHost;
@@ -577,7 +592,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
}
_virtualTagHost = Context.ActorOf(
- VirtualTagHostActor.Props(_opcUaPublishActor, _dependencyMux, _virtualTagEvaluator, _historyWriter),
+ VirtualTagHostActor.Props(_opcUaPublishActor, _dependencyMux, _virtualTagEvaluator, _historyWriter, _telemetryHub),
"virtual-tag-host");
}
@@ -630,7 +645,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
var engine = new ScriptedAlarmEngine(
upstream, store, new ScriptLoggerFactory(_scriptRootLogger.Logger), _scriptRootLogger.Logger);
_scriptedAlarmHost = Context.ActorOf(
- ScriptedAlarmHostActor.Props(_opcUaPublishActor, _dependencyMux, upstream, engine, _localNode),
+ ScriptedAlarmHostActor.Props(
+ _opcUaPublishActor, _dependencyMux, upstream, engine, _localNode, telemetryHub: _telemetryHub),
"scripted-alarm-host");
}
@@ -1402,7 +1418,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null,
ReferencingEquipmentPaths: (IReadOnlyList)Array.Empty());
- _mediator.Tell(new Publish(ScriptedAlarmHostActor.AlertsTopic, new AlarmTransitionEvent(
+ var alertEvent = new AlarmTransitionEvent(
AlarmId: nodeId,
EquipmentPath: meta.EquipmentId,
AlarmName: meta.Name,
@@ -1424,7 +1440,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
HistorizeToAveva: meta.HistorizeToAveva,
// WP4: the Area/Line/Equipment UNS paths that reference this raw condition — the /alerts row
// renders them as the equipment list (empty for a raw condition no equipment references yet).
- ReferencingEquipmentPaths: meta.ReferencingEquipmentPaths)));
+ ReferencingEquipmentPaths: meta.ReferencingEquipmentPaths);
+ _mediator.Tell(new Publish(ScriptedAlarmHostActor.AlertsTopic, alertEvent));
+ // Phase 5: fan the same transition into the node-local live-telemetry hub (no-op until a gRPC
+ // client subscribes). The DPS publish above is unchanged — a strictly additive tap, riding the
+ // same Primary gate (only reached when serviceAlertsAsPrimary).
+ _telemetryHub?.Emit(new TelemetryItem.Alarm(alertEvent));
}
}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs
index 7b538c66..8f7c2074 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs
@@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
@@ -116,6 +117,10 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// alerts-emit gate reads while the role is unknown. Null ⇒ read the live cluster state.
private readonly Func? _driverMemberCountProvider;
+ /// Optional node-local live-telemetry hub (Phase 5): each cluster-wide alerts transition
+ /// is also emitted here for the gRPC streaming service. Null (tests / no hub wired) skips the emit.
+ private readonly ITelemetryLocalHub? _telemetryHub;
+
/// Monotonic load generation, bumped on every . The continuation that
/// pipes back an captures the generation it was started under; a stale
/// completion (an earlier generation arriving after a newer apply) is discarded in
@@ -138,6 +143,10 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// The local cluster node id, used to read this node's
/// from the redundancy-state topic so only the Primary publishes the cluster-wide alerts
/// transition. Null (the default) leaves the role unknown ⇒ default-emit (single-node deploys + tests).
+ /// Test seam (archreview 03/S4): overrides the count of Up
+ /// driver-role cluster members the alerts-emit gate reads while the role is unknown.
+ /// Optional node-local live-telemetry hub (Phase 5) each cluster-wide
+ /// alerts transition is also emitted into; null (tests) skips the emit.
/// The used to instantiate the actor.
public static Props Props(
IActorRef publishActor,
@@ -145,8 +154,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
DependencyMuxTagUpstreamSource upstream,
ScriptedAlarmEngine engine,
NodeId? localNode = null,
- Func? driverMemberCountProvider = null) =>
- Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider));
+ Func? driverMemberCountProvider = null,
+ ITelemetryLocalHub? telemetryHub = null) =>
+ Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));
/// Initializes a new instance of the class.
/// The OPC UA publish actor emissions are bridged to.
@@ -159,13 +169,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// Test seam (archreview 03/S4): overrides the count of Up
/// driver-role cluster members the alerts-emit gate reads while the role is unknown. Null reads
/// the live cluster state (0 on a non-cluster ActorRefProvider ⇒ single-node default-emit).
+ /// Optional node-local live-telemetry hub (Phase 5) each cluster-wide
+ /// alerts transition is also emitted into; null (tests) skips the emit.
public ScriptedAlarmHostActor(
IActorRef publishActor,
IActorRef? mux,
DependencyMuxTagUpstreamSource upstream,
ScriptedAlarmEngine engine,
NodeId? localNode = null,
- Func? driverMemberCountProvider = null)
+ Func? driverMemberCountProvider = null,
+ ITelemetryLocalHub? telemetryHub = null)
{
ArgumentNullException.ThrowIfNull(publishActor);
ArgumentNullException.ThrowIfNull(upstream);
@@ -176,6 +189,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
_engine = engine;
_localNode = localNode;
_driverMemberCountProvider = driverMemberCountProvider;
+ _telemetryHub = telemetryHub;
// OnEvent fires on the engine's worker thread. NEVER touch Context / actor state here —
// marshal onto the actor thread via the thread-safe Self.Tell. Keep the handler in a field
@@ -377,6 +391,10 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
}
_mediator.Tell(new Publish(AlertsTopic, evt));
+ // Phase 5: fan the same transition into the node-local live-telemetry hub (no-op until a gRPC
+ // client subscribes). The DPS publish above is unchanged — a strictly additive tap, and it rides
+ // the same Primary gate as the publish (only the Primary reaches this line).
+ _telemetryHub?.Emit(new TelemetryItem.Alarm(evt));
}
/// Count of Up cluster members carrying the driver role, for the alerts-emit gate. Uses
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Scripting/DpsScriptLogPublisher.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Scripting/DpsScriptLogPublisher.cs
index fa77729c..e5bba132 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Scripting/DpsScriptLogPublisher.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Scripting/DpsScriptLogPublisher.cs
@@ -2,6 +2,7 @@ using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Scripting;
@@ -27,15 +28,20 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Scripting;
public sealed class DpsScriptLogPublisher : IScriptLogPublisher
{
private readonly Func _system;
+ private readonly ITelemetryLocalHub _hub;
/// Initializes a new instance of the class.
///
/// Lazy accessor for the running . Invoked on each
/// so registration does not depend on Akka having started yet.
///
- /// Thrown when is null.
- public DpsScriptLogPublisher(Func system) =>
+ /// The node-local live-telemetry hub each script-log entry is also emitted into (Phase 5).
+ /// Thrown when or is null.
+ public DpsScriptLogPublisher(Func system, ITelemetryLocalHub hub)
+ {
_system = system ?? throw new ArgumentNullException(nameof(system));
+ _hub = hub ?? throw new ArgumentNullException(nameof(hub));
+ }
///
public void Publish(ScriptLogEntry entry)
@@ -44,6 +50,9 @@ public sealed class DpsScriptLogPublisher : IScriptLogPublisher
{
var mediator = DistributedPubSub.Get(_system()).Mediator;
mediator.Tell(new Publish(VirtualTagActor.ScriptLogsTopic, entry));
+ // Phase 5: fan the same entry into the node-local live-telemetry hub (no-op until a gRPC
+ // client subscribes). The DPS publish above is unchanged — the hub is a strictly additive tap.
+ _hub.Emit(new TelemetryItem.Script(entry));
}
catch (Exception ex)
{
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs
index 477a6344..062ff3ad 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs
@@ -28,6 +28,7 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using ZB.MOM.WW.LocalDb;
@@ -70,6 +71,10 @@ public static class ServiceCollectionExtensions
services.TryAddSingleton(NullOpcUaAddressSpaceSink.Instance);
services.TryAddSingleton(NullServiceLevelPublisher.Instance);
services.TryAddSingleton();
+ // Per-cluster mesh Phase 5: the node-local live-telemetry fan-out hub. Feeds this node's OWN
+ // telemetry (never DPS) to the Phase-5 gRPC streaming service central dials in on. AddOtOpcUaRuntime
+ // runs only inside Program.cs's hasDriver block, so this lands on driver-role nodes only.
+ services.TryAddSingleton();
return services;
}
@@ -490,7 +495,11 @@ public static class ServiceCollectionExtensions
replicationPeerHost: replicationPeerHost,
fetchAndCacheMode: fetchAndCacheMode,
artifactFetcher: artifactFetcher,
- alarmStateStore: alarmStateStore),
+ alarmStateStore: alarmStateStore,
+ // Phase 5: the node-local live-telemetry hub (registered in AddOtOpcUaRuntime). The host
+ // emits its Primary-gated native-alarm transitions into it and threads it into the
+ // VirtualTag + ScriptedAlarm hosts it spawns.
+ telemetryHub: resolver.GetService()),
DriverHostActorName);
registry.Register(driverHost);
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/ITelemetryLocalHub.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/ITelemetryLocalHub.cs
new file mode 100644
index 00000000..036b6f3c
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/ITelemetryLocalHub.cs
@@ -0,0 +1,87 @@
+using System.Threading.Channels;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
+
+///
+/// A single live-telemetry item flowing through the node-local hub. A closed union over the four
+/// wire-facing telemetry records — carried as the domain records themselves, NOT as proto (the
+/// gRPC projection happens at the streaming-service seam, one layer out).
+///
+///
+/// Two of the four are snapshot-style (a last-value-per-key view: keyed
+/// by DriverInstanceId, keyed by (DriverInstanceId, HostName))
+/// and are cached so a newly-attaching subscriber is primed immediately. The other two
+/// (, ) are append-style logs and are forwarded live-only.
+///
+public abstract record TelemetryItem
+{
+ private TelemetryItem() { }
+
+ /// An append-style alarm transition. Live-forwarded only; never cached/replayed.
+ public sealed record Alarm(AlarmTransitionEvent E) : TelemetryItem;
+
+ /// An append-style script-log line. Live-forwarded only; never cached/replayed.
+ public sealed record Script(ScriptLogEntry E) : TelemetryItem;
+
+ /// A snapshot-style driver-health change, cached last-value per DriverInstanceId.
+ public sealed record Health(DriverHealthChanged E) : TelemetryItem;
+
+ /// A snapshot-style resilience-status change, cached last-value per (DriverInstanceId, HostName).
+ public sealed record Resilience(DriverResilienceStatusChanged E) : TelemetryItem;
+}
+
+///
+/// One attached reader on the . Disposing detaches it from the hub
+/// and completes its channel; the reader first yields the cached snapshots, then live deltas.
+///
+public interface ITelemetrySubscription : IDisposable
+{
+ /// The bounded reader this subscription drains. Completed on .
+ ChannelReader Reader { get; }
+}
+
+///
+/// Process-wide, node-local fan-out hub sitting between this node's own telemetry producers and the
+/// (Phase-5) gRPC streaming service central dials into.
+///
+///
+///
+/// CRITICAL INVARIANT — this hub carries ONLY this node's own telemetry. It is fed
+/// directly, in-process, by the node's producers and MUST NEVER subscribe to cluster
+/// DistributedPubSub. On the single mesh, DPS delivers every node's events to every node; if the
+/// hub read DPS it would stream peers' events and central — which dials each node individually —
+/// would double-count.
+///
+///
+/// Fan-out is lossy under backpressure by design: each subscriber owns a bounded channel with
+/// , so a slow consumer sheds its own oldest items
+/// and can never block the producer / node-actor threads.
+///
+///
+public interface ITelemetryLocalHub
+{
+ ///
+ /// Fans to every currently-attached subscriber (non-blocking, drop-oldest
+ /// on a full channel) and, for the two snapshot-style items, updates the last-value cache that
+ /// primes future subscribers.
+ ///
+ void Emit(TelemetryItem item);
+
+ ///
+ /// Attaches a new subscriber. Its reader first yields the cached snapshots (all cached
+ /// , then all cached ),
+ /// then live deltas — with no delta lost across the attach boundary. Dispose to detach.
+ ///
+ ///
+ /// Per-subscriber channel capacity (must be positive). The snapshot prelude is written into this
+ /// same bounded/DropOldest channel before returns, so
+ /// should comfortably exceed this node's live driver-instance +
+ /// (instance, host) count — otherwise the earliest cached snapshots are silently evicted during
+ /// priming before the reader ever drains them. The node-local cache holds only THIS node's
+ /// instances (a handful), so a normal large capacity leaves ample headroom.
+ ///
+ ITelemetrySubscription Subscribe(int boundedCapacity);
+}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs
new file mode 100644
index 00000000..aa70c434
--- /dev/null
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Telemetry/TelemetryLocalHub.cs
@@ -0,0 +1,129 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
+
+///
+/// Default : a process-wide singleton on driver-role nodes.
+///
+///
+///
+/// No DPS. This hub is fed only by this node's own in-process producers (a later Phase-5
+/// task taps them). It never subscribes to cluster DistributedPubSub — see the invariant on
+/// .
+///
+///
+/// Snapshot-then-attach ordering (no lost delta). updates the snapshot
+/// cache and fans out under a single gate; writes the cached snapshots
+/// into the new channel and only THEN adds it to the subscriber set — all under that same gate.
+/// Because both operations are serialized by the gate, an racing a
+/// is resolved one of two ways, both correct: it runs entirely before the
+/// subscribe (its value is in the snapshot, and it is not delivered live because the channel is
+/// not yet attached), or entirely after (it is delivered live, strictly after the snapshot). No
+/// item is dropped or duplicated across the boundary. Fan-out is a non-blocking
+/// into bounded/DropOldest channels, so holding the gate
+/// never blocks on a slow consumer.
+///
+///
+public sealed class TelemetryLocalHub : ITelemetryLocalHub
+{
+ private readonly object _gate = new();
+
+ // Plain Dictionary, NOT ConcurrentDictionary: every access is already under _gate, and
+ // ConcurrentDictionary.Values would materialize a fresh List (and take its own internal lock) on
+ // every Emit — the hot node-actor-thread path. Under the gate a plain Dictionary is safe and
+ // allocation-free to iterate.
+ private readonly Dictionary> _subscribers = new();
+
+ // Snapshot last-value caches. Only ever mutated under _gate (kept ConcurrentDictionary so a
+ // Subscribe reading them under the gate is trivially safe even against any future lock-free read).
+ // TODO(mesh-phase5+): evict snapshot cache entries on driver-instance removal (needs a
+ // driver-lifecycle hook); pre-production, accepted for now. Until then a decommissioned driver
+ // instance's last-known Health/Resilience state replays to new subscribers forever.
+ private readonly ConcurrentDictionary _healthCache = new();
+ private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
+
+ ///
+ public void Emit(TelemetryItem item)
+ {
+ ArgumentNullException.ThrowIfNull(item);
+
+ lock (_gate)
+ {
+ switch (item)
+ {
+ case TelemetryItem.Health h:
+ _healthCache[h.E.DriverInstanceId] = h;
+ break;
+ case TelemetryItem.Resilience r:
+ _resilienceCache[(r.E.DriverInstanceId, r.E.HostName)] = r;
+ break;
+ // Alarm / Script are append-style — never cached.
+ }
+
+ foreach (var channel in _subscribers.Values)
+ channel.Writer.TryWrite(item); // bounded + DropOldest ⇒ non-blocking, sheds oldest when full
+ }
+ }
+
+ ///
+ public ITelemetrySubscription Subscribe(int boundedCapacity)
+ {
+ if (boundedCapacity < 1)
+ throw new ArgumentOutOfRangeException(nameof(boundedCapacity), boundedCapacity, "Capacity must be positive.");
+
+ var channel = Channel.CreateBounded(new BoundedChannelOptions(boundedCapacity)
+ {
+ FullMode = BoundedChannelFullMode.DropOldest,
+ SingleReader = true,
+ SingleWriter = false,
+ });
+
+ var id = Guid.NewGuid();
+ lock (_gate)
+ {
+ // Snapshot FIRST (Health then Resilience), attach SECOND — both under the gate so a
+ // concurrent Emit cannot slip a delta between the snapshot read and the attach.
+ foreach (var health in _healthCache.Values)
+ channel.Writer.TryWrite(health);
+ foreach (var resilience in _resilienceCache.Values)
+ channel.Writer.TryWrite(resilience);
+
+ _subscribers[id] = channel;
+ }
+
+ return new Subscription(this, id, channel);
+ }
+
+ private void Detach(Guid id)
+ {
+ lock (_gate)
+ {
+ if (_subscribers.Remove(id, out var channel))
+ channel.Writer.TryComplete();
+ }
+ }
+
+ private sealed class Subscription : ITelemetrySubscription
+ {
+ private readonly TelemetryLocalHub _hub;
+ private readonly Guid _id;
+ private int _disposed;
+
+ public Subscription(TelemetryLocalHub hub, Guid id, Channel channel)
+ {
+ _hub = hub;
+ _id = id;
+ Reader = channel.Reader;
+ }
+
+ public ChannelReader Reader { get; }
+
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ return; // idempotent
+ _hub.Detach(_id);
+ }
+ }
+}
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs
index d733a264..ebc658ed 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs
@@ -6,6 +6,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -61,6 +62,7 @@ public sealed class VirtualTagActor : ReceiveActor
private readonly Func? _publisherFactory;
private readonly IReadOnlyList _dependencyRefs;
private readonly IActorRef? _mux;
+ private readonly ITelemetryLocalHub? _telemetryHub;
private readonly ILoggingAdapter _log = Context.GetLogger();
private readonly Dictionary _dependencies = new(StringComparer.Ordinal);
@@ -78,6 +80,8 @@ public sealed class VirtualTagActor : ReceiveActor
/// Optional factory for creating DPS publishers.
/// Optional list of dependency tag references; defaults to empty.
/// Optional reference to a dependency multiplexer actor.
+ /// Optional node-local live-telemetry hub each script-log entry is also
+ /// emitted into (Phase 5); null (tests) simply skips the emit.
/// A configured for creating the actor.
public static Props Props(
string virtualTagId,
@@ -86,14 +90,16 @@ public sealed class VirtualTagActor : ReceiveActor
string? scriptId = null,
Func? publisherFactory = null,
IReadOnlyList? dependencyRefs = null,
- IActorRef? mux = null) =>
+ IActorRef? mux = null,
+ ITelemetryLocalHub? telemetryHub = null) =>
Akka.Actor.Props.Create(() => new VirtualTagActor(
virtualTagId, expression,
evaluator ?? NullVirtualTagEvaluator.Instance,
scriptId ?? virtualTagId,
publisherFactory,
dependencyRefs ?? Array.Empty(),
- mux));
+ mux,
+ telemetryHub));
/// Initializes a virtual tag actor with the given configuration and dependencies.
/// Unique identifier for the virtual tag.
@@ -103,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor
/// Optional factory for creating DPS publishers.
/// List of dependency tag references that this tag depends on.
/// Optional reference to a dependency multiplexer actor.
+ /// Optional node-local live-telemetry hub each script-log entry is also
+ /// emitted into (Phase 5); null (tests) simply skips the emit.
public VirtualTagActor(
string virtualTagId,
string expression,
@@ -110,7 +118,8 @@ public sealed class VirtualTagActor : ReceiveActor
string scriptId,
Func? publisherFactory,
IReadOnlyList dependencyRefs,
- IActorRef? mux)
+ IActorRef? mux,
+ ITelemetryLocalHub? telemetryHub = null)
{
_virtualTagId = virtualTagId;
_scriptId = scriptId;
@@ -119,6 +128,7 @@ public sealed class VirtualTagActor : ReceiveActor
_publisherFactory = publisherFactory;
_dependencyRefs = dependencyRefs;
_mux = mux;
+ _telemetryHub = telemetryHub;
Receive(OnDependencyChanged);
Receive(_ => OnReassertValue());
@@ -252,6 +262,12 @@ public sealed class VirtualTagActor : ReceiveActor
AlarmId: null,
EquipmentId: null);
+ // Phase 5: fan the same entry into the node-local live-telemetry hub (no-op until a gRPC client
+ // subscribes). Emitted BEFORE the publish branch so BOTH the production DPS path and the test-seam
+ // publisherFactory path (which returns early below) feed the hub. Both taps are fire-and-forget, so
+ // emit-then-publish ordering is immaterial; the DPS/factory publish itself is unchanged.
+ _telemetryHub?.Emit(new TelemetryItem.Script(entry));
+
if (_publisherFactory is not null)
{
_publisherFactory().Publish(ScriptLogsTopic, entry);
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs
index 50592064..7005dfdc 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs
@@ -6,6 +6,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -39,6 +40,7 @@ public sealed class VirtualTagHostActor : ReceiveActor
// Sink for historized VirtualTag results (plans with Historize=true). NullHistoryWriter when no
// durable historian is wired, so OnResult always has a non-null target.
private readonly IHistoryWriter _history;
+ private readonly ITelemetryLocalHub? _telemetryHub;
private readonly ILoggingAdapter _log = Context.GetLogger();
// vtagId -> spawned child VirtualTagActor.
@@ -59,18 +61,22 @@ public sealed class VirtualTagHostActor : ReceiveActor
/// Sink for results whose plan has Historize=true. Null ⇒
/// (no durable historian wired), so existing call sites
/// compile unchanged and never historize.
+ /// Optional node-local live-telemetry hub passed to each spawned child so
+ /// its script-log entries also emit to the Phase-5 hub; null (tests) skips the emit.
/// The used to spawn a .
public static Props Props(IActorRef publishActor, IActorRef? mux, IVirtualTagEvaluator evaluator,
- IHistoryWriter? historyWriter = null) =>
- Akka.Actor.Props.Create(() => new VirtualTagHostActor(publishActor, mux, evaluator, historyWriter));
+ IHistoryWriter? historyWriter = null, ITelemetryLocalHub? telemetryHub = null) =>
+ Akka.Actor.Props.Create(() => new VirtualTagHostActor(publishActor, mux, evaluator, historyWriter, telemetryHub));
/// Initializes a new instance of the class.
/// The OPC UA publish actor results are bridged to.
/// Optional dependency multiplexer passed to each spawned child.
/// The evaluator each child uses to compute its expression.
/// Sink for historized results; null ⇒ .
+ /// Optional node-local live-telemetry hub passed to each spawned child so
+ /// its script-log entries also emit to the Phase-5 hub; null (tests) skips the emit.
public VirtualTagHostActor(IActorRef publishActor, IActorRef? mux, IVirtualTagEvaluator evaluator,
- IHistoryWriter? historyWriter = null)
+ IHistoryWriter? historyWriter = null, ITelemetryLocalHub? telemetryHub = null)
{
ArgumentNullException.ThrowIfNull(publishActor);
ArgumentNullException.ThrowIfNull(evaluator);
@@ -78,6 +84,7 @@ public sealed class VirtualTagHostActor : ReceiveActor
_mux = mux;
_evaluator = evaluator;
_history = historyWriter ?? NullHistoryWriter.Instance;
+ _telemetryHub = telemetryHub;
Receive(OnApply);
Receive(OnResult);
@@ -166,7 +173,8 @@ public sealed class VirtualTagHostActor : ReceiveActor
scriptId: p.VirtualTagId,
publisherFactory: null,
dependencyRefs: p.DependencyRefs,
- mux: _mux));
+ mux: _mux,
+ telemetryHub: _telemetryHub));
Context.Watch(child);
_children[p.VirtualTagId] = child;
newlySpawned.Add(p.VirtualTagId);
diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/TelemetryOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/TelemetryOptionsValidatorTests.cs
new file mode 100644
index 00000000..c55bd59a
--- /dev/null
+++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/TelemetryOptionsValidatorTests.cs
@@ -0,0 +1,157 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+
+namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
+
+///
+/// A Grpc-mode node with no listen port or no key does not error — it simply never serves
+/// telemetry, and whoever is watching sees nothing with no stack trace to point at. These tests
+/// make that misconfiguration a loud host-start failure instead.
+///
+public class TelemetryOptionsValidatorTests
+{
+ private static ValidateOptionsResult Validate(TelemetryOptions o, params string[] roles)
+ {
+ var pairs = new Dictionary();
+ for (var i = 0; i < roles.Length; i++)
+ {
+ pairs[$"Cluster:Roles:{i}"] = roles[i];
+ }
+
+ var configuration = new ConfigurationBuilder().AddInMemoryCollection(pairs).Build();
+
+ return new TelemetryOptionsValidator(configuration).Validate(TelemetryOptions.SectionName, o);
+ }
+
+ [Fact]
+ public void Default_options_are_valid()
+ {
+ Validate(new TelemetryOptions()).Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Unknown_mode_fails()
+ {
+ var result = Validate(new TelemetryOptions { Mode = "stream" });
+
+ result.Failed.ShouldBeTrue();
+ result.FailureMessage.ShouldContain("stream");
+ }
+
+ [Fact]
+ public void Mode_matching_is_case_insensitive()
+ {
+ Validate(new TelemetryOptions
+ {
+ Mode = "grpc",
+ GrpcListenPort = 5100,
+ ApiKey = "k",
+ }, "driver").Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Grpc_driver_node_with_no_listen_port_fails()
+ {
+ var result = Validate(
+ new TelemetryOptions { Mode = TelemetryOptions.ModeGrpc, ApiKey = "k" },
+ "driver");
+
+ result.Failed.ShouldBeTrue();
+ result.FailureMessage.ShouldContain(nameof(TelemetryOptions.GrpcListenPort));
+ }
+
+ [Fact]
+ public void Grpc_with_empty_key_fails()
+ {
+ var result = Validate(
+ new TelemetryOptions { Mode = TelemetryOptions.ModeGrpc, GrpcListenPort = 5100, ApiKey = "" },
+ "driver");
+
+ result.Failed.ShouldBeTrue();
+ result.FailureMessage.ShouldContain(nameof(TelemetryOptions.ApiKey));
+ }
+
+ [Fact]
+ public void Grpc_driver_with_port_and_key_is_valid()
+ {
+ Validate(
+ new TelemetryOptions { Mode = TelemetryOptions.ModeGrpc, GrpcListenPort = 5100, ApiKey = "k" },
+ "driver").Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Grpc_admin_only_node_does_not_require_a_listen_port()
+ {
+ // The listen-port requirement is driver-specific — an admin-only node dialling out (via
+ // TelemetryDialOptions) never serves, so it has nothing to bind.
+ Validate(
+ new TelemetryOptions { Mode = TelemetryOptions.ModeGrpc, ApiKey = "k" },
+ "admin").Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Grpc_with_no_roles_does_not_require_a_key()
+ {
+ // Fail-closed only applies to a node that actually carries a role — a roleless node hosts
+ // nothing to protect.
+ Validate(new TelemetryOptions { Mode = TelemetryOptions.ModeGrpc, ApiKey = "" })
+ .Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Dps_ignores_the_empty_grpc_surface()
+ {
+ Validate(
+ new TelemetryOptions { Mode = TelemetryOptions.ModeDps, GrpcListenPort = 0, ApiKey = "" },
+ "driver").Succeeded.ShouldBeTrue();
+ }
+}
+
+///
+/// Mirrors for the central dial-side options.
+///
+public class TelemetryDialOptionsValidatorTests
+{
+ private static ValidateOptionsResult Validate(TelemetryDialOptions o) =>
+ new TelemetryDialOptionsValidator().Validate(TelemetryDialOptions.SectionName, o);
+
+ [Fact]
+ public void Default_options_are_valid()
+ {
+ Validate(new TelemetryDialOptions()).Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Unknown_mode_fails()
+ {
+ var result = Validate(new TelemetryDialOptions { Mode = "poll" });
+
+ result.Failed.ShouldBeTrue();
+ result.FailureMessage.ShouldContain("poll");
+ }
+
+ [Fact]
+ public void Grpc_with_empty_key_fails()
+ {
+ var result = Validate(new TelemetryDialOptions { Mode = TelemetryDialOptions.ModeGrpc, ApiKey = "" });
+
+ result.Failed.ShouldBeTrue();
+ result.FailureMessage.ShouldContain(nameof(TelemetryDialOptions.ApiKey));
+ }
+
+ [Fact]
+ public void Grpc_with_a_key_is_valid()
+ {
+ Validate(new TelemetryDialOptions { Mode = TelemetryDialOptions.ModeGrpc, ApiKey = "k" })
+ .Succeeded.ShouldBeTrue();
+ }
+
+ [Fact]
+ public void Dps_ignores_the_empty_key()
+ {
+ Validate(new TelemetryDialOptions { Mode = TelemetryDialOptions.ModeDps, ApiKey = "" })
+ .Succeeded.ShouldBeTrue();
+ }
+}
diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests/TelemetryProtoContractTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests/TelemetryProtoContractTests.cs
new file mode 100644
index 00000000..855bb17a
--- /dev/null
+++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Commons.Tests/TelemetryProtoContractTests.cs
@@ -0,0 +1,32 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Protos;
+using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.Commons.Tests;
+
+///
+/// Contract-lock for the Phase 5 telemetry oneof. If a fifth variant is added to
+/// telemetry.proto without adding a matching entry to
+/// , this test goes red — the single guard that
+/// keeps the wire contract and the handled-case set from silently drifting apart.
+///
+public class TelemetryProtoContractTests
+{
+ [Fact]
+ public void EveryOneofVariant_IsAccountedFor()
+ {
+ var variants = System.Enum.GetValues()
+ .Where(c => c != TelemetryEvent.EventOneofCase.None).ToArray();
+
+ variants.ShouldBe(TelemetryProtoContract.HandledCases, ignoreOrder: true);
+ }
+
+ [Fact]
+ public void The_service_base_and_client_types_generate()
+ {
+ // Referencing these types is the whole assertion — GrpcServices="Both" must emit both.
+ typeof(TelemetryStreamService.TelemetryStreamServiceBase).ShouldNotBeNull();
+ typeof(TelemetryStreamService.TelemetryStreamServiceClient).ShouldNotBeNull();
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Hubs/TelemetryModeWiringTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Hubs/TelemetryModeWiringTests.cs
new file mode 100644
index 00000000..bfd3ed4d
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Hubs/TelemetryModeWiringTests.cs
@@ -0,0 +1,134 @@
+using Akka.Actor;
+using Akka.Hosting;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Hubs;
+
+///
+/// Verifies the Phase 5 dark switch in :
+/// TelemetryDial:Mode = Dps (default) spawns the four telemetry DPS bridge actors and NOT the
+/// gRPC dial supervisor; Grpc spawns the dial supervisor and NONE of the four telemetry
+/// bridges. The fleet-status bridge (deferred / out of Phase 5 scope) is spawned in BOTH modes.
+/// The gate is proven on a real from a started Akka host.
+///
+public sealed class TelemetryModeWiringTests
+{
+ /// Dps mode: the four telemetry bridges + fleet bridge are registered; the dial supervisor is not.
+ [Fact]
+ public async Task Dps_mode_spawns_the_four_telemetry_bridges_and_not_the_dial_supervisor()
+ {
+ using var host = BuildBridgeHost(TelemetryDialOptions.ModeDps);
+ await host.StartAsync();
+ try
+ {
+ var registry = host.Services.GetRequiredService();
+
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeTrue();
+
+ registry.TryGet(out _).ShouldBeFalse();
+ }
+ finally
+ {
+ await host.StopAsync();
+ }
+ }
+
+ /// Grpc mode: the dial supervisor + fleet bridge are registered; none of the four telemetry bridges are.
+ [Fact]
+ public async Task Grpc_mode_spawns_the_dial_supervisor_and_none_of_the_four_telemetry_bridges()
+ {
+ using var host = BuildBridgeHost(TelemetryDialOptions.ModeGrpc);
+ await host.StartAsync();
+ try
+ {
+ var registry = host.Services.GetRequiredService();
+
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeTrue();
+
+ registry.TryGet(out _).ShouldBeFalse();
+ registry.TryGet(out _).ShouldBeFalse();
+ registry.TryGet(out _).ShouldBeFalse();
+ registry.TryGet(out _).ShouldBeFalse();
+ }
+ finally
+ {
+ await host.StopAsync();
+ }
+ }
+
+ /// Unset (default) mode is treated as Dps — the four bridges spawn, the supervisor does not.
+ [Fact]
+ public async Task Default_mode_is_dps()
+ {
+ using var host = BuildBridgeHost(mode: null);
+ await host.StartAsync();
+ try
+ {
+ var registry = host.Services.GetRequiredService();
+ registry.TryGet(out _).ShouldBeTrue();
+ registry.TryGet(out _).ShouldBeFalse();
+ }
+ finally
+ {
+ await host.StopAsync();
+ }
+ }
+
+ /// Builds an admin-role host that runs WithOtOpcUaSignalRBridges under the given telemetry mode.
+ /// The TelemetryDial:Mode value, or to leave it at its default.
+ private static IHost BuildBridgeHost(string? mode)
+ => Host.CreateDefaultBuilder()
+ .ConfigureServices((_, services) =>
+ {
+ services.AddSignalR();
+ services.AddOtOpcUaDriverStatusServices();
+ services.AddSingleton>(
+ new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N")));
+
+ var options = new TelemetryDialOptions { ApiKey = "test-key" };
+ if (mode is not null)
+ {
+ options.Mode = mode;
+ }
+
+ services.AddSingleton>(Options.Create(options));
+
+ services.AddAkka("otopcua-test", (ab, _) =>
+ {
+ ab.AddHocon(@"
+ akka.actor.provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
+ akka.remote.dot-netty.tcp.hostname = ""127.0.0.1""
+ akka.remote.dot-netty.tcp.port = 0
+ akka.cluster.seed-nodes = []
+ akka.cluster.roles = [""admin""]
+ ", HoconAddMode.Prepend);
+ ab.WithOtOpcUaSignalRBridges();
+ });
+ })
+ .Build();
+
+ /// An whose contexts share one InMemory database.
+ private sealed class InMemoryConfigDbFactory(string dbName) : IDbContextFactory
+ {
+ public OtOpcUaConfigDbContext CreateDbContext() =>
+ new(new DbContextOptionsBuilder()
+ .UseInMemoryDatabase(dbName)
+ .Options);
+
+ public Task CreateDbContextAsync(CancellationToken cancellationToken = default) =>
+ Task.FromResult(CreateDbContext());
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Telemetry/TelemetryDialSupervisorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Telemetry/TelemetryDialSupervisorTests.cs
new file mode 100644
index 00000000..4de5f16d
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Telemetry/TelemetryDialSupervisorTests.cs
@@ -0,0 +1,471 @@
+using System.Collections.Concurrent;
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Telemetry;
+
+///
+/// Unit tests for — the central-side dial supervisor that
+/// discovers driver nodes, keeps one reconnecting dialer each, and feeds the four AdminUI
+/// in-process sinks. All seams are faked: no real DB, no real gRPC.
+///
+public sealed class TelemetryDialSupervisorTests : TestKit
+{
+ private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
+
+ [Fact]
+ public void Discovery_starts_one_dialer_per_target_with_distinct_correlation_ids()
+ {
+ var dial = new FakeDialLoop();
+ var targets = new[]
+ {
+ new TelemetryDialTarget("node-1", "http://h1:5300"),
+ new TelemetryDialTarget("node-2", "http://h2:5300"),
+ };
+
+ Spawn(dial, () => targets);
+
+ var invocations = dial.WaitForCount(2, Timeout);
+ invocations.Select(i => i.Target.NodeId).ShouldBe(new[] { "node-1", "node-2" }, ignoreOrder: true);
+ invocations.ShouldContain(i => i.CorrelationId == "central-node-1-1");
+ invocations.ShouldContain(i => i.CorrelationId == "central-node-2-1");
+ invocations.ShouldAllBe(i => i.Ct.CanBeCanceled);
+ }
+
+ [Fact]
+ public void Refresh_removes_departed_nodes_and_adds_new_ones()
+ {
+ var dial = new FakeDialLoop();
+ var current = new List
+ {
+ new("node-1", "http://h1:5300"),
+ new("node-2", "http://h2:5300"),
+ };
+
+ var actor = Spawn(dial, () => current.ToArray());
+ dial.WaitForCount(2, Timeout);
+
+ var node1 = dial.Invocations.Single(i => i.Target.NodeId == "node-1");
+
+ // node-1 leaves, node-3 joins.
+ current.Clear();
+ current.Add(new TelemetryDialTarget("node-2", "http://h2:5300"));
+ current.Add(new TelemetryDialTarget("node-3", "http://h3:5300"));
+ actor.Tell(new TelemetryDialSupervisor.RefreshNodes());
+
+ // node-1's dialer is cancelled and dropped.
+ AwaitCondition(() => node1.Ct.IsCancellationRequested, Timeout);
+ // node-3 gets a fresh dialer.
+ var invocations = dial.WaitForCount(3, Timeout);
+ invocations.ShouldContain(i => i.Target.NodeId == "node-3" && i.CorrelationId == "central-node-3-1");
+
+ // node-2 is present in BOTH refreshes with an unchanged endpoint: it must NOT be restarted, or
+ // its live stream would be dropped on every refresh. Exactly one dial invocation for it.
+ ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+ dial.Invocations.Count(i => i.Target.NodeId == "node-2").ShouldBe(1);
+ }
+
+ [Fact]
+ public void Refresh_redials_a_node_whose_endpoint_changed_but_not_an_unchanged_one()
+ {
+ var dial = new FakeDialLoop();
+ var current = new List
+ {
+ new("node-1", "http://old-host:5300"),
+ new("node-2", "http://h2:5300"),
+ };
+
+ var actor = Spawn(dial, () => current.ToArray());
+ dial.WaitForCount(2, Timeout);
+ var node1Original = dial.Invocations.Single(i => i.Target.NodeId == "node-1");
+
+ // node-1 re-provisioned to a new host; node-2 unchanged.
+ current.Clear();
+ current.Add(new TelemetryDialTarget("node-1", "http://new-host:5300"));
+ current.Add(new TelemetryDialTarget("node-2", "http://h2:5300"));
+ actor.Tell(new TelemetryDialSupervisor.RefreshNodes());
+
+ // The old node-1 stream is torn down and a fresh dialer opens at the NEW endpoint (gen 2).
+ AwaitCondition(() => node1Original.Ct.IsCancellationRequested, Timeout);
+ var invocations = dial.WaitForCount(3, Timeout);
+ invocations.ShouldContain(i =>
+ i.Target.NodeId == "node-1"
+ && i.Target.Endpoint == "http://new-host:5300"
+ && i.CorrelationId == "central-node-1-2");
+
+ // node-2's endpoint did not change → still exactly one dial invocation (no churn).
+ ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+ dial.Invocations.Count(i => i.Target.NodeId == "node-2").ShouldBe(1);
+ }
+
+ [Fact]
+ public void Each_record_type_lands_on_its_sink()
+ {
+ var dial = new FakeDialLoop();
+ var sinks = new Sinks();
+ Spawn(dial, () => new[] { new TelemetryDialTarget("node-1", "http://h1:5300") }, sinks);
+
+ var inv = dial.WaitForCount(1, Timeout).Single();
+
+ inv.OnMapped(Alarm("a1"));
+ inv.OnMapped(Script("s1"));
+ inv.OnMapped(Health("d1"));
+ inv.OnMapped(Resilience("d1"));
+
+ AwaitAssert(
+ () =>
+ {
+ sinks.Alarms.ShouldHaveSingleItem().AlarmId.ShouldBe("a1");
+ sinks.Scripts.ShouldHaveSingleItem().ScriptId.ShouldBe("s1");
+ sinks.Health.ShouldHaveSingleItem().DriverInstanceId.ShouldBe("d1");
+ sinks.Resilience.ShouldHaveSingleItem().DriverInstanceId.ShouldBe("d1");
+ },
+ Timeout);
+ }
+
+ [Fact]
+ public void Stream_failure_reconnects_with_an_incremented_generation()
+ {
+ var dial = new FakeDialLoop();
+ Spawn(dial, () => new[] { new TelemetryDialTarget("node-1", "http://h1:5300") });
+
+ var first = dial.WaitForCount(1, Timeout).Single();
+ first.CorrelationId.ShouldBe("central-node-1-1");
+
+ // Transport failure → the client would end the stream after onError, so end the fake loop too.
+ first.OnError(new InvalidOperationException("boom"));
+ first.Complete();
+
+ // First retry is immediate; generation is bumped.
+ var invocations = dial.WaitForCount(2, Timeout);
+ invocations[1].CorrelationId.ShouldBe("central-node-1-2");
+ }
+
+ [Fact]
+ public void Late_event_from_a_superseded_generation_is_dropped()
+ {
+ var dial = new FakeDialLoop();
+ var sinks = new Sinks();
+ Spawn(dial, () => new[] { new TelemetryDialTarget("node-1", "http://h1:5300") }, sinks);
+
+ var first = dial.WaitForCount(1, Timeout).Single();
+ first.OnError(new InvalidOperationException("boom"));
+ first.Complete();
+ dial.WaitForCount(2, Timeout); // gen-2 dialer is live.
+
+ // A late event arriving on the OLD (gen-1) stream must be dropped, not routed.
+ first.OnMapped(Alarm("stale"));
+
+ // Give the message a chance to be (wrongly) routed, then assert it was not.
+ ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+ sinks.Alarms.ShouldBeEmpty();
+ }
+
+ [Fact]
+ public void Late_failure_from_a_superseded_generation_does_not_trigger_a_second_reconnect()
+ {
+ var dial = new FakeDialLoop();
+ Spawn(dial, () => new[] { new TelemetryDialTarget("node-1", "http://h1:5300") });
+
+ var first = dial.WaitForCount(1, Timeout).Single();
+ first.OnError(new InvalidOperationException("boom"));
+ first.Complete();
+ dial.WaitForCount(2, Timeout); // gen-2 dialer.
+
+ // A duplicate/late failure from gen-1 must be ignored (no third dial).
+ first.OnError(new InvalidOperationException("late duplicate"));
+
+ ExpectNoMsg(TimeSpan.FromMilliseconds(400));
+ dial.Invocations.Count.ShouldBe(2);
+ }
+
+ [Fact]
+ public void Pill_flips_true_on_first_connection_and_false_only_when_all_down()
+ {
+ var dial = new FakeDialLoop();
+ var sinks = new Sinks();
+ Spawn(
+ dial,
+ () => new[]
+ {
+ new TelemetryDialTarget("node-1", "http://h1:5300"),
+ new TelemetryDialTarget("node-2", "http://h2:5300"),
+ },
+ sinks);
+
+ var inv = dial.WaitForCount(2, Timeout);
+ var node1 = inv.Single(i => i.Target.NodeId == "node-1");
+ var node2 = inv.Single(i => i.Target.NodeId == "node-2");
+
+ // First node connects → both broadcasters flip to connected.
+ node1.OnMapped(Alarm("a1"));
+ AwaitAssert(() => sinks.PillStates.ToArray().ShouldBe(new[] { true }), Timeout);
+
+ // Second node connects → still connected, no extra transition.
+ node2.OnMapped(Alarm("a2"));
+ ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+ sinks.PillStates.ToArray().ShouldBe(new[] { true });
+
+ // node-1 drops but node-2 is still up → pill stays true.
+ node1.OnError(new InvalidOperationException("down"));
+ node1.Complete();
+ ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+ sinks.PillStates.ToArray().ShouldBe(new[] { true });
+
+ // node-2 drops too → all down → pill flips false.
+ node2.OnError(new InvalidOperationException("down"));
+ node2.Complete();
+ AwaitAssert(() => sinks.PillStates.ToArray().ShouldBe(new[] { true, false }), Timeout);
+ }
+
+ [Fact]
+ public void PostStop_cancels_every_dialer()
+ {
+ var dial = new FakeDialLoop();
+ var actor = Spawn(
+ dial,
+ () => new[]
+ {
+ new TelemetryDialTarget("node-1", "http://h1:5300"),
+ new TelemetryDialTarget("node-2", "http://h2:5300"),
+ });
+
+ var inv = dial.WaitForCount(2, Timeout);
+
+ Sys.Stop(actor);
+
+ AwaitCondition(() => inv.All(i => i.Ct.IsCancellationRequested), Timeout);
+ }
+
+ [Fact]
+ public async Task Production_node_source_skips_null_grpc_port_and_maintenance_rows()
+ {
+ var factory = new InMemoryFactory(Guid.NewGuid().ToString());
+ await using (var db = factory.CreateDbContext())
+ {
+ db.ClusterNodes.AddRange(
+ Node("has-port", "h1", grpcPort: 5300, enabled: true, maintenance: false),
+ Node("null-port", "h2", grpcPort: null, enabled: true, maintenance: false),
+ Node("maintenance", "h3", grpcPort: 5300, enabled: true, maintenance: true),
+ Node("disabled", "h4", grpcPort: 5300, enabled: false, maintenance: false));
+ await db.SaveChangesAsync();
+ }
+
+ var source = TelemetryNodeSource.Create(factory, NullLogger.Instance);
+ var targets = await source();
+
+ targets.ShouldHaveSingleItem();
+ targets[0].NodeId.ShouldBe("has-port");
+ targets[0].Endpoint.ShouldBe("http://h1:5300");
+ }
+
+ private IActorRef Spawn(
+ FakeDialLoop dial,
+ Func nodeSource,
+ Sinks? sinks = null)
+ {
+ sinks ??= new Sinks();
+ var options = new TelemetryDialOptions { ContactRefreshSeconds = 3600 };
+ return Sys.ActorOf(TelemetryDialSupervisor.Props(
+ () => Task.FromResult>(nodeSource()),
+ dial.Loop,
+ sinks.AlarmBroadcaster,
+ sinks.ScriptBroadcaster,
+ sinks.HealthStore,
+ sinks.ResilienceStore,
+ options));
+ }
+
+ private static AlarmTransitionEvent Alarm(string id) =>
+ new(id, "Area/Line/Eq", "hi", "Activated", 500, "msg", "system", DateTime.UtcNow);
+
+ private static ScriptLogEntry Script(string id) =>
+ new(id, "Info", "msg", DateTime.UtcNow, null, null, null);
+
+ private static DriverHealthChanged Health(string id) =>
+ new("C1", id, "Healthy", DateTime.UtcNow, null, 0, DateTime.UtcNow);
+
+ private static DriverResilienceStatusChanged Resilience(string id) =>
+ new(id, "host", false, 0, 0, null, DateTime.UtcNow, DateTime.UtcNow);
+
+ private static ClusterNode Node(string id, string host, int? grpcPort, bool enabled, bool maintenance) =>
+ new()
+ {
+ NodeId = id,
+ ClusterId = "C1",
+ Host = host,
+ GrpcPort = grpcPort,
+ Enabled = enabled,
+ MaintenanceMode = maintenance,
+ ApplicationUri = $"urn:{id}",
+ CreatedBy = "test",
+ };
+
+ /// A captured dial-loop invocation with the callbacks the test drives by hand.
+ private sealed class DialInvocation
+ {
+ private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public required TelemetryDialTarget Target { get; init; }
+ public required string CorrelationId { get; init; }
+ public required Action