Compare commits

...

14 Commits

Author SHA1 Message Date
Joseph Doherty 4af8e65af1 fix(redundancy): PeerProbeSupervisor explicitly ignores co-mingled OpcUaProbeResult (integration review)
v2-ci / build (push) Failing after 34s
v2-ci / unit-tests (tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests) (push) Has been skipped
v2-ci / unit-tests (tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests) (push) Has been skipped
v2-ci / unit-tests (tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests) (push) Has been skipped
v2-ci / unit-tests (tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests) (push) Has been skipped
v2-ci / unit-tests (tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests) (push) Has been skipped
v2-ci / integration (tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests) (push) Has been skipped
v2-ci / integration (tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests) (push) Has been skipped
2026-06-15 13:40:16 -04:00
Joseph Doherty 4c78dcd358 feat(redundancy): wire dbHealth into OpcUaPublishActor + spawn PeerProbeSupervisor per node 2026-06-15 13:33:34 -04:00
Joseph Doherty 393b746d9b docs(redundancy): sync component table with the wired calculator + PeerProbeSupervisor 2026-06-15 13:30:55 -04:00
Joseph Doherty 5a064e086d test(redundancy): lock in stale-Terminated guard + clarify OnTerminated (code-review) 2026-06-15 13:29:58 -04:00
Joseph Doherty 70e6d3d2c0 docs(redundancy): ServiceLevelCalculator is wired into the live publish path 2026-06-15 13:26:34 -04:00
Joseph Doherty f41e957e07 feat(redundancy): PeerProbeSupervisor maintains one peer OPC UA probe per driver peer 2026-06-15 13:22:38 -04:00
Joseph Doherty 37b32a5623 feat(redundancy): periodic HealthTick refreshes DB reachability via Ask/PipeTo 2026-06-15 13:15:26 -04:00
Joseph Doherty 5382eea9b5 test(redundancy): cover stale-probe-not-demoted branch + make _probeFreshnessWindow readonly (code-review) 2026-06-15 13:11:01 -04:00
Joseph Doherty cf278035d2 feat(redundancy): OpcUaProbeOk from peer-probes-me with freshness debounce 2026-06-15 13:04:41 -04:00
Joseph Doherty a9ff1a64b2 fix(redundancy): always publish first ServiceLevel (even 0) + log SafeSelfStatus failures (code-review) 2026-06-15 13:00:25 -04:00
Joseph Doherty 3e609a2b19 feat(redundancy): OpcUaPublishActor computes ServiceLevel via calculator (DB+stale+leader; legacy seam) 2026-06-15 12:51:32 -04:00
Joseph Doherty ff0f62db38 refactor(redundancy): move ServiceLevelCalculator to Core.Cluster (shared, Runtime-reachable) 2026-06-15 12:45:17 -04:00
Joseph Doherty 7605f4d8fd docs(redundancy): Phase 2 implementation plan + tasks (H3 ServiceLevel wiring) 2026-06-15 12:41:51 -04:00
Joseph Doherty 0528353315 docs(redundancy): Phase 2 design — health-aware ServiceLevel (H3) 2026-06-15 12:33:09 -04:00
12 changed files with 1755 additions and 83 deletions
+75 -52
View File
@@ -13,77 +13,100 @@ The runtime pieces live in:
| Component | Project | Role |
|---|---|---|
| `RedundancyStateActor` | `OtOpcUa.ControlPlane.Redundancy` | Admin-role cluster singleton; subscribes to cluster topology events, debounces 250ms, broadcasts `RedundancyStateChanged` on the `redundancy-state` DPS topic. |
| `OpcUaPublishActor` | `OtOpcUa.Runtime.OpcUa` | Per-driver-node; subscribes to the `redundancy-state` topic, maps the local node's role to a ServiceLevel byte (see below), and forwards it to `IServiceLevelPublisher`. |
| `OpcUaPublishActor` | `OtOpcUa.Runtime.OpcUa` | Per-driver-node; subscribes to the `redundancy-state` topic, computes a health-aware ServiceLevel byte via `ServiceLevelCalculator` (see below), and forwards it to `IServiceLevelPublisher`. |
| `IServiceLevelPublisher` / `SdkServiceLevelPublisher` | `OtOpcUa.Commons.OpcUa` / `OtOpcUa.OpcUaServer` | Writes the byte into the SDK's `Server.ServiceLevel` Variable. Production binds `DeferredServiceLevelPublisher`, which swaps in the real `SdkServiceLevelPublisher` once the SDK is up (it needs `IServerInternal`, available only after `StandardServer.Start`); until then writes route through `NullServiceLevelPublisher`. |
| `ServiceLevelCalculator` | `OtOpcUa.ControlPlane.Redundancy` | Pure function `(NodeHealthInputs) → byte` — the fuller DB/probe-aware tiering (see truth table below). Covered by `ServiceLevelCalculatorTests`; **not yet wired into the live driver publish path**, which uses the coarse role mapping in `OpcUaPublishActor`. |
| `DbHealthProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; runs `SELECT 1` against ConfigDb every 5s. Read by health endpoint. |
| `PeerOpcUaProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; pings peer `opc.tcp://peer:4840` with a TCP connect (2s timeout) and publishes the result on the `redundancy-state` topic. A full secure-channel Hello handshake is a possible future upgrade; the TCP connect is the current real probe. |
| `ServiceLevelCalculator` | `OtOpcUa.Cluster.Redundancy` (`Core.Cluster`) | Pure function `(NodeHealthInputs) → byte` — the DB/probe-aware tiering (see truth table below). Covered by `ServiceLevelCalculatorTests`. **Now the live publish path** — `OpcUaPublishActor` calls it on every `HealthTick` and `RedundancyStateChanged` event. Moved to `Core.Cluster` so Runtime can reach it without a Runtime→ControlPlane reference. |
| `DbHealthProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; runs `SELECT 1` against ConfigDb every 5s. Read by the health endpoint AND by `OpcUaPublishActor` (the `DbReachable` ServiceLevel input). |
| `PeerProbeSupervisor` | `OtOpcUa.Runtime.Health` | Per-node; subscribes to the `redundancy-state` topic and maintains one `PeerOpcUaProbeActor` child per OTHER driver-role peer (spawn on join, stop on departure), so every node is continuously probed by its peers. |
| `PeerOpcUaProbeActor` | `OtOpcUa.Runtime.Health` | Spawned by `PeerProbeSupervisor`; pings a peer `opc.tcp://peer:4840` with a TCP connect (2s timeout) and publishes `OpcUaProbeResult` on the `redundancy-state` topic. A full secure-channel Hello handshake is a possible future upgrade; the TCP connect is the current real probe. |
| `ClusterRoleInfo` | `OtOpcUa.Cluster` | Live view of cluster membership + role-leader; exposes `IClusterRoleInfo` to the rest of the host. |
## ServiceLevel tiers
### Live driver-side mapping (current)
### Health-aware tiering (`ServiceLevelCalculator` — live path)
`OpcUaPublishActor.HandleRedundancyStateChanged` maps the local node's role
(from the `RedundancyStateChanged` snapshot) to a ServiceLevel byte and forwards
it through `IServiceLevelPublisher` to the SDK's `Server.ServiceLevel` Variable:
`ServiceLevelCalculator.Compute(NodeHealthInputs)` is the live publish path.
`OpcUaPublishActor` calls it on every `HealthTick` (~5 s) and on each
`RedundancyStateChanged` snapshot, then forwards the result through
`IServiceLevelPublisher` to the SDK's `Server.ServiceLevel` Variable.
| Local role | Byte |
The four inputs are sourced locally per driver node:
| Input | Source |
|---|---|
| `Primary` and `driver` role-leader | 240 |
| `Primary` (not role-leader) | 200 |
| `Secondary` | 100 |
| `Detached` (no `driver` role) | 0 |
| `MemberState` | Local `SelfMember.Status` from the Akka cluster (Up / Joining / Leaving / …). |
| `DbReachable` | Local `DbHealthProbeActor` — `OpcUaPublishActor` Asks it on each `HealthTick`; an Ask timeout is treated as `Reachable=false`. |
| `OpcUaProbeOk` | Result of a peer probing THIS node's OPC UA endpoint: `PeerProbeSupervisor` spawns one `PeerOpcUaProbeActor` per OTHER driver-role peer; each probe publishes `OpcUaProbeResult(probed-node, ok)` on the `redundancy-state` topic; the publish actor consumes only results whose target is itself. Freshness-debounced: absent or stale (>30 s) → `true` (benefit of the doubt — single-node clusters and a departed peer never demote); only an actively-observed RECENT `false` demotes. |
| `Stale` (derived) | `!DbReachable \|\| (now − lastDbHealth.AsOfUtc) > 30 s \|\| (now − snapshotEntry.AsOfUtc) > 30 s`. |
| `IsDriverRoleLeader` | The local node's entry in the `RedundancyStateChanged` snapshot from `RedundancyStateActor`. |
The resulting truth table (all tiers are now reachable at runtime):
| Tier | Byte | Condition |
|---|---|---|
| Down / Detached | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting), OR node has no `driver` role (Detached). Published immediately — a starting or detached node never leaves the SDK default 255. |
| Critically degraded | 100 | ConfigDb unreachable AND data is stale. |
| Stale | 200 | Data stale but ConfigDb reachable. |
| Healthy follower | 240 | DB reachable + OPC UA probe ok + not stale + not role-leader. |
| Healthy leader | 250 | Same as healthy follower + this node is the `driver` role-leader (+10 bonus). |
> **Secondary 100 → 240 (behavior change).** Previously a healthy Secondary
> published 100 (coarse role-only mapping). It now publishes **240** — both
> nodes sit at 240/250 under healthy conditions, with the leader still preferred
> by the +10 bonus. Clients with the standard "pick highest ServiceLevel"
> heuristic continue to prefer the primary.
#### Backward-compatible fallback (legacy seam)
A node with no `DbHealthStatus` wired (e.g. early bootstrap window before the
first `DbHealthProbeActor` reply) falls back to the old role-only mapping:
Primary-leader → 240, Primary → 200, Secondary → 100, Detached → 0. Once the
first `DbHealthStatus` arrives the calculator takes over. The first computed
ServiceLevel (even 0) is always published so no node lingers at the SDK default
255.
Roles come from `RedundancyStateActor.BuildSnapshot`: a node with the `driver`
role is `Primary` when it holds the `driver` role-leader lease, otherwise
`Secondary`; a node without the `driver` role is `Detached`.
### Full health-aware tiering (`ServiceLevelCalculator`)
`ServiceLevelCalculator.Compute(NodeHealthInputs)` is the fuller, DB/probe-aware
calculation. It is unit-tested but **not yet on the live publish path** — the
driver-side mapping above is what actually drives the SDK today.
| Tier | Byte | Condition |
|---|---|---|
| Down | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting). |
| Critically degraded | 100 | ConfigDb unreachable AND data is stale. |
| Stale | 200 | Data stale but ConfigDb reachable. |
| Healthy follower | 240 | DB ok + OPC UA probe ok + not stale. |
| Healthy leader | 250 | Healthy follower (240) + a `+10` bonus when this node is the `driver` role-leader. |
Either way, clients with the standard redundancy heuristic ("pick the highest
ServiceLevel") prefer the `driver` role-leader and fall back to followers on its
degradation.
## Data flow
```
Cluster topology event ──┐
▼
RedundancyStateActor (admin singleton)
│ debounce 250ms
▼
DPS topic "redundancy-state"
│
▼
Driver nodes' OpcUaPublishActor
│ role → byte (240/200/100/0)
▼
Cluster topology event ──────────────────────────────────────────┐
▼
RedundancyStateActor (admin singleton)
│ debounce 250ms
▼
DPS topic "redundancy-state"
│ ▲
┌───────────────────────┘ │
│ │
▼ │
Driver node: OpcUaPublishActor │
┌─────────────────────────────────────────────────────────┐ │
│ Inputs collected per ~5s HealthTick: │ │
│ • MemberState ← Akka SelfMember.Status │ │
│ • DbReachable ← DbHealthProbeActor (Ask, timeout→F) │ │
│ • OpcUaProbeOk ← OpcUaProbeResult about THIS node │──────┘
│ • Stale ← derived from above timestamps │ PeerProbeSupervisor
│ • IsLeader ← RedundancyStateChanged snapshot │ → PeerOpcUaProbeActor(s)
│ │ publish OpcUaProbeResult
│ ServiceLevelCalculator.Compute(NodeHealthInputs) │ on "redundancy-state"
│ → byte (0/100/200/240/250) │
└───────────────────────────────────────────────────────-─┘
│
▼
IServiceLevelPublisher (SdkServiceLevelPublisher)
│
▼
│
▼
OPC UA Server.ServiceLevel Variable
```
Today only cluster topology drives the published ServiceLevel.
`PeerOpcUaProbeActor` and `DbHealthProbeActor` also run per-node — the peer probe
publishes `OpcUaProbeResult` onto the `redundancy-state` topic and the DB probe
backs the health endpoint — but their outputs are not yet consumed by
`RedundancyStateActor` or folded into the published byte. They are the inputs the
fuller `ServiceLevelCalculator` truth table is designed to use once that path goes
live.
Both `DbHealthProbeActor` and `PeerOpcUaProbeActor` feed the live publish path.
The peer probe publishes `OpcUaProbeResult` on the `redundancy-state` topic;
`OpcUaPublishActor` consumes only results whose target is itself and applies
freshness-debouncing before passing them to the calculator. `DbHealthProbeActor`
is queried directly via Ask on each `HealthTick`.
The admin singleton is the cluster's only `RedundancyStateActor`. If the admin leader fails over, the new admin node spins up its replacement, re-subscribes to cluster events, and publishes a fresh snapshot from the current `Cluster.State`. There is no DB-persisted state to recover.
@@ -113,7 +136,7 @@ There is no longer a `Node:NodeId` setting and no `ClusterNode.RedundancyRole` c
> **`RedundancyStateActor` NodeId consistency (fixed).** `RedundancyStateActor` now keys each node's `NodeRedundancyState` entry by the canonical `host:port` node id (via a `ToNodeId(Address)` helper mirroring `ClusterRoleInfo.ToNodeId`). Previously it keyed by `member.Address.Host` (host-only, e.g. `central-2`); since every subscriber matches by the canonical `host:port` form, the mismatch silently meant no node ever matched its own entry — all nodes stayed at the default ServiceLevel 255 and never learned their role. This fix makes `RedundancyStateActor` consistent with the stated contract above. Additionally, `RedundancyStateActor` now **re-publishes the current snapshot on a periodic heartbeat (default 10 s)** so any node that subscribes after the last topology-change publish converges within the interval (DistributedPubSub does not replay to late subscribers).
The `ClusterNode.ServiceLevelBase` column still exists and is editable in the Admin UI (NodeEdit / Cluster Redundancy pages), but it no longer drives the runtime ServiceLevel — that value is computed from cluster role/health and published per the mapping above, independent of this stored preference.
The `ClusterNode.ServiceLevelBase` column still exists and is editable in the Admin UI (NodeEdit / Cluster Redundancy pages), but it no longer drives the runtime ServiceLevel — that value is computed by `ServiceLevelCalculator` from cluster role and live health inputs, independent of this stored preference.
### Peer URI advertising
@@ -0,0 +1,183 @@
# Phase 2 — Health-aware redundancy ServiceLevel (H3) — design
> **Status:** approved 2026-06-15. Parent roadmap: `docs/plans/2026-06-15-stillpending-backlog-design.md`
> (Phase 2). Backlog item **H3** in `stillpending.md` §1.
> **Branch:** `feat/stillpending-phase-2-servicelevel` off master `4bd7180e`.
> **Classification:** high-risk (actor model + redundancy correctness). Live `/run` on the 2-node rig is
> the acceptance gate; unit tests cannot prove the cross-node wiring (per `project_redundancy_state_delivery`).
## Problem (H3)
The SDK's `Server.ServiceLevel` byte is driven **solely by the coarse role map** today
(`OpcUaPublishActor.HandleRedundancyStateChanged`: Primary-leader→240, Primary→200, Secondary→100,
Detached→0). The richer `ServiceLevelCalculator.Compute(NodeHealthInputs)` — which folds in DB
reachability, an OPC-UA liveness probe, and a staleness signal — **is never invoked in production**.
The two health producers are dead or half-wired:
- `DbHealthProbeActor` — **spawned** per node, but only feeds `/health/ready`; its `DbReachable` never
reaches the published byte.
- `PeerOpcUaProbeActor` — **never spawned anywhere**; its `OpcUaProbeResult` has **zero consumers**.
- `NodeHealthInputs.Stale` — **no producer exists** anywhere in the codebase.
Consequence: a **DB-unreachable / OPC-UA-down primary still advertises its full role-based ServiceLevel**,
so redundant clients keep subscribing to a degraded node instead of failing over.
## Goal
Make each driver node publish a ServiceLevel computed by `ServiceLevelCalculator.Compute` from its **real
local health**, so an unhealthy node drops below its role-based level (and below a healthy peer's level,
triggering client failover). Honor the **already-documented** tiers in `docs/Redundancy.md`
(250 / 240 / 200 / 100 / 0) — do not reshape the calculator's truth table. No EF migration; no change to
the `RedundancyStateChanged` / `NodeRedundancyState` message contract.
## Locked decisions (from brainstorming 2026-06-15)
1. **Compute site = local per-node** (`OpcUaPublishActor`), *not* the admin `RedundancyStateActor`
singleton. DB reachability is an inherently local fact; centralizing it would require a new
cluster-wide health-broadcast contract for no benefit. The publish actor already runs per node, already
subscribes to the `redundancy-state` topic, already holds the role snapshot, and already writes the byte.
2. **`Stale` is derived from signal freshness, with DB-unreachable ⟹ stale.** This makes the documented
200/100 graceful-degradation tiers reachable, so a node that loses its DB settles into 100 ("degraded,
still serving cached values") rather than slamming to 0. The 0 tier stays reserved for "not a healthy
cluster member / detached." A richer *driver-input* staleness signal is **deferred** (flagged, not built).
3. **`OpcUaProbeOk` = peer-probes-me**, reusing the existing `PeerOpcUaProbeActor` (finally spawned),
closing the audit's "never spawned / zero consumers" gaps. Absence of a fresh peer result → `true`
(benefit of the doubt: don't penalize a node for its peer being down); only an *actively observed*
recent failure demotes. Single-node / no-peer cluster → `true`.
## Architecture
All changes are **per-driver-node, inside `Runtime`** (`WithOtOpcUaRuntimeActors`). No ControlPlane /
admin-singleton change. No new Commons message types.
```
WithOtOpcUaRuntimeActors (per driver node)
├─ DbHealthProbeActor (exists; now ALSO consulted by the publish actor)
├─ PeerProbeSupervisor (NEW) watches cluster membership →
│ └─ PeerOpcUaProbeActor(peer) one child per OTHER driver member;
│ publishes OpcUaProbeResult(peer, ok) on the redundancy-state topic
└─ OpcUaPublishActor (MODIFIED)
subscribes redundancy-state topic → RedundancyStateChanged (role/leadership)
→ OpcUaProbeResult (peer's verdict on ME)
holds IActorRef dbHealthProbe → Ask<DbHealthStatus> on a periodic HealthTick
Cluster.SelfMember.Status → MemberState (local)
on any trigger: ServiceLevelCalculator.Compute(inputs) → publish-if-changed (existing dedup)
```
### Components
**1. `PeerProbeSupervisor` (NEW, `Runtime/Health/PeerProbeSupervisor.cs`)** — a small per-node actor whose
only job is the probe lifecycle (kept separate so the pinned-dispatcher `OpcUaPublishActor` never parents
network-probe children):
- Subscribes to cluster membership (`IMemberEvent` / `ReachabilityEvent`) and/or the redundancy snapshot.
- Maintains one `PeerOpcUaProbeActor` child per **other driver-role member**, (re)spawning on membership
change, stopping children for departed members. Resolves the peer's OPC UA host from its node id
(`host:port` → host) and the configured OPC UA port.
- Children publish `OpcUaProbeResult` to the `redundancy-state` topic (existing behavior).
- Spawned on the default dispatcher (not the OPC UA pinned one).
**2. `OpcUaPublishActor` (MODIFIED)** — replace the role-only switch in `HandleRedundancyStateChanged`
with a `RecomputeServiceLevel()` that calls the calculator. New Props params (both optional, defaulted so
test Props/harnesses keep working): `IActorRef? dbHealthProbe`, and the freshness windows /
self-OPC-UA-port (injectable for tests).
- New state: `_dbReachable` + `_dbAsOfUtc` (from the DB Ask), `_lastSnapshotAsOfUtc`, a per-peer map of
`(ok, asOfUtc)` for results **about my own node**, and the current `RedundancyStateChanged` local entry.
- New `Receive<OpcUaProbeResult>`: if `msg.NodeId == _localNode`, record `(msg.Ok, now)` for the reporting
peer; recompute.
- New `Receive<HealthTick>` (periodic, e.g. 5 s): `dbHealthProbe.Ask<DbHealthStatus>(GetStatus, 1s).PipeTo(Self)`.
- New `Receive<DbHealthStatus>`: cache `_dbReachable` + `_dbAsOfUtc`; recompute.
- `RecomputeServiceLevel()` builds `NodeHealthInputs` and calls `ServiceLevelCalculator.Compute`, then
`Self.Tell(new ServiceLevelChanged(level))` (the existing handler dedups + publishes + emits the metric).
### Input derivation (the load-bearing logic)
For the **local** node, on each recompute:
| `NodeHealthInputs` field | Source |
|---|---|
| `MemberState` | `Cluster.Get(system).SelfMember.Status` (local; most accurate) |
| `IsDriverRoleLeader` | local entry of the latest `RedundancyStateChanged` snapshot (`IsRoleLeaderForDriver`) |
| `DbReachable` | latest `DbHealthStatus.Reachable` from the local `DbHealthProbeActor` |
| `OpcUaProbeOk` | `true` if **no** fresh peer result about me, else the latest such result's `Ok` (only an actively-observed recent failure → `false`); single-node → `true` |
| `Stale` | `!DbReachable` **OR** `(now − _dbAsOfUtc) > StaleWindow` **OR** `(now − _lastSnapshotAsOfUtc) > StaleWindow` |
**Guards (calculator does not model these):**
- If there is **no local snapshot entry**, or the local entry's `Role == Detached`, publish **0** directly
and skip the calculator (a healthy detached node would otherwise compute 240 because the calculator only
checks `MemberState` + the leader bonus). In steady state a node running `OpcUaPublishActor` always carries
the `driver` role, so this is defensive — it preserves the current Detached→0 behavior during transitions.
### Resulting ServiceLevel truth table (now fully reachable)
| Node condition | Byte | How reached |
|---|---|---|
| Not a healthy member / detached | **0** | `MemberState` not Up/Joining, or Detached guard |
| DB unreachable, sustained | **100** | `DbReachable=false ⇒ Stale=true` → `(false,_,true)` |
| DB reachable but signals stale | **200** | `(true,_,true)` |
| Healthy follower (DB ok + probe ok + fresh) | **240** | `(true,true,false)` |
| Healthy leader | **250** | follower 240 + `IsDriverRoleLeader` +10 |
| DB ok + fresh but peer reports me unreachable | **0** | `(true,false,false)` → falls through |
**Behavior change to be aware of (already documented as the target):** a *healthy Secondary moves
100 → 240*. Both healthy nodes sit at 240/250 with the leader preferred by the +10 bonus; role-leadership is
stable so the bytes do not flap. The "DB ok + probe-fail" → 0 edge is intentionally sharp but **debounced**
by the `OpcUaProbeOk` freshness rule (a single missed probe does not flip it; only a sustained, actively
observed failure does).
## Error handling / edge cases
- **DB Ask timeout** → treat as `DbReachable=false` for that cycle (fail-safe demote); the next tick re-Asks.
- **No peer (single node)** → `OpcUaProbeOk=true`; supervisor spawns no children.
- **Peer departs** → its prior result ages out of the freshness window → `OpcUaProbeOk` reverts to `true`
(we don't penalize a node for its peer's absence). Supervisor stops the child for the departed member.
- **Cluster forming / role-leader not yet resolved** → no local snapshot entry yet → Detached guard → 0,
same as today, until the first snapshot arrives.
- **`HealthTick` before the first snapshot** → no local entry → 0 (no spurious 240 before role is known).
## Testing strategy (xUnit + Shouldly, TDD; NO bUnit)
Unit-testable (the wiring that *can* be proven in a TestKit):
- `OpcUaPublishActor` input-derivation: feed a `RedundancyStateChanged` (Primary-leader) + a cached
`DbHealthStatus(reachable)` + an `OpcUaProbeResult(me, ok)` → assert published byte 250; flip DB
unreachable → 100; stale snapshot → 200; healthy secondary → 240; Detached entry → 0; probe `Ok=false`
about me with DB-ok+fresh → 0. Use `PropsForTests` with a broadcast/serviceLevel capture + injected
`dbHealthProbe` test ref + short windows.
- `OpcUaProbeOk` freshness/debounce: a single stale/absent result → `true`; a recent explicit `false` → `false`.
- `PeerProbeSupervisor`: on a membership snapshot with one peer driver member, spawns exactly one child
targeting that peer; on the peer leaving, stops it; single-node → no children. (TestKit, with a probe
factory injected so no real TCP is attempted.)
- Reuse existing `ServiceLevelCalculatorTests` unchanged (the pure function is not modified).
Not unit-testable (acceptance gate): the live cross-node ServiceLevel on the **2-node rig** — proven by
`/run` (user-driven). A DB-unreachable primary must be observed dropping below the secondary, and the
secondary taking over as the higher-ServiceLevel endpoint. See `project_redundancy_state_delivery`.
## Verification
- `dotnet build` clean (production projects are `TreatWarningsAsErrors`); full `dotnet test` green.
- High-risk review chain (serial spec → code → final integration).
- **Live `/run` on the 2-node / docker-dev rig** (user-driven; agent does not sign in): confirm
steady-state 250/240, then kill a node's DB reachability and confirm its ServiceLevel drops to 100 and the
peer is preferred. Update `docs/Redundancy.md` to mark the calculator path **wired** (remove the "not yet
wired into the live driver publish path" caveats) and document the freshness/peer-probe sourcing.
## Alternatives considered
- **Central compute in `RedundancyStateActor`** — rejected: the admin singleton can't see each node's local
DB health without a new cluster-wide health-broadcast contract; heavier, bigger blast radius, no benefit.
- **Local self-probe** (TCP to own `localhost:4840` / SDK server-state) instead of the peer-probe — simpler
(no membership-tracking supervisor) and a defensible "is my OPC UA serving" signal, but leaves
`PeerOpcUaProbeActor` dead and misses the cross-node vantage. Rejected to honor the locked design and close
the audit's dead-code gap; recorded here so the trade-off is explicit if the supervisor proves heavier than
valued (then it becomes a follow-up swap, not a redesign).
- **Reshape the calculator to role-baseline-demoted-by-health** (keep the wide 240-vs-100 gap; health only
lowers) — rejected: the calculator's 250/240/200/100/0 tiers are already approved, documented, and tested;
reshaping re-opens a settled decision. The +10 leader bonus already preserves client preference for the
primary.
## Hard constraints (carried from the roadmap)
NO Configuration entity / EF migration. Stage by path — never `git add .`; never stage `sql_login.txt`,
`src/Server/.../Host/pki/`, `pending.md`, `current.md`, `docker-dev/docker-compose.yml`, `stillpending.md`.
Never echo or commit secrets. No force-push, no `--no-verify`. Razor/runtime cross-node behavior proven only
by live `/run`, never bUnit.
@@ -0,0 +1,564 @@
# Phase 2 — Health-aware redundancy ServiceLevel (H3) Implementation Plan
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development to implement this plan task-by-task.
**Goal:** Wire the existing-but-never-invoked `ServiceLevelCalculator` into the live per-node OPC UA publish
path so a DB-unreachable / OPC-UA-unreachable node drops below its role-based ServiceLevel (and below a
healthy peer), triggering client failover.
**Architecture:** Per-driver-node, inside `Runtime`. `OpcUaPublishActor` computes its own ServiceLevel from
real local health (local `DbHealthProbeActor`, peer's-probe-of-me from the `redundancy-state` topic, the
role snapshot, signal freshness) via `ServiceLevelCalculator.Compute`, replacing its role-only switch — with
a **backward-compatible seam** (no DB-health source ⇒ legacy role-only). A new tiny `PeerProbeSupervisor`
finally spawns `PeerOpcUaProbeActor` per driver peer. No admin-singleton change, no EF migration, no Commons
message-contract change.
**Tech Stack:** C# / .NET 10, Akka.NET (ReceiveActor, DistributedPubSub, IWithTimers, Ask+PipeTo), xUnit +
Shouldly + Akka.TestKit. Design: `docs/plans/2026-06-15-stillpending-phase-2-servicelevel-design.md`.
**Branch:** `feat/stillpending-phase-2-servicelevel` (already created off master `4bd7180e`; design doc
committed `05283533`).
---
## Hard rules (every task)
- Stage **by path** — never `git add .`. Never stage `sql_login.txt`, `src/Server/.../Host/pki/`,
`pending.md`, `current.md`, `docker-dev/docker-compose.yml`, `stillpending.md`. Never echo/commit secrets.
No force-push, no `--no-verify`.
- **NO** Configuration entity / EF migration. **NO** change to `OpcUaProbeResult` / `RedundancyStateChanged`
/ `NodeRedundancyState` contracts (reuse as-is).
- TDD fail-then-pass. xUnit + Shouldly. **NO bUnit.** Cross-node behavior proven only by live `/run`.
- Production projects are `TreatWarningsAsErrors` — fix all warnings.
## Key facts the implementer needs
- `ServiceLevelCalculator.Compute(NodeHealthInputs)` is pure and **must not be reshaped**. Tiers:
`MemberState not Up/Joining → 0`; `(DbReachable,OpcUaProbeOk,Stale)` → `(true,true,false)=240`,
`(true,_,true)=200`, `(false,_,true)=100`, else `0`; `+10` if `IsDriverRoleLeader` (clamp 255).
- `Runtime` already references `Core.Cluster` (`ZB.MOM.WW.OtOpcUa.Cluster.csproj`); `ControlPlane` too.
`Runtime` does **NOT** reference `ControlPlane` and must not start to (deliberate boundary — see the
comment in `PeerOpcUaProbeActor.cs:22`). So the calculator **moves** to `Core.Cluster` (Task 1).
- `DbHealthProbeActor` (`Runtime/Health/`) answers `GetStatus` → `DbHealthStatus(bool Reachable, DateTime
AsOfUtc, string? LastError)` from cache (cheap). Already spawned in `WithOtOpcUaRuntimeActors` as the local
`dbHealth` `IActorRef`.
- `PeerOpcUaProbeActor` (`Runtime/Health/`) publishes `OpcUaProbeResult(NodeId peer, bool Ok)` to the
`redundancy-state` topic. `Props(peer, interval?, connectTimeout?, opcUaPort=4840, broadcast?)`.
- `OpcUaPublishActor` subscribes to the `redundancy-state` topic in PreStart, so it already receives every
message on it — adding `Receive<OpcUaProbeResult>` makes those visible with no extra subscribe.
- Test base `RuntimeActorTestBase` self-joins a single-node `driver` cluster → `Cluster.Get(Sys).SelfMember.
Status == Up` works in tests. The two existing tests in `OpcUaPublishActorTests.cs`
(`..._primary_leader` → 240, `..._for_secondary_publishes_100` → 100) inject **no** `dbHealthProbe`, so the
legacy seam keeps them green unchanged.
---
### Task 1: Move `ServiceLevelCalculator` to `Core.Cluster` (shared, cycle-free)
**Classification:** standard
**Estimated implement time:** ~3 min
**Parallelizable with:** none (Task 2 depends on this)
**Files:**
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs`
- Delete: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ServiceLevelCalculator.cs`
- Modify: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ServiceLevelCalculatorTests.cs` (using/namespace)
**Step 1: Verify `Core.Cluster` references `Akka.Cluster`** (the calculator uses `MemberStatus`):
Run: `grep -rn "Akka.Cluster" src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj && grep -rln "using Akka.Cluster" src/Core/ZB.MOM.WW.OtOpcUa.Cluster/`
Expected: a PackageReference (or transitive) to `Akka.Cluster` and existing `using Akka.Cluster;` files
(`ClusterRoleInfo` uses it). If the package is **not** referenced there, STOP and surface it — fall back to
adding `Akka.Cluster` as a `PackageReference` on `Core.Cluster` (it already depends on Akka cluster types) or
placing the file in `Commons` only if `Commons` references `Akka.Cluster`. Prefer `Core.Cluster`.
**Step 2: Move the type verbatim**, changing only the namespace:
`namespace ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;` (keep `using Akka.Cluster;`, keep `NodeHealthInputs` +
`ServiceLevelCalculator` together, keep all XML docs). Delete the old ControlPlane file.
**Step 3: Fix references.** `RedundancyStateActor.cs` does not call the calculator (only doc-comment
mentions) — update the doc-comment text if it names the old namespace, no `using` needed. Update the test:
```csharp
// tests/.../ServiceLevelCalculatorTests.cs
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy; // was ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy
```
Also `grep -rn "ControlPlane.Redundancy.ServiceLevelCalculator\|ControlPlane.Redundancy;.*ServiceLevel\|NodeHealthInputs" src tests` and fix any stragglers (e.g. `OpcUaPublishActor.cs`'s `<see cref>` doc-comment, `OtOpcUaConfigDbContext.cs:159` comment text — comment-only, update the prose).
**Step 4: Build + run the moved test.**
Run: `dotnet build ZB.MOM.WW.OtOpcUa.slnx` then
`dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests --filter "FullyQualifiedName~ServiceLevelCalculator"`
Expected: build clean; `ServiceLevelCalculatorTests` PASS (unchanged behavior, new namespace).
**Step 5: Commit**
```bash
git add src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs \
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ServiceLevelCalculator.cs \
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ServiceLevelCalculatorTests.cs
# plus any comment-only files touched in Step 3, by path
git commit -m "refactor(redundancy): move ServiceLevelCalculator to Core.Cluster (shared, Runtime-reachable)"
```
---
### Task 2a: `OpcUaPublishActor` calculator path — DB + Stale + leader + Detached guard (legacy seam)
**Classification:** high-risk
**Estimated implement time:** ~5 min
**Parallelizable with:** none (Task 1 → 2a → 2b → 3 are serial on the same file)
**Files:**
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
**Design contract:** `RecomputeServiceLevel()` resolves the local snapshot entry; **no entry or
`Role==Detached` ⇒ publish `0`**. If **`_dbHealthProbe is null` OR `_lastDbHealth is null`** (no DB-health
source yet) ⇒ **legacy role-only byte** (the old switch — Primary-leader 240 / Primary 200 / Secondary 100 /
Detached 0) — this is the bootstrap + back-compat path. Otherwise build `NodeHealthInputs` and call
`ServiceLevelCalculator.Compute`. `MemberState` ← `Cluster.Get(Context.System).SelfMember.Status`.
`Stale = !DbReachable || (now - _lastDbHealth.AsOfUtc) > _staleWindow || (now - localEntry.AsOfUtc) > _staleWindow`.
`OpcUaProbeOk` is hardcoded `true` in this task (Task 2b wires the real value).
**Step 1: Write failing tests** (add to `OpcUaPublishActorTests.cs`). A tiny stub DB-health probe actor +
calculator-path assertions:
```csharp
using Akka.Actor;
using ZB.MOM.WW.OtOpcUa.Runtime.Health; // DbHealthProbeActor.GetStatus / DbHealthStatus
// helper at class scope:
private sealed class StubDbHealth : ReceiveActor
{
public StubDbHealth(DbHealthProbeActor.DbHealthStatus status) =>
Receive<DbHealthProbeActor.GetStatus>(_ => Sender.Tell(status));
}
[Fact] // healthy primary-leader → 250
public void Calculator_path_healthy_primary_leader_publishes_250()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var db = Sys.ActorOf(Props.Create(() => new StubDbHealth(
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
staleWindow: TimeSpan.FromSeconds(30)));
// Seed DB-health directly (deterministic, no timer): Receive<DbHealthStatus> caches + recomputes.
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new RedundancyStateChanged(new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
}, CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromMilliseconds(500));
}
[Fact] // healthy secondary → 240 (NOT 100 — documented change)
public void Calculator_path_healthy_secondary_publishes_240() { /* Role=Secondary, IsRoleLeader=false → 240 */ }
[Fact] // DB unreachable → 100 (Stale via !DbReachable)
public void Calculator_path_db_unreachable_publishes_100() { /* DbHealthStatus(false,...) + Primary-leader → 100 */ }
[Fact] // stale snapshot but DB ok → 200
public void Calculator_path_stale_snapshot_publishes_200() { /* DbHealthStatus(true, now) but entry AsOfUtc = now-1min, staleWindow=2s → 200 (leader → 200, no +10? see note) */ }
[Fact] // Detached → 0
public void Calculator_path_detached_publishes_0() { /* local entry Role=Detached → 0 */ }
[Fact] // back-compat: no dbHealthProbe → legacy role-only (existing two tests already cover 240 & 100)
public void Legacy_path_when_no_db_probe_keeps_role_only() { /* no dbHealthProbe → Primary-leader → 240 */ }
```
NOTE for the 200 test: with `IsDriverRoleLeader=true`, `Compute` adds +10 → 210. Either assert 210 for a
leader, or build the 200 case with `IsRoleLeaderForDriver:false` (a stale follower) → exactly 200. Use a
**stale follower** so the expected byte is unambiguous (200).
**Step 2: Run, expect FAIL** (`dbHealthProbe`/`staleWindow` params don't exist; calculator not called):
Run: `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~OpcUaPublishActorTests.Calculator_path"`
**Step 3: Implement.** In `OpcUaPublishActor.cs`:
- Add `using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;` and `using ZB.MOM.WW.OtOpcUa.Runtime.Health;` and
`using Akka.Cluster;`.
- Add Props/PropsForTests params (all optional, defaulted): `IActorRef? dbHealthProbe = null`,
`TimeSpan? staleWindow = null` (default e.g. 30 s). Thread through both factory methods + the ctor.
- Add fields: `IActorRef? _dbHealthProbe; TimeSpan _staleWindow; DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
RedundancyStateChanged? _lastSnapshot; Cluster? _cluster;` (resolve `_cluster = Cluster.Get(Context.System)`
lazily/in ctor — safe; tests form a cluster).
- `Receive<DbHealthProbeActor.DbHealthStatus>(s => { _lastDbHealth = s; RecomputeServiceLevel(); });`
- Replace `HandleRedundancyStateChanged` body with `{ _lastSnapshot = msg; RecomputeServiceLevel(); }`.
- New `RecomputeServiceLevel()`:
```csharp
private void RecomputeServiceLevel()
{
if (_localNode is null || _lastSnapshot is null) return;
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
if (entry is null || entry.Role == RedundancyRole.Detached) { Self.Tell(new ServiceLevelChanged(0)); return; }
// Back-compat / bootstrap: no DB-health source → legacy role-only (keeps existing tests + a sane
// value until the first DbHealthStatus arrives).
if (_dbHealthProbe is null || _lastDbHealth is null) { Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry))); return; }
var now = DateTime.UtcNow;
var stale = !_lastDbHealth.Reachable
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|| (now - entry.AsOfUtc) > _staleWindow;
var inputs = new NodeHealthInputs(
MemberState: SafeSelfStatus(),
DbReachable: _lastDbHealth.Reachable,
OpcUaProbeOk: true, // Task 2b replaces with the freshness-debounced value
Stale: stale,
IsDriverRoleLeader: entry.IsRoleLeaderForDriver);
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
}
private static byte LegacyRoleOnly(NodeRedundancyState e) => e.Role switch
{
RedundancyRole.Primary when e.IsRoleLeaderForDriver => 240,
RedundancyRole.Primary => 200,
RedundancyRole.Secondary => 100,
_ => 0,
};
private MemberStatus SafeSelfStatus()
{
try { return _cluster!.SelfMember.Status; } catch { return MemberStatus.Removed; }
}
```
(The `Self.Tell(ServiceLevelChanged)` reuses the existing dedup/publish/metric handler — do not publish
directly.)
**Step 4: Run, expect PASS** (the new `Calculator_path*` tests + the two pre-existing role-only tests).
Run: `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~OpcUaPublishActorTests"`
Expected: all green, including the unchanged `..._primary_leader` (240) and `..._secondary_publishes_100`.
**Step 5: Commit** (stage `OpcUaPublishActor.cs` + the test file by path).
`git commit -m "feat(redundancy): OpcUaPublishActor computes ServiceLevel via calculator (DB+stale+leader; legacy seam)"`
---
### Task 2b: `OpcUaProbeOk` from peer-probes-me (freshness + debounce)
**Classification:** high-risk
**Estimated implement time:** ~4 min
**Parallelizable with:** none
**Files:**
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
**Contract:** `OpcUaProbeOk()` = `true` if **no fresh** `OpcUaProbeResult` about the local node exists, else
the latest such result's `Ok` (only an actively-observed, recent `Ok==false` demotes). "Fresh" = within
`_probeFreshnessWindow` (injectable, default e.g. 30 s). Single-node / no peer → always `true`.
**Step 1: Failing tests:**
```csharp
[Fact] // peer reports me unreachable, DB ok + fresh → (true,false,false) → 0
public void Probe_false_about_me_with_healthy_db_publishes_0() {
// inject dbHealthProbe(true) + staleWindow 30s + probeFreshnessWindow 30s
// Tell DbHealthStatus(true, now); Tell new OpcUaProbeResult(local, Ok:false);
// Tell RedundancyStateChanged(primary-leader) → AwaitAssert Levels contains 0
}
[Fact] // absent/stale probe result → OpcUaProbeOk true → healthy 250 (don't penalize missing peer)
public void No_probe_result_defaults_ok_true_publishes_250() { /* never Tell OpcUaProbeResult → 250 */ }
[Fact] // a later Ok:true supersedes an earlier Ok:false (recovery)
public void Probe_true_supersedes_earlier_false() { /* Tell false then true → recompute → 250 */ }
```
Ignore `OpcUaProbeResult` whose `NodeId != _localNode` (it's about a peer, not me).
**Step 2: Run, expect FAIL.**
**Step 3: Implement.** Add `Receive<OpcUaProbeResult>(HandlePeerProbe);` and:
```csharp
private (bool Ok, DateTime At)? _probeAboutMe; // latest peer verdict on THIS node
private TimeSpan _probeFreshnessWindow;
private void HandlePeerProbe(PeerOpcUaProbeActor.OpcUaProbeResult r)
{
if (_localNode is null || r.NodeId != _localNode.Value) return; // only verdicts about me
_probeAboutMe = (r.Ok, DateTime.UtcNow);
RecomputeServiceLevel();
}
private bool OpcUaProbeOk()
{
if (_probeAboutMe is not { } p) return true; // no peer verdict → benefit of doubt
if (DateTime.UtcNow - p.At > _probeFreshnessWindow) return true; // stale verdict → benefit of doubt
return p.Ok;
}
```
Replace `OpcUaProbeOk: true` in `RecomputeServiceLevel` with `OpcUaProbeOk: OpcUaProbeOk()`. Add the
`probeFreshnessWindow` Props/PropsForTests param + ctor wiring (default ~30 s). Add
`using ZB.MOM.WW.OtOpcUa.Runtime.Health;` (already added in 2a).
**Step 4: Run, expect PASS** (new probe tests + all Task 2a + the two legacy tests).
**Step 5: Commit** by path.
`git commit -m "feat(redundancy): OpcUaProbeOk from peer-probes-me with freshness debounce"`
---
### Task 3: `HealthTick` — periodic DB Ask/PipeTo + immediate PreStart refresh
**Classification:** high-risk
**Estimated implement time:** ~4 min
**Parallelizable with:** none
**Files:**
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
**Contract:** the actor refreshes DB health on its own so production needs no external pump. Implement
`IWithTimers`. In `PreStart` (only when `_dbHealthProbe is not null`): kick an immediate tick + a periodic
timer (`_healthTickInterval`, injectable, default 5 s). `Receive<HealthTick>` → `_dbHealthProbe.Ask<DbHealth
ProbeActor.DbHealthStatus>(DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1)).PipeTo(Self)`; on
Ask failure (timeout) PipeTo a `DbHealthStatus(false, now, "ask-timeout")` (fail-safe demote). The existing
`Receive<DbHealthStatus>` caches + recomputes.
**Step 1: Failing test** — proves the Ask path without the test Telling `DbHealthStatus` directly:
```csharp
[Fact]
public void HealthTick_asks_db_probe_and_publishes_calculator_byte()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var db = Sys.ActorOf(Props.Create(() => new StubDbHealth(
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30),
healthTickInterval: TimeSpan.FromMilliseconds(100)));
actor.Tell(new RedundancyStateChanged(new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
}, CorrelationId.NewId()));
// No direct DbHealthStatus Tell — the periodic HealthTick Ask must populate it → 250.
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromSeconds(2));
}
```
**Step 2: Run, expect FAIL** (`healthTickInterval` param missing; no tick wired).
**Step 3: Implement.** `public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers`; add
`public ITimerScheduler Timers { get; set; } = null!;`. Add `record HealthTick` singleton. Add the
`healthTickInterval` param to both Props factories + ctor (store `_healthTickInterval`). Extend `PreStart`:
```csharp
protected override void PreStart()
{
if (_subscribeRedundancyTopic)
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
if (_dbHealthProbe is not null)
{
Self.Tell(HealthTick.Instance); // immediate refresh
Timers.StartPeriodicTimer("health", HealthTick.Instance, _healthTickInterval);
}
}
```
`Receive<HealthTick>(_ => _dbHealthProbe!.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1))
.ContinueWith(t => t.IsCompletedSuccessfully ? t.Result
: new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "ask-timeout"))
.PipeTo(Self));`
**Step 4: Run, expect PASS** — the HealthTick test + all prior tests. Verify the pinned-dispatcher production
`Props()` still builds (it does not set `dbHealthProbe`, so PreStart skips the timer in pure-Props tests).
**Step 5: Commit** by path.
`git commit -m "feat(redundancy): periodic HealthTick refreshes DB reachability via Ask/PipeTo"`
---
### Task 4: `PeerProbeSupervisor` — spawn one probe per driver peer
**Classification:** high-risk
**Estimated implement time:** ~5 min
**Parallelizable with:** Task 6 (docs) — disjoint files
**Files:**
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Health/PeerProbeSupervisor.cs`
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Health/PeerProbeSupervisorTests.cs`
**Contract:** subscribes to the `redundancy-state` topic; on each `RedundancyStateChanged`, maintain exactly
one child per **other, non-Detached** node (`n.NodeId != _localNode && n.Role != Detached`): spawn for new
peers, `Context.Stop` children for departed peers. Child `Props` come from an injected
`Func<NodeId, Props>` (production default `peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: _opcUaPort)`).
Child actor name = a stable sanitization of the peer node id (so re-adds match). Default dispatcher (NOT the
pinned OPC UA one).
**Step 1: Failing tests** (drive lifecycle with fabricated snapshots + a stub probe factory; assert child
count via `ActorCells`/a TestProbe registry):
```csharp
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
{
private static Props StubChild() => Props.Create(() => new BlackHoleActor()); // or EchoActor
[Fact]
public void Spawns_one_child_per_non_self_non_detached_peer()
{
var local = NodeId.Parse("me:4053");
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(local, _ => StubChild()));
sup.Tell(new RedundancyStateChanged(new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
new NodeRedundancyState(NodeId.Parse("peer:4053"), RedundancyRole.Secondary, false, false, DateTime.UtcNow),
new NodeRedundancyState(NodeId.Parse("adm:4053"), RedundancyRole.Detached, false, false, DateTime.UtcNow),
}, CorrelationId.NewId()));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1)); // peer only; self + detached excluded
}
[Fact] public void Stops_child_for_departed_peer() { /* snapshot with peer, then without → ChildCount 1→0 */ }
[Fact] public void Single_node_snapshot_spawns_no_children() { /* only local entry → 0 */ }
}
```
Expose a test-visible `ChildCount` (e.g. `public int ChildCount => Context.GetChildren().Count();`) OR assert
via `ExpectMsg` on a registry. Prefer `ChildCount` with `ActorOfAsTestActorRef`.
**Step 2: Run, expect FAIL** (type doesn't exist).
**Step 3: Implement** `PeerProbeSupervisor : ReceiveActor`:
```csharp
public static Props Props(NodeId localNode, int opcUaPort = PeerOpcUaProbeActor.DefaultOpcUaPort) =>
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode,
peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: opcUaPort)));
public static Props PropsForTests(NodeId localNode, Func<NodeId, Props> probeFactory) =>
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
private readonly Dictionary<NodeId, IActorRef> _children = new();
// ctor: store localNode + probeFactory; Receive<RedundancyStateChanged>(OnSnapshot); Receive<SubscribeAck>(_=>{});
protected override void PreStart() =>
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe("redundancy-state", Self));
private void OnSnapshot(RedundancyStateChanged s)
{
var want = s.Nodes.Where(n => n.NodeId != _localNode && n.Role != RedundancyRole.Detached)
.Select(n => n.NodeId).ToHashSet();
foreach (var gone in _children.Keys.Where(k => !want.Contains(k)).ToList())
{ Context.Stop(_children[gone]); _children.Remove(gone); }
foreach (var peer in want.Where(p => !_children.ContainsKey(p)))
_children[peer] = Context.ActorOf(_probeFactory(peer), "probe-" + Sanitize(peer.Value));
}
private static string Sanitize(string s) => new(s.Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray());
```
(Use the literal `"redundancy-state"` to match `PeerOpcUaProbeActor.RedundancyStateTopic` — keep the Runtime
"duplicated to avoid a ControlPlane ref" convention. Reference `PeerOpcUaProbeActor.RedundancyStateTopic`
since both are in `Runtime.Health`.)
**Step 4: Run, expect PASS.**
**Step 5: Commit** by path.
`git commit -m "feat(redundancy): PeerProbeSupervisor maintains one peer OPC UA probe per driver peer"`
---
### Task 5: Wire into `WithOtOpcUaRuntimeActors`
**Classification:** high-risk
**Estimated implement time:** ~3 min
**Parallelizable with:** none (depends on 2a–4)
**Files:**
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs`
**Step 1: Implement** (no new unit test — DI wiring is covered by the live `/run` gate + existing
`ServiceCollectionExtensionsTests` smoke if present; if that test asserts a fixed actor set, update it):
- Pass the local `dbHealth` ref into the publish actor:
```csharp
var publishActor = system.ActorOf(
OpcUaPublishActor.Props(
sink: addressSpaceSink, serviceLevel: serviceLevel, localNode: roleInfo.LocalNode,
dbFactory: dbFactory, applier: applier,
dbHealthProbe: dbHealth), // NEW — enables the calculator path in production
OpcUaPublishActorName);
```
- Spawn the supervisor after `publishActor`:
```csharp
var peerProbes = system.ActorOf(PeerProbeSupervisor.Props(roleInfo.LocalNode), PeerProbeSupervisorName);
registry.Register<PeerProbeSupervisorKey>(peerProbes);
```
- Add `public const string PeerProbeSupervisorName = "peer-probe-supervisor";` near the other name consts and
`public sealed class PeerProbeSupervisorKey { }` near the other key classes.
- Confirm `OpcUaPublishActor.Props` signature now accepts `dbHealthProbe` (add the optional param in Task 2a;
if not yet, this task is blocked — order 2a before 5).
**Step 2: Build + full test.**
Run: `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests`
Expected: clean build, all green. If a `ServiceCollectionExtensionsTests` enumerates spawned actors, extend
it for the new supervisor.
**Step 3: Commit** by path (`ServiceCollectionExtensions.cs` only, plus the test if updated).
`git commit -m "feat(redundancy): wire dbHealth into OpcUaPublishActor + spawn PeerProbeSupervisor per node"`
---
### Task 6: Docs — `docs/Redundancy.md` marks the calculator WIRED
**Classification:** small
**Estimated implement time:** ~3 min
**Parallelizable with:** Task 4
**Files:**
- Modify: `docs/Redundancy.md`
**Step 1:** Update the doc to reflect shipped reality:
- The `ServiceLevelCalculator` table row (line ~18) + the "Today only cluster topology drives the published
ServiceLevel" line (~80) + the "designed to use once that path goes…" note (~85): change to **WIRED** —
each driver node now computes its ServiceLevel via `ServiceLevelCalculator` from local `DbHealthProbeActor`
reachability, a peer OPC UA probe (`PeerProbeSupervisor` → `PeerOpcUaProbeActor`, "peer-probes-me"), and
signal-freshness staleness.
- Document the **backward-compat seam**: a node with no DB-health source falls back to role-only.
- Document the **behavior change**: a healthy Secondary now publishes **240** (was 100); both healthy nodes
sit at 240/250 with the leader preferred by +10. Note `Stale = !DbReachable || signals older than the
freshness window`, and that the calculator moved to `Core.Cluster`.
- Keep the truth table (250/240/200/100/0) — it's now reachable.
**Step 2:** No build needed (doc only). Sanity-grep for now-false claims:
`grep -n "not yet wired\|placeholder\|F10b\|coarse role" docs/Redundancy.md` and fix each.
**Step 3: Commit** by path.
`git commit -m "docs(redundancy): ServiceLevelCalculator is wired into the live publish path"`
---
### Task 7: Full build + test + final integration review
**Classification:** high-risk
**Estimated implement time:** ~4 min
**Parallelizable with:** none
**Files:** none (verification only)
**Step 1:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx` — clean (TreatWarningsAsErrors).
**Step 2:** `dotnet test ZB.MOM.WW.OtOpcUa.slnx` — full suite green (esp. `OpcUaPublishActorTests`,
`PeerProbeSupervisorTests`, `ServiceLevelCalculatorTests`, `RedundancyStateActorTests`, `ServiceLevelEndToEndTests`).
**Step 3:** Final integration reviewer subagent: confirm (a) no Runtime→ControlPlane reference was added;
(b) the two pre-existing role-only tests are still green via the legacy seam; (c) no Commons contract / EF
change; (d) staging was by-path only; (e) the Detached guard + probe-freshness debounce + DB-Ask-timeout
fail-safe are all present. Apply any actionable findings as follow-up commits.
**Step 4:** Update `pending.md` (working-tree only, never staged) with the Phase 2 status.
---
### Task 8: Live `/run` on the 2-node rig (acceptance gate)
**Classification:** high-risk (verification)
**Estimated implement time:** ~6 min (agent-driven; docker-dev login is disabled — agent does NOT sign in to anything)
**Files:** none (runtime verification on docker-dev MAIN = central-1/central-2, the 2-node warm pair).
**Recipe:**
1. Rebuild the local rig to this branch: `docker compose -f docker-dev/docker-compose.yml up -d --build`
(do NOT stage compose changes). Confirm both central nodes are healthy and bootstrap-deploy the config.
2. **Steady state:** read each node's `Server.ServiceLevel` via Client.CLI
(`dotnet run --project src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI -- read -u opc.tcp://localhost:4840 -n "i=2267"`
— `ServiceLevel` is NodeId `i=2267`; confirm the actual id) on `:4840` (central-1) and `:4841` (central-2).
Expect the leader at **250** and the follower at **240** (the documented healthy pair).
3. **Demote one node's DB:** isolate ONLY one central node from SQL (they share the SQL container, so do
NOT stop SQL). Use `docker network disconnect <net> otopcua-dev-central-1-1` (or block its SQL route) so
central-1's `DbHealthProbeActor` starts failing while central-2 stays healthy.
4. Within ~`max(healthTickInterval, staleWindow)` confirm central-1's `ServiceLevel` drops to **100** (DB
unreachable ⇒ stale) — i.e. **below** central-2's 240 — proving health-aware demotion + the failover
ordering. Reconnect the network and confirm it climbs back to 240/250.
5. Capture the observed bytes. If the docker network manipulation is not feasible on the rig, record exactly
what blocked and hand that single step to the user; everything else is agent-driven.
**Done when:** build clean + full `dotnet test` green + the live demotion observed (or the precise blocker
recorded). Then `finishing-a-development-branch` → merge to master + push.
@@ -0,0 +1,16 @@
{
"planPath": "docs/plans/2026-06-15-stillpending-phase-2-servicelevel.md",
"branch": "feat/stillpending-phase-2-servicelevel",
"tasks": [
{"id": 414, "subject": "P2 Task 1: Move ServiceLevelCalculator to Core.Cluster", "status": "pending"},
{"id": 415, "subject": "P2 Task 2a: OpcUaPublishActor calculator path (DB+stale+leader+Detached guard, legacy seam)", "status": "pending", "blockedBy": [414]},
{"id": 416, "subject": "P2 Task 2b: OpcUaProbeOk from peer-probes-me (freshness + debounce)", "status": "pending", "blockedBy": [415]},
{"id": 417, "subject": "P2 Task 3: HealthTick — periodic DB Ask/PipeTo + PreStart immediate refresh", "status": "pending", "blockedBy": [416]},
{"id": 418, "subject": "P2 Task 4: PeerProbeSupervisor — one peer probe per driver peer", "status": "pending"},
{"id": 419, "subject": "P2 Task 5: Wire WithOtOpcUaRuntimeActors (dbHealth ref + spawn supervisor)", "status": "pending", "blockedBy": [417, 418]},
{"id": 420, "subject": "P2 Task 6: docs/Redundancy.md — calculator is WIRED", "status": "pending"},
{"id": 421, "subject": "P2 Task 7: Full build + test + final integration review", "status": "pending", "blockedBy": [419, 420]},
{"id": 422, "subject": "P2 Task 8: Live /run on the 2-node rig (acceptance gate)", "status": "pending", "blockedBy": [421]}
],
"lastUpdated": "2026-06-15"
}
@@ -1,6 +1,6 @@
using Akka.Cluster;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
public readonly record struct NodeHealthInputs(
MemberStatus MemberState,
@@ -0,0 +1,134 @@
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
/// <summary>
/// Per-node supervisor that keeps exactly one <see cref="PeerOpcUaProbeActor"/> child per OTHER,
/// non-<see cref="RedundancyRole.Detached"/> driver node named in the latest
/// <see cref="RedundancyStateChanged"/> snapshot. Because every node runs this supervisor, each
/// node is continuously TCP-probed by all of its peers; a node learns its own reachability by
/// consuming the resulting <see cref="PeerOpcUaProbeActor.OpcUaProbeResult"/> messages in
/// <c>OpcUaPublishActor</c>.
///
/// The supervisor reacts to the published <see cref="RedundancyStateChanged"/> snapshot (a plain
/// record on the <see cref="PeerOpcUaProbeActor.RedundancyStateTopic"/> DistributedPubSub topic)
/// rather than to raw Akka <c>Member</c> events, so the desired peer set is derived from one
/// authoritative message instead of being reconstructed from membership churn.
/// </summary>
public sealed class PeerProbeSupervisor : ReceiveActor
{
private readonly NodeId _localNode;
private readonly Func<NodeId, Props> _probeFactory;
private readonly Dictionary<NodeId, IActorRef> _children = new();
private readonly ILoggingAdapter _log = Context.GetLogger();
/// <summary>
/// Monotonic generation counter appended to each spawned child's name. Guarantees a freshly
/// re-added peer never collides on a child name with a same-peer child that is still
/// asynchronously terminating after a <see cref="ActorBase.Context"/> <c>Stop</c>.
/// </summary>
private long _generation;
/// <summary>Creates production props that probe peers on the given OPC UA port.</summary>
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
/// <param name="opcUaPort">The OPC UA port each peer probe connects to.</param>
/// <returns>Props configured to create a <see cref="PeerProbeSupervisor"/>.</returns>
public static Props Props(NodeId localNode, int opcUaPort = PeerOpcUaProbeActor.DefaultOpcUaPort)
{
// Build the factory delegate OUTSIDE the Props.Create expression tree: a named argument
// (opcUaPort:) is not permitted inside an expression tree, so capture it in a local first.
Func<NodeId, Props> probeFactory = peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: opcUaPort);
return Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
}
/// <summary>Creates props with an injected probe factory for tests (e.g. no-op children).</summary>
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
/// <param name="probeFactory">Factory producing the child <see cref="Props"/> for a given peer.</param>
/// <returns>Props configured to create a <see cref="PeerProbeSupervisor"/>.</returns>
public static Props PropsForTests(NodeId localNode, Func<NodeId, Props> probeFactory) =>
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
/// <summary>Initializes a new instance of the <see cref="PeerProbeSupervisor"/> class.</summary>
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
/// <param name="probeFactory">Factory producing the child <see cref="Props"/> for a given peer.</param>
public PeerProbeSupervisor(NodeId localNode, Func<NodeId, Props> probeFactory)
{
_localNode = localNode;
_probeFactory = probeFactory;
Receive<RedundancyStateChanged>(OnSnapshot);
Receive<Terminated>(OnTerminated);
Receive<SubscribeAck>(_ => { });
// The redundancy-state topic also carries OpcUaProbeResult (published by our own
// PeerOpcUaProbeActor children + peers). The supervisor doesn't consume them — only
// OpcUaPublishActor does — so explicitly drop them to avoid unhandled-message noise.
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
}
/// <summary>Gets the current number of live peer-probe children.</summary>
public int ChildCount => _children.Count;
/// <inheritdoc />
protected override void PreStart() =>
DistributedPubSub.Get(Context.System).Mediator.Tell(
new Subscribe(PeerOpcUaProbeActor.RedundancyStateTopic, Self));
/// <summary>
/// Reconciles the live child set against the desired peer set in the snapshot: stops children
/// for peers that are gone (or became Detached / self), and spawns a probe for each newly
/// desired peer.
/// </summary>
/// <param name="s">The latest redundancy-state snapshot.</param>
private void OnSnapshot(RedundancyStateChanged s)
{
var want = s.Nodes
.Where(n => n.NodeId != _localNode && n.Role != RedundancyRole.Detached)
.Select(n => n.NodeId)
.ToHashSet();
foreach (var gone in _children.Keys.Where(k => !want.Contains(k)).ToList())
{
Context.Stop(_children[gone]);
_children.Remove(gone);
}
foreach (var peer in want.Where(p => !_children.ContainsKey(p)))
{
var name = "probe-" + Sanitize(peer.Value) + "-" + _generation++;
var child = Context.ActorOf(_probeFactory(peer), name);
Context.Watch(child);
_children[peer] = child;
}
}
/// <summary>
/// Prunes a child that has terminated for any reason (e.g. crash) so <see cref="ChildCount"/>
/// stays accurate and the next snapshot will respawn it if still desired.
/// </summary>
/// <param name="t">The termination notice for the dead child.</param>
private void OnTerminated(Terminated t)
{
var entry = _children.FirstOrDefault(kvp => kvp.Value.Equals(t.ActorRef));
// No-match yields Value == null because IActorRef is a reference type. A null here means the
// terminated ref isn't our CURRENT child for any peer — either it was already pruned in
// OnSnapshot (deliberate stop pre-removes from _children), or it's a stale old-generation
// child whose peer has since been re-spawned with a fresh child. Either way: no-op, so we
// never evict the fresh same-peer child on a stale Terminated.
if (entry.Value is null) return;
_children.Remove(entry.Key);
_log.Debug("PeerProbeSupervisor: pruned terminated child for peer {Peer}", entry.Key);
}
/// <summary>
/// Sanitizes a <see cref="NodeId"/> value into a legal Akka child-name fragment — Akka actor
/// names may not contain <c>:</c> or <c>.</c>, which appear in <c>host:port</c> node ids.
/// </summary>
/// <param name="s">The raw node-id value.</param>
/// <returns>The value with every non-alphanumeric character replaced by <c>-</c>.</returns>
private static string Sanitize(string s) =>
new string(s.Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray());
}
@@ -1,7 +1,9 @@
using Akka.Actor;
using Akka.Cluster;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
@@ -9,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
@@ -24,11 +27,18 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
/// <c>Opc.Ua.Server</c>. The remaining piece is wiring those bindings to a real
/// <c>StandardServer</c> address space — tracked as F10b.
/// </summary>
public sealed class OpcUaPublishActor : ReceiveActor
public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
{
public const string DispatcherId = "opcua-synchronized-dispatcher";
public const string RedundancyStateTopic = "redundancy-state";
/// <summary>Gets or sets the timer scheduler for the periodic DB-health refresh tick.</summary>
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>Self-tick that drives the periodic Ask of the local <see cref="DbHealthProbeActor"/>
/// for its cached status. Private singleton — the actor pumps this for itself, no external sender.</summary>
private sealed class HealthTick { public static readonly HealthTick Instance = new(); private HealthTick() { } }
public sealed record AttributeValueUpdate(string NodeId, object? Value, OpcUaQuality Quality, DateTime TimestampUtc);
/// <summary>Carries the full Part 9 condition state for a scripted alarm to the sink. The
@@ -54,10 +64,19 @@ public sealed class OpcUaPublishActor : ReceiveActor
private readonly NodeId? _localNode;
private readonly IDbContextFactory<OtOpcUaConfigDbContext>? _dbFactory;
private readonly Phase7Applier? _applier;
private readonly IActorRef? _dbHealthProbe;
private readonly TimeSpan _staleWindow;
private readonly TimeSpan _probeFreshnessWindow;
private readonly TimeSpan _healthTickInterval;
private readonly Akka.Cluster.Cluster _cluster = Akka.Cluster.Cluster.Get(Context.System);
private readonly ILoggingAdapter _log = Context.GetLogger();
private int _writes;
private byte _lastServiceLevel;
private bool _publishedAtLeastOnce;
private DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
private RedundancyStateChanged? _lastSnapshot;
private (bool Ok, DateTime At)? _probeAboutMe;
private Phase7CompositionResult _lastApplied = new(
Array.Empty<UnsAreaProjection>(),
Array.Empty<UnsLineProjection>(),
@@ -74,25 +93,45 @@ public sealed class OpcUaPublishActor : ReceiveActor
/// <c>redundancy-state</c> DPS topic so cluster transitions drive the local ServiceLevel
/// publish path. When <paramref name="dbFactory"/> + <paramref name="applier"/> are supplied,
/// <see cref="RebuildAddressSpace"/> reads the latest deployment artifact + drives the
/// applier through the sink.</summary>
/// applier through the sink. When <paramref name="dbHealthProbe"/> is supplied the local
/// ServiceLevel is computed via <see cref="ServiceLevelCalculator"/> from real DB-health +
/// staleness + role-leader inputs; otherwise the legacy role-only switch is used.</summary>
/// <param name="sink">The OPC UA address space sink.</param>
/// <param name="serviceLevel">The service level publisher.</param>
/// <param name="localNode">The local cluster node ID.</param>
/// <param name="dbFactory">The optional database context factory.</param>
/// <param name="applier">The optional Phase 7 applier.</param>
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
/// considered stale; defaults to 30 seconds.</param>
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
/// defaults to 30 seconds.</param>
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
/// started when <paramref name="dbHealthProbe"/> is null.</param>
public static Props Props(
IOpcUaAddressSpaceSink? sink = null,
IServiceLevelPublisher? serviceLevel = null,
NodeId? localNode = null,
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
Phase7Applier? applier = null) =>
Phase7Applier? applier = null,
IActorRef? dbHealthProbe = null,
TimeSpan? staleWindow = null,
TimeSpan? probeFreshnessWindow = null,
TimeSpan? healthTickInterval = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
subscribeRedundancyTopic: true,
localNode,
dbFactory,
applier)).WithDispatcher(DispatcherId);
applier,
dbHealthProbe,
staleWindow,
probeFreshnessWindow,
healthTickInterval)).WithDispatcher(DispatcherId);
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
@@ -102,20 +141,38 @@ public sealed class OpcUaPublishActor : ReceiveActor
/// <param name="localNode">The local cluster node ID.</param>
/// <param name="dbFactory">The optional database context factory.</param>
/// <param name="applier">The optional Phase 7 applier.</param>
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
/// considered stale; defaults to 30 seconds.</param>
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
/// defaults to 30 seconds.</param>
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
/// started when <paramref name="dbHealthProbe"/> is null.</param>
public static Props PropsForTests(
IOpcUaAddressSpaceSink? sink = null,
IServiceLevelPublisher? serviceLevel = null,
bool subscribeRedundancyTopic = false,
NodeId? localNode = null,
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
Phase7Applier? applier = null) =>
Phase7Applier? applier = null,
IActorRef? dbHealthProbe = null,
TimeSpan? staleWindow = null,
TimeSpan? probeFreshnessWindow = null,
TimeSpan? healthTickInterval = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
subscribeRedundancyTopic,
localNode,
dbFactory,
applier));
applier,
dbHealthProbe,
staleWindow,
probeFreshnessWindow,
healthTickInterval));
/// <summary>Initializes a new instance of the <see cref="OpcUaPublishActor"/> class.</summary>
/// <param name="sink">The OPC UA address space sink.</param>
@@ -124,13 +181,27 @@ public sealed class OpcUaPublishActor : ReceiveActor
/// <param name="localNode">The local cluster node ID.</param>
/// <param name="dbFactory">The optional database context factory.</param>
/// <param name="applier">The optional Phase 7 applier.</param>
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
/// considered stale; defaults to 30 seconds.</param>
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
/// defaults to 30 seconds.</param>
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
/// started when <paramref name="dbHealthProbe"/> is null.</param>
public OpcUaPublishActor(
IOpcUaAddressSpaceSink sink,
IServiceLevelPublisher serviceLevel,
bool subscribeRedundancyTopic,
NodeId? localNode,
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
Phase7Applier? applier = null)
Phase7Applier? applier = null,
IActorRef? dbHealthProbe = null,
TimeSpan? staleWindow = null,
TimeSpan? probeFreshnessWindow = null,
TimeSpan? healthTickInterval = null)
{
_sink = sink;
_serviceLevel = serviceLevel;
@@ -138,12 +209,19 @@ public sealed class OpcUaPublishActor : ReceiveActor
_localNode = localNode;
_dbFactory = dbFactory;
_applier = applier;
_dbHealthProbe = dbHealthProbe;
_staleWindow = staleWindow ?? TimeSpan.FromSeconds(30);
_probeFreshnessWindow = probeFreshnessWindow ?? TimeSpan.FromSeconds(30);
_healthTickInterval = healthTickInterval ?? TimeSpan.FromSeconds(5);
Receive<AttributeValueUpdate>(HandleAttributeUpdate);
Receive<AlarmStateUpdate>(HandleAlarmUpdate);
Receive<RebuildAddressSpace>(HandleRebuild);
Receive<ServiceLevelChanged>(HandleServiceLevelChanged);
Receive<RedundancyStateChanged>(HandleRedundancyStateChanged);
Receive<DbHealthProbeActor.DbHealthStatus>(HandleDbHealthStatus);
Receive<HealthTick>(_ => OnHealthTick());
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
}
@@ -154,6 +232,16 @@ public sealed class OpcUaPublishActor : ReceiveActor
{
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
}
// Production delivery of DB-health: when a probe is wired, kick an immediate refresh + start the
// periodic tick so _lastDbHealth gets populated and kept fresh without any external pump. Gated
// ONLY on the probe being present (independent of the DPS subscribe) so the calculator path works
// in tests too. No probe → stay on the legacy role-only seam (no timer).
if (_dbHealthProbe is not null)
{
Self.Tell(HealthTick.Instance); // immediate first refresh
Timers.StartPeriodicTimer("db-health", HealthTick.Instance, _healthTickInterval);
}
}
private void HandleAttributeUpdate(AttributeValueUpdate msg)
@@ -304,11 +392,15 @@ public sealed class OpcUaPublishActor : ReceiveActor
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
{
if (msg.ServiceLevel == _lastServiceLevel) return;
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
_lastServiceLevel = msg.ServiceLevel;
try
{
_serviceLevel.Publish(msg.ServiceLevel);
_publishedAtLeastOnce = true;
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
@@ -319,30 +411,134 @@ public sealed class OpcUaPublishActor : ReceiveActor
}
}
/// <summary>
/// Compute a coarse ServiceLevel from the cluster snapshot and forward to the
/// <see cref="IServiceLevelPublisher"/>. This is a placeholder for F10b's full health
/// aggregation — for now we surface "primary-leader → 240, secondary → 100, detached → 0"
/// so the local SDK at least reflects role state. The full <see cref="ServiceLevelCalculator"/>
/// path (with DB-reachable, OPC UA probe inputs) lives in <c>RedundancyStateActor</c> on
/// admin nodes; this driver-side mirror exists so each node's own SDK exposes a sensible
/// ServiceLevel without round-tripping back through the admin singleton.
/// </summary>
/// <summary>Caches the latest redundancy snapshot and recomputes the local ServiceLevel.
/// The actual byte is produced by <see cref="RecomputeServiceLevel"/> — either via the
/// health-aware <see cref="ServiceLevelCalculator"/> (once a DB-health probe + sample are
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
{
if (_localNode is null) return;
_lastSnapshot = msg;
RecomputeServiceLevel();
}
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
if (local is null) return;
/// <summary>Caches the latest DB-health sample and recomputes the local ServiceLevel. The
/// probe pushes these (or the actor Asks for them); either way the freshest sample feeds the
/// calculator's <c>DbReachable</c>/<c>Stale</c> inputs.</summary>
private void HandleDbHealthStatus(DbHealthProbeActor.DbHealthStatus msg)
{
_lastDbHealth = msg;
RecomputeServiceLevel();
}
byte level = local.Role switch
/// <summary>Periodic self-tick: Asks the local <see cref="DbHealthProbeActor"/> for its cached
/// status and pipes the reply back to <see cref="Self"/>, where <see cref="HandleDbHealthStatus"/>
/// caches it + recomputes. A hung/late probe (Ask timeout) pipes a <c>Reachable=false</c> status so
/// the node FAIL-SAFE-DEMOTES rather than freezing the last-known-good. The continuation only
/// CONSTRUCTS a record off the actor thread — the actual state mutation happens on the actor thread
/// when the piped <see cref="DbHealthProbeActor.DbHealthStatus"/> is received, so no actor field is
/// touched here.</summary>
private void OnHealthTick()
{
_dbHealthProbe!.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1))
.ContinueWith(t => t.IsCompletedSuccessfully
? t.Result
: new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "db-health ask timeout"))
.PipeTo(Self);
}
/// <summary>Records a peer's OPC UA probe verdict about THIS node and recomputes the local
/// ServiceLevel. The probe's <see cref="PeerOpcUaProbeActor.OpcUaProbeResult.NodeId"/> is the
/// target that was probed, so a result whose <c>NodeId</c> is not this node is about a peer and
/// is ignored. A matching result is stamped with the receive time so <see cref="OpcUaProbeOk"/>
/// can debounce stale verdicts.</summary>
private void HandlePeerProbe(PeerOpcUaProbeActor.OpcUaProbeResult r)
{
// The result targets the probed node. If it isn't me, it's about a peer — ignore it.
if (_localNode is null || r.NodeId != _localNode.Value) return;
_probeAboutMe = (r.Ok, DateTime.UtcNow);
RecomputeServiceLevel();
}
/// <summary>The OPC UA self-probe input for the calculator: "did a peer recently observe MY OPC UA
/// endpoint as reachable?" Returns <c>true</c> (benefit of the doubt) when no peer verdict has
/// arrived yet (single-node / no peer) or when the latest verdict is older than
/// <see cref="_probeFreshnessWindow"/> (the peer went away — don't penalise this node for that).
/// Only an actively-observed, RECENT <c>Ok==false</c> demotes.</summary>
private bool OpcUaProbeOk()
{
if (_probeAboutMe is not { } verdict) return true;
if (DateTime.UtcNow - verdict.At > _probeFreshnessWindow) return true;
return verdict.Ok;
}
/// <summary>
/// Computes the local OPC UA ServiceLevel and routes it through <see cref="ServiceLevelChanged"/>
/// (the dedup/publish/metric handler). The full <see cref="ServiceLevelCalculator"/> path is
/// used once a DB-health probe is wired AND a sample has arrived; until then (and when no probe
/// is supplied at all) a legacy role-only seam keeps the historical "primary-leader → 240,
/// secondary → 100, detached → 0" behaviour. The calculator does not model Detached, so a
/// detached local node is guarded to 0 before either path runs.
/// </summary>
private void RecomputeServiceLevel()
{
if (_localNode is null || _lastSnapshot is null) return;
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
// The calculator does NOT model Detached — a healthy detached node would wrongly compute
// 240, so guard it (and the missing-entry case) to 0 here.
if (entry is null || entry.Role == RedundancyRole.Detached)
{
RedundancyRole.Primary when local.IsRoleLeaderForDriver => 240,
RedundancyRole.Primary => 200,
RedundancyRole.Secondary => 100,
RedundancyRole.Detached => 0,
_ => 0,
};
Self.Tell(new ServiceLevelChanged(level));
Self.Tell(new ServiceLevelChanged(0));
return;
}
// Legacy / back-compat seam: with no DB-health probe wired (or before the first sample
// arrives) fall back to the old role-only switch. This preserves historical behaviour and
// is the bootstrap value until the first DbHealthStatus lands.
if (_dbHealthProbe is null || _lastDbHealth is null)
{
Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry)));
return;
}
var now = DateTime.UtcNow;
var inputs = new NodeHealthInputs(
MemberState: SafeSelfStatus(),
DbReachable: _lastDbHealth.Reachable,
OpcUaProbeOk: OpcUaProbeOk(),
Stale: !_lastDbHealth.Reachable
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|| (now - entry.AsOfUtc) > _staleWindow,
IsDriverRoleLeader: entry.IsRoleLeaderForDriver);
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
}
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
{
RedundancyRole.Primary when entry.IsRoleLeaderForDriver => 240,
RedundancyRole.Primary => 200,
RedundancyRole.Secondary => 100,
_ => 0,
};
/// <summary>Reads this node's cluster <see cref="MemberStatus"/>, returning
/// <see cref="MemberStatus.Removed"/> if the cluster is unavailable (so the calculator treats it
/// as untrusted → 0 rather than throwing).</summary>
private MemberStatus SafeSelfStatus()
{
try
{
return _cluster.SelfMember.Status;
}
catch (Exception ex)
{
_log.Debug(ex, "OpcUaPublish: SelfMember status unavailable; treating as Removed (ServiceLevel→0)");
return MemberStatus.Removed;
}
}
}
@@ -32,6 +32,7 @@ public static class ServiceCollectionExtensions
public const string HistorianAdapterActorName = "historian-adapter";
public const string DependencyMuxActorName = "dependency-mux";
public const string OpcUaPublishActorName = "opcua-publish";
public const string PeerProbeSupervisorName = "peer-probe-supervisor";
/// <summary>
/// Registers shared runtime services. Currently binds <see cref="IAlarmHistorianSink"/>
@@ -213,10 +214,19 @@ public static class ServiceCollectionExtensions
serviceLevel: serviceLevel,
localNode: roleInfo.LocalNode,
dbFactory: dbFactory,
applier: applier),
applier: applier,
dbHealthProbe: dbHealth),
OpcUaPublishActorName);
registry.Register<OpcUaPublishActorKey>(publishActor);
// Per-node peer-probe supervisor — keeps one OPC UA TCP probe per OTHER non-Detached
// driver node, so every node is continuously probed by all its peers. OpcUaPublishActor
// consumes the resulting probe verdicts to learn this node's own reachability.
var peerProbes = system.ActorOf(
PeerProbeSupervisor.Props(roleInfo.LocalNode),
PeerProbeSupervisorName);
registry.Register<PeerProbeSupervisorKey>(peerProbes);
var driverHost = system.ActorOf(
DriverHostActor.Props(dbFactory, roleInfo.LocalNode, coordinator: null,
driverFactory: driverFactory, localRoles: roleInfo.LocalRoles,
@@ -246,3 +256,6 @@ public sealed class DbHealthProbeActorKey { }
public sealed class HistorianAdapterActorKey { }
public sealed class DependencyMuxActorKey { }
public sealed class OpcUaPublishActorKey { }
/// <summary>Marker key for the per-node PeerProbeSupervisor.</summary>
public sealed class PeerProbeSupervisorKey { }
@@ -1,7 +1,7 @@
using Akka.Cluster;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
@@ -0,0 +1,158 @@
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
/// <summary>
/// Tests for <see cref="PeerProbeSupervisor"/>: it maintains exactly one peer-OPC-UA-probe child
/// per OTHER, non-Detached driver node named in the latest <see cref="RedundancyStateChanged"/>
/// snapshot, spawning/stopping children as the topology changes.
/// </summary>
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
{
private static readonly NodeId Local = NodeId.Parse("local:4053");
private static readonly NodeId Peer = NodeId.Parse("peer:4053");
private static readonly NodeId Adm = NodeId.Parse("adm:4053");
/// <summary>No-op child actor stub so we can count children without real TCP probes.</summary>
private sealed class NoopActor : ReceiveActor { }
private static Props NoopProps() => Akka.Actor.Props.Create(() => new NoopActor());
/// <summary>
/// No-op child stub that records its own <see cref="ActorBase.Self"/> into a shared list on
/// start, in spawn order — lets a test grab a specific (e.g. the FIRST, old-generation) child
/// ref so it can deliver a synthetic <see cref="Terminated"/> for it.
/// </summary>
private sealed class RecordingNoopActor : ReceiveActor
{
public RecordingNoopActor(List<IActorRef> spawned) => spawned.Add(Self);
}
private static Props RecordingProps(List<IActorRef> spawned) =>
Akka.Actor.Props.Create(() => new RecordingNoopActor(spawned));
private static NodeRedundancyState State(NodeId id, RedundancyRole role) =>
new(id, role, IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow);
private static RedundancyStateChanged Snapshot(params NodeRedundancyState[] nodes) =>
new(nodes, CorrelationId.NewId());
/// <summary>Verifies one child is spawned per non-self, non-Detached peer — self and Detached
/// nodes are excluded.</summary>
[Fact]
public void Spawns_one_child_per_non_self_non_detached_peer()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary),
State(Adm, RedundancyRole.Detached)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies the child for a departed peer is stopped when the next snapshot omits it.</summary>
[Fact]
public void Stops_child_for_departed_peer()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies a single-node snapshot (just the local node) spawns no children.</summary>
[Fact]
public void Single_node_snapshot_spawns_no_children()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies a previously-removed peer is respawned when it re-appears, without an
/// "actor name not unique" collision on the sanitized child name.</summary>
[Fact]
public void Re_adding_a_previously_removed_peer_respawns_it()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Locks in the stale-Terminated guard: when an OLD (already-replaced) child's
/// <see cref="Terminated"/> for a peer arrives AFTER a NEW child for the SAME peer was spawned,
/// the fresh child must NOT be evicted (removal is keyed by current-child ref-equality, not by
/// peer key). Without this guard a late stale Terminated would silently drop a live probe.</summary>
[Fact]
public void Stale_terminated_for_old_child_does_not_evict_fresh_peer_child()
{
var spawned = new List<IActorRef>();
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => RecordingProps(spawned)));
// First add of the peer -> child #0 (the OLD ref).
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
AwaitAssert(() => spawned.Count.ShouldBe(1), duration: TimeSpan.FromMilliseconds(500));
var oldRef = spawned[0];
// Drop the peer -> child #0 stopped, ChildCount back to 0.
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
// Re-add the SAME peer -> a NEW child #1 (the FRESH ref) is spawned.
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
AwaitAssert(() => spawned.Count.ShouldBe(2), duration: TimeSpan.FromMilliseconds(500));
// Now deliver a STALE Terminated for the OLD ref. The current child for Peer is the fresh
// child #1, so ref-equality finds no match and the supervisor must leave ChildCount at 1.
sup.Tell(new Terminated(oldRef, existenceConfirmed: true, addressTerminated: false));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
}
@@ -5,6 +5,7 @@ using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
@@ -112,6 +113,31 @@ public sealed class OpcUaPublishActorTests : RuntimeActorTestBase
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the very first computed ServiceLevel is always published even when it is
/// 0 — a node starting Detached (or role-less) must publish 0 rather than let the SDK default
/// (255 = full service) stand. The dedup against the <c>byte</c>-default <c>0</c> must not swallow
/// the first publish. Drives a Detached local entry (first computed level = 0) with no DB-health
/// probe and asserts the publisher saw exactly [0].</summary>
[Fact]
public void First_service_level_zero_is_published_not_deduped()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("detached-node");
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Detached,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 0 }),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that RedundancyStateChanged drives local ServiceLevel publish for primary leader.</summary>
[Fact]
public void RedundancyStateChanged_drives_local_ServiceLevel_publish_for_primary_leader()
@@ -157,6 +183,362 @@ public sealed class OpcUaPublishActorTests : RuntimeActorTestBase
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the calculator path computes 250 for a healthy primary role-leader
/// (basis 240 from DB-reachable + probe-ok + fresh, +10 driver-role-leader bonus).</summary>
[Fact]
public void Calculator_path_healthy_primary_leader_publishes_250()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the calculator path computes 240 for a healthy non-leader secondary
/// (basis 240 from DB-reachable + probe-ok + fresh, no leader bonus). Documented change from the
/// legacy role-only path, which mapped Secondary → 100.</summary>
[Fact]
public void Calculator_path_healthy_secondary_publishes_240()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("secondary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Secondary,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the calculator path computes 100 when the DB is unreachable
/// ((false,_,true) basis, stale via !DbReachable) for a non-leader secondary (no bonus).</summary>
[Fact]
public void Calculator_path_db_unreachable_publishes_100()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("secondary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "down"))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "down"));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Secondary,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)100),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the calculator path computes 200 for a stale snapshot when the DB is
/// reachable + fresh but the redundancy entry's AsOfUtc is older than the stale window
/// ((true,_,true) basis) for a non-leader secondary (no bonus).</summary>
[Fact]
public void Calculator_path_stale_snapshot_publishes_200()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("secondary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(2)));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Secondary,
IsClusterLeader: false, IsRoleLeaderForDriver: false,
DateTime.UtcNow - TimeSpan.FromMinutes(1)),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)200),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that a detached local node publishes 0 (the calculator does not model
/// Detached, so the handler guards it before the calculator path). The node first goes healthy
/// (250) so the dedup'd transition down to 0 is observable.</summary>
[Fact]
public void Calculator_path_detached_publishes_0()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("detached-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
// First go healthy primary-leader (250) so the transition down to 0 is not dedup'd against
// the initial 0.
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
duration: TimeSpan.FromMilliseconds(500));
// Now detach — expect the guard to drive ServiceLevel down to 0.
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Detached,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that an actively-observed, recent peer probe of MY endpoint that came back
/// <c>Ok==false</c> demotes the calculator basis to 0. A non-leader Secondary entry (no +10 bonus)
/// is used so the assertion is unambiguous: inputs (DbReachable=true, OpcUaProbeOk=false,
/// Stale=false) match Compute's <c>_ => 0</c> arm → 0, +0 (non-leader) → 0.</summary>
[Fact]
public void Probe_false_about_me_with_healthy_db_publishes_0()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("secondary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Secondary,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies branch (3) of <c>OpcUaProbeOk()</c>: a peer's NEGATIVE verdict about this node
/// that has aged past <c>_probeFreshnessWindow</c> is given the benefit of the doubt (the peer's
/// verdict aged out → don't demote), so the node still publishes the HEALTHY level. With a 1ms
/// freshness window and a 30ms wait before the recompute, the cached <c>Ok==false</c> verdict is
/// reliably stale by the time the level is computed, so <c>OpcUaProbeOk()</c> returns true and a
/// healthy primary-leader computes inputs (true, true, false) → 240, +10 leader → 250 — NOT the 0
/// that a still-fresh negative verdict would produce (cf. <see cref="Probe_false_about_me_with_healthy_db_publishes_0"/>).</summary>
[Fact]
public void Stale_probe_verdict_is_not_demoted_publishes_healthy()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(30),
probeFreshnessWindow: TimeSpan.FromMilliseconds(1)));
// Seed a healthy DB sample and a NEGATIVE peer verdict about me. (No RedundancyStateChanged yet,
// so no level is computed from these — the verdict just gets stamped with the receive time.)
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
// Let the verdict age WELL past the 1ms freshness window before any level is computed. The 30ms
// wait is >> 1ms, so the verdict is reliably stale by the time OpcUaProbeOk() runs — no reliance
// on sub-ms clock ties.
ExpectNoMsg(TimeSpan.FromMilliseconds(30));
// Now trigger the recompute with a fresh healthy primary-leader snapshot. By now _probeAboutMe is
// >1ms old, so branch (3) ignores the negative verdict and OpcUaProbeOk() returns true.
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
// Healthy (250), NOT 0 — proves an aged negative verdict does not demote.
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that with no peer probe result ever received, <c>OpcUaProbeOk()</c> defaults
/// to <c>true</c> (benefit of the doubt / single-node). A healthy primary-leader thus computes
/// inputs (true, true, false) → 240, +10 leader → 250.</summary>
[Fact]
public void No_probe_result_defaults_ok_true_publishes_250()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that a later <c>Ok==true</c> peer probe supersedes an earlier <c>Ok==false</c>
/// (recovery). A non-leader Secondary entry is used (no +10 bonus): after the true supersedes,
/// inputs (true, true, false) → 240, +0 → 240.</summary>
[Fact]
public void Probe_true_supersedes_earlier_false_publishes_240()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("secondary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: true));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Secondary,
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that a peer probe result about a DIFFERENT node is ignored — it does not
/// affect MY <c>OpcUaProbeOk()</c>, which stays at its default <c>true</c>. A healthy
/// primary-leader thus still computes (true, true, false) → 240, +10 → 250.</summary>
[Fact]
public void Probe_about_a_different_node_is_ignored()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(NodeId.Parse("someone-else"), Ok: false));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies the legacy back-compat seam: with no DB-health probe wired, the handler
/// falls back to the old role-only switch (Primary + leader → 240).</summary>
[Fact]
public void Legacy_path_no_db_probe_keeps_role_only()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local));
actor.Tell(new RedundancyStateChanged(
Nodes: new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
},
CorrelationId.NewId()));
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the periodic <c>HealthTick</c> Asks the local <see cref="DbHealthProbeActor"/>
/// for status and pipes the reply back to itself — populating <c>_lastDbHealth</c> in production WITHOUT
/// any external pump (no direct <see cref="DbHealthProbeActor.DbHealthStatus"/> Tell). With a healthy DB
/// reply and a primary-leader snapshot, the calculator path therefore reaches 250.</summary>
[Fact]
public void HealthTick_asks_db_probe_and_publishes_calculator_byte()
{
var publisher = new RecordingPublisher();
var local = NodeId.Parse("primary-node");
var db = Sys.ActorOf(Akka.Actor.Props.Create(() => new StubDbHealth(
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30),
healthTickInterval: TimeSpan.FromMilliseconds(100)));
actor.Tell(new RedundancyStateChanged(new[]
{
new NodeRedundancyState(local, RedundancyRole.Primary,
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
}, CorrelationId.NewId()));
// No direct DbHealthStatus Tell — the periodic HealthTick Ask must populate it → 250.
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromSeconds(2));
}
/// <summary>Stub DB-health probe actor that answers <see cref="DbHealthProbeActor.GetStatus"/>
/// with a fixed status, so the calculator path is deterministic without a real timer.</summary>
private sealed class StubDbHealth : Akka.Actor.ReceiveActor
{
/// <summary>Initializes a new instance of the <see cref="StubDbHealth"/> class.</summary>
/// <param name="status">The fixed DB-health status to reply with.</param>
public StubDbHealth(DbHealthProbeActor.DbHealthStatus status) =>
Receive<DbHealthProbeActor.GetStatus>(_ => Sender.Tell(status));
}
/// <summary>Test implementation of IOpcUaAddressSpaceSink that records calls.</summary>
private sealed class RecordingSink : IOpcUaAddressSpaceSink
{
@@ -50,17 +50,20 @@ public sealed class ServiceCollectionExtensionsTests
var historian = host.Services.GetRequiredService<IRequiredActor<HistorianAdapterActorKey>>();
var mux = host.Services.GetRequiredService<IRequiredActor<DependencyMuxActorKey>>();
var publish = host.Services.GetRequiredService<IRequiredActor<OpcUaPublishActorKey>>();
var peerProbes = host.Services.GetRequiredService<IRequiredActor<PeerProbeSupervisorKey>>();
driverHost.ActorRef.ShouldNotBeNull();
dbHealth.ActorRef.ShouldNotBeNull();
historian.ActorRef.ShouldNotBeNull();
mux.ActorRef.ShouldNotBeNull();
publish.ActorRef.ShouldNotBeNull();
peerProbes.ActorRef.ShouldNotBeNull();
driverHost.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DriverHostActorName);
dbHealth.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DbHealthProbeActorName);
historian.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.HistorianAdapterActorName);
mux.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DependencyMuxActorName);
publish.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.OpcUaPublishActorName);
peerProbes.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.PeerProbeSupervisorName);
}
finally
{