Compare commits
14 Commits
4bd7180e7f
...
4af8e65af1
| Author | SHA1 | Date | |
|---|---|---|---|
| 4af8e65af1 | |||
| 4c78dcd358 | |||
| 393b746d9b | |||
| 5a064e086d | |||
| 70e6d3d2c0 | |||
| f41e957e07 | |||
| 37b32a5623 | |||
| 5382eea9b5 | |||
| cf278035d2 | |||
| a9ff1a64b2 | |||
| 3e609a2b19 | |||
| ff0f62db38 | |||
| 7605f4d8fd | |||
| 0528353315 |
+75
-52
@@ -13,77 +13,100 @@ The runtime pieces live in:
|
||||
| Component | Project | Role |
|
||||
|---|---|---|
|
||||
| `RedundancyStateActor` | `OtOpcUa.ControlPlane.Redundancy` | Admin-role cluster singleton; subscribes to cluster topology events, debounces 250ms, broadcasts `RedundancyStateChanged` on the `redundancy-state` DPS topic. |
|
||||
| `OpcUaPublishActor` | `OtOpcUa.Runtime.OpcUa` | Per-driver-node; subscribes to the `redundancy-state` topic, maps the local node's role to a ServiceLevel byte (see below), and forwards it to `IServiceLevelPublisher`. |
|
||||
| `OpcUaPublishActor` | `OtOpcUa.Runtime.OpcUa` | Per-driver-node; subscribes to the `redundancy-state` topic, computes a health-aware ServiceLevel byte via `ServiceLevelCalculator` (see below), and forwards it to `IServiceLevelPublisher`. |
|
||||
| `IServiceLevelPublisher` / `SdkServiceLevelPublisher` | `OtOpcUa.Commons.OpcUa` / `OtOpcUa.OpcUaServer` | Writes the byte into the SDK's `Server.ServiceLevel` Variable. Production binds `DeferredServiceLevelPublisher`, which swaps in the real `SdkServiceLevelPublisher` once the SDK is up (it needs `IServerInternal`, available only after `StandardServer.Start`); until then writes route through `NullServiceLevelPublisher`. |
|
||||
| `ServiceLevelCalculator` | `OtOpcUa.ControlPlane.Redundancy` | Pure function `(NodeHealthInputs) → byte` — the fuller DB/probe-aware tiering (see truth table below). Covered by `ServiceLevelCalculatorTests`; **not yet wired into the live driver publish path**, which uses the coarse role mapping in `OpcUaPublishActor`. |
|
||||
| `DbHealthProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; runs `SELECT 1` against ConfigDb every 5s. Read by health endpoint. |
|
||||
| `PeerOpcUaProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; pings peer `opc.tcp://peer:4840` with a TCP connect (2s timeout) and publishes the result on the `redundancy-state` topic. A full secure-channel Hello handshake is a possible future upgrade; the TCP connect is the current real probe. |
|
||||
| `ServiceLevelCalculator` | `OtOpcUa.Cluster.Redundancy` (`Core.Cluster`) | Pure function `(NodeHealthInputs) → byte` — the DB/probe-aware tiering (see truth table below). Covered by `ServiceLevelCalculatorTests`. **Now the live publish path** — `OpcUaPublishActor` calls it on every `HealthTick` and `RedundancyStateChanged` event. Moved to `Core.Cluster` so Runtime can reach it without a Runtime→ControlPlane reference. |
|
||||
| `DbHealthProbeActor` | `OtOpcUa.Runtime.Health` | Per-node; runs `SELECT 1` against ConfigDb every 5s. Read by the health endpoint AND by `OpcUaPublishActor` (the `DbReachable` ServiceLevel input). |
|
||||
| `PeerProbeSupervisor` | `OtOpcUa.Runtime.Health` | Per-node; subscribes to the `redundancy-state` topic and maintains one `PeerOpcUaProbeActor` child per OTHER driver-role peer (spawn on join, stop on departure), so every node is continuously probed by its peers. |
|
||||
| `PeerOpcUaProbeActor` | `OtOpcUa.Runtime.Health` | Spawned by `PeerProbeSupervisor`; pings a peer `opc.tcp://peer:4840` with a TCP connect (2s timeout) and publishes `OpcUaProbeResult` on the `redundancy-state` topic. A full secure-channel Hello handshake is a possible future upgrade; the TCP connect is the current real probe. |
|
||||
| `ClusterRoleInfo` | `OtOpcUa.Cluster` | Live view of cluster membership + role-leader; exposes `IClusterRoleInfo` to the rest of the host. |
|
||||
|
||||
## ServiceLevel tiers
|
||||
|
||||
### Live driver-side mapping (current)
|
||||
### Health-aware tiering (`ServiceLevelCalculator` — live path)
|
||||
|
||||
`OpcUaPublishActor.HandleRedundancyStateChanged` maps the local node's role
|
||||
(from the `RedundancyStateChanged` snapshot) to a ServiceLevel byte and forwards
|
||||
it through `IServiceLevelPublisher` to the SDK's `Server.ServiceLevel` Variable:
|
||||
`ServiceLevelCalculator.Compute(NodeHealthInputs)` is the live publish path.
|
||||
`OpcUaPublishActor` calls it on every `HealthTick` (~5 s) and on each
|
||||
`RedundancyStateChanged` snapshot, then forwards the result through
|
||||
`IServiceLevelPublisher` to the SDK's `Server.ServiceLevel` Variable.
|
||||
|
||||
| Local role | Byte |
|
||||
The four inputs are sourced locally per driver node:
|
||||
|
||||
| Input | Source |
|
||||
|---|---|
|
||||
| `Primary` and `driver` role-leader | 240 |
|
||||
| `Primary` (not role-leader) | 200 |
|
||||
| `Secondary` | 100 |
|
||||
| `Detached` (no `driver` role) | 0 |
|
||||
| `MemberState` | Local `SelfMember.Status` from the Akka cluster (Up / Joining / Leaving / …). |
|
||||
| `DbReachable` | Local `DbHealthProbeActor` — `OpcUaPublishActor` Asks it on each `HealthTick`; an Ask timeout is treated as `Reachable=false`. |
|
||||
| `OpcUaProbeOk` | Result of a peer probing THIS node's OPC UA endpoint: `PeerProbeSupervisor` spawns one `PeerOpcUaProbeActor` per OTHER driver-role peer; each probe publishes `OpcUaProbeResult(probed-node, ok)` on the `redundancy-state` topic; the publish actor consumes only results whose target is itself. Freshness-debounced: absent or stale (>30 s) → `true` (benefit of the doubt — single-node clusters and a departed peer never demote); only an actively-observed RECENT `false` demotes. |
|
||||
| `Stale` (derived) | `!DbReachable \|\| (now − lastDbHealth.AsOfUtc) > 30 s \|\| (now − snapshotEntry.AsOfUtc) > 30 s`. |
|
||||
| `IsDriverRoleLeader` | The local node's entry in the `RedundancyStateChanged` snapshot from `RedundancyStateActor`. |
|
||||
|
||||
The resulting truth table (all tiers are now reachable at runtime):
|
||||
|
||||
| Tier | Byte | Condition |
|
||||
|---|---|---|
|
||||
| Down / Detached | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting), OR node has no `driver` role (Detached). Published immediately — a starting or detached node never leaves the SDK default 255. |
|
||||
| Critically degraded | 100 | ConfigDb unreachable AND data is stale. |
|
||||
| Stale | 200 | Data stale but ConfigDb reachable. |
|
||||
| Healthy follower | 240 | DB reachable + OPC UA probe ok + not stale + not role-leader. |
|
||||
| Healthy leader | 250 | Same as healthy follower + this node is the `driver` role-leader (+10 bonus). |
|
||||
|
||||
> **Secondary 100 → 240 (behavior change).** Previously a healthy Secondary
|
||||
> published 100 (coarse role-only mapping). It now publishes **240** — both
|
||||
> nodes sit at 240/250 under healthy conditions, with the leader still preferred
|
||||
> by the +10 bonus. Clients with the standard "pick highest ServiceLevel"
|
||||
> heuristic continue to prefer the primary.
|
||||
|
||||
#### Backward-compatible fallback (legacy seam)
|
||||
|
||||
A node with no `DbHealthStatus` wired (e.g. early bootstrap window before the
|
||||
first `DbHealthProbeActor` reply) falls back to the old role-only mapping:
|
||||
Primary-leader → 240, Primary → 200, Secondary → 100, Detached → 0. Once the
|
||||
first `DbHealthStatus` arrives the calculator takes over. The first computed
|
||||
ServiceLevel (even 0) is always published so no node lingers at the SDK default
|
||||
255.
|
||||
|
||||
Roles come from `RedundancyStateActor.BuildSnapshot`: a node with the `driver`
|
||||
role is `Primary` when it holds the `driver` role-leader lease, otherwise
|
||||
`Secondary`; a node without the `driver` role is `Detached`.
|
||||
|
||||
### Full health-aware tiering (`ServiceLevelCalculator`)
|
||||
|
||||
`ServiceLevelCalculator.Compute(NodeHealthInputs)` is the fuller, DB/probe-aware
|
||||
calculation. It is unit-tested but **not yet on the live publish path** — the
|
||||
driver-side mapping above is what actually drives the SDK today.
|
||||
|
||||
| Tier | Byte | Condition |
|
||||
|---|---|---|
|
||||
| Down | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting). |
|
||||
| Critically degraded | 100 | ConfigDb unreachable AND data is stale. |
|
||||
| Stale | 200 | Data stale but ConfigDb reachable. |
|
||||
| Healthy follower | 240 | DB ok + OPC UA probe ok + not stale. |
|
||||
| Healthy leader | 250 | Healthy follower (240) + a `+10` bonus when this node is the `driver` role-leader. |
|
||||
|
||||
Either way, clients with the standard redundancy heuristic ("pick the highest
|
||||
ServiceLevel") prefer the `driver` role-leader and fall back to followers on its
|
||||
degradation.
|
||||
|
||||
## Data flow
|
||||
|
||||
```
|
||||
Cluster topology event ──┐
|
||||
▼
|
||||
RedundancyStateActor (admin singleton)
|
||||
│ debounce 250ms
|
||||
▼
|
||||
DPS topic "redundancy-state"
|
||||
│
|
||||
▼
|
||||
Driver nodes' OpcUaPublishActor
|
||||
│ role → byte (240/200/100/0)
|
||||
▼
|
||||
Cluster topology event ──────────────────────────────────────────┐
|
||||
▼
|
||||
RedundancyStateActor (admin singleton)
|
||||
│ debounce 250ms
|
||||
▼
|
||||
DPS topic "redundancy-state"
|
||||
│ ▲
|
||||
┌───────────────────────┘ │
|
||||
│ │
|
||||
▼ │
|
||||
Driver node: OpcUaPublishActor │
|
||||
┌─────────────────────────────────────────────────────────┐ │
|
||||
│ Inputs collected per ~5s HealthTick: │ │
|
||||
│ • MemberState ← Akka SelfMember.Status │ │
|
||||
│ • DbReachable ← DbHealthProbeActor (Ask, timeout→F) │ │
|
||||
│ • OpcUaProbeOk ← OpcUaProbeResult about THIS node │──────┘
|
||||
│ • Stale ← derived from above timestamps │ PeerProbeSupervisor
|
||||
│ • IsLeader ← RedundancyStateChanged snapshot │ → PeerOpcUaProbeActor(s)
|
||||
│ │ publish OpcUaProbeResult
|
||||
│ ServiceLevelCalculator.Compute(NodeHealthInputs) │ on "redundancy-state"
|
||||
│ → byte (0/100/200/240/250) │
|
||||
└───────────────────────────────────────────────────────-─┘
|
||||
│
|
||||
▼
|
||||
IServiceLevelPublisher (SdkServiceLevelPublisher)
|
||||
│
|
||||
▼
|
||||
│
|
||||
▼
|
||||
OPC UA Server.ServiceLevel Variable
|
||||
```
|
||||
|
||||
Today only cluster topology drives the published ServiceLevel.
|
||||
`PeerOpcUaProbeActor` and `DbHealthProbeActor` also run per-node — the peer probe
|
||||
publishes `OpcUaProbeResult` onto the `redundancy-state` topic and the DB probe
|
||||
backs the health endpoint — but their outputs are not yet consumed by
|
||||
`RedundancyStateActor` or folded into the published byte. They are the inputs the
|
||||
fuller `ServiceLevelCalculator` truth table is designed to use once that path goes
|
||||
live.
|
||||
Both `DbHealthProbeActor` and `PeerOpcUaProbeActor` feed the live publish path.
|
||||
The peer probe publishes `OpcUaProbeResult` on the `redundancy-state` topic;
|
||||
`OpcUaPublishActor` consumes only results whose target is itself and applies
|
||||
freshness-debouncing before passing them to the calculator. `DbHealthProbeActor`
|
||||
is queried directly via Ask on each `HealthTick`.
|
||||
|
||||
The admin singleton is the cluster's only `RedundancyStateActor`. If the admin leader fails over, the new admin node spins up its replacement, re-subscribes to cluster events, and publishes a fresh snapshot from the current `Cluster.State`. There is no DB-persisted state to recover.
|
||||
|
||||
@@ -113,7 +136,7 @@ There is no longer a `Node:NodeId` setting and no `ClusterNode.RedundancyRole` c
|
||||
|
||||
> **`RedundancyStateActor` NodeId consistency (fixed).** `RedundancyStateActor` now keys each node's `NodeRedundancyState` entry by the canonical `host:port` node id (via a `ToNodeId(Address)` helper mirroring `ClusterRoleInfo.ToNodeId`). Previously it keyed by `member.Address.Host` (host-only, e.g. `central-2`); since every subscriber matches by the canonical `host:port` form, the mismatch silently meant no node ever matched its own entry — all nodes stayed at the default ServiceLevel 255 and never learned their role. This fix makes `RedundancyStateActor` consistent with the stated contract above. Additionally, `RedundancyStateActor` now **re-publishes the current snapshot on a periodic heartbeat (default 10 s)** so any node that subscribes after the last topology-change publish converges within the interval (DistributedPubSub does not replay to late subscribers).
|
||||
|
||||
The `ClusterNode.ServiceLevelBase` column still exists and is editable in the Admin UI (NodeEdit / Cluster Redundancy pages), but it no longer drives the runtime ServiceLevel — that value is computed from cluster role/health and published per the mapping above, independent of this stored preference.
|
||||
The `ClusterNode.ServiceLevelBase` column still exists and is editable in the Admin UI (NodeEdit / Cluster Redundancy pages), but it no longer drives the runtime ServiceLevel — that value is computed by `ServiceLevelCalculator` from cluster role and live health inputs, independent of this stored preference.
|
||||
|
||||
### Peer URI advertising
|
||||
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# Phase 2 — Health-aware redundancy ServiceLevel (H3) — design
|
||||
|
||||
> **Status:** approved 2026-06-15. Parent roadmap: `docs/plans/2026-06-15-stillpending-backlog-design.md`
|
||||
> (Phase 2). Backlog item **H3** in `stillpending.md` §1.
|
||||
> **Branch:** `feat/stillpending-phase-2-servicelevel` off master `4bd7180e`.
|
||||
> **Classification:** high-risk (actor model + redundancy correctness). Live `/run` on the 2-node rig is
|
||||
> the acceptance gate; unit tests cannot prove the cross-node wiring (per `project_redundancy_state_delivery`).
|
||||
|
||||
## Problem (H3)
|
||||
|
||||
The SDK's `Server.ServiceLevel` byte is driven **solely by the coarse role map** today
|
||||
(`OpcUaPublishActor.HandleRedundancyStateChanged`: Primary-leader→240, Primary→200, Secondary→100,
|
||||
Detached→0). The richer `ServiceLevelCalculator.Compute(NodeHealthInputs)` — which folds in DB
|
||||
reachability, an OPC-UA liveness probe, and a staleness signal — **is never invoked in production**.
|
||||
The two health producers are dead or half-wired:
|
||||
|
||||
- `DbHealthProbeActor` — **spawned** per node, but only feeds `/health/ready`; its `DbReachable` never
|
||||
reaches the published byte.
|
||||
- `PeerOpcUaProbeActor` — **never spawned anywhere**; its `OpcUaProbeResult` has **zero consumers**.
|
||||
- `NodeHealthInputs.Stale` — **no producer exists** anywhere in the codebase.
|
||||
|
||||
Consequence: a **DB-unreachable / OPC-UA-down primary still advertises its full role-based ServiceLevel**,
|
||||
so redundant clients keep subscribing to a degraded node instead of failing over.
|
||||
|
||||
## Goal
|
||||
|
||||
Make each driver node publish a ServiceLevel computed by `ServiceLevelCalculator.Compute` from its **real
|
||||
local health**, so an unhealthy node drops below its role-based level (and below a healthy peer's level,
|
||||
triggering client failover). Honor the **already-documented** tiers in `docs/Redundancy.md`
|
||||
(250 / 240 / 200 / 100 / 0) — do not reshape the calculator's truth table. No EF migration; no change to
|
||||
the `RedundancyStateChanged` / `NodeRedundancyState` message contract.
|
||||
|
||||
## Locked decisions (from brainstorming 2026-06-15)
|
||||
|
||||
1. **Compute site = local per-node** (`OpcUaPublishActor`), *not* the admin `RedundancyStateActor`
|
||||
singleton. DB reachability is an inherently local fact; centralizing it would require a new
|
||||
cluster-wide health-broadcast contract for no benefit. The publish actor already runs per node, already
|
||||
subscribes to the `redundancy-state` topic, already holds the role snapshot, and already writes the byte.
|
||||
2. **`Stale` is derived from signal freshness, with DB-unreachable ⟹ stale.** This makes the documented
|
||||
200/100 graceful-degradation tiers reachable, so a node that loses its DB settles into 100 ("degraded,
|
||||
still serving cached values") rather than slamming to 0. The 0 tier stays reserved for "not a healthy
|
||||
cluster member / detached." A richer *driver-input* staleness signal is **deferred** (flagged, not built).
|
||||
3. **`OpcUaProbeOk` = peer-probes-me**, reusing the existing `PeerOpcUaProbeActor` (finally spawned),
|
||||
closing the audit's "never spawned / zero consumers" gaps. Absence of a fresh peer result → `true`
|
||||
(benefit of the doubt: don't penalize a node for its peer being down); only an *actively observed*
|
||||
recent failure demotes. Single-node / no-peer cluster → `true`.
|
||||
|
||||
## Architecture
|
||||
|
||||
All changes are **per-driver-node, inside `Runtime`** (`WithOtOpcUaRuntimeActors`). No ControlPlane /
|
||||
admin-singleton change. No new Commons message types.
|
||||
|
||||
```
|
||||
WithOtOpcUaRuntimeActors (per driver node)
|
||||
├─ DbHealthProbeActor (exists; now ALSO consulted by the publish actor)
|
||||
├─ PeerProbeSupervisor (NEW) watches cluster membership →
|
||||
│ └─ PeerOpcUaProbeActor(peer) one child per OTHER driver member;
|
||||
│ publishes OpcUaProbeResult(peer, ok) on the redundancy-state topic
|
||||
└─ OpcUaPublishActor (MODIFIED)
|
||||
subscribes redundancy-state topic → RedundancyStateChanged (role/leadership)
|
||||
→ OpcUaProbeResult (peer's verdict on ME)
|
||||
holds IActorRef dbHealthProbe → Ask<DbHealthStatus> on a periodic HealthTick
|
||||
Cluster.SelfMember.Status → MemberState (local)
|
||||
on any trigger: ServiceLevelCalculator.Compute(inputs) → publish-if-changed (existing dedup)
|
||||
```
|
||||
|
||||
### Components
|
||||
|
||||
**1. `PeerProbeSupervisor` (NEW, `Runtime/Health/PeerProbeSupervisor.cs`)** — a small per-node actor whose
|
||||
only job is the probe lifecycle (kept separate so the pinned-dispatcher `OpcUaPublishActor` never parents
|
||||
network-probe children):
|
||||
- Subscribes to cluster membership (`IMemberEvent` / `ReachabilityEvent`) and/or the redundancy snapshot.
|
||||
- Maintains one `PeerOpcUaProbeActor` child per **other driver-role member**, (re)spawning on membership
|
||||
change, stopping children for departed members. Resolves the peer's OPC UA host from its node id
|
||||
(`host:port` → host) and the configured OPC UA port.
|
||||
- Children publish `OpcUaProbeResult` to the `redundancy-state` topic (existing behavior).
|
||||
- Spawned on the default dispatcher (not the OPC UA pinned one).
|
||||
|
||||
**2. `OpcUaPublishActor` (MODIFIED)** — replace the role-only switch in `HandleRedundancyStateChanged`
|
||||
with a `RecomputeServiceLevel()` that calls the calculator. New Props params (both optional, defaulted so
|
||||
test Props/harnesses keep working): `IActorRef? dbHealthProbe`, and the freshness windows /
|
||||
self-OPC-UA-port (injectable for tests).
|
||||
- New state: `_dbReachable` + `_dbAsOfUtc` (from the DB Ask), `_lastSnapshotAsOfUtc`, a per-peer map of
|
||||
`(ok, asOfUtc)` for results **about my own node**, and the current `RedundancyStateChanged` local entry.
|
||||
- New `Receive<OpcUaProbeResult>`: if `msg.NodeId == _localNode`, record `(msg.Ok, now)` for the reporting
|
||||
peer; recompute.
|
||||
- New `Receive<HealthTick>` (periodic, e.g. 5 s): `dbHealthProbe.Ask<DbHealthStatus>(GetStatus, 1s).PipeTo(Self)`.
|
||||
- New `Receive<DbHealthStatus>`: cache `_dbReachable` + `_dbAsOfUtc`; recompute.
|
||||
- `RecomputeServiceLevel()` builds `NodeHealthInputs` and calls `ServiceLevelCalculator.Compute`, then
|
||||
`Self.Tell(new ServiceLevelChanged(level))` (the existing handler dedups + publishes + emits the metric).
|
||||
|
||||
### Input derivation (the load-bearing logic)
|
||||
|
||||
For the **local** node, on each recompute:
|
||||
|
||||
| `NodeHealthInputs` field | Source |
|
||||
|---|---|
|
||||
| `MemberState` | `Cluster.Get(system).SelfMember.Status` (local; most accurate) |
|
||||
| `IsDriverRoleLeader` | local entry of the latest `RedundancyStateChanged` snapshot (`IsRoleLeaderForDriver`) |
|
||||
| `DbReachable` | latest `DbHealthStatus.Reachable` from the local `DbHealthProbeActor` |
|
||||
| `OpcUaProbeOk` | `true` if **no** fresh peer result about me, else the latest such result's `Ok` (only an actively-observed recent failure → `false`); single-node → `true` |
|
||||
| `Stale` | `!DbReachable` **OR** `(now − _dbAsOfUtc) > StaleWindow` **OR** `(now − _lastSnapshotAsOfUtc) > StaleWindow` |
|
||||
|
||||
**Guards (calculator does not model these):**
|
||||
- If there is **no local snapshot entry**, or the local entry's `Role == Detached`, publish **0** directly
|
||||
and skip the calculator (a healthy detached node would otherwise compute 240 because the calculator only
|
||||
checks `MemberState` + the leader bonus). In steady state a node running `OpcUaPublishActor` always carries
|
||||
the `driver` role, so this is defensive — it preserves the current Detached→0 behavior during transitions.
|
||||
|
||||
### Resulting ServiceLevel truth table (now fully reachable)
|
||||
|
||||
| Node condition | Byte | How reached |
|
||||
|---|---|---|
|
||||
| Not a healthy member / detached | **0** | `MemberState` not Up/Joining, or Detached guard |
|
||||
| DB unreachable, sustained | **100** | `DbReachable=false ⇒ Stale=true` → `(false,_,true)` |
|
||||
| DB reachable but signals stale | **200** | `(true,_,true)` |
|
||||
| Healthy follower (DB ok + probe ok + fresh) | **240** | `(true,true,false)` |
|
||||
| Healthy leader | **250** | follower 240 + `IsDriverRoleLeader` +10 |
|
||||
| DB ok + fresh but peer reports me unreachable | **0** | `(true,false,false)` → falls through |
|
||||
|
||||
**Behavior change to be aware of (already documented as the target):** a *healthy Secondary moves
|
||||
100 → 240*. Both healthy nodes sit at 240/250 with the leader preferred by the +10 bonus; role-leadership is
|
||||
stable so the bytes do not flap. The "DB ok + probe-fail" → 0 edge is intentionally sharp but **debounced**
|
||||
by the `OpcUaProbeOk` freshness rule (a single missed probe does not flip it; only a sustained, actively
|
||||
observed failure does).
|
||||
|
||||
## Error handling / edge cases
|
||||
|
||||
- **DB Ask timeout** → treat as `DbReachable=false` for that cycle (fail-safe demote); the next tick re-Asks.
|
||||
- **No peer (single node)** → `OpcUaProbeOk=true`; supervisor spawns no children.
|
||||
- **Peer departs** → its prior result ages out of the freshness window → `OpcUaProbeOk` reverts to `true`
|
||||
(we don't penalize a node for its peer's absence). Supervisor stops the child for the departed member.
|
||||
- **Cluster forming / role-leader not yet resolved** → no local snapshot entry yet → Detached guard → 0,
|
||||
same as today, until the first snapshot arrives.
|
||||
- **`HealthTick` before the first snapshot** → no local entry → 0 (no spurious 240 before role is known).
|
||||
|
||||
## Testing strategy (xUnit + Shouldly, TDD; NO bUnit)
|
||||
|
||||
Unit-testable (the wiring that *can* be proven in a TestKit):
|
||||
- `OpcUaPublishActor` input-derivation: feed a `RedundancyStateChanged` (Primary-leader) + a cached
|
||||
`DbHealthStatus(reachable)` + an `OpcUaProbeResult(me, ok)` → assert published byte 250; flip DB
|
||||
unreachable → 100; stale snapshot → 200; healthy secondary → 240; Detached entry → 0; probe `Ok=false`
|
||||
about me with DB-ok+fresh → 0. Use `PropsForTests` with a broadcast/serviceLevel capture + injected
|
||||
`dbHealthProbe` test ref + short windows.
|
||||
- `OpcUaProbeOk` freshness/debounce: a single stale/absent result → `true`; a recent explicit `false` → `false`.
|
||||
- `PeerProbeSupervisor`: on a membership snapshot with one peer driver member, spawns exactly one child
|
||||
targeting that peer; on the peer leaving, stops it; single-node → no children. (TestKit, with a probe
|
||||
factory injected so no real TCP is attempted.)
|
||||
- Reuse existing `ServiceLevelCalculatorTests` unchanged (the pure function is not modified).
|
||||
|
||||
Not unit-testable (acceptance gate): the live cross-node ServiceLevel on the **2-node rig** — proven by
|
||||
`/run` (user-driven). A DB-unreachable primary must be observed dropping below the secondary, and the
|
||||
secondary taking over as the higher-ServiceLevel endpoint. See `project_redundancy_state_delivery`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `dotnet build` clean (production projects are `TreatWarningsAsErrors`); full `dotnet test` green.
|
||||
- High-risk review chain (serial spec → code → final integration).
|
||||
- **Live `/run` on the 2-node / docker-dev rig** (user-driven; agent does not sign in): confirm
|
||||
steady-state 250/240, then kill a node's DB reachability and confirm its ServiceLevel drops to 100 and the
|
||||
peer is preferred. Update `docs/Redundancy.md` to mark the calculator path **wired** (remove the "not yet
|
||||
wired into the live driver publish path" caveats) and document the freshness/peer-probe sourcing.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Central compute in `RedundancyStateActor`** — rejected: the admin singleton can't see each node's local
|
||||
DB health without a new cluster-wide health-broadcast contract; heavier, bigger blast radius, no benefit.
|
||||
- **Local self-probe** (TCP to own `localhost:4840` / SDK server-state) instead of the peer-probe — simpler
|
||||
(no membership-tracking supervisor) and a defensible "is my OPC UA serving" signal, but leaves
|
||||
`PeerOpcUaProbeActor` dead and misses the cross-node vantage. Rejected to honor the locked design and close
|
||||
the audit's dead-code gap; recorded here so the trade-off is explicit if the supervisor proves heavier than
|
||||
valued (then it becomes a follow-up swap, not a redesign).
|
||||
- **Reshape the calculator to role-baseline-demoted-by-health** (keep the wide 240-vs-100 gap; health only
|
||||
lowers) — rejected: the calculator's 250/240/200/100/0 tiers are already approved, documented, and tested;
|
||||
reshaping re-opens a settled decision. The +10 leader bonus already preserves client preference for the
|
||||
primary.
|
||||
|
||||
## Hard constraints (carried from the roadmap)
|
||||
|
||||
NO Configuration entity / EF migration. Stage by path — never `git add .`; never stage `sql_login.txt`,
|
||||
`src/Server/.../Host/pki/`, `pending.md`, `current.md`, `docker-dev/docker-compose.yml`, `stillpending.md`.
|
||||
Never echo or commit secrets. No force-push, no `--no-verify`. Razor/runtime cross-node behavior proven only
|
||||
by live `/run`, never bUnit.
|
||||
@@ -0,0 +1,564 @@
|
||||
# Phase 2 — Health-aware redundancy ServiceLevel (H3) Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Wire the existing-but-never-invoked `ServiceLevelCalculator` into the live per-node OPC UA publish
|
||||
path so a DB-unreachable / OPC-UA-unreachable node drops below its role-based ServiceLevel (and below a
|
||||
healthy peer), triggering client failover.
|
||||
|
||||
**Architecture:** Per-driver-node, inside `Runtime`. `OpcUaPublishActor` computes its own ServiceLevel from
|
||||
real local health (local `DbHealthProbeActor`, peer's-probe-of-me from the `redundancy-state` topic, the
|
||||
role snapshot, signal freshness) via `ServiceLevelCalculator.Compute`, replacing its role-only switch — with
|
||||
a **backward-compatible seam** (no DB-health source ⇒ legacy role-only). A new tiny `PeerProbeSupervisor`
|
||||
finally spawns `PeerOpcUaProbeActor` per driver peer. No admin-singleton change, no EF migration, no Commons
|
||||
message-contract change.
|
||||
|
||||
**Tech Stack:** C# / .NET 10, Akka.NET (ReceiveActor, DistributedPubSub, IWithTimers, Ask+PipeTo), xUnit +
|
||||
Shouldly + Akka.TestKit. Design: `docs/plans/2026-06-15-stillpending-phase-2-servicelevel-design.md`.
|
||||
|
||||
**Branch:** `feat/stillpending-phase-2-servicelevel` (already created off master `4bd7180e`; design doc
|
||||
committed `05283533`).
|
||||
|
||||
---
|
||||
|
||||
## Hard rules (every task)
|
||||
|
||||
- Stage **by path** — never `git add .`. Never stage `sql_login.txt`, `src/Server/.../Host/pki/`,
|
||||
`pending.md`, `current.md`, `docker-dev/docker-compose.yml`, `stillpending.md`. Never echo/commit secrets.
|
||||
No force-push, no `--no-verify`.
|
||||
- **NO** Configuration entity / EF migration. **NO** change to `OpcUaProbeResult` / `RedundancyStateChanged`
|
||||
/ `NodeRedundancyState` contracts (reuse as-is).
|
||||
- TDD fail-then-pass. xUnit + Shouldly. **NO bUnit.** Cross-node behavior proven only by live `/run`.
|
||||
- Production projects are `TreatWarningsAsErrors` — fix all warnings.
|
||||
|
||||
## Key facts the implementer needs
|
||||
|
||||
- `ServiceLevelCalculator.Compute(NodeHealthInputs)` is pure and **must not be reshaped**. Tiers:
|
||||
`MemberState not Up/Joining → 0`; `(DbReachable,OpcUaProbeOk,Stale)` → `(true,true,false)=240`,
|
||||
`(true,_,true)=200`, `(false,_,true)=100`, else `0`; `+10` if `IsDriverRoleLeader` (clamp 255).
|
||||
- `Runtime` already references `Core.Cluster` (`ZB.MOM.WW.OtOpcUa.Cluster.csproj`); `ControlPlane` too.
|
||||
`Runtime` does **NOT** reference `ControlPlane` and must not start to (deliberate boundary — see the
|
||||
comment in `PeerOpcUaProbeActor.cs:22`). So the calculator **moves** to `Core.Cluster` (Task 1).
|
||||
- `DbHealthProbeActor` (`Runtime/Health/`) answers `GetStatus` → `DbHealthStatus(bool Reachable, DateTime
|
||||
AsOfUtc, string? LastError)` from cache (cheap). Already spawned in `WithOtOpcUaRuntimeActors` as the local
|
||||
`dbHealth` `IActorRef`.
|
||||
- `PeerOpcUaProbeActor` (`Runtime/Health/`) publishes `OpcUaProbeResult(NodeId peer, bool Ok)` to the
|
||||
`redundancy-state` topic. `Props(peer, interval?, connectTimeout?, opcUaPort=4840, broadcast?)`.
|
||||
- `OpcUaPublishActor` subscribes to the `redundancy-state` topic in PreStart, so it already receives every
|
||||
message on it — adding `Receive<OpcUaProbeResult>` makes those visible with no extra subscribe.
|
||||
- Test base `RuntimeActorTestBase` self-joins a single-node `driver` cluster → `Cluster.Get(Sys).SelfMember.
|
||||
Status == Up` works in tests. The two existing tests in `OpcUaPublishActorTests.cs`
|
||||
(`..._primary_leader` → 240, `..._for_secondary_publishes_100` → 100) inject **no** `dbHealthProbe`, so the
|
||||
legacy seam keeps them green unchanged.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Move `ServiceLevelCalculator` to `Core.Cluster` (shared, cycle-free)
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** none (Task 2 depends on this)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs`
|
||||
- Delete: `src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ServiceLevelCalculator.cs`
|
||||
- Modify: `tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ServiceLevelCalculatorTests.cs` (using/namespace)
|
||||
|
||||
**Step 1: Verify `Core.Cluster` references `Akka.Cluster`** (the calculator uses `MemberStatus`):
|
||||
Run: `grep -rn "Akka.Cluster" src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj && grep -rln "using Akka.Cluster" src/Core/ZB.MOM.WW.OtOpcUa.Cluster/`
|
||||
Expected: a PackageReference (or transitive) to `Akka.Cluster` and existing `using Akka.Cluster;` files
|
||||
(`ClusterRoleInfo` uses it). If the package is **not** referenced there, STOP and surface it — fall back to
|
||||
adding `Akka.Cluster` as a `PackageReference` on `Core.Cluster` (it already depends on Akka cluster types) or
|
||||
placing the file in `Commons` only if `Commons` references `Akka.Cluster`. Prefer `Core.Cluster`.
|
||||
|
||||
**Step 2: Move the type verbatim**, changing only the namespace:
|
||||
`namespace ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;` (keep `using Akka.Cluster;`, keep `NodeHealthInputs` +
|
||||
`ServiceLevelCalculator` together, keep all XML docs). Delete the old ControlPlane file.
|
||||
|
||||
**Step 3: Fix references.** `RedundancyStateActor.cs` does not call the calculator (only doc-comment
|
||||
mentions) — update the doc-comment text if it names the old namespace, no `using` needed. Update the test:
|
||||
```csharp
|
||||
// tests/.../ServiceLevelCalculatorTests.cs
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy; // was ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy
|
||||
```
|
||||
Also `grep -rn "ControlPlane.Redundancy.ServiceLevelCalculator\|ControlPlane.Redundancy;.*ServiceLevel\|NodeHealthInputs" src tests` and fix any stragglers (e.g. `OpcUaPublishActor.cs`'s `<see cref>` doc-comment, `OtOpcUaConfigDbContext.cs:159` comment text — comment-only, update the prose).
|
||||
|
||||
**Step 4: Build + run the moved test.**
|
||||
Run: `dotnet build ZB.MOM.WW.OtOpcUa.slnx` then
|
||||
`dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests --filter "FullyQualifiedName~ServiceLevelCalculator"`
|
||||
Expected: build clean; `ServiceLevelCalculatorTests` PASS (unchanged behavior, new namespace).
|
||||
|
||||
**Step 5: Commit**
|
||||
```bash
|
||||
git add src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs \
|
||||
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ServiceLevelCalculator.cs \
|
||||
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ServiceLevelCalculatorTests.cs
|
||||
# plus any comment-only files touched in Step 3, by path
|
||||
git commit -m "refactor(redundancy): move ServiceLevelCalculator to Core.Cluster (shared, Runtime-reachable)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2a: `OpcUaPublishActor` calculator path — DB + Stale + leader + Detached guard (legacy seam)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (Task 1 → 2a → 2b → 3 are serial on the same file)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
|
||||
|
||||
**Design contract:** `RecomputeServiceLevel()` resolves the local snapshot entry; **no entry or
|
||||
`Role==Detached` ⇒ publish `0`**. If **`_dbHealthProbe is null` OR `_lastDbHealth is null`** (no DB-health
|
||||
source yet) ⇒ **legacy role-only byte** (the old switch — Primary-leader 240 / Primary 200 / Secondary 100 /
|
||||
Detached 0) — this is the bootstrap + back-compat path. Otherwise build `NodeHealthInputs` and call
|
||||
`ServiceLevelCalculator.Compute`. `MemberState` ← `Cluster.Get(Context.System).SelfMember.Status`.
|
||||
`Stale = !DbReachable || (now - _lastDbHealth.AsOfUtc) > _staleWindow || (now - localEntry.AsOfUtc) > _staleWindow`.
|
||||
`OpcUaProbeOk` is hardcoded `true` in this task (Task 2b wires the real value).
|
||||
|
||||
**Step 1: Write failing tests** (add to `OpcUaPublishActorTests.cs`). A tiny stub DB-health probe actor +
|
||||
calculator-path assertions:
|
||||
```csharp
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health; // DbHealthProbeActor.GetStatus / DbHealthStatus
|
||||
|
||||
// helper at class scope:
|
||||
private sealed class StubDbHealth : ReceiveActor
|
||||
{
|
||||
public StubDbHealth(DbHealthProbeActor.DbHealthStatus status) =>
|
||||
Receive<DbHealthProbeActor.GetStatus>(_ => Sender.Tell(status));
|
||||
}
|
||||
|
||||
[Fact] // healthy primary-leader → 250
|
||||
public void Calculator_path_healthy_primary_leader_publishes_250()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var db = Sys.ActorOf(Props.Create(() => new StubDbHealth(
|
||||
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
|
||||
staleWindow: TimeSpan.FromSeconds(30)));
|
||||
|
||||
// Seed DB-health directly (deterministic, no timer): Receive<DbHealthStatus> caches + recomputes.
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new RedundancyStateChanged(new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
|
||||
}, CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
[Fact] // healthy secondary → 240 (NOT 100 — documented change)
|
||||
public void Calculator_path_healthy_secondary_publishes_240() { /* Role=Secondary, IsRoleLeader=false → 240 */ }
|
||||
|
||||
[Fact] // DB unreachable → 100 (Stale via !DbReachable)
|
||||
public void Calculator_path_db_unreachable_publishes_100() { /* DbHealthStatus(false,...) + Primary-leader → 100 */ }
|
||||
|
||||
[Fact] // stale snapshot but DB ok → 200
|
||||
public void Calculator_path_stale_snapshot_publishes_200() { /* DbHealthStatus(true, now) but entry AsOfUtc = now-1min, staleWindow=2s → 200 (leader → 200, no +10? see note) */ }
|
||||
|
||||
[Fact] // Detached → 0
|
||||
public void Calculator_path_detached_publishes_0() { /* local entry Role=Detached → 0 */ }
|
||||
|
||||
[Fact] // back-compat: no dbHealthProbe → legacy role-only (existing two tests already cover 240 & 100)
|
||||
public void Legacy_path_when_no_db_probe_keeps_role_only() { /* no dbHealthProbe → Primary-leader → 240 */ }
|
||||
```
|
||||
NOTE for the 200 test: with `IsDriverRoleLeader=true`, `Compute` adds +10 → 210. Either assert 210 for a
|
||||
leader, or build the 200 case with `IsRoleLeaderForDriver:false` (a stale follower) → exactly 200. Use a
|
||||
**stale follower** so the expected byte is unambiguous (200).
|
||||
|
||||
**Step 2: Run, expect FAIL** (`dbHealthProbe`/`staleWindow` params don't exist; calculator not called):
|
||||
Run: `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~OpcUaPublishActorTests.Calculator_path"`
|
||||
|
||||
**Step 3: Implement.** In `OpcUaPublishActor.cs`:
|
||||
- Add `using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;` and `using ZB.MOM.WW.OtOpcUa.Runtime.Health;` and
|
||||
`using Akka.Cluster;`.
|
||||
- Add Props/PropsForTests params (all optional, defaulted): `IActorRef? dbHealthProbe = null`,
|
||||
`TimeSpan? staleWindow = null` (default e.g. 30 s). Thread through both factory methods + the ctor.
|
||||
- Add fields: `IActorRef? _dbHealthProbe; TimeSpan _staleWindow; DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
|
||||
RedundancyStateChanged? _lastSnapshot; Cluster? _cluster;` (resolve `_cluster = Cluster.Get(Context.System)`
|
||||
lazily/in ctor — safe; tests form a cluster).
|
||||
- `Receive<DbHealthProbeActor.DbHealthStatus>(s => { _lastDbHealth = s; RecomputeServiceLevel(); });`
|
||||
- Replace `HandleRedundancyStateChanged` body with `{ _lastSnapshot = msg; RecomputeServiceLevel(); }`.
|
||||
- New `RecomputeServiceLevel()`:
|
||||
```csharp
|
||||
private void RecomputeServiceLevel()
|
||||
{
|
||||
if (_localNode is null || _lastSnapshot is null) return;
|
||||
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
|
||||
if (entry is null || entry.Role == RedundancyRole.Detached) { Self.Tell(new ServiceLevelChanged(0)); return; }
|
||||
|
||||
// Back-compat / bootstrap: no DB-health source → legacy role-only (keeps existing tests + a sane
|
||||
// value until the first DbHealthStatus arrives).
|
||||
if (_dbHealthProbe is null || _lastDbHealth is null) { Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry))); return; }
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var stale = !_lastDbHealth.Reachable
|
||||
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|
||||
|| (now - entry.AsOfUtc) > _staleWindow;
|
||||
var inputs = new NodeHealthInputs(
|
||||
MemberState: SafeSelfStatus(),
|
||||
DbReachable: _lastDbHealth.Reachable,
|
||||
OpcUaProbeOk: true, // Task 2b replaces with the freshness-debounced value
|
||||
Stale: stale,
|
||||
IsDriverRoleLeader: entry.IsRoleLeaderForDriver);
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
|
||||
}
|
||||
|
||||
private static byte LegacyRoleOnly(NodeRedundancyState e) => e.Role switch
|
||||
{
|
||||
RedundancyRole.Primary when e.IsRoleLeaderForDriver => 240,
|
||||
RedundancyRole.Primary => 200,
|
||||
RedundancyRole.Secondary => 100,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
private MemberStatus SafeSelfStatus()
|
||||
{
|
||||
try { return _cluster!.SelfMember.Status; } catch { return MemberStatus.Removed; }
|
||||
}
|
||||
```
|
||||
(The `Self.Tell(ServiceLevelChanged)` reuses the existing dedup/publish/metric handler — do not publish
|
||||
directly.)
|
||||
|
||||
**Step 4: Run, expect PASS** (the new `Calculator_path*` tests + the two pre-existing role-only tests).
|
||||
Run: `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~OpcUaPublishActorTests"`
|
||||
Expected: all green, including the unchanged `..._primary_leader` (240) and `..._secondary_publishes_100`.
|
||||
|
||||
**Step 5: Commit** (stage `OpcUaPublishActor.cs` + the test file by path).
|
||||
`git commit -m "feat(redundancy): OpcUaPublishActor computes ServiceLevel via calculator (DB+stale+leader; legacy seam)"`
|
||||
|
||||
---
|
||||
|
||||
### Task 2b: `OpcUaProbeOk` from peer-probes-me (freshness + debounce)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
|
||||
|
||||
**Contract:** `OpcUaProbeOk()` = `true` if **no fresh** `OpcUaProbeResult` about the local node exists, else
|
||||
the latest such result's `Ok` (only an actively-observed, recent `Ok==false` demotes). "Fresh" = within
|
||||
`_probeFreshnessWindow` (injectable, default e.g. 30 s). Single-node / no peer → always `true`.
|
||||
|
||||
**Step 1: Failing tests:**
|
||||
```csharp
|
||||
[Fact] // peer reports me unreachable, DB ok + fresh → (true,false,false) → 0
|
||||
public void Probe_false_about_me_with_healthy_db_publishes_0() {
|
||||
// inject dbHealthProbe(true) + staleWindow 30s + probeFreshnessWindow 30s
|
||||
// Tell DbHealthStatus(true, now); Tell new OpcUaProbeResult(local, Ok:false);
|
||||
// Tell RedundancyStateChanged(primary-leader) → AwaitAssert Levels contains 0
|
||||
}
|
||||
[Fact] // absent/stale probe result → OpcUaProbeOk true → healthy 250 (don't penalize missing peer)
|
||||
public void No_probe_result_defaults_ok_true_publishes_250() { /* never Tell OpcUaProbeResult → 250 */ }
|
||||
[Fact] // a later Ok:true supersedes an earlier Ok:false (recovery)
|
||||
public void Probe_true_supersedes_earlier_false() { /* Tell false then true → recompute → 250 */ }
|
||||
```
|
||||
Ignore `OpcUaProbeResult` whose `NodeId != _localNode` (it's about a peer, not me).
|
||||
|
||||
**Step 2: Run, expect FAIL.**
|
||||
|
||||
**Step 3: Implement.** Add `Receive<OpcUaProbeResult>(HandlePeerProbe);` and:
|
||||
```csharp
|
||||
private (bool Ok, DateTime At)? _probeAboutMe; // latest peer verdict on THIS node
|
||||
private TimeSpan _probeFreshnessWindow;
|
||||
|
||||
private void HandlePeerProbe(PeerOpcUaProbeActor.OpcUaProbeResult r)
|
||||
{
|
||||
if (_localNode is null || r.NodeId != _localNode.Value) return; // only verdicts about me
|
||||
_probeAboutMe = (r.Ok, DateTime.UtcNow);
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
|
||||
private bool OpcUaProbeOk()
|
||||
{
|
||||
if (_probeAboutMe is not { } p) return true; // no peer verdict → benefit of doubt
|
||||
if (DateTime.UtcNow - p.At > _probeFreshnessWindow) return true; // stale verdict → benefit of doubt
|
||||
return p.Ok;
|
||||
}
|
||||
```
|
||||
Replace `OpcUaProbeOk: true` in `RecomputeServiceLevel` with `OpcUaProbeOk: OpcUaProbeOk()`. Add the
|
||||
`probeFreshnessWindow` Props/PropsForTests param + ctor wiring (default ~30 s). Add
|
||||
`using ZB.MOM.WW.OtOpcUa.Runtime.Health;` (already added in 2a).
|
||||
|
||||
**Step 4: Run, expect PASS** (new probe tests + all Task 2a + the two legacy tests).
|
||||
|
||||
**Step 5: Commit** by path.
|
||||
`git commit -m "feat(redundancy): OpcUaProbeOk from peer-probes-me with freshness debounce"`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `HealthTick` — periodic DB Ask/PipeTo + immediate PreStart refresh
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorTests.cs`
|
||||
|
||||
**Contract:** the actor refreshes DB health on its own so production needs no external pump. Implement
|
||||
`IWithTimers`. In `PreStart` (only when `_dbHealthProbe is not null`): kick an immediate tick + a periodic
|
||||
timer (`_healthTickInterval`, injectable, default 5 s). `Receive<HealthTick>` → `_dbHealthProbe.Ask<DbHealth
|
||||
ProbeActor.DbHealthStatus>(DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1)).PipeTo(Self)`; on
|
||||
Ask failure (timeout) PipeTo a `DbHealthStatus(false, now, "ask-timeout")` (fail-safe demote). The existing
|
||||
`Receive<DbHealthStatus>` caches + recomputes.
|
||||
|
||||
**Step 1: Failing test** — proves the Ask path without the test Telling `DbHealthStatus` directly:
|
||||
```csharp
|
||||
[Fact]
|
||||
public void HealthTick_asks_db_probe_and_publishes_calculator_byte()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var db = Sys.ActorOf(Props.Create(() => new StubDbHealth(
|
||||
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30),
|
||||
healthTickInterval: TimeSpan.FromMilliseconds(100)));
|
||||
|
||||
actor.Tell(new RedundancyStateChanged(new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
|
||||
}, CorrelationId.NewId()));
|
||||
|
||||
// No direct DbHealthStatus Tell — the periodic HealthTick Ask must populate it → 250.
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromSeconds(2));
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run, expect FAIL** (`healthTickInterval` param missing; no tick wired).
|
||||
|
||||
**Step 3: Implement.** `public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers`; add
|
||||
`public ITimerScheduler Timers { get; set; } = null!;`. Add `record HealthTick` singleton. Add the
|
||||
`healthTickInterval` param to both Props factories + ctor (store `_healthTickInterval`). Extend `PreStart`:
|
||||
```csharp
|
||||
protected override void PreStart()
|
||||
{
|
||||
if (_subscribeRedundancyTopic)
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
|
||||
if (_dbHealthProbe is not null)
|
||||
{
|
||||
Self.Tell(HealthTick.Instance); // immediate refresh
|
||||
Timers.StartPeriodicTimer("health", HealthTick.Instance, _healthTickInterval);
|
||||
}
|
||||
}
|
||||
```
|
||||
`Receive<HealthTick>(_ => _dbHealthProbe!.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1))
|
||||
.ContinueWith(t => t.IsCompletedSuccessfully ? t.Result
|
||||
: new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "ask-timeout"))
|
||||
.PipeTo(Self));`
|
||||
|
||||
**Step 4: Run, expect PASS** — the HealthTick test + all prior tests. Verify the pinned-dispatcher production
|
||||
`Props()` still builds (it does not set `dbHealthProbe`, so PreStart skips the timer in pure-Props tests).
|
||||
|
||||
**Step 5: Commit** by path.
|
||||
`git commit -m "feat(redundancy): periodic HealthTick refreshes DB reachability via Ask/PipeTo"`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `PeerProbeSupervisor` — spawn one probe per driver peer
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 6 (docs) — disjoint files
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Health/PeerProbeSupervisor.cs`
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Health/PeerProbeSupervisorTests.cs`
|
||||
|
||||
**Contract:** subscribes to the `redundancy-state` topic; on each `RedundancyStateChanged`, maintain exactly
|
||||
one child per **other, non-Detached** node (`n.NodeId != _localNode && n.Role != Detached`): spawn for new
|
||||
peers, `Context.Stop` children for departed peers. Child `Props` come from an injected
|
||||
`Func<NodeId, Props>` (production default `peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: _opcUaPort)`).
|
||||
Child actor name = a stable sanitization of the peer node id (so re-adds match). Default dispatcher (NOT the
|
||||
pinned OPC UA one).
|
||||
|
||||
**Step 1: Failing tests** (drive lifecycle with fabricated snapshots + a stub probe factory; assert child
|
||||
count via `ActorCells`/a TestProbe registry):
|
||||
```csharp
|
||||
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
|
||||
{
|
||||
private static Props StubChild() => Props.Create(() => new BlackHoleActor()); // or EchoActor
|
||||
|
||||
[Fact]
|
||||
public void Spawns_one_child_per_non_self_non_detached_peer()
|
||||
{
|
||||
var local = NodeId.Parse("me:4053");
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(local, _ => StubChild()));
|
||||
sup.Tell(new RedundancyStateChanged(new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary, true, true, DateTime.UtcNow),
|
||||
new NodeRedundancyState(NodeId.Parse("peer:4053"), RedundancyRole.Secondary, false, false, DateTime.UtcNow),
|
||||
new NodeRedundancyState(NodeId.Parse("adm:4053"), RedundancyRole.Detached, false, false, DateTime.UtcNow),
|
||||
}, CorrelationId.NewId()));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1)); // peer only; self + detached excluded
|
||||
}
|
||||
|
||||
[Fact] public void Stops_child_for_departed_peer() { /* snapshot with peer, then without → ChildCount 1→0 */ }
|
||||
[Fact] public void Single_node_snapshot_spawns_no_children() { /* only local entry → 0 */ }
|
||||
}
|
||||
```
|
||||
Expose a test-visible `ChildCount` (e.g. `public int ChildCount => Context.GetChildren().Count();`) OR assert
|
||||
via `ExpectMsg` on a registry. Prefer `ChildCount` with `ActorOfAsTestActorRef`.
|
||||
|
||||
**Step 2: Run, expect FAIL** (type doesn't exist).
|
||||
|
||||
**Step 3: Implement** `PeerProbeSupervisor : ReceiveActor`:
|
||||
```csharp
|
||||
public static Props Props(NodeId localNode, int opcUaPort = PeerOpcUaProbeActor.DefaultOpcUaPort) =>
|
||||
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode,
|
||||
peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: opcUaPort)));
|
||||
public static Props PropsForTests(NodeId localNode, Func<NodeId, Props> probeFactory) =>
|
||||
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
|
||||
|
||||
private readonly Dictionary<NodeId, IActorRef> _children = new();
|
||||
// ctor: store localNode + probeFactory; Receive<RedundancyStateChanged>(OnSnapshot); Receive<SubscribeAck>(_=>{});
|
||||
protected override void PreStart() =>
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe("redundancy-state", Self));
|
||||
|
||||
private void OnSnapshot(RedundancyStateChanged s)
|
||||
{
|
||||
var want = s.Nodes.Where(n => n.NodeId != _localNode && n.Role != RedundancyRole.Detached)
|
||||
.Select(n => n.NodeId).ToHashSet();
|
||||
foreach (var gone in _children.Keys.Where(k => !want.Contains(k)).ToList())
|
||||
{ Context.Stop(_children[gone]); _children.Remove(gone); }
|
||||
foreach (var peer in want.Where(p => !_children.ContainsKey(p)))
|
||||
_children[peer] = Context.ActorOf(_probeFactory(peer), "probe-" + Sanitize(peer.Value));
|
||||
}
|
||||
private static string Sanitize(string s) => new(s.Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray());
|
||||
```
|
||||
(Use the literal `"redundancy-state"` to match `PeerOpcUaProbeActor.RedundancyStateTopic` — keep the Runtime
|
||||
"duplicated to avoid a ControlPlane ref" convention. Reference `PeerOpcUaProbeActor.RedundancyStateTopic`
|
||||
since both are in `Runtime.Health`.)
|
||||
|
||||
**Step 4: Run, expect PASS.**
|
||||
|
||||
**Step 5: Commit** by path.
|
||||
`git commit -m "feat(redundancy): PeerProbeSupervisor maintains one peer OPC UA probe per driver peer"`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Wire into `WithOtOpcUaRuntimeActors`
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** none (depends on 2a–4)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs`
|
||||
|
||||
**Step 1: Implement** (no new unit test — DI wiring is covered by the live `/run` gate + existing
|
||||
`ServiceCollectionExtensionsTests` smoke if present; if that test asserts a fixed actor set, update it):
|
||||
- Pass the local `dbHealth` ref into the publish actor:
|
||||
```csharp
|
||||
var publishActor = system.ActorOf(
|
||||
OpcUaPublishActor.Props(
|
||||
sink: addressSpaceSink, serviceLevel: serviceLevel, localNode: roleInfo.LocalNode,
|
||||
dbFactory: dbFactory, applier: applier,
|
||||
dbHealthProbe: dbHealth), // NEW — enables the calculator path in production
|
||||
OpcUaPublishActorName);
|
||||
```
|
||||
- Spawn the supervisor after `publishActor`:
|
||||
```csharp
|
||||
var peerProbes = system.ActorOf(PeerProbeSupervisor.Props(roleInfo.LocalNode), PeerProbeSupervisorName);
|
||||
registry.Register<PeerProbeSupervisorKey>(peerProbes);
|
||||
```
|
||||
- Add `public const string PeerProbeSupervisorName = "peer-probe-supervisor";` near the other name consts and
|
||||
`public sealed class PeerProbeSupervisorKey { }` near the other key classes.
|
||||
- Confirm `OpcUaPublishActor.Props` signature now accepts `dbHealthProbe` (add the optional param in Task 2a;
|
||||
if not yet, this task is blocked — order 2a before 5).
|
||||
|
||||
**Step 2: Build + full test.**
|
||||
Run: `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests`
|
||||
Expected: clean build, all green. If a `ServiceCollectionExtensionsTests` enumerates spawned actors, extend
|
||||
it for the new supervisor.
|
||||
|
||||
**Step 3: Commit** by path (`ServiceCollectionExtensions.cs` only, plus the test if updated).
|
||||
`git commit -m "feat(redundancy): wire dbHealth into OpcUaPublishActor + spawn PeerProbeSupervisor per node"`
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Docs — `docs/Redundancy.md` marks the calculator WIRED
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/Redundancy.md`
|
||||
|
||||
**Step 1:** Update the doc to reflect shipped reality:
|
||||
- The `ServiceLevelCalculator` table row (line ~18) + the "Today only cluster topology drives the published
|
||||
ServiceLevel" line (~80) + the "designed to use once that path goes…" note (~85): change to **WIRED** —
|
||||
each driver node now computes its ServiceLevel via `ServiceLevelCalculator` from local `DbHealthProbeActor`
|
||||
reachability, a peer OPC UA probe (`PeerProbeSupervisor` → `PeerOpcUaProbeActor`, "peer-probes-me"), and
|
||||
signal-freshness staleness.
|
||||
- Document the **backward-compat seam**: a node with no DB-health source falls back to role-only.
|
||||
- Document the **behavior change**: a healthy Secondary now publishes **240** (was 100); both healthy nodes
|
||||
sit at 240/250 with the leader preferred by +10. Note `Stale = !DbReachable || signals older than the
|
||||
freshness window`, and that the calculator moved to `Core.Cluster`.
|
||||
- Keep the truth table (250/240/200/100/0) — it's now reachable.
|
||||
|
||||
**Step 2:** No build needed (doc only). Sanity-grep for now-false claims:
|
||||
`grep -n "not yet wired\|placeholder\|F10b\|coarse role" docs/Redundancy.md` and fix each.
|
||||
|
||||
**Step 3: Commit** by path.
|
||||
`git commit -m "docs(redundancy): ServiceLevelCalculator is wired into the live publish path"`
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Full build + test + final integration review
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
**Step 1:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx` — clean (TreatWarningsAsErrors).
|
||||
**Step 2:** `dotnet test ZB.MOM.WW.OtOpcUa.slnx` — full suite green (esp. `OpcUaPublishActorTests`,
|
||||
`PeerProbeSupervisorTests`, `ServiceLevelCalculatorTests`, `RedundancyStateActorTests`, `ServiceLevelEndToEndTests`).
|
||||
**Step 3:** Final integration reviewer subagent: confirm (a) no Runtime→ControlPlane reference was added;
|
||||
(b) the two pre-existing role-only tests are still green via the legacy seam; (c) no Commons contract / EF
|
||||
change; (d) staging was by-path only; (e) the Detached guard + probe-freshness debounce + DB-Ask-timeout
|
||||
fail-safe are all present. Apply any actionable findings as follow-up commits.
|
||||
**Step 4:** Update `pending.md` (working-tree only, never staged) with the Phase 2 status.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Live `/run` on the 2-node rig (acceptance gate)
|
||||
|
||||
**Classification:** high-risk (verification)
|
||||
**Estimated implement time:** ~6 min (agent-driven; docker-dev login is disabled — agent does NOT sign in to anything)
|
||||
|
||||
**Files:** none (runtime verification on docker-dev MAIN = central-1/central-2, the 2-node warm pair).
|
||||
|
||||
**Recipe:**
|
||||
1. Rebuild the local rig to this branch: `docker compose -f docker-dev/docker-compose.yml up -d --build`
|
||||
(do NOT stage compose changes). Confirm both central nodes are healthy and bootstrap-deploy the config.
|
||||
2. **Steady state:** read each node's `Server.ServiceLevel` via Client.CLI
|
||||
(`dotnet run --project src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI -- read -u opc.tcp://localhost:4840 -n "i=2267"`
|
||||
— `ServiceLevel` is NodeId `i=2267`; confirm the actual id) on `:4840` (central-1) and `:4841` (central-2).
|
||||
Expect the leader at **250** and the follower at **240** (the documented healthy pair).
|
||||
3. **Demote one node's DB:** isolate ONLY one central node from SQL (they share the SQL container, so do
|
||||
NOT stop SQL). Use `docker network disconnect <net> otopcua-dev-central-1-1` (or block its SQL route) so
|
||||
central-1's `DbHealthProbeActor` starts failing while central-2 stays healthy.
|
||||
4. Within ~`max(healthTickInterval, staleWindow)` confirm central-1's `ServiceLevel` drops to **100** (DB
|
||||
unreachable ⇒ stale) — i.e. **below** central-2's 240 — proving health-aware demotion + the failover
|
||||
ordering. Reconnect the network and confirm it climbs back to 240/250.
|
||||
5. Capture the observed bytes. If the docker network manipulation is not feasible on the rig, record exactly
|
||||
what blocked and hand that single step to the user; everything else is agent-driven.
|
||||
|
||||
**Done when:** build clean + full `dotnet test` green + the live demotion observed (or the precise blocker
|
||||
recorded). Then `finishing-a-development-branch` → merge to master + push.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-06-15-stillpending-phase-2-servicelevel.md",
|
||||
"branch": "feat/stillpending-phase-2-servicelevel",
|
||||
"tasks": [
|
||||
{"id": 414, "subject": "P2 Task 1: Move ServiceLevelCalculator to Core.Cluster", "status": "pending"},
|
||||
{"id": 415, "subject": "P2 Task 2a: OpcUaPublishActor calculator path (DB+stale+leader+Detached guard, legacy seam)", "status": "pending", "blockedBy": [414]},
|
||||
{"id": 416, "subject": "P2 Task 2b: OpcUaProbeOk from peer-probes-me (freshness + debounce)", "status": "pending", "blockedBy": [415]},
|
||||
{"id": 417, "subject": "P2 Task 3: HealthTick — periodic DB Ask/PipeTo + PreStart immediate refresh", "status": "pending", "blockedBy": [416]},
|
||||
{"id": 418, "subject": "P2 Task 4: PeerProbeSupervisor — one peer probe per driver peer", "status": "pending"},
|
||||
{"id": 419, "subject": "P2 Task 5: Wire WithOtOpcUaRuntimeActors (dbHealth ref + spawn supervisor)", "status": "pending", "blockedBy": [417, 418]},
|
||||
{"id": 420, "subject": "P2 Task 6: docs/Redundancy.md — calculator is WIRED", "status": "pending"},
|
||||
{"id": 421, "subject": "P2 Task 7: Full build + test + final integration review", "status": "pending", "blockedBy": [419, 420]},
|
||||
{"id": 422, "subject": "P2 Task 8: Live /run on the 2-node rig (acceptance gate)", "status": "pending", "blockedBy": [421]}
|
||||
],
|
||||
"lastUpdated": "2026-06-15"
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
using Akka.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
|
||||
|
||||
public readonly record struct NodeHealthInputs(
|
||||
MemberStatus MemberState,
|
||||
@@ -0,0 +1,134 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Per-node supervisor that keeps exactly one <see cref="PeerOpcUaProbeActor"/> child per OTHER,
|
||||
/// non-<see cref="RedundancyRole.Detached"/> driver node named in the latest
|
||||
/// <see cref="RedundancyStateChanged"/> snapshot. Because every node runs this supervisor, each
|
||||
/// node is continuously TCP-probed by all of its peers; a node learns its own reachability by
|
||||
/// consuming the resulting <see cref="PeerOpcUaProbeActor.OpcUaProbeResult"/> messages in
|
||||
/// <c>OpcUaPublishActor</c>.
|
||||
///
|
||||
/// The supervisor reacts to the published <see cref="RedundancyStateChanged"/> snapshot (a plain
|
||||
/// record on the <see cref="PeerOpcUaProbeActor.RedundancyStateTopic"/> DistributedPubSub topic)
|
||||
/// rather than to raw Akka <c>Member</c> events, so the desired peer set is derived from one
|
||||
/// authoritative message instead of being reconstructed from membership churn.
|
||||
/// </summary>
|
||||
public sealed class PeerProbeSupervisor : ReceiveActor
|
||||
{
|
||||
private readonly NodeId _localNode;
|
||||
private readonly Func<NodeId, Props> _probeFactory;
|
||||
private readonly Dictionary<NodeId, IActorRef> _children = new();
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <summary>
|
||||
/// Monotonic generation counter appended to each spawned child's name. Guarantees a freshly
|
||||
/// re-added peer never collides on a child name with a same-peer child that is still
|
||||
/// asynchronously terminating after a <see cref="ActorBase.Context"/> <c>Stop</c>.
|
||||
/// </summary>
|
||||
private long _generation;
|
||||
|
||||
/// <summary>Creates production props that probe peers on the given OPC UA port.</summary>
|
||||
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
|
||||
/// <param name="opcUaPort">The OPC UA port each peer probe connects to.</param>
|
||||
/// <returns>Props configured to create a <see cref="PeerProbeSupervisor"/>.</returns>
|
||||
public static Props Props(NodeId localNode, int opcUaPort = PeerOpcUaProbeActor.DefaultOpcUaPort)
|
||||
{
|
||||
// Build the factory delegate OUTSIDE the Props.Create expression tree: a named argument
|
||||
// (opcUaPort:) is not permitted inside an expression tree, so capture it in a local first.
|
||||
Func<NodeId, Props> probeFactory = peer => PeerOpcUaProbeActor.Props(peer, opcUaPort: opcUaPort);
|
||||
return Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
|
||||
}
|
||||
|
||||
/// <summary>Creates props with an injected probe factory for tests (e.g. no-op children).</summary>
|
||||
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
|
||||
/// <param name="probeFactory">Factory producing the child <see cref="Props"/> for a given peer.</param>
|
||||
/// <returns>Props configured to create a <see cref="PeerProbeSupervisor"/>.</returns>
|
||||
public static Props PropsForTests(NodeId localNode, Func<NodeId, Props> probeFactory) =>
|
||||
Akka.Actor.Props.Create(() => new PeerProbeSupervisor(localNode, probeFactory));
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="PeerProbeSupervisor"/> class.</summary>
|
||||
/// <param name="localNode">This node's identifier; excluded from the probed peer set.</param>
|
||||
/// <param name="probeFactory">Factory producing the child <see cref="Props"/> for a given peer.</param>
|
||||
public PeerProbeSupervisor(NodeId localNode, Func<NodeId, Props> probeFactory)
|
||||
{
|
||||
_localNode = localNode;
|
||||
_probeFactory = probeFactory;
|
||||
|
||||
Receive<RedundancyStateChanged>(OnSnapshot);
|
||||
Receive<Terminated>(OnTerminated);
|
||||
Receive<SubscribeAck>(_ => { });
|
||||
// The redundancy-state topic also carries OpcUaProbeResult (published by our own
|
||||
// PeerOpcUaProbeActor children + peers). The supervisor doesn't consume them — only
|
||||
// OpcUaPublishActor does — so explicitly drop them to avoid unhandled-message noise.
|
||||
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
|
||||
}
|
||||
|
||||
/// <summary>Gets the current number of live peer-probe children.</summary>
|
||||
public int ChildCount => _children.Count;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart() =>
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(
|
||||
new Subscribe(PeerOpcUaProbeActor.RedundancyStateTopic, Self));
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles the live child set against the desired peer set in the snapshot: stops children
|
||||
/// for peers that are gone (or became Detached / self), and spawns a probe for each newly
|
||||
/// desired peer.
|
||||
/// </summary>
|
||||
/// <param name="s">The latest redundancy-state snapshot.</param>
|
||||
private void OnSnapshot(RedundancyStateChanged s)
|
||||
{
|
||||
var want = s.Nodes
|
||||
.Where(n => n.NodeId != _localNode && n.Role != RedundancyRole.Detached)
|
||||
.Select(n => n.NodeId)
|
||||
.ToHashSet();
|
||||
|
||||
foreach (var gone in _children.Keys.Where(k => !want.Contains(k)).ToList())
|
||||
{
|
||||
Context.Stop(_children[gone]);
|
||||
_children.Remove(gone);
|
||||
}
|
||||
|
||||
foreach (var peer in want.Where(p => !_children.ContainsKey(p)))
|
||||
{
|
||||
var name = "probe-" + Sanitize(peer.Value) + "-" + _generation++;
|
||||
var child = Context.ActorOf(_probeFactory(peer), name);
|
||||
Context.Watch(child);
|
||||
_children[peer] = child;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prunes a child that has terminated for any reason (e.g. crash) so <see cref="ChildCount"/>
|
||||
/// stays accurate and the next snapshot will respawn it if still desired.
|
||||
/// </summary>
|
||||
/// <param name="t">The termination notice for the dead child.</param>
|
||||
private void OnTerminated(Terminated t)
|
||||
{
|
||||
var entry = _children.FirstOrDefault(kvp => kvp.Value.Equals(t.ActorRef));
|
||||
// No-match yields Value == null because IActorRef is a reference type. A null here means the
|
||||
// terminated ref isn't our CURRENT child for any peer — either it was already pruned in
|
||||
// OnSnapshot (deliberate stop pre-removes from _children), or it's a stale old-generation
|
||||
// child whose peer has since been re-spawned with a fresh child. Either way: no-op, so we
|
||||
// never evict the fresh same-peer child on a stale Terminated.
|
||||
if (entry.Value is null) return;
|
||||
_children.Remove(entry.Key);
|
||||
_log.Debug("PeerProbeSupervisor: pruned terminated child for peer {Peer}", entry.Key);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes a <see cref="NodeId"/> value into a legal Akka child-name fragment — Akka actor
|
||||
/// names may not contain <c>:</c> or <c>.</c>, which appear in <c>host:port</c> node ids.
|
||||
/// </summary>
|
||||
/// <param name="s">The raw node-id value.</param>
|
||||
/// <returns>The value with every non-alphanumeric character replaced by <c>-</c>.</returns>
|
||||
private static string Sanitize(string s) =>
|
||||
new string(s.Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray());
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
@@ -9,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
|
||||
@@ -24,11 +27,18 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
/// <c>Opc.Ua.Server</c>. The remaining piece is wiring those bindings to a real
|
||||
/// <c>StandardServer</c> address space — tracked as F10b.
|
||||
/// </summary>
|
||||
public sealed class OpcUaPublishActor : ReceiveActor
|
||||
public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
public const string DispatcherId = "opcua-synchronized-dispatcher";
|
||||
public const string RedundancyStateTopic = "redundancy-state";
|
||||
|
||||
/// <summary>Gets or sets the timer scheduler for the periodic DB-health refresh tick.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>Self-tick that drives the periodic Ask of the local <see cref="DbHealthProbeActor"/>
|
||||
/// for its cached status. Private singleton — the actor pumps this for itself, no external sender.</summary>
|
||||
private sealed class HealthTick { public static readonly HealthTick Instance = new(); private HealthTick() { } }
|
||||
|
||||
public sealed record AttributeValueUpdate(string NodeId, object? Value, OpcUaQuality Quality, DateTime TimestampUtc);
|
||||
|
||||
/// <summary>Carries the full Part 9 condition state for a scripted alarm to the sink. The
|
||||
@@ -54,10 +64,19 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
private readonly NodeId? _localNode;
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext>? _dbFactory;
|
||||
private readonly Phase7Applier? _applier;
|
||||
private readonly IActorRef? _dbHealthProbe;
|
||||
private readonly TimeSpan _staleWindow;
|
||||
private readonly TimeSpan _probeFreshnessWindow;
|
||||
private readonly TimeSpan _healthTickInterval;
|
||||
private readonly Akka.Cluster.Cluster _cluster = Akka.Cluster.Cluster.Get(Context.System);
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
private int _writes;
|
||||
private byte _lastServiceLevel;
|
||||
private bool _publishedAtLeastOnce;
|
||||
private DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
|
||||
private RedundancyStateChanged? _lastSnapshot;
|
||||
private (bool Ok, DateTime At)? _probeAboutMe;
|
||||
private Phase7CompositionResult _lastApplied = new(
|
||||
Array.Empty<UnsAreaProjection>(),
|
||||
Array.Empty<UnsLineProjection>(),
|
||||
@@ -74,25 +93,45 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
/// <c>redundancy-state</c> DPS topic so cluster transitions drive the local ServiceLevel
|
||||
/// publish path. When <paramref name="dbFactory"/> + <paramref name="applier"/> are supplied,
|
||||
/// <see cref="RebuildAddressSpace"/> reads the latest deployment artifact + drives the
|
||||
/// applier through the sink.</summary>
|
||||
/// applier through the sink. When <paramref name="dbHealthProbe"/> is supplied the local
|
||||
/// ServiceLevel is computed via <see cref="ServiceLevelCalculator"/> from real DB-health +
|
||||
/// staleness + role-leader inputs; otherwise the legacy role-only switch is used.</summary>
|
||||
/// <param name="sink">The OPC UA address space sink.</param>
|
||||
/// <param name="serviceLevel">The service level publisher.</param>
|
||||
/// <param name="localNode">The local cluster node ID.</param>
|
||||
/// <param name="dbFactory">The optional database context factory.</param>
|
||||
/// <param name="applier">The optional Phase 7 applier.</param>
|
||||
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
|
||||
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
|
||||
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
|
||||
/// considered stale; defaults to 30 seconds.</param>
|
||||
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
|
||||
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
|
||||
/// defaults to 30 seconds.</param>
|
||||
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
|
||||
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
|
||||
/// started when <paramref name="dbHealthProbe"/> is null.</param>
|
||||
public static Props Props(
|
||||
IOpcUaAddressSpaceSink? sink = null,
|
||||
IServiceLevelPublisher? serviceLevel = null,
|
||||
NodeId? localNode = null,
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
|
||||
Phase7Applier? applier = null) =>
|
||||
Phase7Applier? applier = null,
|
||||
IActorRef? dbHealthProbe = null,
|
||||
TimeSpan? staleWindow = null,
|
||||
TimeSpan? probeFreshnessWindow = null,
|
||||
TimeSpan? healthTickInterval = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
subscribeRedundancyTopic: true,
|
||||
localNode,
|
||||
dbFactory,
|
||||
applier)).WithDispatcher(DispatcherId);
|
||||
applier,
|
||||
dbHealthProbe,
|
||||
staleWindow,
|
||||
probeFreshnessWindow,
|
||||
healthTickInterval)).WithDispatcher(DispatcherId);
|
||||
|
||||
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
|
||||
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
|
||||
@@ -102,20 +141,38 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
/// <param name="localNode">The local cluster node ID.</param>
|
||||
/// <param name="dbFactory">The optional database context factory.</param>
|
||||
/// <param name="applier">The optional Phase 7 applier.</param>
|
||||
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
|
||||
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
|
||||
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
|
||||
/// considered stale; defaults to 30 seconds.</param>
|
||||
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
|
||||
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
|
||||
/// defaults to 30 seconds.</param>
|
||||
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
|
||||
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
|
||||
/// started when <paramref name="dbHealthProbe"/> is null.</param>
|
||||
public static Props PropsForTests(
|
||||
IOpcUaAddressSpaceSink? sink = null,
|
||||
IServiceLevelPublisher? serviceLevel = null,
|
||||
bool subscribeRedundancyTopic = false,
|
||||
NodeId? localNode = null,
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
|
||||
Phase7Applier? applier = null) =>
|
||||
Phase7Applier? applier = null,
|
||||
IActorRef? dbHealthProbe = null,
|
||||
TimeSpan? staleWindow = null,
|
||||
TimeSpan? probeFreshnessWindow = null,
|
||||
TimeSpan? healthTickInterval = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
subscribeRedundancyTopic,
|
||||
localNode,
|
||||
dbFactory,
|
||||
applier));
|
||||
applier,
|
||||
dbHealthProbe,
|
||||
staleWindow,
|
||||
probeFreshnessWindow,
|
||||
healthTickInterval));
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OpcUaPublishActor"/> class.</summary>
|
||||
/// <param name="sink">The OPC UA address space sink.</param>
|
||||
@@ -124,13 +181,27 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
/// <param name="localNode">The local cluster node ID.</param>
|
||||
/// <param name="dbFactory">The optional database context factory.</param>
|
||||
/// <param name="applier">The optional Phase 7 applier.</param>
|
||||
/// <param name="dbHealthProbe">The optional <see cref="DbHealthProbeActor"/> ref; when null the
|
||||
/// legacy role-only ServiceLevel seam is used until a <see cref="DbHealthProbeActor.DbHealthStatus"/> arrives.</param>
|
||||
/// <param name="staleWindow">The window beyond which a DB-health sample or redundancy snapshot is
|
||||
/// considered stale; defaults to 30 seconds.</param>
|
||||
/// <param name="probeFreshnessWindow">The window beyond which a peer's OPC UA probe verdict about
|
||||
/// this node is considered stale (and thus given the benefit of the doubt rather than demoting);
|
||||
/// defaults to 30 seconds.</param>
|
||||
/// <param name="healthTickInterval">The period between self-driven DB-health refresh ticks (each
|
||||
/// Asks <paramref name="dbHealthProbe"/> for its cached status); defaults to 5 seconds. No timer is
|
||||
/// started when <paramref name="dbHealthProbe"/> is null.</param>
|
||||
public OpcUaPublishActor(
|
||||
IOpcUaAddressSpaceSink sink,
|
||||
IServiceLevelPublisher serviceLevel,
|
||||
bool subscribeRedundancyTopic,
|
||||
NodeId? localNode,
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory = null,
|
||||
Phase7Applier? applier = null)
|
||||
Phase7Applier? applier = null,
|
||||
IActorRef? dbHealthProbe = null,
|
||||
TimeSpan? staleWindow = null,
|
||||
TimeSpan? probeFreshnessWindow = null,
|
||||
TimeSpan? healthTickInterval = null)
|
||||
{
|
||||
_sink = sink;
|
||||
_serviceLevel = serviceLevel;
|
||||
@@ -138,12 +209,19 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
_localNode = localNode;
|
||||
_dbFactory = dbFactory;
|
||||
_applier = applier;
|
||||
_dbHealthProbe = dbHealthProbe;
|
||||
_staleWindow = staleWindow ?? TimeSpan.FromSeconds(30);
|
||||
_probeFreshnessWindow = probeFreshnessWindow ?? TimeSpan.FromSeconds(30);
|
||||
_healthTickInterval = healthTickInterval ?? TimeSpan.FromSeconds(5);
|
||||
|
||||
Receive<AttributeValueUpdate>(HandleAttributeUpdate);
|
||||
Receive<AlarmStateUpdate>(HandleAlarmUpdate);
|
||||
Receive<RebuildAddressSpace>(HandleRebuild);
|
||||
Receive<ServiceLevelChanged>(HandleServiceLevelChanged);
|
||||
Receive<RedundancyStateChanged>(HandleRedundancyStateChanged);
|
||||
Receive<DbHealthProbeActor.DbHealthStatus>(HandleDbHealthStatus);
|
||||
Receive<HealthTick>(_ => OnHealthTick());
|
||||
Receive<PeerOpcUaProbeActor.OpcUaProbeResult>(HandlePeerProbe);
|
||||
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
|
||||
}
|
||||
|
||||
@@ -154,6 +232,16 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
{
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(RedundancyStateTopic, Self));
|
||||
}
|
||||
|
||||
// Production delivery of DB-health: when a probe is wired, kick an immediate refresh + start the
|
||||
// periodic tick so _lastDbHealth gets populated and kept fresh without any external pump. Gated
|
||||
// ONLY on the probe being present (independent of the DPS subscribe) so the calculator path works
|
||||
// in tests too. No probe → stay on the legacy role-only seam (no timer).
|
||||
if (_dbHealthProbe is not null)
|
||||
{
|
||||
Self.Tell(HealthTick.Instance); // immediate first refresh
|
||||
Timers.StartPeriodicTimer("db-health", HealthTick.Instance, _healthTickInterval);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleAttributeUpdate(AttributeValueUpdate msg)
|
||||
@@ -304,11 +392,15 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
|
||||
private void HandleServiceLevelChanged(ServiceLevelChanged msg)
|
||||
{
|
||||
if (msg.ServiceLevel == _lastServiceLevel) return;
|
||||
// Always publish the FIRST computed level, even if it equals the byte-default 0. Otherwise a
|
||||
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
|
||||
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
|
||||
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
|
||||
_lastServiceLevel = msg.ServiceLevel;
|
||||
try
|
||||
{
|
||||
_serviceLevel.Publish(msg.ServiceLevel);
|
||||
_publishedAtLeastOnce = true;
|
||||
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
|
||||
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
|
||||
_log.Debug("OpcUaPublish: ServiceLevel={Level}", msg.ServiceLevel);
|
||||
@@ -319,30 +411,134 @@ public sealed class OpcUaPublishActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compute a coarse ServiceLevel from the cluster snapshot and forward to the
|
||||
/// <see cref="IServiceLevelPublisher"/>. This is a placeholder for F10b's full health
|
||||
/// aggregation — for now we surface "primary-leader → 240, secondary → 100, detached → 0"
|
||||
/// so the local SDK at least reflects role state. The full <see cref="ServiceLevelCalculator"/>
|
||||
/// path (with DB-reachable, OPC UA probe inputs) lives in <c>RedundancyStateActor</c> on
|
||||
/// admin nodes; this driver-side mirror exists so each node's own SDK exposes a sensible
|
||||
/// ServiceLevel without round-tripping back through the admin singleton.
|
||||
/// </summary>
|
||||
/// <summary>Caches the latest redundancy snapshot and recomputes the local ServiceLevel.
|
||||
/// The actual byte is produced by <see cref="RecomputeServiceLevel"/> — either via the
|
||||
/// health-aware <see cref="ServiceLevelCalculator"/> (once a DB-health probe + sample are
|
||||
/// wired) or via the legacy role-only seam (back-compat / bootstrap).</summary>
|
||||
private void HandleRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
if (_localNode is null) return;
|
||||
_lastSnapshot = msg;
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
|
||||
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
|
||||
if (local is null) return;
|
||||
/// <summary>Caches the latest DB-health sample and recomputes the local ServiceLevel. The
|
||||
/// probe pushes these (or the actor Asks for them); either way the freshest sample feeds the
|
||||
/// calculator's <c>DbReachable</c>/<c>Stale</c> inputs.</summary>
|
||||
private void HandleDbHealthStatus(DbHealthProbeActor.DbHealthStatus msg)
|
||||
{
|
||||
_lastDbHealth = msg;
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
|
||||
byte level = local.Role switch
|
||||
/// <summary>Periodic self-tick: Asks the local <see cref="DbHealthProbeActor"/> for its cached
|
||||
/// status and pipes the reply back to <see cref="Self"/>, where <see cref="HandleDbHealthStatus"/>
|
||||
/// caches it + recomputes. A hung/late probe (Ask timeout) pipes a <c>Reachable=false</c> status so
|
||||
/// the node FAIL-SAFE-DEMOTES rather than freezing the last-known-good. The continuation only
|
||||
/// CONSTRUCTS a record off the actor thread — the actual state mutation happens on the actor thread
|
||||
/// when the piped <see cref="DbHealthProbeActor.DbHealthStatus"/> is received, so no actor field is
|
||||
/// touched here.</summary>
|
||||
private void OnHealthTick()
|
||||
{
|
||||
_dbHealthProbe!.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(1))
|
||||
.ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? t.Result
|
||||
: new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "db-health ask timeout"))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
|
||||
/// <summary>Records a peer's OPC UA probe verdict about THIS node and recomputes the local
|
||||
/// ServiceLevel. The probe's <see cref="PeerOpcUaProbeActor.OpcUaProbeResult.NodeId"/> is the
|
||||
/// target that was probed, so a result whose <c>NodeId</c> is not this node is about a peer and
|
||||
/// is ignored. A matching result is stamped with the receive time so <see cref="OpcUaProbeOk"/>
|
||||
/// can debounce stale verdicts.</summary>
|
||||
private void HandlePeerProbe(PeerOpcUaProbeActor.OpcUaProbeResult r)
|
||||
{
|
||||
// The result targets the probed node. If it isn't me, it's about a peer — ignore it.
|
||||
if (_localNode is null || r.NodeId != _localNode.Value) return;
|
||||
_probeAboutMe = (r.Ok, DateTime.UtcNow);
|
||||
RecomputeServiceLevel();
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA self-probe input for the calculator: "did a peer recently observe MY OPC UA
|
||||
/// endpoint as reachable?" Returns <c>true</c> (benefit of the doubt) when no peer verdict has
|
||||
/// arrived yet (single-node / no peer) or when the latest verdict is older than
|
||||
/// <see cref="_probeFreshnessWindow"/> (the peer went away — don't penalise this node for that).
|
||||
/// Only an actively-observed, RECENT <c>Ok==false</c> demotes.</summary>
|
||||
private bool OpcUaProbeOk()
|
||||
{
|
||||
if (_probeAboutMe is not { } verdict) return true;
|
||||
if (DateTime.UtcNow - verdict.At > _probeFreshnessWindow) return true;
|
||||
return verdict.Ok;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the local OPC UA ServiceLevel and routes it through <see cref="ServiceLevelChanged"/>
|
||||
/// (the dedup/publish/metric handler). The full <see cref="ServiceLevelCalculator"/> path is
|
||||
/// used once a DB-health probe is wired AND a sample has arrived; until then (and when no probe
|
||||
/// is supplied at all) a legacy role-only seam keeps the historical "primary-leader → 240,
|
||||
/// secondary → 100, detached → 0" behaviour. The calculator does not model Detached, so a
|
||||
/// detached local node is guarded to 0 before either path runs.
|
||||
/// </summary>
|
||||
private void RecomputeServiceLevel()
|
||||
{
|
||||
if (_localNode is null || _lastSnapshot is null) return;
|
||||
|
||||
var entry = _lastSnapshot.Nodes.FirstOrDefault(n => n.NodeId == _localNode.Value);
|
||||
|
||||
// The calculator does NOT model Detached — a healthy detached node would wrongly compute
|
||||
// 240, so guard it (and the missing-entry case) to 0 here.
|
||||
if (entry is null || entry.Role == RedundancyRole.Detached)
|
||||
{
|
||||
RedundancyRole.Primary when local.IsRoleLeaderForDriver => 240,
|
||||
RedundancyRole.Primary => 200,
|
||||
RedundancyRole.Secondary => 100,
|
||||
RedundancyRole.Detached => 0,
|
||||
_ => 0,
|
||||
};
|
||||
Self.Tell(new ServiceLevelChanged(level));
|
||||
Self.Tell(new ServiceLevelChanged(0));
|
||||
return;
|
||||
}
|
||||
|
||||
// Legacy / back-compat seam: with no DB-health probe wired (or before the first sample
|
||||
// arrives) fall back to the old role-only switch. This preserves historical behaviour and
|
||||
// is the bootstrap value until the first DbHealthStatus lands.
|
||||
if (_dbHealthProbe is null || _lastDbHealth is null)
|
||||
{
|
||||
Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry)));
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var inputs = new NodeHealthInputs(
|
||||
MemberState: SafeSelfStatus(),
|
||||
DbReachable: _lastDbHealth.Reachable,
|
||||
OpcUaProbeOk: OpcUaProbeOk(),
|
||||
Stale: !_lastDbHealth.Reachable
|
||||
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|
||||
|| (now - entry.AsOfUtc) > _staleWindow,
|
||||
IsDriverRoleLeader: entry.IsRoleLeaderForDriver);
|
||||
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
|
||||
}
|
||||
|
||||
/// <summary>The legacy role-only ServiceLevel switch (primary-leader → 240, primary → 200,
|
||||
/// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
|
||||
private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
|
||||
{
|
||||
RedundancyRole.Primary when entry.IsRoleLeaderForDriver => 240,
|
||||
RedundancyRole.Primary => 200,
|
||||
RedundancyRole.Secondary => 100,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
/// <summary>Reads this node's cluster <see cref="MemberStatus"/>, returning
|
||||
/// <see cref="MemberStatus.Removed"/> if the cluster is unavailable (so the calculator treats it
|
||||
/// as untrusted → 0 rather than throwing).</summary>
|
||||
private MemberStatus SafeSelfStatus()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _cluster.SelfMember.Status;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: SelfMember status unavailable; treating as Removed (ServiceLevel→0)");
|
||||
return MemberStatus.Removed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ public static class ServiceCollectionExtensions
|
||||
public const string HistorianAdapterActorName = "historian-adapter";
|
||||
public const string DependencyMuxActorName = "dependency-mux";
|
||||
public const string OpcUaPublishActorName = "opcua-publish";
|
||||
public const string PeerProbeSupervisorName = "peer-probe-supervisor";
|
||||
|
||||
/// <summary>
|
||||
/// Registers shared runtime services. Currently binds <see cref="IAlarmHistorianSink"/>
|
||||
@@ -213,10 +214,19 @@ public static class ServiceCollectionExtensions
|
||||
serviceLevel: serviceLevel,
|
||||
localNode: roleInfo.LocalNode,
|
||||
dbFactory: dbFactory,
|
||||
applier: applier),
|
||||
applier: applier,
|
||||
dbHealthProbe: dbHealth),
|
||||
OpcUaPublishActorName);
|
||||
registry.Register<OpcUaPublishActorKey>(publishActor);
|
||||
|
||||
// Per-node peer-probe supervisor — keeps one OPC UA TCP probe per OTHER non-Detached
|
||||
// driver node, so every node is continuously probed by all its peers. OpcUaPublishActor
|
||||
// consumes the resulting probe verdicts to learn this node's own reachability.
|
||||
var peerProbes = system.ActorOf(
|
||||
PeerProbeSupervisor.Props(roleInfo.LocalNode),
|
||||
PeerProbeSupervisorName);
|
||||
registry.Register<PeerProbeSupervisorKey>(peerProbes);
|
||||
|
||||
var driverHost = system.ActorOf(
|
||||
DriverHostActor.Props(dbFactory, roleInfo.LocalNode, coordinator: null,
|
||||
driverFactory: driverFactory, localRoles: roleInfo.LocalRoles,
|
||||
@@ -246,3 +256,6 @@ public sealed class DbHealthProbeActorKey { }
|
||||
public sealed class HistorianAdapterActorKey { }
|
||||
public sealed class DependencyMuxActorKey { }
|
||||
public sealed class OpcUaPublishActorKey { }
|
||||
|
||||
/// <summary>Marker key for the per-node PeerProbeSupervisor.</summary>
|
||||
public sealed class PeerProbeSupervisorKey { }
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Akka.Cluster;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="PeerProbeSupervisor"/>: it maintains exactly one peer-OPC-UA-probe child
|
||||
/// per OTHER, non-Detached driver node named in the latest <see cref="RedundancyStateChanged"/>
|
||||
/// snapshot, spawning/stopping children as the topology changes.
|
||||
/// </summary>
|
||||
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId Local = NodeId.Parse("local:4053");
|
||||
private static readonly NodeId Peer = NodeId.Parse("peer:4053");
|
||||
private static readonly NodeId Adm = NodeId.Parse("adm:4053");
|
||||
|
||||
/// <summary>No-op child actor stub so we can count children without real TCP probes.</summary>
|
||||
private sealed class NoopActor : ReceiveActor { }
|
||||
|
||||
private static Props NoopProps() => Akka.Actor.Props.Create(() => new NoopActor());
|
||||
|
||||
/// <summary>
|
||||
/// No-op child stub that records its own <see cref="ActorBase.Self"/> into a shared list on
|
||||
/// start, in spawn order — lets a test grab a specific (e.g. the FIRST, old-generation) child
|
||||
/// ref so it can deliver a synthetic <see cref="Terminated"/> for it.
|
||||
/// </summary>
|
||||
private sealed class RecordingNoopActor : ReceiveActor
|
||||
{
|
||||
public RecordingNoopActor(List<IActorRef> spawned) => spawned.Add(Self);
|
||||
}
|
||||
|
||||
private static Props RecordingProps(List<IActorRef> spawned) =>
|
||||
Akka.Actor.Props.Create(() => new RecordingNoopActor(spawned));
|
||||
|
||||
private static NodeRedundancyState State(NodeId id, RedundancyRole role) =>
|
||||
new(id, role, IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow);
|
||||
|
||||
private static RedundancyStateChanged Snapshot(params NodeRedundancyState[] nodes) =>
|
||||
new(nodes, CorrelationId.NewId());
|
||||
|
||||
/// <summary>Verifies one child is spawned per non-self, non-Detached peer — self and Detached
|
||||
/// nodes are excluded.</summary>
|
||||
[Fact]
|
||||
public void Spawns_one_child_per_non_self_non_detached_peer()
|
||||
{
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
||||
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary),
|
||||
State(Adm, RedundancyRole.Detached)));
|
||||
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies the child for a departed peer is stopped when the next snapshot omits it.</summary>
|
||||
[Fact]
|
||||
public void Stops_child_for_departed_peer()
|
||||
{
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
||||
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies a single-node snapshot (just the local node) spawns no children.</summary>
|
||||
[Fact]
|
||||
public void Single_node_snapshot_spawns_no_children()
|
||||
{
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
||||
|
||||
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
||||
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies a previously-removed peer is respawned when it re-appears, without an
|
||||
/// "actor name not unique" collision on the sanitized child name.</summary>
|
||||
[Fact]
|
||||
public void Re_adding_a_previously_removed_peer_respawns_it()
|
||||
{
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
||||
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Locks in the stale-Terminated guard: when an OLD (already-replaced) child's
|
||||
/// <see cref="Terminated"/> for a peer arrives AFTER a NEW child for the SAME peer was spawned,
|
||||
/// the fresh child must NOT be evicted (removal is keyed by current-child ref-equality, not by
|
||||
/// peer key). Without this guard a late stale Terminated would silently drop a live probe.</summary>
|
||||
[Fact]
|
||||
public void Stale_terminated_for_old_child_does_not_evict_fresh_peer_child()
|
||||
{
|
||||
var spawned = new List<IActorRef>();
|
||||
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
||||
PeerProbeSupervisor.PropsForTests(Local, _ => RecordingProps(spawned)));
|
||||
|
||||
// First add of the peer -> child #0 (the OLD ref).
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
AwaitAssert(() => spawned.Count.ShouldBe(1), duration: TimeSpan.FromMilliseconds(500));
|
||||
var oldRef = spawned[0];
|
||||
|
||||
// Drop the peer -> child #0 stopped, ChildCount back to 0.
|
||||
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// Re-add the SAME peer -> a NEW child #1 (the FRESH ref) is spawned.
|
||||
sup.Tell(Snapshot(
|
||||
State(Local, RedundancyRole.Primary),
|
||||
State(Peer, RedundancyRole.Secondary)));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
AwaitAssert(() => spawned.Count.ShouldBe(2), duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// Now deliver a STALE Terminated for the OLD ref. The current child for Peer is the fresh
|
||||
// child #1, so ref-equality finds no match and the supervisor must leave ChildCount at 1.
|
||||
sup.Tell(new Terminated(oldRef, existenceConfirmed: true, addressTerminated: false));
|
||||
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
@@ -112,6 +113,31 @@ public sealed class OpcUaPublishActorTests : RuntimeActorTestBase
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the very first computed ServiceLevel is always published even when it is
|
||||
/// 0 — a node starting Detached (or role-less) must publish 0 rather than let the SDK default
|
||||
/// (255 = full service) stand. The dedup against the <c>byte</c>-default <c>0</c> must not swallow
|
||||
/// the first publish. Drives a Detached local entry (first computed level = 0) with no DB-health
|
||||
/// probe and asserts the publisher saw exactly [0].</summary>
|
||||
[Fact]
|
||||
public void First_service_level_zero_is_published_not_deduped()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("detached-node");
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local));
|
||||
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Detached,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldBe(new byte[] { 0 }),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that RedundancyStateChanged drives local ServiceLevel publish for primary leader.</summary>
|
||||
[Fact]
|
||||
public void RedundancyStateChanged_drives_local_ServiceLevel_publish_for_primary_leader()
|
||||
@@ -157,6 +183,362 @@ public sealed class OpcUaPublishActorTests : RuntimeActorTestBase
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the calculator path computes 250 for a healthy primary role-leader
|
||||
/// (basis 240 from DB-reachable + probe-ok + fresh, +10 driver-role-leader bonus).</summary>
|
||||
[Fact]
|
||||
public void Calculator_path_healthy_primary_leader_publishes_250()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the calculator path computes 240 for a healthy non-leader secondary
|
||||
/// (basis 240 from DB-reachable + probe-ok + fresh, no leader bonus). Documented change from the
|
||||
/// legacy role-only path, which mapped Secondary → 100.</summary>
|
||||
[Fact]
|
||||
public void Calculator_path_healthy_secondary_publishes_240()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("secondary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Secondary,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the calculator path computes 100 when the DB is unreachable
|
||||
/// ((false,_,true) basis, stale via !DbReachable) for a non-leader secondary (no bonus).</summary>
|
||||
[Fact]
|
||||
public void Calculator_path_db_unreachable_publishes_100()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("secondary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "down"))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(false, DateTime.UtcNow, "down"));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Secondary,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)100),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the calculator path computes 200 for a stale snapshot when the DB is
|
||||
/// reachable + fresh but the redundancy entry's AsOfUtc is older than the stale window
|
||||
/// ((true,_,true) basis) for a non-leader secondary (no bonus).</summary>
|
||||
[Fact]
|
||||
public void Calculator_path_stale_snapshot_publishes_200()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("secondary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(2)));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Secondary,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false,
|
||||
DateTime.UtcNow - TimeSpan.FromMinutes(1)),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)200),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a detached local node publishes 0 (the calculator does not model
|
||||
/// Detached, so the handler guards it before the calculator path). The node first goes healthy
|
||||
/// (250) so the dedup'd transition down to 0 is observable.</summary>
|
||||
[Fact]
|
||||
public void Calculator_path_detached_publishes_0()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("detached-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
// First go healthy primary-leader (250) so the transition down to 0 is not dedup'd against
|
||||
// the initial 0.
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// Now detach — expect the guard to drive ServiceLevel down to 0.
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Detached,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that an actively-observed, recent peer probe of MY endpoint that came back
|
||||
/// <c>Ok==false</c> demotes the calculator basis to 0. A non-leader Secondary entry (no +10 bonus)
|
||||
/// is used so the assertion is unambiguous: inputs (DbReachable=true, OpcUaProbeOk=false,
|
||||
/// Stale=false) match Compute's <c>_ => 0</c> arm → 0, +0 (non-leader) → 0.</summary>
|
||||
[Fact]
|
||||
public void Probe_false_about_me_with_healthy_db_publishes_0()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("secondary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Secondary,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)0),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies branch (3) of <c>OpcUaProbeOk()</c>: a peer's NEGATIVE verdict about this node
|
||||
/// that has aged past <c>_probeFreshnessWindow</c> is given the benefit of the doubt (the peer's
|
||||
/// verdict aged out → don't demote), so the node still publishes the HEALTHY level. With a 1ms
|
||||
/// freshness window and a 30ms wait before the recompute, the cached <c>Ok==false</c> verdict is
|
||||
/// reliably stale by the time the level is computed, so <c>OpcUaProbeOk()</c> returns true and a
|
||||
/// healthy primary-leader computes inputs (true, true, false) → 240, +10 leader → 250 — NOT the 0
|
||||
/// that a still-fresh negative verdict would produce (cf. <see cref="Probe_false_about_me_with_healthy_db_publishes_0"/>).</summary>
|
||||
[Fact]
|
||||
public void Stale_probe_verdict_is_not_demoted_publishes_healthy()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(30),
|
||||
probeFreshnessWindow: TimeSpan.FromMilliseconds(1)));
|
||||
|
||||
// Seed a healthy DB sample and a NEGATIVE peer verdict about me. (No RedundancyStateChanged yet,
|
||||
// so no level is computed from these — the verdict just gets stamped with the receive time.)
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
|
||||
|
||||
// Let the verdict age WELL past the 1ms freshness window before any level is computed. The 30ms
|
||||
// wait is >> 1ms, so the verdict is reliably stale by the time OpcUaProbeOk() runs — no reliance
|
||||
// on sub-ms clock ties.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(30));
|
||||
|
||||
// Now trigger the recompute with a fresh healthy primary-leader snapshot. By now _probeAboutMe is
|
||||
// >1ms old, so branch (3) ignores the negative verdict and OpcUaProbeOk() returns true.
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
// Healthy (250), NOT 0 — proves an aged negative verdict does not demote.
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that with no peer probe result ever received, <c>OpcUaProbeOk()</c> defaults
|
||||
/// to <c>true</c> (benefit of the doubt / single-node). A healthy primary-leader thus computes
|
||||
/// inputs (true, true, false) → 240, +10 leader → 250.</summary>
|
||||
[Fact]
|
||||
public void No_probe_result_defaults_ok_true_publishes_250()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a later <c>Ok==true</c> peer probe supersedes an earlier <c>Ok==false</c>
|
||||
/// (recovery). A non-leader Secondary entry is used (no +10 bonus): after the true supersedes,
|
||||
/// inputs (true, true, false) → 240, +0 → 240.</summary>
|
||||
[Fact]
|
||||
public void Probe_true_supersedes_earlier_false_publishes_240()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("secondary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: false));
|
||||
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(local, Ok: true));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Secondary,
|
||||
IsClusterLeader: false, IsRoleLeaderForDriver: false, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a peer probe result about a DIFFERENT node is ignored — it does not
|
||||
/// affect MY <c>OpcUaProbeOk()</c>, which stays at its default <c>true</c>. A healthy
|
||||
/// primary-leader thus still computes (true, true, false) → 240, +10 → 250.</summary>
|
||||
[Fact]
|
||||
public void Probe_about_a_different_node_is_ignored()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var probe = Sys.ActorOf(Akka.Actor.Props.Create(() =>
|
||||
new StubDbHealth(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: probe,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30)));
|
||||
|
||||
actor.Tell(new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null));
|
||||
actor.Tell(new PeerOpcUaProbeActor.OpcUaProbeResult(NodeId.Parse("someone-else"), Ok: false));
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies the legacy back-compat seam: with no DB-health probe wired, the handler
|
||||
/// falls back to the old role-only switch (Primary + leader → 240).</summary>
|
||||
[Fact]
|
||||
public void Legacy_path_no_db_probe_keeps_role_only()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local));
|
||||
|
||||
actor.Tell(new RedundancyStateChanged(
|
||||
Nodes: new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)240),
|
||||
duration: TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the periodic <c>HealthTick</c> Asks the local <see cref="DbHealthProbeActor"/>
|
||||
/// for status and pipes the reply back to itself — populating <c>_lastDbHealth</c> in production WITHOUT
|
||||
/// any external pump (no direct <see cref="DbHealthProbeActor.DbHealthStatus"/> Tell). With a healthy DB
|
||||
/// reply and a primary-leader snapshot, the calculator path therefore reaches 250.</summary>
|
||||
[Fact]
|
||||
public void HealthTick_asks_db_probe_and_publishes_calculator_byte()
|
||||
{
|
||||
var publisher = new RecordingPublisher();
|
||||
var local = NodeId.Parse("primary-node");
|
||||
var db = Sys.ActorOf(Akka.Actor.Props.Create(() => new StubDbHealth(
|
||||
new DbHealthProbeActor.DbHealthStatus(true, DateTime.UtcNow, null))));
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
||||
serviceLevel: publisher, localNode: local, dbHealthProbe: db,
|
||||
staleWindow: TimeSpan.FromSeconds(30), probeFreshnessWindow: TimeSpan.FromSeconds(30),
|
||||
healthTickInterval: TimeSpan.FromMilliseconds(100)));
|
||||
actor.Tell(new RedundancyStateChanged(new[]
|
||||
{
|
||||
new NodeRedundancyState(local, RedundancyRole.Primary,
|
||||
IsClusterLeader: true, IsRoleLeaderForDriver: true, DateTime.UtcNow),
|
||||
}, CorrelationId.NewId()));
|
||||
// No direct DbHealthStatus Tell — the periodic HealthTick Ask must populate it → 250.
|
||||
AwaitAssert(() => publisher.Levels.ShouldContain((byte)250), TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
/// <summary>Stub DB-health probe actor that answers <see cref="DbHealthProbeActor.GetStatus"/>
|
||||
/// with a fixed status, so the calculator path is deterministic without a real timer.</summary>
|
||||
private sealed class StubDbHealth : Akka.Actor.ReceiveActor
|
||||
{
|
||||
/// <summary>Initializes a new instance of the <see cref="StubDbHealth"/> class.</summary>
|
||||
/// <param name="status">The fixed DB-health status to reply with.</param>
|
||||
public StubDbHealth(DbHealthProbeActor.DbHealthStatus status) =>
|
||||
Receive<DbHealthProbeActor.GetStatus>(_ => Sender.Tell(status));
|
||||
}
|
||||
|
||||
/// <summary>Test implementation of IOpcUaAddressSpaceSink that records calls.</summary>
|
||||
private sealed class RecordingSink : IOpcUaAddressSpaceSink
|
||||
{
|
||||
|
||||
@@ -50,17 +50,20 @@ public sealed class ServiceCollectionExtensionsTests
|
||||
var historian = host.Services.GetRequiredService<IRequiredActor<HistorianAdapterActorKey>>();
|
||||
var mux = host.Services.GetRequiredService<IRequiredActor<DependencyMuxActorKey>>();
|
||||
var publish = host.Services.GetRequiredService<IRequiredActor<OpcUaPublishActorKey>>();
|
||||
var peerProbes = host.Services.GetRequiredService<IRequiredActor<PeerProbeSupervisorKey>>();
|
||||
|
||||
driverHost.ActorRef.ShouldNotBeNull();
|
||||
dbHealth.ActorRef.ShouldNotBeNull();
|
||||
historian.ActorRef.ShouldNotBeNull();
|
||||
mux.ActorRef.ShouldNotBeNull();
|
||||
publish.ActorRef.ShouldNotBeNull();
|
||||
peerProbes.ActorRef.ShouldNotBeNull();
|
||||
driverHost.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DriverHostActorName);
|
||||
dbHealth.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DbHealthProbeActorName);
|
||||
historian.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.HistorianAdapterActorName);
|
||||
mux.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DependencyMuxActorName);
|
||||
publish.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.OpcUaPublishActorName);
|
||||
peerProbes.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.PeerProbeSupervisorName);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user