From f817c77e0200f0fcc934d24600227a176baea91b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 13 Nov 2026 03:39:49 -0500 Subject: [PATCH] test(717): deferred-event boundary, ACL-filtered promotion refresh, native-ack refusal, effective-URI XML load; docs + T13 overlay - Wire: an event reported while Active but deferred (#838) is delivered after demotion; the promotion refresh withholds an ACL-denied condition (positive control alongside); ConditionRefresh2 and model-change events on the Standby; a native Acknowledge is refused and never routed; Warm validates and advertises the EFFECTIVE ApplicationUri from an ApplicationConfigPath XML load. - docs/Redundancy.md: Warm section, the Unset/None i=11314 behaviour (orphan readable by NodeId under Unset, deleted under None), instruments; OpcUaServer.md, Configuration.md, security.md, cluster-topology rule. - docker-dev/docker-compose.warm.yml for the T13 gate (central pair, Mode=Warm). - Plan: implementation status and the test gaps left for T13/follow-up. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi --- .claude/rules/cluster-topology.md | 2 +- docker-dev/docker-compose.warm.yml | 15 ++ docs/Configuration.md | 4 +- docs/OpcUaServer.md | 4 +- docs/Redundancy.md | 146 +++++++++++++++- .../2026-11-11-warm-redundancy-717-plan.md | 27 ++- docs/security.md | 6 + .../ServerEventRig.cs | 3 + .../WarmRedundancyWireTests.cs | 164 +++++++++++++++++- 9 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 docker-dev/docker-compose.warm.yml diff --git a/.claude/rules/cluster-topology.md b/.claude/rules/cluster-topology.md index 9d7ce0b72..88689705e 100644 --- a/.claude/rules/cluster-topology.md +++ b/.claude/rules/cluster-topology.md @@ -19,7 +19,7 @@ paths: ## Redundancy -The server supports non-transparent warm/hot redundancy via the `Redundancy` section in `appsettings.json`. Two instances share the same Galaxy DB and the same mxaccessgw (under distinct `MxAccess.ClientName` values) but have unique `ApplicationUri` values. Each exposes a dynamic `ServiceLevel` based on role and runtime health and publishes self + peer URIs in `Server.ServerArray` (`i=2254`); the primary advertises a higher ServiceLevel than the secondary. ⚠️ It does **not** yet set `ServerRedundancy.RedundancySupport` (`i=3709`, reads the SDK default) or expose `ServerUriArray` (`i=11314`) — #717: feasible on the pinned SDK without an upgrade, but blocked on the owner-selected "Warm with a degraded standby" contract (today's standby serves Good data at 240), so do not "just set it to Warm". See `docs/Redundancy.md` for the full guide. +The server supports non-transparent warm/hot redundancy via the `Redundancy` section in `appsettings.json`. Two instances share the same Galaxy DB and the same mxaccessgw (under distinct `MxAccess.ClientName` values) but have unique `ApplicationUri` values. Each exposes a dynamic `ServiceLevel` based on role and runtime health and publishes self + peer URIs in `Server.ServerArray` (`i=2254`); the primary advertises a higher ServiceLevel than the secondary. **Standard redundancy metadata is opt-in via `OpcUa:Redundancy:Mode` (#717, default `Unset` = today's server exactly).** `Unset` writes nothing (`i=3709` reads the SDK default; the SDK's orphan `i=11314` reads Good `[]` by NodeId but is not browsable); `None` advertises `RedundancySupport=None` and deletes that orphan; `Warm` installs `NonTransparentRedundancyType` + read-only `ServerUriArray [self, peers]` and the degraded-standby contract — Standby ServiceLevel 90/50 (Active 250/200/100, boot 1), ordinary-tag values `BadOutOfService`/null while acquisition continues, new alarm events suppressed (stamped, refreshed on promotion), writes and Part 9 methods refused `BadInvalidState`. Landmines: (1) the Warm verdict is NOT `PrimaryGatePolicy` — it needs a **revisioned** snapshot (`RedundancyStateChanged.Revision`), so a pre-#717 publisher can never make a Warm node Active: upgrade both nodes before flipping either; (2) D4 (no Good first-delivered after demotion) is keyed by `(SubscriptionId, SequenceNumber)` and an `OrdinaryTagMarker` on `MonitoredItemNotification.Message` — never by object, because SDK 2.0.1 clones every delivery and pools payloads; `WarmStandbyUnitTests` pins the SDK field contract, so an SDK bump that breaks it fails there; (3) `Member.UpNumber` is internal in Akka.NET and read by reflection (pinned by a test); (4) no tier sets `Warm` until ScadaBridge#138 (ServiceLevel-aware failover) is deployed. See `docs/Redundancy.md` → "Warm non-transparent redundancy (#717)". **Two corrections landed 2026-07-21 — both were total-outage or wrong-node defects, and both are worth knowing before touching this area.** diff --git a/docker-dev/docker-compose.warm.yml b/docker-dev/docker-compose.warm.yml new file mode 100644 index 000000000..1ccc2cf5e --- /dev/null +++ b/docker-dev/docker-compose.warm.yml @@ -0,0 +1,15 @@ +# Gitea #717 T13 live gate: the MAIN central pair under Warm non-transparent redundancy. +# +# docker-dev/compose.sh -f docker-dev/docker-compose.warm.yml up -d --build central-1 central-2 +# +# The base file already gives each central node its own ApplicationUri (urn:OtOpcUa:central-1|2, equal to the seeded +# ClusterNode.ApplicationUri) and lists its partner in OpcUa:PeerApplicationUris (lmxopcua#801), which is exactly what +# Warm validates; this overlay only flips the mode. Both nodes must run a revision-capable (#717) build before either is +# flipped. Rollback: drop this overlay and recreate central-1/2 (Mode defaults to Unset). +services: + central-1: + environment: + OpcUa__Redundancy__Mode: "Warm" + central-2: + environment: + OpcUa__Redundancy__Mode: "Warm" diff --git a/docs/Configuration.md b/docs/Configuration.md index 5c512b885..eaaab7193 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -73,7 +73,9 @@ Until #678.6 the overlay never loaded on a node carrying a `cluster-*` role (the | `PkiStoreRoot` | string | `pki` | Root of the PKI hierarchy (`own`/`issuer`/`trusted`/`rejected` substores created under it). Required. See [`security.md`](security.md). | | `EnabledSecurityProfiles` | list of `OpcUaSecurityProfile` | _(absent)_ = `[None, Basic256Sha256Sign, Basic256Sha256SignAndEncrypt]` | Transport-security profiles, one endpoint per entry. **A configured list replaces the default** (#879); the default applies only when the key is absent. A set-but-empty list fails startup validation. Profile detail in [`security.md`](security.md). | | `AutoAcceptUntrustedClientCertificates` | bool | `false` | Auto-trust unknown client certs on first connect (dev convenience). Not validated. See [`security.md`](security.md). | -| `PeerApplicationUris` | list of string | `[]` (empty) | Partner node `ApplicationUri`s published in `Server.ServerArray` for redundancy discovery. See [`Redundancy.md`](Redundancy.md). | +| `PeerApplicationUris` | list of string | `[]` (empty) | Partner node `ApplicationUri`s published in `Server.ServerArray` for redundancy discovery, and (under `Redundancy:Mode=Warm`) after self in `ServerUriArray`. Required (≥ 1, distinct, not this node's own effective URI) under `Warm`; must be empty under `None`. See [`Redundancy.md`](Redundancy.md). | +| `Redundancy:Mode` | string | `Unset` | #717. `Unset` = today's server exactly; `None` = standalone (`RedundancySupport=None`); `Warm` = 2-node non-transparent pair with a degraded standby. `Hot` and other values fail host start; `Warm` needs the `driver` role. Startup-static — flip both pair nodes in one window. See [`Redundancy.md`](Redundancy.md) → "Warm non-transparent redundancy (#717)". | +| `Redundancy:PromoteWhilePrimaryUnreachable` | bool | `true` | Warm only: the survivor goes Active while the Primary is unreachable but not yet downed (owner answer Q2). | | `DiscoveryServerUrl` | string? | `null` | OPC UA discovery server (LDS/GDS) to register with via `RegisterServer2`, e.g. `opc.tcp://lds-host:4840`. **Unset = no registration** (#876 — the SDK default registered with `opc.tcp://localhost:4840`, i.e. the server itself, and logged `BadServiceUnsupported` forever). Must be an absolute `opc.tcp://` URL when set (validated). Ignored when `ApplicationConfigPath` is set (the XML's own `RegistrationEndpoint`/`MaxRegistrationInterval` apply). | | `DiscoveryRegistrationIntervalMs` | int | `30000` | Maximum interval between `RegisterServer2` calls while `DiscoveryServerUrl` is set (SDK default). Must be `> 0` while `DiscoveryServerUrl` is set (validated); ignored otherwise. | | `CredentialLockout:MaxFailedAttempts` | int | `5` | Failed UserName (API-key) activations for ONE credential identity (key id, else username) from one client before that client is refused that identity `BadUserAccessDenied` (#833, per-identity since #844 — a success clears only its own identity's count). Only definite bad credentials count; store timeouts / miss-budget refusals / authenticator faults do not (#845). `≤ 0` disables the whole lockout. **Never refuses Anonymous.** See [`security.md`](security.md#failed-credential-lockout-opcuacredentiallockout). | diff --git a/docs/OpcUaServer.md b/docs/OpcUaServer.md index f6872d862..23375c564 100644 --- a/docs/OpcUaServer.md +++ b/docs/OpcUaServer.md @@ -67,7 +67,7 @@ Node-level authorization is backed by a permission trie under `src/Core/ZB.MOM.W ## Redundancy -`Redundancy.Enabled = true` on the `ServerInstance` activates the `RedundancyStateActor` + `ServiceLevelCalculator` (`src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/`). The OPC UA `Server/ServiceLevel` node (`VariableIds.Server_ServiceLevel`) is recomputed and republished via `SdkServiceLevelPublisher` (`src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/SdkServiceLevelPublisher.cs`, wired as `IServiceLevelPublisher`) whenever role or driver-health changes; `ServiceLevelCalculator` produces a 0–255 value where higher means more authoritative, so the primary advertises a higher ServiceLevel than the secondary. Peer discovery is `Server/ServerArray` (`i=2254`, self + `PeerApplicationUris`); the standard `Server/ServerRedundancy/RedundancySupport` is not set (SDK default) and `ServerUriArray` is not exposed — see `docs/Redundancy.md` "Standard redundancy metadata" (#717). An apply-lease prevents two instances from concurrently applying a generation. See `docs/Redundancy.md`. +`Redundancy.Enabled = true` on the `ServerInstance` activates the `RedundancyStateActor` + `ServiceLevelCalculator` (`src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/`). The OPC UA `Server/ServiceLevel` node (`VariableIds.Server_ServiceLevel`) is recomputed and republished via `SdkServiceLevelPublisher` (`src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/SdkServiceLevelPublisher.cs`, wired as `IServiceLevelPublisher`) whenever role or driver-health changes; `ServiceLevelCalculator` produces a 0–255 value where higher means more authoritative, so the primary advertises a higher ServiceLevel than the secondary. Peer discovery is `Server/ServerArray` (`i=2254`, self + `PeerApplicationUris`). The standard redundancy metadata is opt-in through `OpcUa:Redundancy:Mode` (#717): `Unset` (default) leaves `RedundancySupport` at the SDK default and the SDK's orphan `ServerUriArray` (`i=11314`) readable by NodeId but not browsable; `None` advertises `RedundancySupport=None` and deletes that orphan; `Warm` installs `NonTransparentRedundancyType` with a read-only `ServerUriArray` `[self, peers…]` and the degraded-standby contract (Standby ServiceLevel 90/50, ordinary-tag values `BadOutOfService`, new alarm events suppressed, writes and Part 9 methods refused `BadInvalidState`). See `docs/Redundancy.md` → "Warm non-transparent redundancy (#717)". An apply-lease prevents two instances from concurrently applying a generation. See `docs/Redundancy.md`. Peer endpoints are advertised through the standard `Server.ServerArray` property: `OpcUaApplicationHost` appends `OpcUaApplicationHostOptions.PeerApplicationUris` to `IServerInternal.ServerUris` after start so warm-redundancy clients can discover the partner. @@ -77,6 +77,8 @@ Peer endpoints are advertised through the standard `Server.ServerArray` property - **`CreateMasterNodeManager`** — Constructs the single `OtOpcUaNodeManager` and wraps it in a `MasterNodeManager` with no extra core managers. - **`NodeManager`** — Public accessor exposing the live `OtOpcUaNodeManager` once the SDK has bootstrapped (null until `CreateMasterNodeManager` runs). +- **`ConfigureRedundancy` / `OnNodeManagerStartedAsync`** (#717) — the startup-static redundancy metadata per `OpcUa:Redundancy:Mode`; `Unset` touches nothing. +- **`PublishAsync` / `RepublishAsync`** (#717) — under `Warm` only, apply the per-`(SubscriptionId, SequenceNumber)` first-delivery decision (`FirstDeliveryRecord`) and rewrite marked ordinary-tag notifications on a Standby delivery (D4); otherwise they return the base response unchanged. `ApplicationName`, `ApplicationUri` (`urn:OtOpcUa`), and `ProductUri` (`https://zb.com/otopcua`) come from `OpcUaApplicationHostOptions`, which the `ApplicationConfiguration` is built from in `OpcUaApplicationHost`. diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 652dbdfa3..691f588d7 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -6,7 +6,17 @@ OtOpcUa runs as a 2-node **non-transparent** redundant pair (client-side failove > **Discovery surface.** The `ServerArray` path on the `Server` object is what each node populates with self + peer `ApplicationUri`s — see `OpcUaApplicationHost.PopulateServerArray` and the per-node `PeerApplicationUris` option below. Clients should read `Server.ServerArray` for peer discovery today. > -> **Standard redundancy metadata — not yet advertised (#717).** Nothing in `src/` sets `Server.ServerRedundancy.RedundancySupport` (`i=3709`), so it reads the SDK default regardless of the cluster's `RedundancyMode`, and `Server.ServerRedundancy` is the base `ServerRedundancyType` with **no** `ServerUriArray` (`i=11314`, mandatory only on `NonTransparentRedundancyType`). `ServerArray` is **not** a substitute for it. This is **not** blocked on an SDK upgrade: hermetic spikes on the pinned SDK 1.5.378.106 built the correct startup-static graph (object `i=2296` typed `NonTransparentRedundancyType` `i=2039`, `RedundancySupport`, `ServerUriArray` = effective self URI then peers) through `StandardServer.CreateMainNodeManagerFactory` + `ConfigurationNodeManager.AddBehaviourToPredefinedNode`. What blocks it is the contract: the owner-selected target is **Warm with an explicit degraded standby** (standby ServiceLevel in the degraded 2–199 band, ordinary standby tag Reads/subscriptions degraded while internal acquisition continues, standby alarm events suppressed and refreshed on promotion) and **None** for standalone deployments. Today's Secondary serves live Good data at ServiceLevel 240, so advertising `Warm` without that standby behaviour would misdescribe the server; the design, spec and plan for it are pending under #717. Interop gates A4/A5b in `docs/plans/phase-6-3-redundancy-interop-plan.md` stay unmet until then. +> **Standard redundancy metadata (#717) — opt-in, default off.** `OpcUa:Redundancy:Mode` selects the contract. The +> default `Unset` is byte-for-byte the pre-#717 server: nothing writes `Server.ServerRedundancy.RedundancySupport` +> (`i=3709`, so it reads the SDK default `None`), `Server.ServerRedundancy` stays the base `ServerRedundancyType`, the +> Secondary serves live Good data at ServiceLevel 240, and interop gates A4/A5b stay unmet. Under `Unset` the SDK's own +> predefined `ServerUriArray` node (`i=11314`) is an **orphan**: it is not referenced from `ServerRedundancy`, so Browse +> and TranslateBrowsePaths do not reach it, but a Read **by NodeId** returns Good with an empty `String[]` (T0a, +> `docs/plans/evidence/2026-11-13-717-sdk2/RESULTS.md`). That is left as the SDK builds it, because `Unset` changes +> nothing. `None` (standalone) advertises `RedundancySupport=None` and deletes the orphan, so `i=11314` is absent by +> NodeId as well (`BadNodeIdUnknown`). `Warm` (a pair) advertises the full non-transparent graph and the degraded-standby +> contract; see [Warm redundancy](#warm-non-transparent-redundancy-717). No tier sets `Warm` until ScadaBridge#138 +> (ServiceLevel-aware failover) is deployed there. > **v2 change.** v1's operator-managed `ClusterNode.RedundancyRole` column + `RedundancyCoordinator` / `ApplyLeaseRegistry` / `PeerHttpProbeLoop` are gone. Primary/secondary is now derived from **Akka cluster membership** — the oldest Up member carrying the `driver` role. The operator no longer writes a role into the DB; cluster topology drives ServiceLevel automatically. @@ -280,6 +290,132 @@ Each node advertises its partner via `OpcUaApplicationHostOptions.PeerApplicatio Node A lists Node B's `ApplicationUri` and vice-versa. Validated by `DualEndpointTests` in `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/` — boots two `OpcUaApplicationHost` instances on loopback, asserts a real OPCFoundation client `Session` reading `Server.ServerArray` from Node A sees both URIs. +## Warm non-transparent redundancy (#717) + +Spec: `docs/plans/2026-11-11-warm-redundancy-717-design.md`; plan: `docs/plans/2026-11-11-warm-redundancy-717-plan.md`. +Everything below applies only under `OpcUa:Redundancy:Mode=Warm`. `Unset` (the default) and `None` keep every behaviour +described elsewhere on this page. + +### Modes + +| Mode | `RedundancySupport` (`i=3709`) | `ServerRedundancy` type | `ServerUriArray` (`i=11314`) | Standby behaviour | +|---|---|---|---|---| +| `Unset` (default) | SDK default (`None`), never written | `ServerRedundancyType` | orphan, Good `[]` by NodeId, not browsable | none; Secondary serves Good at 240 | +| `None` | `None` | `ServerRedundancyType` | absent (orphan deleted) | never | +| `Warm` | `Warm` | `NonTransparentRedundancyType` (`i=2039`) | read-only, `[effectiveSelfUri, peers…]` | degraded standby (below) | + +`Hot` and any other value fail host start (continuous acquisition alone is not Part 4 Hot). The mode is +**startup-static**. `Warm` needs at least one `OpcUa:PeerApplicationUris` entry, distinct URIs, the node's own +**effective** `ApplicationUri` (the one an `ApplicationConfigPath` XML load produces, not the options default) not +listed as a peer, and the `driver` role. `None` needs no peers. The metadata is installed in +`OtOpcUaSdkServer.OnNodeManagerStartedAsync` through the SDK 2.0.1 `IPredefinedNodeSubtypeReplacer`, which keeps the +well-known `i=11314`. Under `Warm` the server boots at ServiceLevel **1** (NoData) until the first computed level. + +At each deploy the node compares the artifact's `ServerCluster.RedundancyMode` for its cluster with its local mode +(`None` or `Warm` only). A mismatch, `Hot` included, logs an Error and counts `otopcua.redundancy.mode_mismatch`; the +node keeps its local mode. + +### Role verdict and bands + +A Warm node is **Active** iff it is Up with a listening endpoint, its own cluster state names it the oldest Up `driver` +member, and the latest **accepted** redundancy snapshot names it Primary. Otherwise it is **Standby**, including before +the first accepted snapshot. With `OpcUa:Redundancy:PromoteWhilePrimaryUnreachable` (default `true`) the oldest +**reachable** driver member also goes Active while the snapshot's Primary is unreachable (a SIGKILLed Primary before +`auto-down`); the cost is dual-Active during a real partition, which serves the same Good data on both nodes. The +verdict is separate from `PrimaryGatePolicy`, which is unchanged for the data-plane gates below. + +Snapshots are ordered: `RedundancyStateChanged.Revision` = (publisher up-number, incarnation tick, sequence). Under +`Warm`, `OpcUaPublishActor` and `DriverHostActor` drop a snapshot whose revision is not newer than the last accepted +one (a late heartbeat of an old singleton incarnation) and count `otopcua.redundancy.snapshot_rejected`. A snapshot +without a revision (a pre-#717 publisher) still feeds the health inputs but can never make a Warm node Active, so +upgrade both nodes before flipping either. `Unset`/`None` keep the unconditional replace. + +| Warm state | ServiceLevel | +|---|---| +| Down / Detached / endpoint failed | 0 | +| Boot, before the first computed level | 1 | +| Standby, stale or DB unreachable | 50 | +| Standby, healthy | 90 | +| Active, critically degraded | 100 | +| Active, stale | 200 | +| Active, healthy | 250 | + +Every Standby level is below every level a data-serving Active node publishes (`ServiceLevelCalculator.ComputeWarm`). + +### What the Standby does + +- **Ordinary tags** (the `Value` of an `EquipmentVariableState`: raw tag, UNS reference, virtual tag, Calculation tag): + Read, the initial sample, change fan-out, ResendData and AggregateFilter input all answer `BadOutOfService`/null. The + stored value keeps updating from acquisition, which never stops on the Standby. Other attributes, condition children + and ns=0 (ServiceLevel, ServerStatus, ServerRedundancy) are unchanged. ACL decisions run first, so a denied node is + still `BadUserAccessDenied`. +- **Writes** to an ordinary tag are refused `BadInvalidState` synchronously, after authorization and before anything + reaches the driver (nothing to revert). `Unset`/`None` keep the #5 optimistic Good + revert. +- **Part 9 condition methods** (Acknowledge, Confirm, AddComment, Enable/Disable, shelving) are refused + `BadInvalidState` before any state change, native and scripted. `Unset`/`None` keep today's behaviour. +- **New alarm events** are not delivered on any item kind (Server object, folders, direct), ACL on or off. The + condition node is still stamped (EventId, Time), so a ConditionRefresh on the Standby returns RefreshStart, the + current retained conditions, RefreshEnd, and a Standby-stamped EventId acknowledges after promotion. The decision is + made at report time: an event reported while Active is delivered even if delivery finishes after demotion. + Model-change and audit events are not suppressed. +- HistoryRead, HistoryUpdate (Primary-gated, #674), Browse and sessions are unchanged. + +### Strict first delivery (D4) + +After demotion no Publish response delivered **for the first time** carries a Good ordinary-tag value, including +samples queued in an item and messages the subscription built before the demotion. SDK 2.0.1 clones the cached +message on every delivery and pools notification payloads, so the mechanism is keyed by value, not by object (T0): + +- Ordinary-tag `Value` items are created as `OrdinaryTagMonitoredItem`, which sets + `MonitoredItemNotification.Message` (a helper the encoder never writes and the SDK clone copies) to an + `OrdinaryTagMarker(ClientHandle, DataValue)`. A notification counts as ordinary-tag only when the marker matches its + own handle and value, so a stale marker on a recycled payload is ignored. `WarmStandbyUnitTests` pins both SDK + behaviours this relies on. +- `OtOpcUaSdkServer` overrides `PublishAsync`/`RepublishAsync` and records one decision per + `(SubscriptionId, SequenceNumber)` on first delivery. A Standby decision rewrites the SDK's fresh clone; the cached + copy is never mutated. Republish returns the recorded decision, so a message first delivered while Active + republishes Good. The record is pruned by `AvailableSequenceNumbers` and bounded. +- Responses already in flight at the demotion step are out of scope. A Republish of a prebuilt, never-sent message is + `BadMessageNotAvailable` in 2.0.1, so first delivery through Republish cannot happen. + +### Transitions + +`OpcUaPublishActor` owns the verdict and drives the node manager through `IStandbyProjection` (late-bound by +`DeferredStandbyProjection`, which replays the latest state) in a fixed order: + +- **Promotion:** open the read projection → queue the current value to each live ordinary-tag item (ignoring filters) + → first deliveries Active → un-suppress alarms → ConditionRefresh2 on every live event item (RefreshStart, retained, + RefreshEnd) → publish the Active ServiceLevel **last**. If that publish fails, the data plane is Active and the level is + retried every second; the Standby level stays visible meanwhile. +- **Demotion:** publish the Standby ServiceLevel **first** (continue even if it fails) → first deliveries Standby (the D4 + boundary) → suppress alarms → close the read projection → queue `BadOutOfService` to each live ordinary-tag item. + +Invariant: an Active-band ServiceLevel is never visible while the projection is closed. + +### Configuration + +```jsonc +{ + "OpcUa": { + "ApplicationUri": "urn:OtOpcUa:node-a", // unique within the pair + "PeerApplicationUris": ["urn:OtOpcUa:node-b"], + "Redundancy": { + "Mode": "Warm", // Unset (default) | None | Warm + "PromoteWhilePrimaryUnreachable": true + } + } +} +``` + +Flip both pair nodes in one window, after both run a revision-capable build. Rollback: set `Mode=Unset` on both and +restart; nothing persistent changes. + +### Not covered + +TransferSubscriptions across the pair (non-transparent: clients re-create), durable subscription restore (the product +does not enable durable subscriptions; an SDK-restored item would not carry the marker), Hot/Transparent redundancy, +dynamic peers. CTT (G8) and production-client cutover (G9) stay open under #615. + ## Split-brain / downing **The default downing strategy is `auto-down` (changed 2026-07-21).** It is selected by @@ -577,7 +713,7 @@ Before archreview 03/S4 the unknown-role case defaulted to *allow* unconditional Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and evaluate scripted alarms, keeping each node's address space and engine state warm for instant failover. To avoid duplicate rows on `/alerts` and duplicate historian writes, only the Primary node publishes externally: - **`alerts` topic emission** — `ScriptedAlarmHostActor` and `DriverHostActor.ForwardNativeAlarm` subscribe to the `redundancy-state` DPS topic and cache the local node's `RedundancyRole`, then gate the cluster `alerts` publish through `PrimaryGatePolicy` (table above). The OPC UA condition-node write and inbound ack/shelve command processing remain **ungated** on both nodes so the secondary is always ready to serve clients after a failover. -- **`HistorianAdapterActor` historization** — likewise Primary-gated so alarm historization is exactly-once across all alarm sources. The actor subscribes to the `alerts` DPS topic and translates each `AlarmTransitionEvent` → `AlarmHistorianEvent` before enqueuing it on the sink; scripted alarms therefore historize exactly once regardless of cluster size. +- **`HistorianAdapterActor` historization** — likewise Primary-gated so each alarm transition historizes once in steady state (at-least-once across a failover, see the drain below). The actor subscribes to the `alerts` DPS topic and translates each `AlarmTransitionEvent` → `AlarmHistorianEvent` before enqueuing it on the sink; scripted alarms therefore historize exactly once regardless of cluster size. - **The alarm sink's DRAIN** — a second, independent gate on the same decision. The enqueue gate above is not sufficient any more: since Phase 2 the store-and-forward buffer lives in the replicated LocalDb, so the Secondary holds a full copy of the Primary's queued rows and an ungated drain there would re-deliver every event continuously. `LocalDbStoreAndForwardSink` takes a `drainGate` fed from `IRedundancyRoleView` — a singleton `DriverHostActor` publishes its `PrimaryGatePolicy` verdict to, because the drain runs on a timer and cannot receive `RedundancyStateChanged` itself. A gated-off node reports `HistorianDrainState.NotPrimary`. The gate applies only to a node whose queue a peer actually holds — `LocalDbReplicationPolicy.SharesRowsWithPeer`: `LocalDb:Replication:ApiKey` AND (`PeerAddress` OR `LocalDb:SyncListenPort` > 0); any other node always drains (review R1 m3). Within a pair BOTH halves stand down for their partner: the dialling half knows it by the `PeerAddress` host, the passive half (no `PeerAddress`) as its per-cluster mesh partner (review R6 MINOR-4 — before that the passive half never stood down, so both halves drained and every event reached the historian twice; a passive half with no `cluster-{ClusterId}` role still cannot identify its partner and keeps draining). Delivery is **at-least-once across a failover** by design: rows the old Primary delivered just before dying may not have replicated their deletes yet. See [AlarmHistorian.md](AlarmHistorian.md). - **Continuous-historization value CAPTURE** (Gitea #537) — a third gate on the same role rule, and the @@ -782,6 +918,12 @@ The OtOpcUa Client CLI at `src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI` supports `-F | Metric | Type | Notes | |---|---|---| | `otopcua.redundancy.service_level_change` | Counter (`{change}`) | OPC UA `Server.ServiceLevel` transitions emitted by the redundancy state. Tagged with `level` = the new byte. | +| `otopcua.redundancy.warm_role` | Gauge | Warm only: 1 Active, 0 Standby, −1 not Warm. | +| `otopcua.redundancy.warm_transition` | Counter (`{transition}`) | Warm role transitions; `direction` = `promote` \| `demote`, `reason` = `boot` \| `membership` \| `snapshot` \| `endpoint` \| `unreachable-primary` \| `agreed`. | +| `otopcua.redundancy.snapshot_rejected` | Counter (`{snapshot}`) | Warm only: snapshots dropped by a subscriber; `reason` = `older_revision` \| `unrevisioned`. | +| `otopcua.redundancy.mode_mismatch` | Counter (`{apply}`) | The deployed cluster `RedundancyMode` disagrees with `OpcUa:Redundancy:Mode` (`local`, `cluster` tags). | +| `otopcua.opcua.standby.projected_notifications` | Counter (`{notification}`) | Ordinary-tag notifications rewritten `BadOutOfService` at a Warm Standby's first delivery (D4). | +| `otopcua.opcua.standby.suppressed_alarm_events` | Counter (`{event}`) | New alarm events not delivered on a Warm Standby. | | `otopcua.redundancy.primary_gate_denied` | Counter (`{denial}`) | Operations denied by the Primary data-plane gate (archreview 03/S4). Tagged `site` = `write` \| `ack` \| `alarm-emit` and `reason` = `secondary` \| `detached` \| `role-unknown`. A non-zero `role-unknown` rate on a healthy multi-node cluster flags a stuck boot window (redundancy snapshot not delivering). | | `otopcua.opcua.apply.failed` | Counter (`{apply}`) | Address-space apply/materialise passes that swallowed at least one sink failure (archreview 01/S-1). Tagged `kind` = `rebuild` (the rebuild threw) \| `nodes` (per-node materialise failures). A non-zero rate means the running server holds a stale or partial address space despite a reported-successful deploy — `OpcUaPublishActor` also logs the degraded apply at **Error**. | | `galaxy.writes.advise_failed` | Counter (`{write}`) | Galaxy writes short-circuited to `Bad` because `AdviseSupervisory` failed — the value could not have committed, so the write **fails closed** (archreview 06/S-1) and the Bad status fires the #5 node revert instead of leaving a phantom-Good node. On the Galaxy driver meter `ZB.MOM.WW.OtOpcUa.Driver.Galaxy`. | diff --git a/docs/plans/2026-11-11-warm-redundancy-717-plan.md b/docs/plans/2026-11-11-warm-redundancy-717-plan.md index 4b45b74cf..6c9752a98 100644 --- a/docs/plans/2026-11-11-warm-redundancy-717-plan.md +++ b/docs/plans/2026-11-11-warm-redundancy-717-plan.md @@ -255,13 +255,9 @@ The T0-adopted alternative (owner, 2026-11-13). The 1.5 object-keyed provenance ## T13: Live gate on docker-dev (central pair, `:4840` / `:4841`) -**Overlay:** new `docker-dev/docker-compose.warm.yml`, central-1/2 only. It sets: - -- `OpcUa__ApplicationUri=urn:OtOpcUa:central-1|2` (matching `seed-clusters.sql`); -- `OpcUa__PeerApplicationUris__0=`; -- `OpcUa__Redundancy__Mode=Warm`. - -Today both centrals advertise the default `urn:OtOpcUa` (spec §2). Login is disabled on the rig, so drive +**Overlay:** `docker-dev/docker-compose.warm.yml`, central-1/2 only. It sets `OpcUa__Redundancy__Mode=Warm`. The base +compose file already gives each central its own `OpcUa__ApplicationUri=urn:OtOpcUa:central-1|2` (matching +`seed-clusters.sql`) and its partner in `OpcUa__PeerApplicationUris__0` (lmxopcua#801, after spec §2 was written). Login is disabled on the rig, so drive everything yourself. Use `otopcua-cli` from `src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI` against both ports. | Leg | Procedure | Pass | @@ -300,3 +296,20 @@ Set `OpcUa:Redundancy:Mode=Unset` on both nodes and restart them. The setting is nothing persistent to undo: no schema changes and no artifact changes. The `Revision` field stays and is harmless, because unrevisioned handling is preserved. Rolling back ScadaBridge is independent; its ServiceLevel-aware failover also works against `Unset` servers. + +## Implementation status (2026-11-13) + +T1–T10 and T12 are implemented on `master`, default-off (`OpcUa:Redundancy:Mode` defaults to `Unset`, pinned by +`WarmRedundancyWireTests.Unset_is_todays_server_exactly` and `OpcUaPublishActorWarmTests.Unset_never_drives_the_projection_and_keeps_todays_levels`). +T11 is ScadaBridge#138 (separate). T13 has not run: `docker-dev/docker-compose.warm.yml` is ready (see T13). + +Test homes: `WarmStandbyUnitTests` (OpcUaServer.Tests: marker, rewrite, decision record, the SDK-field regression pin, +option rules), `WarmRedundancyWireTests` (OpcUaServer.IntegrationTests: metadata per mode, read projection, write and +Part 9 refusal, the three T0 demotion timings, live-item queue, alarm suppression/refresh, effective-URI XML load), +`WarmRedundancyRevisionTests` + `WarmServiceLevelBandTests` (ControlPlane.Tests), `WarmRoleAuthorityTests` + +`OpcUaPublishActorWarmTests` (Runtime.Tests), `OpcUaRedundancyValidationTests` (Host.IntegrationTests). + +Not yet covered by a test (left for T13 or a follow-up): the real 2-node singleton-handover test and the +"removed node cannot rejoin" pin (T3; ordering is covered at gate and actor level), a DataEncoding read, a +deleted/reused monitored item between queue and publish and the ExtensionObject/DiagnosticInfo clone matrix (T7), the +ACL-stood-down item on transition (T8), and per-counter MeterListener tests beyond `mode_mismatch` (T10). diff --git a/docs/security.md b/docs/security.md index d1f8a1713..3a126ce7a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -634,6 +634,12 @@ takes no realm and no node. > data; every raw/UNS node, event subscription and HistoryRead is still gated. The boot warning says so > ("can still connect and read the standard server nodes ... but sees no plant data") — earlier wording > ("can connect and see nothing") overstated it. +> +> **Warm standby projection vs. ACL (#717).** Under `OpcUa:Redundancy:Mode=Warm` the redundancy metadata +> (`ServerRedundancy`, `ServerUriArray`, `ServiceLevel`) is ns=0 and stays ungated, like the nodes above. On the +> Standby the ACL decision runs **first**: a denied ordinary tag is still `BadUserAccessDenied` (the projection never +> reveals that a denied node exists), and a write or Part 9 method is authorized before it is refused +> `BadInvalidState`, so a caller who may not write learns nothing about the node's role. | Key | Default | Effect | |---|---|---| diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs index 725fe1376..9aea4faa9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServerEventRig.cs @@ -44,6 +44,9 @@ internal sealed class ServerEventRig : IAsyncDisposable public string EndpointUrl => $"opc.tcp://127.0.0.1:{_port}/OtOpcUa"; + /// The SDK application configuration the host built (or loaded). + internal ApplicationConfiguration HostConfiguration => _host.ApplicationInstance!.ApplicationConfiguration!; + /// The configured OpcUa:ApplicationUri (the effective one: the rig loads no XML). public string ApplicationUri => _hostOptions.ApplicationUri; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs index 6f3afe48f..c401f3a53 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs @@ -293,12 +293,45 @@ public sealed class WarmRedundancyWireTests } } - private static async Task ReadEventIdAsync(ISession session, CancellationToken ct) + private static async Task ReadEventIdAsync(ISession session, CancellationToken ct, bool native = false) { - var eventId = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.EventId, ct); + var eventId = await TranslateAsync(session, ConditionNodeId(session, native), BrowseNames.EventId, ct); return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!; } + /// Owner answer Q4 for a NATIVE condition: refused on the Warm Standby ahead of the native-ack Primary gate, + /// nothing routed to the driver; accepted once Active (positive control). + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task A_native_Acknowledge_on_a_Warm_standby_is_refused_and_never_routed(bool standby) + { + var ct = Deadline(); + await using var rig = await StartModeAsync($"Warm717NativeAck{standby}", "Warm", ct); + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + rig.MaterialiseNative(); + var routed = 0; + rig.Nm.NativeAlarmAckRouter = _ => Interlocked.Increment(ref routed); + var projection = rig.Server.CreateStandbyProjection()!; + Promote(projection); + rig.Project(native: true, ConditionMethodEventTests.PreState("Acknowledge", native: true) with { Message = "base" }); + if (standby) Demote(projection); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var eventId = await ReadEventIdAsync(session, ct, native: true); + var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: true), "Acknowledge", eventId, ct); + if (standby) + { + (await Should.ThrowAsync(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState); + routed.ShouldBe(0); + } + else + { + await call; + routed.ShouldBe(1); + } + } + private static async Task TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct) { var elements = path.Split('/').Select(name => new RelativePathElement @@ -349,6 +382,16 @@ public sealed class WarmRedundancyWireTests events.Has(item, "flash").ShouldBeFalse(); } + // Model-change events are not alarm events: still delivered on the Standby. + rig.Sink.RaiseNodesAddedModelChange(EqA, AddressSpaceRealm.Uns); + await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.GeneralModelChangeEventType)); + + // D5: ConditionRefresh2 on one item, then ConditionRefresh on the subscription, on the Standby: Start, the + // retained condition, End. + await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, + subscription.Id, folderItem.Status.Id); + await RecordedEvents.WaitUntilAsync(() => events.HasType(folderItem, ObjectTypeIds.RefreshEndEventType)); + events.Has(folderItem, "while-standby").ShouldBeTrue("ConditionRefresh2 on the Standby returns the retained condition"); // D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End. await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id); await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType)); @@ -397,6 +440,123 @@ public sealed class WarmRedundancyWireTests lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge"); } + /// Spec §4.5 boundary: an event REPORTED while Active but still deferred (#838) at demotion is delivered. + [Fact] + public async Task An_event_reported_while_Active_but_deferred_is_still_delivered_after_demotion() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717Deferred", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); + rig.MaterialiseNative(); + Promote(projection); + + using var session = await rig.ConnectAsync(anonymous: false, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var onA = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA)); + await subscription.ApplyChangesAsync(ct); + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "seed")); + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed")); + + using (var stall = NotifierStall.Hold(rig.Nm, EqA)) + { + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 710, Message: "reported-active")); + rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the report must actually be deferred"); + Demote(projection); + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 720, Message: "reported-standby")); + stall.Release(); + } + + await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "reported-active")); + await Task.Delay(500, ct); + events.Has(onA, "reported-standby").ShouldBeFalse("a report made on the Standby is suppressed"); + } + + /// The promotion refresh goes through the per-item ACL decision: a condition the session may not see is + /// not replayed, while one it may see is (positive control). + [Fact] + public async Task The_promotion_refresh_withholds_an_ACL_denied_condition() + { + var ct = Deadline(); + await using var rig = await StartModeAsync("Warm717AclRefresh", "Warm", ct); + var projection = rig.Server.CreateStandbyProjection()!; + rig.MaterialiseNative(); + rig.MaterialiseScripted(); + rig.Enforce(true, [Key()], EquipmentGrant(EqB, + NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead)); + rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "native-visible")); + rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, + Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "scripted-denied")); + + using var session = await rig.ConnectAsync(anonymous: true, ct); + var events = new RecordedEvents(); + var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); + var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); + await subscription.ApplyChangesAsync(ct); + + Promote(projection); + await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType)); + events.Has(serverItem, "native-visible").ShouldBeTrue("positive control: a visible retained condition is replayed"); + events.Has(serverItem, "scripted-denied").ShouldBeFalse("an ACL-denied condition is withheld from the promotion refresh"); + } + + /// T1: the redundancy check uses the EFFECTIVE ApplicationUri an ApplicationConfigPath XML load produces, not + /// the options value (c7787): listing the options URI as a peer is fine, listing the XML URI fails start. + [Fact] + public async Task The_effective_application_uri_from_a_loaded_xml_configuration_is_what_Warm_validates_and_advertises() + { + var ct = Deadline(); + var seed = await ServerEventRig.StartAsync("Warm717XmlSeed", encrypted: false, ct); + var saved = seed.HostConfiguration; + var dir = Path.Combine(Path.GetTempPath(), $"warm717-xml-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + var file = Path.Combine(dir, "OtOpcUa.Config.xml"); + var xmlUri = saved.ApplicationUri; + try + { + var port = OpcUaHostStart.AllocateFreePort(); + saved.ServerConfiguration!.BaseAddresses = [$"opc.tcp://127.0.0.1:{port}/OtOpcUa"]; + saved.SaveToFile(file); + + OpcUaApplicationHostOptions Options(string peer) => new() + { + ApplicationName = saved.ApplicationName ?? "OtOpcUa.Warm717Xml", + ApplicationUri = "urn:OtOpcUa:options-value", + ApplicationConfigPath = file, + PublicHostname = "127.0.0.1", + OpcUaPort = port, + PkiStoreRoot = dir, + AutoAcceptUntrustedClientCertificates = true, + PeerApplicationUris = [peer], + Redundancy = new OpcUaRedundancyOptions { Mode = "Warm" }, + }; + + var refused = new OpcUaApplicationHost(Options(xmlUri!), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + (await Should.ThrowAsync(refused.StartAsync(new OtOpcUaSdkServer(), ct))) + .Message.ShouldContain(xmlUri!); + + await using var host = new OpcUaApplicationHost(Options("urn:OtOpcUa:options-value"), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var server = new OtOpcUaSdkServer(); + await host.StartAsync(server, ct); + var redundancy = server.CurrentInstance!.ServerObject.ServerRedundancy.ShouldBeOfType(); + var uris = redundancy.ServerUriArray!.Value.ToArray()!; + uris.ShouldBe(new[] { xmlUri!, "urn:OtOpcUa:options-value" }, "self is the effective (XML) URI"); + } + finally + { + // The seed stays up until here: the saved XML points at its PKI store (and certificate thumbprint). + await seed.DisposeAsync(); + try { Directory.Delete(dir, recursive: true); } catch (IOException) { } + } + } + // ------------------------------------------------------------------ T8 live items on transition [Fact]