From 3fff439b117ad00e4767ce43fbc774d8d78be8b1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 12 Nov 2026 11:11:11 -0500 Subject: [PATCH] fix(redundancy): advertise ServiceLevel 0 until the node has a redundancy level (#885) The SDK starts Server.ServiceLevel at 255 (full service), and OpcUaPublishActor publishes nothing until the node's first redundancy snapshot arrives, so a restarting pair node advertised 255 for the ~10-13 s it took to join, out-ranking the real Primary (250) for any client that picks the highest ServiceLevel. OtOpcUaSdkServer now writes StartupServiceLevel = 0 in OnNodeManagerStartedAsync: after the SDK's CreateServerObjectAsync wrote 255 and before the server goes Running, so no session can ever read 255. 0 is the design's existing Down / Detached tier (ServiceLevelCalculator returns 0 for a member that is not Up/Joining, and a node with no snapshot has no role yet). It is below the SDK 2 session-headroom floor (200), so it stands through session churn before the #884 claim, and the claim's bind replays any level already requested. Single-node deployments are unaffected beyond the window: the first snapshot publishes 250 as before. New Node_with_no_requested_level_advertises_the_startup_level_until_one_is_published: reads 0 before the bind, 0 after a bind with nothing to replay, then 250 once published. Without the startup write it reads 255 on the first session (mutation-checked). docs/Redundancy.md updated for #884 and #885. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi --- docs/Redundancy.md | 16 +++-- .../OtOpcUaSdkServer.cs | 19 ++++++ .../ServiceLevelBindingReplayTests.cs | 62 +++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/docs/Redundancy.md b/docs/Redundancy.md index db8b6f525..652dbdfa3 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -16,7 +16,7 @@ The runtime pieces live in: |---|---|---| | `RedundancyStateActor` | `OtOpcUa.ControlPlane.Redundancy` | Admin-role cluster singleton; subscribes to cluster topology events, debounces 250ms, broadcasts `RedundancyStateChanged` on the `redundancy-state` DPS topic. | | `OpcUaPublishActor` | `OtOpcUa.Runtime.OpcUa` | Per-driver-node; subscribes to the `redundancy-state` topic, computes a health-aware ServiceLevel byte via `ServiceLevelCalculator` (see below), and forwards it to `IServiceLevelPublisher`. | -| `IServiceLevelPublisher` / `SdkServiceLevelPublisher` | `OtOpcUa.Commons.OpcUa` / `OtOpcUa.OpcUaServer` | Writes the byte into the SDK's `Server.ServiceLevel` Variable. Production binds `DeferredServiceLevelPublisher`, which swaps in the real `SdkServiceLevelPublisher` once the SDK is up (it needs `IServerInternal`, available only after `StandardServer.Start`); until then writes route through `NullServiceLevelPublisher`. | +| `IServiceLevelPublisher` / `SdkServiceLevelPublisher` | `OtOpcUa.Commons.OpcUa` / `OtOpcUa.OpcUaServer` | Writes the byte into the SDK's `Server.ServiceLevel` Variable. Production binds `DeferredServiceLevelPublisher`, which swaps in the real `SdkServiceLevelPublisher` once the SDK is up (it needs `IServerInternal`, available only after `StandardServer.Start`); until then it retains the latest request and replays it on bind. Before any request the wire reads `OtOpcUaSdkServer.StartupServiceLevel` (0, #885). On SDK 2 the publisher claims the value so the SDK stops overwriting it (#884). | | `ServiceLevelCalculator` | `OtOpcUa.Cluster.Redundancy` (`Core.Cluster`) | Pure function `(NodeHealthInputs) → byte` — the DB/probe-aware tiering (see truth table below). Covered by `ServiceLevelCalculatorTests`. **Now the live publish path** — `OpcUaPublishActor` calls it on every `HealthTick` and `RedundancyStateChanged` event. Moved to `Core.Cluster` so Runtime can reach it without a Runtime→ControlPlane reference. | | `DbHealthProbeActor` | `OtOpcUa.Runtime.Health` | Runs `SELECT 1` against ConfigDb every 5s. Read by the health endpoint AND by `OpcUaPublishActor` (the `DbReachable` ServiceLevel input). **Spawned only when a ConfigDb is present** (admin-role or fused admin+driver nodes) — a driver-only node has no `IDbContextFactory` to probe, so this actor is not spawned there at all; see "DB-less nodes" below. | | `PeerProbeSupervisor` | `OtOpcUa.Runtime.Health` | Per-node; subscribes to the `redundancy-state` topic and maintains one `PeerOpcUaProbeActor` child per OTHER driver-role peer (spawn on join, stop on departure), so every node is continuously probed by its peers. | @@ -47,7 +47,7 @@ The resulting truth table (all tiers are now reachable at runtime): | Tier | Byte | Condition | |---|---|---| -| Down / Detached | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting), OR node has no `driver` role (Detached). Published immediately — a starting or detached node never leaves the SDK default 255. | +| Down / Detached | 0 | Member status is not `Up` or `Joining` (leaving, removed, exiting), OR node has no `driver` role (Detached). Published immediately. A node that has no redundancy snapshot yet (cold start, rejoin after a restart) also advertises **0**: `OtOpcUaSdkServer` replaces the SDK's built-in 255 before the endpoint accepts sessions (`StartupServiceLevel`, #885), so a joining node never out-ranks the real Primary. | | Endpoint failed | 0 | This node's OPC UA SDK failed to start (`EndpointListening=false`). Wins over every other input, including the +10 Primary bonus; the legacy role-only seam applies it too. See [Endpoint start failure: Primary step-down](#endpoint-start-failure-primary-step-down). | | Critically degraded | 100 | ConfigDb unreachable AND data is stale. | | Stale | 200 | Data stale but ConfigDb reachable. | @@ -93,8 +93,16 @@ A node with no `DbHealthStatus` wired (e.g. early bootstrap window before the first `DbHealthProbeActor` reply) falls back to the old role-only mapping: Primary-leader → 240, Primary → 200, Secondary → 100, Detached → 0. Once the first `DbHealthStatus` arrives the calculator takes over. The first computed -ServiceLevel (even 0) is always published so no node lingers at the SDK default -255. +ServiceLevel (even 0) is always published. Before that first level the server +advertises 0, not the SDK's built-in 255 (#885). + +> **SDK 2.0 ownership (#884).** SDK 2.0.1 recomputes `Server.ServiceLevel` from +> session headroom on every session create/close and overwrites any value ≥ 200 +> (255 when lightly loaded) unless a provider has claimed the value. +> `SdkServiceLevelPublisher` claims it through +> `IServerServiceLevelControl.ClaimServiceLevelControl()` when the host binds it +> and writes through the returned writer, so 250 / 240 / 200 survive client +> sessions opening and closing. Values below 200 were never touched by the SDK. Roles come from `RedundancyStateActor.BuildSnapshot`: a node with the `driver` role is `Primary` when it is the **oldest Up member carrying the `driver` role**, otherwise diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs index 7db6cbfdd..ceef01ed2 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs @@ -305,11 +305,30 @@ public sealed class OtOpcUaSdkServer : StandardServer } } + /// + /// The Server.ServiceLevel a node advertises from the moment its endpoint accepts sessions until the + /// redundancy layer publishes a real level (lmxopcua#885): 0, the design's Down / Detached tier + /// (ServiceLevelCalculator returns 0 for a member that is not Up/Joining, and a node that has not yet + /// received its first redundancy snapshot has no role). The SDK starts the value at 255 — full service — so + /// a restarting pair node out-ranked the real Primary (250) for the ~10 s it took to join. Below the SDK's + /// 200 floor, so the session-headroom calculator never lifts it either. + /// + public const byte StartupServiceLevel = 0; + /// protected override async ValueTask OnNodeManagerStartedAsync( IServerInternal server, CancellationToken cancellationToken = default) { await base.OnNodeManagerStartedAsync(server, cancellationToken).ConfigureAwait(false); + // Runs after CreateServerObjectAsync wrote the SDK's 255 and before the server goes Running, so no session can + // ever read 255. SdkServiceLevelPublisher claims the value once the host binds and replays any level the + // redundancy layer already requested; until one is requested this value stands. + if (server.ServerObject?.ServiceLevel is { } serviceLevel) + { + serviceLevel.Value = StartupServiceLevel; + serviceLevel.Timestamp = DateTime.UtcNow; + serviceLevel.ClearChangeMasks(server.DefaultSystemContext, includeChildren: false); + } var auditType = server.DiagnosticsNodeManager? .FindPredefinedNode(ObjectTypeIds.AuditWriteUpdateEventType) ?? throw new InvalidOperationException("AuditWriteUpdateEventType is unavailable during server startup."); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServiceLevelBindingReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServiceLevelBindingReplayTests.cs index 0a5704574..97bf1b41c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServiceLevelBindingReplayTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ServiceLevelBindingReplayTests.cs @@ -11,6 +11,8 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// Local loopback evidence that late binding replaces the SDK default on the wire. public sealed class ServiceLevelBindingReplayTests { + // Since #885 the server starts at 0, so the 0 case no longer distinguishes "replayed" from "not replayed" — it pins + // that a requested 0 still reads 0. The 100 case is the replay evidence. [Theory] [InlineData((byte)100)] [InlineData((byte)0)] @@ -135,6 +137,66 @@ public sealed class ServiceLevelBindingReplayTests } } + /// + /// lmxopcua#885: a node that has not yet received its first redundancy snapshot has requested no level. The SDK starts + /// Server.ServiceLevel at 255, so a restarting pair node advertised full service, out-ranking the real Primary + /// (250), for the ~10 s it took to join. It must read (0) from its + /// first session, through the bind with nothing to replay and across session churn, and still settle to the level + /// the redundancy layer publishes once it has one (a single-node deployment's 250). + /// + [Fact] + public async Task Node_with_no_requested_level_advertises_the_startup_level_until_one_is_published() + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(30)); + var ct = deadline.Token; + var pki = Path.Combine(Path.GetTempPath(), $"otopcua-startup-{Guid.NewGuid():N}"); + var clientPki = TestClientSecurity.AllocatePkiRoot(); + var port = 0; + try + { + var started = await OpcUaHostStart.StartAsync(p => + { + port = p; + return new OpcUaApplicationHost(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.Startup", + ApplicationUri = $"urn:OtOpcUa.Startup:{Path.GetFileName(pki)}", + OpcUaPort = p, PublicHostname = "localhost", PkiStoreRoot = pki, + }, NullLogger.Instance); + }, () => new OtOpcUaSdkServer(), ct); + await using var host = started.Host; + var url = $"opc.tcp://localhost:{port}/OtOpcUa"; + + // Before the host binds a publisher: the SDK-start window. + using (var early = await OpenAsync(url, clientPki, ct)) + { + (await ReadServiceLevelAsync(early, ct)).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel, + "an endpoint that has not been told its redundancy level must not advertise the SDK's 255"); + await early.CloseAsync(5_000, true, ct); + } + + // The bind with nothing to replay (no snapshot yet), then session churn under the claim. + var deferred = new DeferredServiceLevelPublisher(); + deferred.SetInner(new SdkServiceLevelPublisher(started.Server.CurrentInstance, + NullLogger.Instance)); + using var session = await OpenAsync(url, clientPki, ct); + (await ReadServiceLevelAsync(session, ct)).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel, + "binding with no requested level must leave the startup level standing"); + + // The first snapshot arrives: a single-node deployment settles to its steady Primary value. + deferred.Publish(250); + using (var later = await OpenAsync(url, clientPki, ct)) + await later.CloseAsync(5_000, true, ct); + (await ReadServiceLevelAsync(session, ct)).ShouldBe((byte)250, "the published level must replace the startup level"); + } + finally + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + if (Directory.Exists(clientPki)) Directory.Delete(clientPki, recursive: true); + } + } + private static async Task OpenAsync(string url, string clientPki, CancellationToken ct) { var config = new ApplicationConfiguration(DefaultTelemetry.Create(_ => { }))