From a801e2aefbb69e34cb7c819f329e8890a1d9918f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:32:24 -0400 Subject: [PATCH 01/45] =?UTF-8?q?fix(runtime):=20comms-loss=20grace=20is?= =?UTF-8?q?=20per=20connection=20=E2=80=94=20max(LastSuccessfulRead,=20con?= =?UTF-8?q?nectedSince)=20(archreview=2003/S17)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...R3-03-restart-resubscribe-watchdog-plan.md | 35 ++++++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverInstanceActor.cs | 24 +++++- ...iverInstanceActorCommsLossWatchdogTests.cs | 83 +++++++++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index e872aefd..4a9966cb 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -452,3 +452,38 @@ T8 anytime; T9 last (after T3/T5/T6/T7 merge-ready) ``` Overall effort: **~0.75–1 dev-day code+unit, plus the rig session.** The one HIGH-RISK production slice (T3, which changes what every child receives at spawn) lands behind the full 25-suite `DriverHostActor` regression before any new test stacks on it, and behind T2's explicitly behaviour-preserving refactor. + +--- + +## Execution notes + +Branch `fix/r3-03-restart-resubscribe-watchdog`, worktree +`.claude/worktrees/agent-a5bcdd61bfba22e10`, off master `5fd65d19`. TDD throughout: every +behaviour task's test was written first, RUN, and observed red for the predicted reason before the +production edit landed. Red observations are recorded per task below. + +### T1 — 03/S17 watchdog grace (`??` → max) + +**RED observed (as predicted).** With the two new tests in place against unmodified +`DriverInstanceActor`: + +``` +DriverInstanceActorCommsLossWatchdogTests.A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again [FAIL] + Failed: Expected no messages during 00:00:00.4000000, instead we received + after 00:00:00.0496590 +Failed! - Failed: 1, Passed: 9, Total: 10 +``` + +The re-trip landed **49.7 ms** after the reconnect — one health tick (`FastPoll` = 50 ms) — against a +`Grace` of 1500 ms, i.e. exactly the "churn at health-tick cadence instead of the designed grace" +the finding describes. `A_half_broken_device_is_still_declared_lost_once_the_grace_elapses` passed +in the same run, which is what makes it the anti-vacuity control: it cannot distinguish the fix from +the bug, so the delay assertion is the one carrying the finding. + +**GREEN after the one-line max at `:574`** (plus the "grace is per connection" remarks paragraph and +the reciprocal falsification-partner xmldoc on the existing +`A_degraded_driver_that_is_still_reading_successfully_is_left_alone`): 10/10 passed, 20 s. +`A_reconnect_attempt_tears_the_dead_transport_down_before_re_initialising` stayed green — as the plan +anticipated, each of its trip cycles now costs a full `FastTimeout` (250 ms) instead of one 50 ms +tick, and its `AwaitCondition(InitializeCount >= 2, 5 s)` still has ample headroom. No budget change +was needed. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index b14841de..2464790a 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -5,7 +5,7 @@ { "id": "T1", "subject": "Comms-loss grace is per connection: ?? -> max(LastSuccessfulRead, _connectedSinceUtc) + falsification pair (03/S17)", - "status": "pending", + "status": "done", "blockedBy": [] }, { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs index 7c8f5b3d..81852667 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs @@ -561,6 +561,19 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers /// state on sustained failure: the nine poll drivers via Degraded, MQTT via its live connection /// state, Galaxy via its reconnect supervisor. /// + /// + /// The grace is per CONNECTION (03/S17). The silence baseline is + /// max(LastSuccessfulRead, _connectedSinceUtc), not LastSuccessfulRead ?? _connectedSinceUtc. + /// Because LastSuccessfulRead is preserved across Degraded/Faulted writes (see above), + /// a device that accepts a connection but serves no data reconnects carrying a stamp that is already + /// older than the threshold, so a ?? baseline re-trips on the FIRST health tick of every new + /// connection — a reconnect cycle at health-tick cadence rather than the designed threshold. Taking + /// the later of the two gives each connection its own full window, which is exactly what re-stamping + /// _connectedSinceUtc on every Connected entry was for. Note this paces the cycle, it + /// does not end it: a permanently half-broken device still reconnects once per threshold, and a + /// driver that stops reading within a stable connection still trips on schedule (its read + /// stamp is the later term, exactly as before). + /// /// /// The health observed on this tick, or null when /// threw — a failed observation is not evidence of comms loss, so it never advances the watchdog. @@ -570,8 +583,15 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers if (health is not { } observed) return; if (observed.State == DriverState.Healthy) return; - // A driver that has never reported a successful read is measured from the current Connected entry. - var lastEvidence = observed.LastSuccessfulRead ?? _connectedSinceUtc; + // Measure silence from the LATER of the last successful read and THIS connect — the grace is per + // connection. `??` was wrong: drivers PRESERVE LastSuccessfulRead across Degraded/Faulted writes (that + // preservation is the #527 design), so after a trip + reconnect to a device that accepts the connection + // but still serves no data, the stamp is already older than the threshold and the FIRST tick of the new + // connection re-trips — churn at health-tick cadence instead of the designed grace. `??` only ever helped + // a driver that had never read at all. + var lastEvidence = observed.LastSuccessfulRead is { } read && read > _connectedSinceUtc + ? read + : _connectedSinceUtc; var silence = DateTime.UtcNow - lastEvidence; if (silence < _commsLossTimeout) return; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorCommsLossWatchdogTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorCommsLossWatchdogTests.cs index f12f3b54..8010dd02 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorCommsLossWatchdogTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorCommsLossWatchdogTests.cs @@ -40,6 +40,10 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50); private static readonly TimeSpan FastTimeout = TimeSpan.FromMilliseconds(250); + /// The threshold used by the per-connection grace pair (03/S17). Deliberately an order of + /// magnitude above so the absence window below is a decision, not a race. + private static readonly TimeSpan Grace = TimeSpan.FromMilliseconds(1500); + /// Drives the actor to Connected and returns once the connect has been announced. private (IActorRef Actor, Akka.TestKit.TestProbe Parent) ConnectDriver( HealthControlledStubDriver driver, @@ -87,6 +91,14 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest /// ModbusDriver.ReadAsync leaves latched whenever the bad tag is read last — but it is still /// reading successfully, so LastSuccessfulRead keeps advancing. It must be left alone. /// A state-only watchdog would tear this driver down on every threshold, forever. + /// + /// Falsification partner (03/S17): + /// . + /// The silence baseline is max(LastSuccessfulRead, _connectedSinceUtc), and each test kills + /// one term of that max — this one goes red if the LastSuccessfulRead term is dropped (its + /// connect ages far past the threshold while its reads keep succeeding), that one goes red if the + /// _connectedSinceUtc term is dropped (i.e. if the original ?? is restored). + /// /// [Fact] public void A_degraded_driver_that_is_still_reading_successfully_is_left_alone() @@ -258,6 +270,77 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } + + /// + /// 03/S17 — the grace is per connection. Drivers deliberately preserve + /// across Degraded/Faulted health writes (that + /// preservation is the #527 design), so for a device that accepts a TCP connection but never serves + /// data the stamp stays frozen from before the outage — already older than the threshold. Measuring + /// silence from that stamp alone (??) makes the first health tick of every new + /// connection re-trip, so the reconnect cycle runs at health-tick cadence (~30 s in production) + /// instead of the designed grace. Measuring from max(LastSuccessfulRead, _connectedSinceUtc) + /// gives each connection its own full window, which is exactly what the _connectedSinceUtc + /// re-stamp on every Connected entry already promises in its comment. + /// + /// Falsification partner: + /// — see its + /// remarks for how the pair covers both terms of the max. + /// + /// + /// The absence window carries the house positive control: a proven number of health-poll ticks + /// processed after the reconnect, so "the watchdog did not re-trip" cannot pass because + /// the actor stopped ticking. It is also deliberately far shorter than , so + /// it asserts a delay, not a disablement — + /// is the anti-vacuity half. + /// + /// + [Fact] + public void A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again() + { + var driver = new HealthControlledStubDriver + { + // The half-broken device: the socket connects, every read fails, and the last successful read is + // frozen well before the outage — preserved across the Degraded writes, as every driver does. + Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"), + }; + var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100)); + + // The first trip is correct and expected — the device really has been silent for five minutes. + parent.ExpectMsg(PresenceBudget) + .Connected.ShouldBeFalse(); + // The retry re-initialises successfully (the socket still accepts), so a NEW connection starts here. + parent.ExpectMsg(PresenceBudget) + .Connected.ShouldBeTrue(); + + var ticksAtReconnect = driver.HealthReadCount; + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(400)); + (driver.HealthReadCount - ticksAtReconnect).ShouldBeGreaterThanOrEqualTo( + 4, + "health-poll ticks did not run after the reconnect — the absence assertion above would be vacuous"); + } + + /// + /// The anti-vacuity half of the 03/S17 pair: the per-connection grace delays the watchdog, it + /// does not disable it. The same half-broken device is still declared lost once the grace elapses — + /// without this, a "fix" that simply stopped evaluating the watchdog after the first connection would + /// pass + /// and look correct. + /// + [Fact] + public void A_half_broken_device_is_still_declared_lost_once_the_grace_elapses() + { + var driver = new HealthControlledStubDriver + { + Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"), + }; + var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100)); + + parent.ExpectMsg(PresenceBudget).Connected.ShouldBeFalse(); + parent.ExpectMsg(PresenceBudget).Connected.ShouldBeTrue(); + + // The second connection serves no data either, so once ITS grace elapses the watchdog trips again. + parent.ExpectMsg(PresenceBudget).Connected.ShouldBeFalse(); + } } /// From bdc81a13726f8389a40f55be0620cf1a86821ea9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:34:49 -0400 Subject: [PATCH 02/45] refactor(runtime): one desired-subscription payload authority (_desiredByDriver) --- ...R3-03-restart-resubscribe-watchdog-plan.md | 17 +++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverHostActor.cs | 68 ++++++++++++++++--- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index 4a9966cb..c74e839d 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -487,3 +487,20 @@ the reciprocal falsification-partner xmldoc on the existing anticipated, each of its trip cycles now costs a full `FastTimeout` (250 ms) instead of one 50 ms tick, and its `AwaitCondition(InitializeCount >= 2, 5 s)` still has ample headroom. No budget change was needed. + +### T2 — desired-subscription payload authority + +Behaviour-preserving refactor, so there is no red observation to record by design — the gate is that +**no test moves**. `--filter "FullyQualifiedName~DriverHostActor"` → **114/114 passed**, unchanged +from the base commit. + +⚠️ One run of that filter reported `DriverHostActorNativeAlarmAckRoutingTests.RouteNativeAlarmAck_on_non_primary_is_dropped` +failing on an `AwaitAssert`. Re-run in isolation: **3/3 passed**; re-run of the whole 114-test filter: +**114/114 passed**. Treated as the known #500 flake set for this assembly (~13 %/run), not a +regression — it touches none of the code this task edits (the alarm-ack Primary gate is a different +method entirely). + +Deviation from the plan's literal step text (cosmetic, recorded for the reviewer): the plan's Finding-1 +step 1 says `_desiredByDriver` *replaces* `_lastComposition` at `:347-349` while T3 step 2 says T3 +deletes `_lastComposition`. Keeping T2 compilable means the two fields coexist for one commit; +`_lastComposition` and its assignment are deleted in T3 as T3's own step text specifies. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 2464790a..582bcf35 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -11,7 +11,7 @@ { "id": "T2", "subject": "Desired-subscription payload authority: _desiredByDriver + SendDesiredSubscriptions (behaviour-preserving refactor)", - "status": "pending", + "status": "done", "blockedBy": [] }, { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index e95ca092..ff212655 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -348,6 +348,22 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// ). Null until the first apply. private AddressSpaceComposition? _lastComposition; + /// One driver child's desired subscription payload — exactly the + /// the apply's bulk push sent it. + /// The non-alarm raw tags' RawPaths this driver should keep subscribed. + /// The alarm-bearing raw tags' RawPaths that un-gate this driver's + /// feed. + private readonly record struct DesiredSubscriptionSet( + IReadOnlyList ValueRefs, + IReadOnlyList AlarmRefs); + + /// Per-driver desired subscription sets from the most-recent apply. THE authority: the bulk + /// push, and the supervision-reprovision handler all send from here, so a + /// child obtained by ANY route carries the same set. Cleared and repopulated whole on every apply, + /// alongside the routing maps, so renames/removals can never leave a stale entry. Empty until the + /// first apply — a child spawned before any apply is handed nothing, which is what it should have. + private readonly Dictionary _desiredByDriver = new(StringComparer.Ordinal); + /// /// Cached local from the latest /// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary @@ -2652,23 +2668,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS } } - // One authored-only push (value refs + alarm refs from the maps built above), shared by the bulk loop AND - // the dropped-driver fallback so the two CANNOT drift: the fallback's correctness depends on sending the - // SAME payload the bulk loop would have, so it's a shared helper (structural), not a comment-maintained - // invariant. An EMPTY set is valid — the child's Connected handler routes it to Unsubscribe (dropping a - // stale handle) rather than a spurious subscribe. Returns the value-ref count for the bulk-loop log total. - int SendAuthoredOnly(IActorRef actor, string driverId) + // Cache the per-driver payload as THE authority, in the same clear-and-repopulate pass as the routing + // maps above so a rename/removal can never leave a stale entry behind. Keyed over the UNION of the two + // ref maps AND the running children: a child with no authored tags must get an explicit EMPTY set (which + // it routes to Unsubscribe), not be absent from the authority — absence and empty must not differ here. + _desiredByDriver.Clear(); + foreach (var driverId in refsByDriver.Keys + .Concat(alarmRefsByDriver.Keys) + .Concat(_children.Keys) + .Distinct(StringComparer.Ordinal)) { - var refs = refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty(); - var alarmRefs = alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty(); - actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(refs, SubscriptionPublishingInterval, alarmRefs)); - return refs.Count; + _desiredByDriver[driverId] = new DesiredSubscriptionSet( + refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty(), + alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty()); } var total = 0; foreach (var (driverId, entry) in _children) { - total += SendAuthoredOnly(entry.Actor, driverId); + total += SendDesiredSubscriptions(entry.Actor, driverId); } if (total > 0) @@ -2710,6 +2728,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS } + /// + /// Send 's desired subscription payload to . + /// + /// + /// The ONE place a is built, so the apply's + /// bulk push, a respawn () and a supervision reprovision + /// (HandleReprovisionRequested) cannot drift. That is structural rather than + /// comment-maintained on purpose: a second sender that assembled its own payload is exactly how + /// 03/S16 happened — the restart path got a child without the set that makes a child useful. + /// + /// An EMPTY set is valid and MEANINGFUL — the child routes it to Unsubscribe (dropping a + /// stale handle) rather than to a spurious subscribe — so a driver missing from + /// is sent an empty set, never skipped. + /// + /// + /// The driver child to hand the set to. + /// The driver instance whose set to send. + /// The value-ref count, for the bulk loop's log total. + private int SendDesiredSubscriptions(IActorRef actor, string driverId) + { + var set = _desiredByDriver.TryGetValue(driverId, out var s) + ? s + : new DesiredSubscriptionSet(Array.Empty(), Array.Empty()); + actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions( + set.ValueRefs, SubscriptionPublishingInterval, set.AlarmRefs)); + return set.ValueRefs.Count; + } + private void SpawnChild(DriverInstanceSpec spec) { var stub = DriverInstanceActor.ShouldStub(spec.DriverType, _localRoles); From 894444376e2c5c3df1c430e9a00f003c678dafcd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:36:04 -0400 Subject: [PATCH 03/45] feat(security): register TimeProvider + S18/S19 option knobs (archreview 03/S18,S19) --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 25 +++++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../Authorization/AclOptions.cs | 19 ++++ .../Authorization/ApiKeyOptions.cs | 25 +++++ .../DataPlaneAuthRegistration.cs | 100 ++++++++++++++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 53 +--------- .../Security/DataPlaneAuthorizer.cs | 9 ++ .../MeshRegistrationRoleScopingTests.cs | 10 +- .../OpcUa/ApiKeyStoreRegistrationTests.cs | 77 ++++++++++++++ 9 files changed, 266 insertions(+), 54 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 2577b436..0ecd9c86 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -578,3 +578,28 @@ T10 anytime after T5/T8; T11 last (after T5 + T9 merge-ready); T12 anytime ``` Overall effort: **~1 dev-day code+unit, plus the rig session.** The two HIGH-RISK slices are T2/T3 (authentication path — every new branch must land on deny) and T11 (the gate that proves the clock axis actually reaches a real client). T9 is the largest single task and is the only one that can falsify T8 end to end. + +--- + +## Execution notes + +Branch `fix/r3-05-acl-hot-path-expiry`, executed 2026-08-20 off master `dd7f4954`. TDD per task: +failing test first, red observed and recorded, then green. T11 (live gate) deliberately left `pending` +for the orchestrator. + +### T1 — TimeProvider registration + option knobs + +- **RED observed:** `error CS1061: 'ServiceCollection' does not contain a definition for + 'AddOtOpcUaDataPlaneAuth'` and `'DataPlaneAuthorizer' does not contain a definition for 'Clock'` — + the seam and the assertion surface both absent, as predicted. +- **DEVIATION (structural, required by the plan's own test):** the plan says the guard test "builds the + service collection for an admin node". `Program.cs` is **top-level statements in an executable**, so no + test can drive its wiring. The `DataPlaneAuthorizer` registration and the three-branch key-store / + authenticator conditional were therefore **moved verbatim** into a new + `Host/Configuration/DataPlaneAuthRegistration.AddOtOpcUaDataPlaneAuth(hasAdmin, hasDriver)` — the same + shape `LocalDbRegistration` already uses — and `Program.cs` now calls it. No branch changed. +- **DEVIATION (small):** `DataPlaneAuthorizer` gained a public `Clock` property. The two-clock hazard the + class already documents can only be pinned by reference, and `_time` was private. +- **Collateral:** `MeshRegistrationRoleScopingTests.HasAdminMarkerScan_MatchesTheKnownShapeOfProgramCs` + pins the `if (hasAdmin)` marker count in `Program.cs`; the move took one braced marker with it, so the + pin went 6 → 5 (the test's own message asks for exactly that when a marker moves). diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index 77023481..46fc793e 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -5,7 +5,7 @@ { "id": "T1", "subject": "Register TimeProvider in DI (shared by authorizer, key store and revalidation timer) + ApiKeyOptions.StoreTimeoutMs/StoreCacheSeconds + AclOptions.RevalidationIntervalSeconds (03/S18, 03/S19)", - "status": "pending", + "status": "done", "blockedBy": [] }, { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/AclOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/AclOptions.cs index 1efe641c..524fa93b 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/AclOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/AclOptions.cs @@ -42,4 +42,23 @@ public sealed class AclOptions /// false, which preserves the pre-stage-5 role gates byte-for-byte. /// public bool Enabled { get; set; } + + /// + /// How often, in seconds, live monitored items are re-authorized against the clock — in addition + /// to on every deployment (arch-review 03/S19). + /// + /// + /// + /// A subscription is authorized once and then streams, so before this a key that expired by the + /// CLOCK kept feeding a live subscription until a deployment happened to advance the generation + /// — indefinitely on a fleet that has been deployed once and left running. Every other seam + /// re-asks per request and honours ExpiresUtc the moment it passes. + /// + /// + /// 0 or negative disables the timer, leaving revalidation deploy-triggered only (the + /// pre-R3-05 behaviour). The timer is inert while is false: the + /// pass early-returns when ACL enforcement is off. + /// + /// + public int RevalidationIntervalSeconds { get; set; } = 60; } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/ApiKeyOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/ApiKeyOptions.cs index 47d122e4..ea2d64ee 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/ApiKeyOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Authorization/ApiKeyOptions.cs @@ -27,4 +27,29 @@ public sealed class ApiKeyOptions /// ApiKeySecret.ComputeHash throws on one and this stays empty to make that fire early. /// public string Pepper { get; set; } = string.Empty; + + /// + /// Hard per-call deadline, in milliseconds, on the API-key store lookup that runs inline on OPC UA + /// session activation (arch-review 03/S18). Exceeding it denies the activation and emits a + /// distinct Warning plus otopcua.opcua.apikey_store.bounded{outcome=timeout}. + /// + /// + /// 0 or negative disables the deadline, restoring the pre-R3-05 unbounded behaviour. Do not + /// do that on a node with a ConfigDb: the SDK raises impersonation synchronously under a + /// SessionManager-wide lock, so an unanswered lookup stalls every session activation + /// in the server, not just its own. + /// + public int StoreTimeoutMs { get; set; } = 2000; + + /// + /// TTL, in seconds, of the read-through cache in front of the API-key store (arch-review 03/S18). + /// Bounds how long a revocation authored in the ConfigDb can take to reach a new session + /// activation. + /// + /// + /// 0 or negative disables the cache, restoring a store read per activation. The window it + /// introduces is strictly smaller than the one that already exists for already-open sessions + /// on every node kind, which is the next deployment — see docs/security.md. + /// + public int StoreCacheSeconds { get; set; } = 5; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs new file mode 100644 index 00000000..cb04f265 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -0,0 +1,100 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Composition of the OPC UA data-plane authentication + authorization seams (#520), extracted from +/// Program.cs so the role-conditional branches are reachable from a test. +/// +/// +/// A store decorator registered in one branch of a role conditional is exactly the +/// "built but never wired" shape, and Program.cs's top-level statements cannot be driven by a +/// test — so the wiring moved here rather than the guard being written against a comment. The +/// branches are unchanged from the ones that lived inline; see ApiKeyStoreRegistrationTests. +/// +public static class DataPlaneAuthRegistration +{ + /// + /// Registers the shared clock, the node-scoped ACL gate and the role-appropriate API-key store + + /// user authenticator. + /// + /// The service collection. + /// Whether this node carries the admin role (and therefore a ConfigDb). + /// Whether this node carries the driver role. + /// The same collection, for chaining. + public static IServiceCollection AddOtOpcUaDataPlaneAuth( + this IServiceCollection services, bool hasAdmin, bool hasDriver) + { + ArgumentNullException.ThrowIfNull(services); + + // ONE clock for the whole authorization surface. The authorizer, the key-store bound and the + // revalidation timer all read time, and DataPlaneAuthorizer already documents what a second, + // diverging clock costs: every request denied, with a verdict indistinguishable from "no grant + // matched". TryAdd so a test (or a future host) can substitute one before this runs. + services.TryAddSingleton(TimeProvider.System); + + // #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because + // OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves + // constructors by whether every parameter is registered, not by whether the parameter type is nullable. + // That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations below + // both carry a comment about; an absent registration is a node that will not start. + services.AddSingleton(sp => + new TriePermissionEvaluator(sp.GetRequiredService())); + services.AddSingleton(sp => new DataPlaneAuthorizer( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>().Value.Enabled, + sp.GetRequiredService().CreateLogger(), + sp.GetRequiredService())); + + if (hasAdmin) + { + // A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect + // on the very next activation rather than on the next deployment. Preferred wherever it is + // available, including on a fused admin+driver node. + // The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds + // the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which + // ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration), + // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services + // rule is what keeps the redundancy-singleton registration in Program.cs from recursing into + // ActorSystem construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter. + services.AddSingleton(sp => new ConfigDbApiKeyStore( + sp.GetRequiredService>(), + ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService>().Value))); + services.AddSingleton(); + + // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. + services.AddSingleton(sp => new ApiKeyAdminService( + sp.GetRequiredService>(), + sp.GetRequiredService>().Value.Pepper)); + } + else if (hasDriver) + { + // Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact. + // Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName + // token — safe, but it meant no machine client could authenticate against a site node at all. + // Deliberately NOT wrapped by the S18 bound: the artifact store is two dictionary probes on an + // in-memory snapshot, so it can neither hang nor be worth caching. + services.AddSingleton(); + services.AddSingleton(); + } + else + { + // Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent — + // OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so + // an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start. + services.AddSingleton(NullOpcUaUserAuthenticator.Instance); + } + + return services; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 12eeee6b..e6e0fbbb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -398,55 +398,10 @@ if (hasDriver) builder.Services.AddSingleton(); } - // #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because - // OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves - // constructors by whether every parameter is registered, not by whether the parameter type is nullable. - // That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations above - // both carry a comment about; an absent registration is a node that will not start. - builder.Services.AddSingleton(sp => - new TriePermissionEvaluator(sp.GetRequiredService())); - builder.Services.AddSingleton(sp => new DataPlaneAuthorizer( - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService>().Value.Enabled, - sp.GetRequiredService().CreateLogger())); - - if (hasAdmin) - { - // A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect - // on the very next request rather than on the next deployment. Preferred wherever it is available, - // including on a fused admin+driver node. - // The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds - // the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which - // ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration), - // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services - // rule is what keeps the redundancy-singleton registration below from recursing into ActorSystem - // construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter (legacy behaviour). - builder.Services.AddSingleton(sp => new ConfigDbApiKeyStore( - sp.GetRequiredService>(), - ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService>().Value))); - builder.Services.AddSingleton(); - - // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. - builder.Services.AddSingleton(sp => new ApiKeyAdminService( - sp.GetRequiredService>(), - sp.GetRequiredService>().Value.Pepper)); - } - else if (hasDriver) - { - // Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact. - // Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName - // token — safe, but it meant no machine client could authenticate against a site node at all. - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - } - else - { - // Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent — - // OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so - // an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start. - builder.Services.AddSingleton(NullOpcUaUserAuthenticator.Instance); - } + // #520 + arch-review 03/S18 — the node-scoped ACL gate, the shared clock and the role-appropriate + // API-key store + user authenticator, all in one extension so the role-conditional branches are + // reachable from a test (Program.cs's top-level statements are not). See DataPlaneAuthRegistration. + builder.Services.AddOtOpcUaDataPlaneAuth(hasAdmin, hasDriver); // Bind + validate the OPC UA host options the same way (fail-fast at start via ValidateOnStart) // and let OtOpcUaServerHostedService consume the validated IOptions instance rather than diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/DataPlaneAuthorizer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/DataPlaneAuthorizer.cs index 3f9a64a1..127e5778 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/DataPlaneAuthorizer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/DataPlaneAuthorizer.cs @@ -108,6 +108,15 @@ public sealed class DataPlaneAuthorizer /// public bool Enabled { get; } + /// The clock this gate evaluates key expiry against. + /// + /// Exposed so the composition root's wiring can be asserted by reference. The remarks on + /// UserAuthorizationState record what a second, diverging clock costs: every request denied, + /// with a verdict indistinguishable from "no grant matched". One registered singleton threaded into + /// every consumer is the fix, and a test that cannot see which instance landed here cannot pin it. + /// + public TimeProvider Clock => _time; + /// /// Memoized grant walks for the request-rate seams (#520 stage 6). See /// for what it is and is not allowed to cache. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/MeshRegistrationRoleScopingTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/MeshRegistrationRoleScopingTests.cs index 58d5d93f..b3e4d7b2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/MeshRegistrationRoleScopingTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/MeshRegistrationRoleScopingTests.cs @@ -236,8 +236,10 @@ public sealed class MeshRegistrationRoleScopingTests } /// - /// Pins the scanner's expected result shape against today's Program.cs — 6 markers, - /// 5 braced blocks + the 1 braceless ConfigDb gate, so 6 scanned ranges. Without this, a + /// Pins the scanner's expected result shape against today's Program.cs — 5 markers, + /// 4 braced blocks + the 1 braceless ConfigDb gate, so 5 scanned ranges (the data-plane + /// key-store branch moved into DataPlaneAuthRegistration for arch-review 03/S18, taking + /// one braced marker with it). Without this, a /// scanner bug that silently drops or merges a marker (as the pre-fix version did to the /// braceless marker — it fabricated a phantom range spanning an unrelated if-block and swallowed /// a real nested 'if (hasAdmin)' block inside it, yielding 5 ranges instead of 6) would not @@ -248,13 +250,13 @@ public sealed class MeshRegistrationRoleScopingTests { var source = ReadProgramSource(); - CountOccurrences(source, HasAdminMarker).ShouldBe(6, + CountOccurrences(source, HasAdminMarker).ShouldBe(5, "Program.cs's 'if (hasAdmin)' marker count changed. If a marker was added or removed, " + "update this pin — and check whether the new/removed occurrence is braced or braceless — " + "rather than letting the scanner silently mis-shape its result."); var ranges = FindHasAdminBlockRanges(source); - ranges.Count.ShouldBe(6, + ranges.Count.ShouldBe(5, "expected one scanned range per 'if (hasAdmin)' marker, including the braceless ConfigDb " + "gate. A mismatch means the scanner adopted an unrelated brace or otherwise merged/dropped " + "a marker instead of handling each one independently."); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs new file mode 100644 index 00000000..7aa19673 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs @@ -0,0 +1,77 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// Guards the role-conditional data-plane authentication wiring (arch-review 03/S18). A decorator +/// registered in one branch of a role conditional is the "built but never wired" shape, so the branch +/// each node role lands in is asserted rather than assumed. +/// +public sealed class ApiKeyStoreRegistrationTests +{ + /// + /// The authorizer's clock and the registered clock must be the SAME instance. + /// + /// + /// already documents that a divergence between two clocks denies + /// every request with a verdict indistinguishable from "no grant matched" — found once by a test + /// clock two hours ahead. One registered singleton threaded into every consumer makes the + /// divergence unrepresentable instead of merely unlikely, so the registration is what is pinned. + /// + [Fact] + public void The_authorizer_and_the_key_store_share_one_TimeProvider() + { + using var sp = BuildNode(hasAdmin: true, hasDriver: true); + + var registered = sp.GetRequiredService(); + var authorizer = sp.GetRequiredService(); + + registered.ShouldBeSameAs(Probe.Instance); + authorizer.Clock.ShouldBeSameAs(registered); + } + + /// Builds the data-plane authentication wiring for one node role combination. + /// Whether the node carries the admin role. + /// Whether the node carries the driver role. + /// The built provider; the caller disposes it. + internal static ServiceProvider BuildNode(bool hasAdmin, bool hasDriver) + { + var services = new ServiceCollection(); + services.AddLogging(b => b.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(Probe.Instance); + services.Configure(_ => { }); + services.Configure(_ => { }); + services.Configure(o => o.Roles = ["admin", "driver", "cluster-MAIN"]); + services.AddSingleton(EmptyDataPlaneAuthoritySource.Instance); + services.AddSingleton(); + services.AddSingleton(); + + if (hasAdmin) + { + services.AddDbContextFactory( + o => o.UseInMemoryDatabase($"apikey-reg-{Guid.NewGuid():N}")); + } + + services.AddOtOpcUaDataPlaneAuth(hasAdmin, hasDriver); + return services.BuildServiceProvider(); + } + + /// A clock distinguishable from by reference. + private sealed class Probe : TimeProvider + { + public static readonly Probe Instance = new(); + + /// + public override DateTimeOffset GetUtcNow() => new(2026, 8, 20, 12, 0, 0, TimeSpan.Zero); + } +} From daaeb854d62befa135f5c5cb4177b5843910cdb3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:36:52 -0400 Subject: [PATCH 04/45] fix(runtime): every spawn hands the child its desired subscriptions (archreview 03/S16) --- ...R3-03-restart-resubscribe-watchdog-plan.md | 17 +++++++++++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverHostActor.cs | 24 ++++++++++++------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index c74e839d..b008f6bb 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -504,3 +504,20 @@ Deviation from the plan's literal step text (cosmetic, recorded for the reviewer step 1 says `_desiredByDriver` *replaces* `_lastComposition` at `:347-349` while T3 step 2 says T3 deletes `_lastComposition`. Keeping T2 compilable means the two fields coexist for one commit; `_lastComposition` and its assignment are deleted in T3 as T3's own step text specifies. + +### T3 — `SpawnChild` always sends the desired set; `_lastComposition` deleted + +The falsification for this production slice is **T5**, written and observed red afterwards (below) — +T3 lands the fix first because the plan sequences it that way and the ordering guard needs the whole +25-suite regression before any new test is trusted on top of it. + +**HIGH-RISK regression gate (every child's spawn-time state changes, deploy path included):** the +FULL `Runtime.Tests` project, not just the `DriverHostActor` filter — +**639 passed, 3 skipped, 0 failed, 55 s**. The 3 skips are pre-existing (`DeploymentArtifact*` parity +tests, skipped on the base commit too). + +`_lastComposition` is gone: `grep -c _lastComposition DriverHostActor.cs` → **0**. Its xmldoc had also +drifted (it claimed the field was set at the end of `PushDesiredSubscriptions`; the assignment was in +`ApplyDesiredSubscriptions`) — deleted with the field rather than corrected. The stale +"dropped-driver fallback" comment on the old `SendAuthoredOnly` local function was rewritten as the +`SendDesiredSubscriptions` remarks naming the three real consumers. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 582bcf35..2a3500fc 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -17,7 +17,7 @@ { "id": "T3", "subject": "SpawnChild always sends the desired set; delete the write-only _lastComposition (03/S16)", - "status": "pending", + "status": "done", "blockedBy": [ "T2" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index ff212655..84bf70b6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -344,10 +344,6 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// the config (#516). Bounded by the number of driver children with a delta in flight. private readonly Dictionary _pendingDelta = new(); - /// The composition from the most-recent apply (set at the END of - /// ). Null until the first apply. - private AddressSpaceComposition? _lastComposition; - /// One driver child's desired subscription payload — exactly the /// the apply's bulk push sent it. /// The non-alarm raw tags' RawPaths this driver should keep subscribed. @@ -2721,11 +2717,6 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS _log.Info("DriverHost {Node}: applied {Count} Equipment ScriptedAlarm(s) to the ScriptedAlarm host", _localNode, composition.EquipmentScriptedAlarms.Count); } - - // Cache the applied composition LAST. Set here (not in HandleArtifactForApply) so both the fresh-apply and - // bootstrap-restore paths — which both route through this method — leave a current composition. - _lastComposition = composition; - } /// @@ -2827,6 +2818,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS _children[spec.DriverInstanceId] = new ChildEntry(child, spec, stub); + // EVERY spawn hands the child its desired set — deploy reconcile, operator Restart (03/S16), and the + // supervision reprovision. Sending it HERE rather than at each call site is the point: a child that + // exists without its subscription set is a dead driver, so the two facts are established together. + // Before this, the only sender was the apply's bulk push, so an operator's Restart produced a child + // that connected and then read nothing until the next deployment — the #488 periodic reconcile cannot + // help, because it early-returns on an empty desired set. + // + // ORDERING, deliberately: ApplyDriverSpecs (which spawns) runs BEFORE ApplyDesiredSubscriptions (which + // rebuilds _desiredByDriver), so a newly-spawned child is handed the PREVIOUS apply's set — or nothing, + // on the first deploy. That transient is superseded milliseconds later by the bulk push on the same + // sender→receiver pair (Akka guarantees per-pair ordering), and the child is still Connecting, where + // SetDesiredSubscriptions only STORES; the subscribe happens at Connected entry. So the stale set is + // never acted on. Sent after InitializeRequested above, which keeps init-before-subscribe ordering. + SendDesiredSubscriptions(child, spec.DriverInstanceId); + // v3 (WP7): if the (real, non-stub) driver consumes other tags' live values, spawn a mux-adapter // child that registers its dependency refs on the per-node dependency mux and forwards changes // into the driver. Requires a wired mux (dev/None deployments have none — then calc inputs simply From 3b56e5bd01a25f5822e4d94da6e5f133c3abc08c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:38:41 -0400 Subject: [PATCH 05/45] fix(security): bound the API-key store call under the SessionManager lock (archreview 03/S18) --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 15 ++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../OpcUa/BoundedApiKeyStore.cs | 115 +++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 244 ++++++++++++++++++ 4 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 0ecd9c86..48b81e82 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -603,3 +603,18 @@ for the orchestrator. - **Collateral:** `MeshRegistrationRoleScopingTests.HasAdminMarkerScan_MatchesTheKnownShapeOfProgramCs` pins the `if (hasAdmin)` marker count in `Program.cs`; the move took one braced marker with it, so the pin went 6 → 5 (the test's own message asks for exactly that when a marker moves). + +### T2 — `BoundedApiKeyStore` deadline + +- **RED observed:** `error CS0246: The type or namespace name 'BoundedApiKeyStore' could not be found` — + type absent, as predicted. +- **DEVIATION (API):** the plan's snippet calls `cts.CancelAfter(TimeSpan, TimeProvider)`. **No such + overload exists** (`error CS1501: No overload for method 'CancelAfter' takes 2 arguments`). The + clock-aware equivalent is the `CancellationTokenSource(TimeSpan, TimeProvider)` **constructor**, linked + with the caller's token — same behaviour, same injected clock, and the linked source still lets the + `when (!ct.IsCancellationRequested)` filter tell a deadline from a caller cancellation. +- Added beyond the plan's list: `A_zero_timeout_disables_the_deadline` (the documented rollback, and the + in-class control proving the *bound* is what denies rather than the fake) and + `Anonymous_grant_resolution_is_bounded_too` (moved forward from the plan's T3 list — the deadline half + reaches `FindGrantsAsync` in the same change, so it is asserted in the same change). +- 5 tests, green. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index 46fc793e..f7855020 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -11,7 +11,7 @@ { "id": "T2", "subject": "BoundedApiKeyStore: hard per-call deadline on the key store under the SessionManager lock, fail-closed (03/S18)", - "status": "pending", + "status": "done", "blockedBy": [ "T1" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs new file mode 100644 index 00000000..cd085541 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// Bounds what an can cost the OPC UA impersonation callback (arch-review +/// 03/S18). Wraps on a node that has a ConfigDb; the in-memory +/// needs neither half and is deliberately left unwrapped. +/// +/// +/// +/// Why a deadline is the fix and the cache is not. The SDK raises impersonation +/// synchronously under a SessionManager-wide lock, so a store that does not answer does not +/// stall one activation — it stalls every activation in the server. A cache lowers how OFTEN the +/// store is asked; only a deadline bounds how LONG one ask may hold that lock. +/// +/// +/// ⚠️ Every bounded outcome DENIES. A deadline exceeded, a faulted inner store and a +/// cancelled call all surface as an exception into +/// 's catch-all, which returns the same opaque denial as +/// a bad credential. Nothing here can admit a session the inner store did not authorize — the +/// decorator can only return what the inner store returned, or throw. +/// +/// +public sealed class BoundedApiKeyStore : IApiKeyStore +{ + private readonly IApiKeyStore _inner; + private readonly IOptions _options; + private readonly ILogger _log; + private readonly TimeProvider _time; + + /// Initializes a new instance of the class. + /// The store being bounded. + /// The Security:ApiKeys options carrying the deadline and TTL. + /// The logger. + /// + /// The shared clock. The same instance the authorizer reads — see + /// DataPlaneAuthRegistration for why a second clock here would be a hazard rather than a + /// detail. + /// + public BoundedApiKeyStore( + IApiKeyStore inner, + IOptions options, + ILogger log, + TimeProvider time) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(log); + ArgumentNullException.ThrowIfNull(time); + + _inner = inner; + _options = options; + _log = log; + _time = time; + } + + /// + public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) => + BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct); + + /// + public Task> FindGrantsAsync(string principal, CancellationToken ct) => + BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct); + + /// Runs one inner call under the configured deadline. + /// The inner call's result type. + /// The inner call, taking the deadline-linked token. + /// The caller's token — in production always None, see remarks. + /// Whatever the inner store returned. + /// + /// + /// The deadline is created HERE rather than received from the caller because there is no caller + /// to receive one from: OpcUaApplicationHost block-bridges the SDK's synchronous + /// impersonation callback with , so the ct the store + /// sees in production is always uncancellable. + /// + /// + /// ⚠️ It re-throws rather than returning . null means + /// "no such usable key", which the authenticator logs as denied — no enabled, unexpired key + /// with id X. A timeout is not that, and an operator reading that line during a ConfigDb + /// outage would go hunting for a key problem that does not exist. The distinct Warning below is + /// the honest signal; the denial itself is unchanged either way. + /// + /// + private async Task BoundAsync(Func> call, CancellationToken ct) + { + var timeoutMs = _options.Value.StoreTimeoutMs; + if (timeoutMs <= 0) return await call(ct).ConfigureAwait(false); + + // The deadline runs on the INJECTED clock (the TimeProvider ctor overload), not on a plain + // CancelAfter, so it is the same TimeProvider the authorizer and the revalidation timer read. + // Linked with the caller's token so a genuine caller cancellation is still honoured — and so the + // catch below can tell the two apart. + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(timeoutMs), _time); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct, deadline.Token); + + try + { + return await call(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + _log.LogWarning( + "ApiKeyStore: the key store did not answer within {TimeoutMs} ms — denying this activation. " + + "Every OPC UA session activation on this server is serialized behind this call; " + + "check ConfigDb health.", + timeoutMs); + throw; + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs new file mode 100644 index 00000000..f448aa4c --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -0,0 +1,244 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// Covers — the deadline and TTL cache the OPC UA impersonation +/// callback's key lookup runs under (arch-review 03/S18). +/// +/// +/// +/// The deadline cases are driven through the real +/// rather than the store directly, because the fail-closed conversion — a throw becoming the +/// standard opaque denial — is part of what is being asserted, and it lives in the authenticator's +/// catch-all rather than in the decorator. +/// +/// +/// ⚠️ Deadline cases use a REAL short timeout; TTL cases use the fake clock. The hand-rolled +/// overrides GetUtcNow() only, so CreateTimer falls through to +/// the base implementation and produces a real system timer — advancing the fake clock does not +/// fire a CancelAfter. A deadline test "improved" to advance the fake clock would pass for +/// the wrong reason: the inner task simply never completing. +/// +/// +public sealed class BoundedApiKeyStoreTests +{ + private const string Pepper = "unit-test-pepper"; + private const string Principal = "scadabridge-dcl"; + + /// + /// A store that never answers must not hold the activation open: the deadline fires and the + /// activation is denied. + /// + /// + /// The SDK raises impersonation under a SessionManager-wide lock, so a lookup that does not + /// return stalls every session activation in the server — not just this one. That is the whole of + /// 03/S18, and it is why the bound is a deadline rather than a cache. + /// + [Fact] + public async Task A_blocked_store_denies_without_hanging_the_activation() + { + var (presented, inner) = SetUp(); + inner.BlockForever = true; + var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0)); + + var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(5)); + + result.Success.ShouldBeFalse(); + result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage); + } + + /// + /// The positive control for : + /// the same fake, answering normally, still authenticates. + /// + /// + /// Without it a build whose decorator denied everything would pass every deadline case in + /// this class, and the bound would look correct while having locked the fleet out. + /// + [Fact] + public async Task A_reachable_store_still_authenticates() + { + var (presented, inner) = SetUp(); + var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0)); + + var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None); + + result.Success.ShouldBeTrue(); + result.Principal.ShouldBe(Principal); + } + + /// + /// A lookup that completes with a perfectly valid key after the deadline still denies — + /// nothing races back in past the bound. + /// + [Fact] + public async Task A_deadline_never_admits() + { + var (presented, inner) = SetUp(); + var gate = new TaskCompletionSource(); + inner.Gate = gate.Task; + var auth = Auth(Bounded(inner, timeoutMs: 100, cacheSeconds: 0)); + + var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None); + + // Let the deadline pass, THEN hand the inner store its (valid) answer. + await Task.Delay(400); + gate.SetResult(); + + var result = await pending.WaitAsync(TimeSpan.FromSeconds(5)); + result.Success.ShouldBeFalse(); + result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage); + } + + /// + /// A non-positive StoreTimeoutMs disables the deadline, restoring the unbounded behaviour + /// byte-for-byte — the documented rollback. + /// + [Fact] + public async Task A_zero_timeout_disables_the_deadline() + { + var (presented, inner) = SetUp(); + var gate = new TaskCompletionSource(); + inner.Gate = gate.Task; + var auth = Auth(Bounded(inner, timeoutMs: 0, cacheSeconds: 0)); + + var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None); + + // Far longer than any deadline this class configures; an active bound would have denied by now. + await Task.Delay(300); + pending.IsCompleted.ShouldBeFalse("a disabled deadline must not cancel a slow lookup"); + + gate.SetResult(); + (await pending.WaitAsync(TimeSpan.FromSeconds(5))).Success.ShouldBeTrue(); + } + + /// The anonymous activation path gets the same deadline, and the same denial. + /// + /// It runs on every unauthenticated connect, so leaving it unbounded would leave the + /// server-wide-stall hazard fully open on the busiest activation path. + /// + [Fact] + public async Task Anonymous_grant_resolution_is_bounded_too() + { + var (_, inner) = SetUp(); + inner.BlockForever = true; + var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0)); + + var result = await auth.AuthenticateAnonymousAsync(CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(5)); + + result.Success.ShouldBeFalse(); + } + + // ---------- helpers ---------- + + private static BoundedApiKeyStore Bounded( + IApiKeyStore inner, int timeoutMs, int cacheSeconds, TimeProvider? clock = null) => new( + inner, + Options.Create(new ApiKeyOptions + { + Username = "apikey", + Pepper = Pepper, + StoreTimeoutMs = timeoutMs, + StoreCacheSeconds = cacheSeconds, + }), + NullLogger.Instance, + clock ?? TimeProvider.System); + + private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new( + store, + Options.Create(new ApiKeyOptions { Username = "apikey", Pepper = Pepper }), + NullLogger.Instance); + + private static (string Presented, GatedApiKeyStore Store) SetUp() + { + var (presented, apiKeyId, hash) = ApiKeySecret.Generate(Pepper); + var key = new ApiKey + { + ApiKeyId = apiKeyId, + Name = "test key", + SecretHash = hash, + Principal = Principal, + Enabled = true, + CreatedUtc = DateTime.UtcNow, + }; + return (presented, new GatedApiKeyStore(key)); + } + + /// + /// An inner store whose answer can be withheld — a hang the EF in-memory provider cannot + /// represent — and whose calls are counted. + /// + private sealed class GatedApiKeyStore(ApiKey key) : IApiKeyStore + { + /// When set, every call awaits this before answering. + public Task? Gate { get; set; } + + /// When true, no call ever completes. + public bool BlockForever { get; set; } + + /// How many calls reached this store (i.e. were not served from the cache). + public int Calls => _calls; + + /// When true, the key lookup answers "no usable key" — a revocation. + public bool Revoked { get; set; } + + /// Number of leading calls that throw before the store starts answering. + public int ThrowsRemaining { get; set; } + + private int _calls; + + public async Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + await WaitAsync(ct).ConfigureAwait(false); + if (Revoked) return null; + return string.Equals(apiKeyId, key.ApiKeyId, StringComparison.Ordinal) + ? new ApiKeyLookup(key, []) + : null; + } + + public async Task> FindGrantsAsync(string principal, CancellationToken ct) + { + await WaitAsync(ct).ConfigureAwait(false); + return []; + } + + private async Task WaitAsync(CancellationToken ct) + { + Interlocked.Increment(ref _calls); + + if (ThrowsRemaining > 0) + { + ThrowsRemaining--; + throw new InvalidOperationException("store is down"); + } + + if (BlockForever) await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + if (Gate is { } gate) await gate.WaitAsync(ct).ConfigureAwait(false); + } + } + + /// + /// A hand-advanced clock. Overrides GetUtcNow() only — see the class remarks for why that + /// confines it to the TTL cases. + /// + private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _now = utcNow; + + /// + public override DateTimeOffset GetUtcNow() => _now; + + /// Moves the clock forward. + /// How far forward to move. + public void Advance(TimeSpan by) => _now = _now.Add(by); + } +} From 37288d4df509d4f4c9a8614fa664154e45ea4f6f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:40:56 -0400 Subject: [PATCH 06/45] feat(security): short TTL read-through cache in front of the ConfigDb key store --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 14 ++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../OpcUa/BoundedApiKeyStore.cs | 106 +++++++++++++- .../OpcUa/BoundedApiKeyStoreTests.cs | 135 ++++++++++++++++++ 4 files changed, 252 insertions(+), 5 deletions(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 48b81e82..7689c090 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -618,3 +618,17 @@ for the orchestrator. `Anonymous_grant_resolution_is_bounded_too` (moved forward from the plan's T3 list — the deadline half reaches `FindGrantsAsync` in the same change, so it is asserted in the same change). - 5 tests, green. + +### T3 — TTL read-through cache + the falsification control + +- **RED observed:** `error CS0117: 'BoundedApiKeyStore' does not contain a definition for + 'MaxCacheEntries'` — the cache and its ceiling absent. +- **DEVIATION (fail-closed refinement, 2 lines):** the plan keys `FindAsync`'s cache on `apiKeyId` alone. + Taken literally that lets the cache **resurrect a key that expired by the clock** for up to a TTL, which + is exactly the axis 03/S19 exists to close. A cached hit is therefore re-checked against the caller's + `utcNow` before it is served (`IsExpired`) — so the TTL bounds how stale an `Enabled` **revocation** may + be, and nothing else. `utcNow` is still not part of the key (it changes every call and would defeat the + cache entirely). +- The counter's `cache_hit` / `timeout` `Add` calls are deliberately NOT here — they are T4, and writing + them early would have made T4's red vacuous. +- 11 tests in the class, green. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index f7855020..cf158b9e 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -19,7 +19,7 @@ { "id": "T3", "subject": "BoundedApiKeyStore: short TTL read-through cache + falsification control (a revoked key is denied once the TTL has passed)", - "status": "pending", + "status": "done", "blockedBy": [ "T2" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index cd085541..63c03710 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; @@ -24,13 +25,34 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; /// a bad credential. Nothing here can admit a session the inner store did not authorize — the /// decorator can only return what the inner store returned, or throw. /// +/// +/// ⚠️ Faults and timeouts are NEVER cached. Caching a deny would be fail-closed and is +/// still wrong: it would extend a ConfigDb outage past the outage, denying valid keys for a +/// further TTL after the database came back. Only a COMPLETED lookup — a hit or an authoritative +/// miss — is stored. +/// /// public sealed class BoundedApiKeyStore : IApiKeyStore { + /// + /// Entries either cache may hold before it is cleared wholesale. + /// + /// + /// Wholesale rather than per-entry eviction, for the reason SessionAclDecisionCache already + /// documents: picking WHICH entry to drop needs knowledge the cache does not have, and a + /// repopulate is cheap and cannot be wrong. The ceiling exists so a key-id guessing flood is a + /// bounded memory event — negative results are cached deliberately, so that a flood of unknown ids + /// does not become a SQL flood. + /// + public const int MaxCacheEntries = 1024; + private readonly IApiKeyStore _inner; private readonly IOptions _options; private readonly ILogger _log; private readonly TimeProvider _time; + private readonly ConcurrentDictionary> _keys = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary>> _grants = + new(StringComparer.Ordinal); /// Initializes a new instance of the class. /// The store being bounded. @@ -59,12 +81,88 @@ public sealed class BoundedApiKeyStore : IApiKeyStore } /// - public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) => - BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct); + /// + /// Keyed by alone. is deliberately not part + /// of the key — it changes on every call, so including it would defeat the cache entirely — but a + /// cached hit IS re-checked against it before being served, so the TTL can never resurrect a key + /// that has expired by the clock in the meantime. The TTL therefore bounds how stale an + /// Enabled revocation may be, and nothing else. + /// + public async Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + var ttl = Ttl; + if (ttl > TimeSpan.Zero + && TryServe(_keys, apiKeyId, ttl, out var cached) + && !IsExpired(cached, utcNow)) + { + return cached; + } + + var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct) + .ConfigureAwait(false); + if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh); + return fresh; + } /// - public Task> FindGrantsAsync(string principal, CancellationToken ct) => - BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct); + /// Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect. + public async Task> FindGrantsAsync(string principal, CancellationToken ct) + { + var ttl = Ttl; + if (ttl > TimeSpan.Zero && TryServe(_grants, principal, ttl, out var cached)) return cached; + + var fresh = await BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct) + .ConfigureAwait(false); + if (ttl > TimeSpan.Zero) Store(_grants, principal, fresh); + return fresh; + } + + /// The configured TTL; non-positive means the cache is off. + private TimeSpan Ttl => TimeSpan.FromSeconds(_options.Value.StoreCacheSeconds); + + /// Whether a cached lookup has since expired by the clock. + /// The cached lookup, possibly null (an authoritative miss). + /// The caller's current time. + /// when the cached key must not be served. + private static bool IsExpired(ApiKeyLookup? cached, DateTime utcNow) => + cached?.Key.ExpiresUtc is { } expires && expires <= utcNow; + + /// Serves an entry if one is present and still inside the TTL. + /// The cached value type. + /// The cache to read. + /// The cache key. + /// How long an entry stays servable. + /// The cached value, when this returns true. + /// on a hit. + private bool TryServe( + ConcurrentDictionary> cache, string key, TimeSpan ttl, out T value) + { + if (cache.TryGetValue(key, out var entry) && _time.GetUtcNow() - entry.StampedUtc < ttl) + { + value = entry.Value; + return true; + } + + value = default!; + return false; + } + + /// Stores a COMPLETED lookup, clearing wholesale first if the ceiling has been reached. + /// The cached value type. + /// The cache to write. + /// The cache key. + /// The value to cache. + private void Store(ConcurrentDictionary> cache, string key, T value) + { + if (cache.Count >= MaxCacheEntries) cache.Clear(); + cache[key] = new Entry(_time.GetUtcNow(), value); + } + + /// One cached lookup and the instant it was taken. + /// The cached value type. + /// When the inner store answered. + /// What it answered. + private readonly record struct Entry(DateTimeOffset StampedUtc, T Value); /// Runs one inner call under the configured deadline. /// The inner call's result type. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index f448aa4c..80a7fa4c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -5,6 +5,7 @@ using Xunit; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Core.Authorization; using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; @@ -138,6 +139,137 @@ public sealed class BoundedApiKeyStoreTests result.Success.ShouldBeFalse(); } + // ---------- TTL read-through cache ---------- + + /// A second activation inside the TTL is served from the cache and never reaches the store. + [Fact] + public async Task A_second_activation_within_the_TTL_does_not_reach_the_store() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock); + + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + clock.Advance(TimeSpan.FromSeconds(2)); + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + + inner.Calls.ShouldBe(1); + } + + /// + /// FALSIFICATION CONTROL. A key revoked in the store is denied once the TTL has passed — the + /// cache expires rather than merely existing. + /// + /// + /// A cache with no expiry at all passes every other case in this class: the hit test, the + /// read-through test and the fault test are all satisfied by "store once, serve forever". This is + /// the one that makes StoreCacheSeconds a bound rather than a claim. + /// + [Fact] + public async Task A_revoked_key_is_denied_once_the_TTL_has_passed() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock); + + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)) + .ShouldNotBeNull("non-vacuity: the key really was usable before the revocation"); + + inner.Revoked = true; + + // Still inside the TTL — the documented, accepted staleness window. + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + + // Past it — the store is re-read and the revocation lands. + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.Calls.ShouldBe(2); + } + + /// A non-positive TTL disables the cache: every call reads through, today's behaviour. + [Fact] + public async Task TTL_zero_reads_through_every_time() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 0, clock); + + for (var i = 0; i < 3; i++) + await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default); + + inner.Calls.ShouldBe(3); + } + + /// + /// A fault is never cached: the next call reaches the store, so an outage cannot outlive itself. + /// + /// + /// Caching a deny would be fail-closed and is still wrong — it would go on denying valid keys for + /// a further TTL after the database came back. + /// + [Fact] + public async Task A_faulted_store_is_not_cached() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 60, clock); + inner.ThrowsRemaining = 1; + + await Should.ThrowAsync( + () => sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)); + + // Same instant — a cached fault would still be "fresh" here, so the clock cannot rescue this. + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.Calls.ShouldBe(2); + } + + /// The anonymous grant lookup is cached on the same terms, keyed by principal. + [Fact] + public async Task Anonymous_grants_are_cached_too() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock); + + await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default); + await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default); + inner.Calls.ShouldBe(1); + + clock.Advance(TimeSpan.FromSeconds(6)); + await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default); + inner.Calls.ShouldBe(2); + } + + /// + /// Past the entry ceiling the cache clears wholesale and keeps answering — a key-id guessing flood + /// is a bounded memory event, not an unbounded one. + /// + /// + /// Wholesale rather than per-entry eviction for the reason SessionAclDecisionCache already + /// documents: choosing WHICH entry to drop needs knowledge the cache does not have, and a + /// repopulate is cheap and cannot be wrong. + /// + [Fact] + public async Task Cache_overflow_clears_wholesale_and_keeps_serving() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock); + + // Seed the real key, then flood past the ceiling with ids that do not exist. + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + for (var i = 0; i < BoundedApiKeyStore.MaxCacheEntries + 1; i++) + await sut.FindAsync($"ghost-{i}", clock.GetUtcNow().UtcDateTime, default); + + var callsAfterFlood = inner.Calls; + + // Still correct after the clear — and the real key was evicted, so it is re-read rather than + // served from a cache the flood was supposed to have emptied. + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.Calls.ShouldBe(callsAfterFlood + 1); + } + // ---------- helpers ---------- private static BoundedApiKeyStore Bounded( @@ -179,6 +311,9 @@ public sealed class BoundedApiKeyStoreTests /// private sealed class GatedApiKeyStore(ApiKey key) : IApiKeyStore { + /// The id of the key this store holds. + public string KeyId => key.ApiKeyId; + /// When set, every call awaits this before answering. public Task? Gate { get; set; } From b3a71f6655bb8103ee6ba1da5497ca2880212bb8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:42:33 -0400 Subject: [PATCH 07/45] test(runtime): recording subscribable driver factory + enabled-driver v3 seed --- ...R3-03-restart-resubscribe-watchdog-plan.md | 16 ++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/StubDrivers.cs | 193 ++++++++++++++++++ .../Harness/V3ArtifactSeeder.cs | 32 ++- 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index b008f6bb..9d0de32d 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -521,3 +521,19 @@ drifted (it claimed the field was set at the end of `PushDesiredSubscriptions`; `ApplyDesiredSubscriptions`) — deleted with the field rather than corrected. The stale "dropped-driver fallback" comment on the old `SendAuthoredOnly` local function was rewritten as the `SendDesiredSubscriptions` remarks naming the three real consumers. + +### T4 — test harness (seeder `enabled` flag + recording driver factory) + +Test-only, additive, source-compatible: every pre-existing `SeedV3Deployment` / `SpawnHostAndApply` +call site compiles and behaves identically (the new parameters are trailing and default to today's +values). Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **179/179 passed** (the plan +named `--filter "…~V3"`, which matches nothing — the seeder's four consumers are all `DriverHostActor*` +classes; the wider filter is the same set plus more). + +**Deviation (recorded).** The plan lists one new seeder parameter (`enabled`); a second was needed and +added: `IReadOnlyDictionary? tagConfigByTag = null`. The seeder hard-codes +`TagConfig = "{}"` on every tag, and a raw tag's alarm is derived *only* from `TagConfig.alarm` via +`TagConfigIntent.Parse` (`AddressSpaceComposer.cs:769-772`) — so without it there is no way to seed the +alarm-bearing raw tag that `Operator_restart_re_subscribes_alarm_refs_too` (a test the plan itself +specifies) requires. It is trailing and optional, so it is the same source-compatible shape as +`enabled`. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 2a3500fc..42819372 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -25,7 +25,7 @@ { "id": "T4", "subject": "Test harness: V3ArtifactSeeder enabled flag + RecordingSubscribableDriver(Factory) with a crash seam", - "status": "pending", + "status": "done", "blockedBy": [] }, { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs index 25d459f4..30a51113 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs @@ -188,3 +188,196 @@ internal sealed class StubHandle : ISubscriptionHandle /// Gets the diagnostic ID of the subscription. public string DiagnosticId => "stub-sub"; } + +/// +/// A per-instance recording driver for the restart / re-subscribe suite (03/S16, U6): unlike +/// it carries the spec's +/// ( hard-codes a constant on a non-virtual member), and it counts and records the +/// alarm subscription alongside the value one — the two travel in a single +/// SetDesiredSubscriptions but through two different child code paths, so both need observing. +/// +/// +/// +/// The falsification bar these counters exist for: a restart test must assert a FUNCTION the new +/// incarnation performs — it subscribed, it re-initialised — never that the incarnation exists. +/// Existence is what +/// DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision +/// asserts, and that test passes against the very defect 03/S16 names. +/// +/// +/// is the supervision crash seam — see +/// DriverHostActorRestartTests's file remarks for why the obvious alternatives do not work. +/// +/// +internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAlarmSource +{ + private readonly Lock _gate = new(); + private readonly StubHandle _handle = new(); + private readonly RecordingAlarmHandle _alarmHandle = new(); + private int _initializeCount; + private int _shutdownCount; + private int _subscribeCount; + private int _subscribeAlarmsCount; + private IReadOnlyList? _lastSubscribedRefs; + private IReadOnlyList? _lastSubscribedAlarmRefs; + + /// Initializes a new recording driver stamped with the spec's identity. + /// The driver instance id from the spec that created it. + /// The driver type from the spec that created it. + public RecordingSubscribableDriver(string driverInstanceId, string driverType) + { + DriverInstanceId = driverInstanceId; + DriverType = driverType; + } + + /// + public string DriverInstanceId { get; } + + /// + public string DriverType { get; } + + /// Number of calls. A supervision restart re-uses the SAME driver + /// object (the Props closure captured it), so this going 1 to 2 is how the reprovision is observed. + public int InitializeCount => Volatile.Read(ref _initializeCount); + + /// Number of calls — a restart's PostStop raises it. + public int ShutdownCount => Volatile.Read(ref _shutdownCount); + + /// Number of calls. + public int SubscribeCount => Volatile.Read(ref _subscribeCount); + + /// Number of calls. + public int SubscribeAlarmsCount => Volatile.Read(ref _subscribeAlarmsCount); + + /// The refs passed to the most recent , or null if never called. + public IReadOnlyList? LastSubscribedRefs + { + get { lock (_gate) return _lastSubscribedRefs; } + } + + /// The refs passed to the most recent , or null if never called. + public IReadOnlyList? LastSubscribedAlarmRefs + { + get { lock (_gate) return _lastSubscribedAlarmRefs; } + } + + /// + public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _initializeCount); + return Task.CompletedTask; + } + + /// + public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task ShutdownAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _shutdownCount); + return Task.CompletedTask; + } + + /// + public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null); + + /// + public event EventHandler? OnDataChange; + + /// + public Task SubscribeAsync( + IReadOnlyList fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _subscribeCount); + lock (_gate) _lastSubscribedRefs = fullReferences.ToArray(); + return Task.FromResult(_handle); + } + + /// + public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public event EventHandler? OnAlarmEvent; + + /// + public Task SubscribeAlarmsAsync( + IReadOnlyList sourceNodeIds, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _subscribeAlarmsCount); + lock (_gate) _lastSubscribedAlarmRefs = sourceNodeIds.ToArray(); + return Task.FromResult(_alarmHandle); + } + + /// + public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task AcknowledgeAsync( + IReadOnlyList acknowledgements, CancellationToken cancellationToken) => Task.CompletedTask; + + /// Fires a well-formed data change (the ordinary production path). + /// The reference whose value changed. + /// The new value. + public void FireDataChange(string fullRef, object? value) => + OnDataChange?.Invoke(this, new DataChangeEventArgs( + _handle, fullRef, new DataValueSnapshot(value, 0u, DateTime.UtcNow, DateTime.UtcNow))); + + /// + /// Raises with a NULL Snapshot — the supervision crash seam. + /// DataChangeEventArgs is a positional record with no null validation, and the actor's + /// OnDataChangeForward dereferences msg.Snapshot unguarded ON THE ACTOR THREAD, so this + /// produces a genuine unhandled exception inside the child, which Akka's default decider turns into a + /// Restart. The fault itself is arbitrary; what is under test is the RECOVERY. + /// + /// The reference to name in the malformed event. + public void FireMalformedDataChange(string fullRef) => + OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!)); + + /// Raises ; present so the event is not merely written-to-never-read. + /// The condition id to raise. + public void FireAlarm(string conditionId) => + OnAlarmEvent?.Invoke(this, new AlarmEventArgs( + _alarmHandle, conditionId, conditionId, "AlarmCondition", "test", AlarmSeverity.High, DateTime.UtcNow)); + + private sealed class RecordingAlarmHandle : IAlarmSubscriptionHandle + { + /// + public string DiagnosticId => "recording-alarm-sub"; + } +} + +/// +/// An that hands out s and KEEPS +/// every one it created, in creation order. The list — not a single "the driver" reference — is the point: +/// an operator Restart tears the child down and spawns a new one around a NEW driver object, so the +/// assertion subject is Created[1], while a SUPERVISION restart re-uses Created[0] (the Props +/// closure captured it) and part of the assertion is that Created.Count stayed 1. +/// +internal sealed class RecordingSubscribableDriverFactory : IDriverFactory +{ + private readonly string _supportedType; + private readonly Lock _gate = new(); + private readonly List _created = new(); + + /// Initializes a factory that materialises exactly one driver type. + /// The driver type this factory answers for. + public RecordingSubscribableDriverFactory(string supportedType) => _supportedType = supportedType; + + /// Every driver this factory created, in creation order. + public IReadOnlyList Created + { + get { lock (_gate) return _created.ToArray(); } + } + + /// + public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) + { + if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null; + var driver = new RecordingSubscribableDriver(driverInstanceId, driverType); + lock (_gate) _created.Add(driver); + return driver; + } + + /// + public IReadOnlyCollection SupportedTypes => new[] { _supportedType }; +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs index 4ffc9a1d..4831fdd2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; @@ -36,16 +37,28 @@ public static class V3ArtifactSeeder /// The revision hash the deployment is sealed under. /// The raw tags to materialise, one per (driver, device, tag). /// UNS references projecting a raw tag (by BackingTag) into an equipment. + /// Whether the seeded DriverInstances are Enabled — i.e. whether applying this + /// artifact actually SPAWNS driver children. Defaults to false, the shape every pre-existing caller + /// relies on: the value-routing suites inject values straight into the host and want no children at all. + /// The restart/re-subscribe suite (03/S16) passes true, because its whole subject is what a spawned + /// child is handed. + /// Optional per-tag TagConfig JSON, keyed by tag name; tags not named + /// here keep the default "{}". The only way to seed an alarm-bearing raw tag (the alarm intent + /// is parsed out of TagConfig.alarm by TagConfigIntent), which the alarm-ref half of the + /// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried + /// in the alarm one instead, so the two travel through different child code paths. /// The seeded deployment's id. public static DeploymentId SeedV3Deployment( IDbContextFactory db, RevisionHash rev, (string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags, - (string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs) + (string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs, + bool enabled = false, + IReadOnlyDictionary? tagConfigByTag = null) { var drivers = rawTags .Select(t => (t.Driver, t.DriverName)) .Distinct() - .Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = false }) + .Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }) .ToArray(); var devices = rawTags .Select(t => (t.Driver, t.Device)) @@ -60,7 +73,7 @@ public static class V3ArtifactSeeder Name = t.Tag, DataType = t.DataType, AccessLevel = t.Writable ? 1 : 0, // TagAccessLevel.ReadWrite = 1 - TagConfig = "{}", + TagConfig = tagConfigByTag is not null && tagConfigByTag.TryGetValue(t.Tag, out var tc) ? tc : "{}", }).ToArray(); // UNS topology: one area/line per distinct (Area, Line); one equipment per (Area, Line, Equip). @@ -114,13 +127,17 @@ public static class V3ArtifactSeeder /// The host's local node id. /// The deployment to dispatch. /// The revision hash the deployment was sealed under. + /// Optional driver factory forwarded to DriverHostActor.Props. Null (the + /// default, preserving every existing call site) leaves the host on its own default, which materialises no + /// real drivers. Supply one — with enabled: true on the seed — to actually spawn driver children. /// The host actor plus the publish and mux probes wired into it. public static (IActorRef Actor, TestProbe Publish, TestProbe Mux) SpawnHostAndApply( TestKitBase kit, IDbContextFactory db, - NodeId localNode, DeploymentId deploymentId, RevisionHash rev) + NodeId localNode, DeploymentId deploymentId, RevisionHash rev, + IDriverFactory? driverFactory = null) { var publish = kit.CreateTestProbe(); - var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref); + var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory); Dispatch(actor, coordinator, deploymentId, rev); publish.ExpectMsg(RuntimeActorTestBase.PresenceBudget); @@ -140,14 +157,14 @@ public static class V3ArtifactSeeder TestKitBase kit, IDbContextFactory db, NodeId localNode, DeploymentId deploymentId, RevisionHash rev) { - var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null); + var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null); Dispatch(actor, coordinator, deploymentId, rev); return (actor, mux); } private static (IActorRef Actor, TestProbe Mux, TestProbe Coordinator) Spawn( TestKitBase kit, IDbContextFactory db, NodeId localNode, - IActorRef? opcUaPublishActor) + IActorRef? opcUaPublishActor, IDriverFactory? driverFactory) { var coordinator = kit.CreateTestProbe(); var mux = kit.CreateTestProbe(); @@ -155,6 +172,7 @@ public static class V3ArtifactSeeder var actor = kit.Sys.ActorOf(DriverHostActor.Props( db, localNode, coordinator.Ref, + driverFactory: driverFactory, localRoles: new HashSet { "driver" }, dependencyMux: mux.Ref, opcUaPublishActor: opcUaPublishActor, From 6dab72a47963d4eda72db56fc698e850712c0cdc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:42:49 -0400 Subject: [PATCH 08/45] refactor(host): extract MapOtOpcUaMeshGrpcServices (one authority for the five mesh maps) Behaviour-preserving: Program.cs:743-784 collapses to a single call, same gates, same order, every existing per-map rationale comment carried over. A new endpoint-metadata census (MeshGrpcAnonymousMetadataTests) drives the SAME production mapper across all three role shapes (admin-only / driver-only / fused), so the guard can never assert a shape production no longer has - archreview 03/S14, T1. --- .../Configuration/MeshGrpcEndpoints.cs | 97 +++++++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 48 +----- .../MeshGrpcAnonymousMetadataTests.cs | 150 ++++++++++++++++++ 3 files changed, 253 insertions(+), 42 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs new file mode 100644 index 00000000..c99c1194 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs @@ -0,0 +1,97 @@ +using ZB.MOM.WW.LocalDb.Replication; +using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc; +using ZB.MOM.WW.OtOpcUa.Host.Grpc; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Maps the mesh gRPC ingress endpoints (artifact serve, central ack, telemetry stream, node +/// command) and the LocalDb sync endpoint. +/// +/// +/// +/// Do not "tidy" this into Program.cs. A standing census guard +/// (MeshGrpcAnonymousMetadataTests) drives THIS method, so a map that moves back +/// inline leaves the guard asserting a shape production no longer has — which is exactly +/// how the AdminUI cookie FallbackPolicy silently blocked every mesh gRPC call on a +/// login-enabled admin node for seven phases of live gates (archreview 03/S14). +/// +/// +public static class MeshGrpcEndpoints +{ + /// + /// Maps the mesh gRPC ingress endpoints and the LocalDb sync endpoint, gated on the same + /// role + dedicated-port checks Program.cs used inline before this extraction. + /// + /// The built application to map endpoints on. + /// Whether this node carries the admin role. + /// Whether this node carries the driver role. + /// + /// The dedicated h2c port bound for the LocalDb passive sync endpoint, or 0 when it is + /// not bound. + /// + /// + /// The dedicated h2c port bound for the ConfigServe artifact + central command endpoints, or + /// 0 when it is not bound. + /// + /// + /// The dedicated h2c port bound for the telemetry stream + node command endpoints, or + /// 0 when it is not bound. + /// + public static void MapOtOpcUaMeshGrpcServices( + this WebApplication app, + bool hasAdmin, + bool hasDriver, + int syncListenPort, + int configServeGrpcPort, + int telemetryListenPort) + { + ArgumentNullException.ThrowIfNull(app); + + // Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a + // default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be + // safe even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey + // configured — but not mapping it at all is the stronger default. + if (hasDriver && syncListenPort > 0) + { + app.MapZbLocalDbSync(); + } + + // Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on + // admin-role nodes (which hold the central SQL connection) and only when a dedicated h2c + // port is bound. The ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes + // with no ConfigServe:ApiKey. + if (hasAdmin && configServeGrpcPort > 0) + { + app.MapGrpcService(); + + // Central command ingress (mesh gRPC command plane): the node→central deployment ack. It + // rides the SAME listener rather than claiming a port of its own — a second cleartext h2c + // port would buy nothing but another firewall rule, and both endpoints are admin-role and + // dialled by the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping + // it under Dps costs nothing (nothing dials it) whereas gating it would mean flipping the + // mode on a running fleet requires a central restart before any node can ack — the same + // reasoning that registers CentralCommunicationActor in both modes. + // CentralCommandAuthInterceptor (added at AddGrpc in Program.cs) is the ONLY inbound auth + // and fail-closes per cluster. + app.MapGrpcService(); + } + + // Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by + // TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey). + // + // The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME + // driver-role h2c listener rather than claiming a fourth port: central already dials this + // node here, both directions are per-cluster-keyed, and a separate port would be one more + // thing to open on every site firewall for no isolation gain. Mapping it is what makes + // MeshCommandAuthInterceptor live — it is registered in Program.cs and is a pass-through + // until a call matches this service's method prefix. It fail-closes with no + // MeshTransport:GrpcPsk, so an operator who maps the service without setting a key gets a + // closed endpoint, not an open one. + if (hasDriver && telemetryListenPort > 0) + { + app.MapGrpcService(); + app.MapGrpcService(); + } + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 12eeee6b..40ae23ab 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -740,48 +740,12 @@ if (hasAdmin) app.MapOtOpcUaDeployApi(app.Configuration); } -// Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a -// default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be safe -// even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey configured — but -// not mapping it at all is the stronger default. -if (hasDriver && syncListenPort > 0) -{ - app.MapZbLocalDbSync(); -} - -// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on admin-role -// nodes (which hold the central SQL connection) and only when a dedicated h2c port is bound. The -// ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes with no ConfigServe:ApiKey. -if (hasAdmin && configServeGrpcPort > 0) -{ - app.MapGrpcService(); - - // Central command ingress (mesh gRPC command plane): the node→central deployment ack. It - // rides the SAME listener rather than claiming a port of its own — a second cleartext h2c port - // would buy nothing but another firewall rule, and both endpoints are admin-role and dialled by - // the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping it under Dps - // costs nothing (nothing dials it) whereas gating it would mean flipping the mode on a running - // fleet requires a central restart before any node can ack — the same reasoning that registers - // CentralCommunicationActor in both modes. CentralCommandAuthInterceptor (added at AddGrpc - // above) is the ONLY inbound auth and fail-closes per cluster. - app.MapGrpcService(); -} - -// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by -// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey). -// -// The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME driver-role -// h2c listener rather than claiming a fourth port: central already dials this node here, both -// directions are per-cluster-keyed, and a separate port would be one more thing to open on every -// site firewall for no isolation gain. Mapping it is what makes MeshCommandAuthInterceptor live — -// it is registered above and is a pass-through until a call matches this service's method prefix. -// It fail-closes with no MeshTransport:GrpcPsk, so an operator who maps the service without setting -// a key gets a closed endpoint, not an open one. -if (hasDriver && telemetryListenPort > 0) -{ - app.MapGrpcService(); - app.MapGrpcService(); -} +// Mesh gRPC ingress: the LocalDb passive sync endpoint, the ConfigServe artifact + central +// command endpoints, and the telemetry-stream + node-command endpoints. Extracted to +// MeshGrpcEndpoints so a standing census test drives the SAME production mapping (see that +// file's remarks — archreview 03/S14). +app.MapOtOpcUaMeshGrpcServices( + hasAdmin, hasDriver, syncListenPort, configServeGrpcPort, telemetryListenPort); app.MapOtOpcUaHealth(); app.MapOtOpcUaMetrics(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs new file mode 100644 index 00000000..1c734779 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs @@ -0,0 +1,150 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// R3-01 (T1-T3) — an endpoint-metadata census over MapOtOpcUaMeshGrpcServices, driven in +/// all three role shapes a node can actually carry (admin-only, driver-only, fused). +/// +/// +/// +/// Why this drives the PRODUCTION mapper, not a hand-copied shape. The Critical this +/// plan fixes (archreview 03/S14) survived seven phases of live gates precisely because every +/// prior test built its own pipeline by hand — MeshGrpcBoundaryTests, +/// ConfigServeListenerTests and LocalDbSyncListenerTests all reproduce the +/// author's mental model of the mapping rather than the host's actual composition. This +/// census calls — the SAME method +/// Program.cs calls — so it is structurally impossible for the guard to assert a shape +/// production no longer has. +/// +/// +/// Filters out grpc-dotnet's own catch-all infrastructure. Every MapGrpcService +/// call also registers a per-service "Unimplemented method for X" fallback, and the FIRST +/// such call in an app additionally registers one shared "Unimplemented service" catch-all — +/// neither carries a route pattern beginning with /, and neither ever reaches real +/// logic (they always answer Unimplemented regardless of caller identity), so their +/// authorization metadata carries no security meaning. +/// keeps only the five real, callable RPC methods. +/// +/// +/// Guard-of-the-guard. Every assertion below pins an EXACT expected count and the +/// exact set of route patterns, never just "not empty" — a census that silently found zero +/// endpoints (a typo'd role gate, a helper that never got called) would pass a bare +/// non-empty check trivially. It cannot pass an exact-count-and-set check by accident. +/// +/// +public sealed class MeshGrpcAnonymousMetadataTests +{ + private const string ArtifactFetch = "/deployment_artifact.v1.DeploymentArtifactService/Fetch"; + private const string CentralReportApply = "/mesh_transport.v1.CentralCommandService/ReportApply"; + private const string TelemetrySubscribe = "/telemetry.v1.TelemetryStreamService/Subscribe"; + private const string NodeExecute = "/mesh_transport.v1.NodeCommandService/Execute"; + private const string LocalDbSyncSync = "/localdb_sync.v1.LocalDbSync/Sync"; + + /// The five known mesh gRPC method prefixes — the ONLY endpoints this map may anonymise. + private static readonly string[] AllFivePrefixes = + [LocalDbSyncSync, ArtifactFetch, CentralReportApply, TelemetrySubscribe, NodeExecute]; + + [Fact] + public void AdminOnly_MapsExactlyTheConfigServeEndpoints() + { + // Admin-only: no driver role, so neither LocalDbSync nor the telemetry/node-command pair are + // mapped even though a port is configured for them below (0 here, but the shape is the point). + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: true, hasDriver: false, + syncListenPort: 0, configServeGrpcPort: 4055, telemetryListenPort: 0)); + + endpoints.Count.ShouldBe(2); + endpoints.Select(RoutePatternOf).ShouldBe([ArtifactFetch, CentralReportApply], ignoreOrder: true); + } + + [Fact] + public void DriverOnly_WithNoSyncPort_MapsExactlyTheTelemetryAndNodeCommandEndpoints() + { + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: false, hasDriver: true, + syncListenPort: 0, configServeGrpcPort: 0, telemetryListenPort: 4056)); + + endpoints.Count.ShouldBe(2); + endpoints.Select(RoutePatternOf).ShouldBe([TelemetrySubscribe, NodeExecute], ignoreOrder: true); + } + + [Fact] + public void DriverOnly_WithASyncPort_AlsoMapsTheLocalDbSyncEndpoint() + { + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: false, hasDriver: true, + syncListenPort: 4054, configServeGrpcPort: 0, telemetryListenPort: 4056)); + + endpoints.Count.ShouldBe(3); + endpoints.Select(RoutePatternOf).ShouldBe( + [LocalDbSyncSync, TelemetrySubscribe, NodeExecute], ignoreOrder: true); + } + + [Fact] + public void Fused_WithAllThreePorts_MapsAllFiveEndpoints() + { + // The docker-dev central pair's shape (admin,driver,cluster-MAIN) once all three dedicated + // h2c ports are configured — the shape 03/S14 called out as breaking the deploy plane for + // the whole fleet, not only the four sites. + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: true, hasDriver: true, + syncListenPort: 4054, configServeGrpcPort: 4055, telemetryListenPort: 4056)); + + endpoints.Count.ShouldBe(5); + endpoints.Select(RoutePatternOf).ShouldBe(AllFivePrefixes, ignoreOrder: true); + } + + [Fact] + public void NeitherRoleNorAnyPort_MapsNothing() + { + // The default-OFF shape: every dedicated port is 0. Nothing about the mesh gRPC surface + // should exist on a node that configured none of it. + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: false, hasDriver: false, + syncListenPort: 0, configServeGrpcPort: 0, telemetryListenPort: 0)); + + endpoints.ShouldBeEmpty(); + } + + // --------------------------------------------------------------------------------------- + // harness + // --------------------------------------------------------------------------------------- + + private static WebApplication BuildApp( + bool hasAdmin, + bool hasDriver, + int syncListenPort, + int configServeGrpcPort, + int telemetryListenPort) + { + var builder = WebApplication.CreateBuilder(); + builder.Environment.ApplicationName = typeof(MeshGrpcAnonymousMetadataTests).Assembly.GetName().Name!; + builder.Services.AddGrpc(); + + var app = builder.Build(); + app.MapOtOpcUaMeshGrpcServices( + hasAdmin, hasDriver, syncListenPort, configServeGrpcPort, telemetryListenPort); + return app; + } + + /// + /// The real, callable RPC method endpoints — excludes grpc-dotnet's per-app "Unimplemented + /// service" catch-all and per-service "Unimplemented method for X" fallback (see class + /// remarks). Both carry a route pattern that never starts with /, which is what this + /// filters on. + /// + private static List RealMethodEndpoints(WebApplication app) => + ((IEndpointRouteBuilder)app).DataSources + .SelectMany(d => d.Endpoints) + .OfType() + .Where(e => e.RoutePattern.RawText?.StartsWith('/') == true) + .ToList(); + + private static string RoutePatternOf(RouteEndpoint e) => e.RoutePattern.RawText!; +} From ac9a71bcbde2fb7b75836fcba80bd9cee2d85ba0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:43:06 -0400 Subject: [PATCH 09/45] docs(plans): mark R3-01 T1 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 35776954..91648629 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -5,7 +5,7 @@ { "id": "T1", "subject": "Extract MapOtOpcUaMeshGrpcServices from Program.cs:743-784 (behaviour-preserving) + endpoint census scaffold (03/S14)", - "status": "pending", + "status": "done", "blockedBy": [] }, { From 6048257dfe3fb118f791b30abe106b273c7c0f83 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:43:07 -0400 Subject: [PATCH 10/45] feat(observability): meter bounded API-key store lookups --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 13 ++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../Observability/OtOpcUaTelemetry.cs | 14 +++ .../OpcUa/BoundedApiKeyStore.cs | 5 + .../OpcUa/BoundedApiKeyStoreTests.cs | 116 ++++++++++++++++++ 5 files changed, 149 insertions(+), 1 deletion(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 7689c090..90dbebeb 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -632,3 +632,16 @@ for the orchestrator. - The counter's `cache_hit` / `timeout` `Add` calls are deliberately NOT here — they are T4, and writing them early would have made T4's red vacuous. - 11 tests in the class, green. + +### T4 — `otopcua.opcua.apikey_store.bounded` counter + +- **RED observed (behavioural, not compile):** `A_timeout_meters_outcome_timeout` and + `A_cache_hit_meters_outcome_cache_hit` both `[FAIL]` with `should be` — the instrument existed nowhere, + so the listener saw nothing. `A_plain_read_through_meters_neither` passed vacuously at red, by design: + its job starts once the counter exists. +- **NOTE on the metrics-leak memory:** the recorder filters on **instrument name only**. That is normally + the leak (a parallel test's measurements land in it), and it is safe here for a checkable reason + recorded on the helper: `BoundedApiKeyStore` is the only writer of this instrument, only this class + constructs one, and xUnit never runs two tests of one class in parallel. Adding a second class that + exercises the decorator means adding a discriminating tag. +- 14 tests in the class, green. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index cf158b9e..f253cd64 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -27,7 +27,7 @@ { "id": "T4", "subject": "otopcua.opcua.apikey_store.bounded counter (outcome=timeout|cache_hit) + MeterListener assertions", - "status": "pending", + "status": "done", "blockedBy": [ "T3" ] diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs index 09950d24..ba3a0441 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs @@ -110,6 +110,20 @@ public static class OtOpcUaTelemetry Meter.CreateCounter("otopcua.redundancy.primary_gate_denied", unit: "{denial}", description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown)."); + /// + /// API-key store lookups the 03/S18 bound intervened on (outcome=timeout|cache_hit). + /// + /// + /// One counter with an outcome tag rather than two: timeout is the alarm signal that + /// the ConfigDb is not answering the OPC UA impersonation callback, and cache_hit is the + /// ratio that says whether the TTL is doing anything. A missing cache_hit stream on a busy + /// fused central is how you notice Security:ApiKeys:StoreCacheSeconds was set to 0 by + /// accident. A plain read-through meters neither — the counter names interventions, not traffic. + /// + public static readonly Counter ApiKeyStoreBounded = + Meter.CreateCounter("otopcua.opcua.apikey_store.bounded", unit: "{lookup}", + description: "API-key store lookups the bound intervened on (outcome=timeout|cache_hit)."); + // ---------------- Convenience helpers ---------------- /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index 63c03710..21e112d5 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -139,6 +140,8 @@ public sealed class BoundedApiKeyStore : IApiKeyStore { if (cache.TryGetValue(key, out var entry) && _time.GetUtcNow() - entry.StampedUtc < ttl) { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add( + 1, new KeyValuePair("outcome", "cache_hit")); value = entry.Value; return true; } @@ -202,6 +205,8 @@ public sealed class BoundedApiKeyStore : IApiKeyStore } catch (OperationCanceledException) when (!ct.IsCancellationRequested) { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add( + 1, new KeyValuePair("outcome", "timeout")); _log.LogWarning( "ApiKeyStore: the key store did not answer within {TimeoutMs} ms — denying this activation. " + "Every OPC UA session activation on this server is serialized behind this call; " diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index 80a7fa4c..74210583 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -1,7 +1,9 @@ +using System.Diagnostics.Metrics; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Shouldly; using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Core.Authorization; using ZB.MOM.WW.OtOpcUa.Host.OpcUa; @@ -270,6 +272,61 @@ public sealed class BoundedApiKeyStoreTests inner.Calls.ShouldBe(callsAfterFlood + 1); } + // ---------- metering ---------- + + /// A deadline that fires meters outcome=timeout — the alarm signal for a sick ConfigDb. + [Fact] + public async Task A_timeout_meters_outcome_timeout() + { + var (_, inner) = SetUp(); + inner.BlockForever = true; + using var meter = new MeterRecorder(); + var sut = Bounded(inner, timeoutMs: 100, cacheSeconds: 0); + + await Should.ThrowAsync( + () => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default).WaitAsync(TimeSpan.FromSeconds(5))); + + meter.WithOutcome("timeout").ShouldBe(1); + meter.WithOutcome("cache_hit").ShouldBe(0); + } + + /// + /// A cache hit meters outcome=cache_hit — the ratio that tells an operator the TTL is + /// doing something. A missing stream on a busy fused central is how you notice + /// StoreCacheSeconds was set to 0 by accident. + /// + [Fact] + public async Task A_cache_hit_meters_outcome_cache_hit() + { + var (_, inner) = SetUp(); + var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); + using var meter = new MeterRecorder(); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock); + + await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default); + meter.WithOutcome("cache_hit").ShouldBe(0, "the first call is a miss"); + + await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default); + meter.WithOutcome("cache_hit").ShouldBe(1); + } + + /// A plain read-through — no deadline hit, no cache hit — meters nothing at all. + /// + /// The counter names lookups the bound INTERVENED on. Metering every lookup would bury the two + /// outcomes that matter under the ordinary traffic they have to be spotted in. + /// + [Fact] + public async Task A_plain_read_through_meters_neither() + { + var (_, inner) = SetUp(); + using var meter = new MeterRecorder(); + var sut = Bounded(inner, timeoutMs: 2000, cacheSeconds: 0); + + await sut.FindAsync(inner.KeyId, DateTime.UtcNow, default); + + meter.Total.ShouldBe(0); + } + // ---------- helpers ---------- private static BoundedApiKeyStore Bounded( @@ -361,6 +418,65 @@ public sealed class BoundedApiKeyStoreTests } } + /// Collects otopcua.opcua.apikey_store.bounded measurements by their outcome tag. + /// + /// ⚠️ Filtered by instrument name only, which is normally the metrics-test leak (a listener sees + /// a PARALLEL test's measurements too). It is safe HERE for a checkable reason: + /// is the only thing in the product that writes this instrument, + /// and this class is the only one in the assembly that constructs one — and xUnit never runs two + /// tests of the same class in parallel. Adding a class that exercises the decorator means adding + /// a discriminating tag here. + /// + private sealed class MeterRecorder : IDisposable + { + private readonly MeterListener _listener; + private readonly List[]> _tagSets = []; + private readonly Lock _gate = new(); + private long _total; + + public MeterRecorder() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName + && instrument.Name == "otopcua.opcua.apikey_store.bounded") + { + listener.EnableMeasurementEvents(instrument); + } + }, + }; + _listener.SetMeasurementEventCallback((_, value, tags, _) => + { + lock (_gate) + { + _total += value; + _tagSets.Add(tags.ToArray()); + } + }); + _listener.Start(); + } + + /// Every measurement seen, regardless of tags. + public long Total + { + get { lock (_gate) return _total; } + } + + /// How many measurements carried outcome=. + /// The outcome tag value to count. + /// The measurement count. + public int WithOutcome(string outcome) + { + lock (_gate) + return _tagSets.Count(set => set.Any(t => t.Key == "outcome" && Equals(t.Value, outcome))); + } + + /// + public void Dispose() => _listener.Dispose(); + } + /// /// A hand-advanced clock. Overrides GetUtcNow() only — see the class remarks for why that /// confines it to the TTL cases. From 39a10a277a929fc5c8ad3fccc9e9c1b139081625 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:45:08 -0400 Subject: [PATCH 11/45] fix(security): wrap the ConfigDb key store on admin nodes + registration guard --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 12 +++++++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../DataPlaneAuthRegistration.cs | 17 +++++++-- .../OpcUa/ApiKeyOpcUaUserAuthenticator.cs | 15 ++++++-- .../OpcUa/BoundedApiKeyStore.cs | 12 +++++++ .../OpcUa/ConfigDbApiKeyStore.cs | 15 ++++++-- .../OpcUa/ApiKeyStoreRegistrationTests.cs | 36 +++++++++++++++++++ 7 files changed, 100 insertions(+), 9 deletions(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 90dbebeb..1220328e 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -645,3 +645,15 @@ for the orchestrator. constructs one, and xUnit never runs two tests of one class in parallel. Adding a second class that exercises the decorator means adding a discriminating tag. - 14 tests in the class, green. + +### T5 — wire the decorator + registration guard + +- **RED observed:** `error CS1061: 'BoundedApiKeyStore' does not contain a definition for 'Inner'` / + `… for 'Clock'` — the guard could not see inside the wrapper because nothing was wrapping yet. +- **DEVIATION (small):** `BoundedApiKeyStore` gained public `Inner` and `Clock` properties, for the same + reason `DataPlaneAuthorizer.Clock` did in T1 — a guard that cannot see what is inside the wrapper cannot + tell a correctly-wired node from one wrapping the wrong store. +- Doc-drift half of the finding landed here: the authenticator's "no network I/O on this path at all" + remark is now qualified by node role and points at the bound, and `ConfigDbApiKeyStore` records both + that it is wrapped in production and that "revocation on the next request" is **activation**-scoped. +- Host.Tests: 123 green (was 106 at base). diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index f253cd64..1928adba 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -35,7 +35,7 @@ { "id": "T5", "subject": "Wire BoundedApiKeyStore on admin/fused nodes in Program.cs + registration guard test; correct the authenticator's 'no network I/O at all' remark", - "status": "pending", + "status": "done", "blockedBy": [ "T4" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs index cb04f265..19d5385d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -67,9 +67,20 @@ public static class DataPlaneAuthRegistration // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services // rule is what keeps the redundancy-singleton registration in Program.cs from recursing into // ActorSystem construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter. - services.AddSingleton(sp => new ConfigDbApiKeyStore( - sp.GetRequiredService>(), - ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService>().Value))); + // + // arch-review 03/S18 — and it is WRAPPED. The lookup runs inline in the SDK's synchronous + // impersonation callback, under a SessionManager-wide lock, with CancellationToken.None: an + // unanswered ConfigDb query therefore stalls every session activation in the server. The + // decorator adds the deadline that bounds that, plus a short TTL that turns a burst of + // activations into one round trip. It cannot admit anything the inner store did not. + services.AddSingleton(sp => new BoundedApiKeyStore( + new ConfigDbApiKeyStore( + sp.GetRequiredService>(), + ConfigDbApiKeyStore.ResolveClusterId( + sp.GetRequiredService>().Value)), + sp.GetRequiredService>(), + sp.GetRequiredService().CreateLogger(), + sp.GetRequiredService())); services.AddSingleton(); // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyOpcUaUserAuthenticator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyOpcUaUserAuthenticator.cs index dc0d0f85..b4e67ee7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyOpcUaUserAuthenticator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyOpcUaUserAuthenticator.cs @@ -18,9 +18,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; /// This runs inline on session activation. The SDK raises ImpersonateUser /// synchronously while holding a SessionManager-wide event lock, so a slow authenticator does not /// stall one request — it serializes every session activation server-wide. That is why -/// verification is a single indexed lookup plus one HMAC, and why there is no directory call, -/// no retry, and no network I/O on this path at all. It is the main practical advantage of keys -/// over the LDAP bind this replaces. +/// verification is a single indexed lookup plus one HMAC, with no directory call and no retry — +/// the main practical advantage of keys over the LDAP bind this replaces. +/// +/// +/// ⚠️ "No network I/O" is TRUE ONLY ON A DRIVER-ONLY NODE (arch-review 03/S18). The claim +/// describes , which is two dictionary probes on an in-memory +/// snapshot. On an admin or fused node is +/// , and a UserName activation costs two EF round trips +/// (the key row, then the principal's NodeAcl rows) — inside this same lock. What bounds +/// that is , which wraps it in production; the deadline it applies +/// surfaces here as an ordinary throw and is denied by the catch-all below, like any other store +/// failure. /// /// /// Fail closed, and say little. Every rejection returns the same opaque message. The diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index 21e112d5..cbf4cf76 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -81,6 +81,18 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _time = time; } + /// The store being bounded. + /// + /// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch + /// of a role conditional is the "built but never wired" shape, and a guard that cannot see what is + /// inside the wrapper cannot tell a correctly-wired node from one wrapping the wrong store. + /// + public IApiKeyStore Inner => _inner; + + /// The clock this bound's deadline and TTL are measured on. + /// Same reason as : the shared-clock invariant is only pinnable by reference. + public TimeProvider Clock => _time; + /// /// /// Keyed by alone. is deliberately not part diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs index 09178426..e3cdc004 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs @@ -14,8 +14,19 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; /// A driver-only node has no ConfigDb since per-cluster mesh Phase 4, so this store cannot be /// registered there; that node uses instead (#520 stage 4). /// -/// Reading per request is what makes revocation immediate here, which is why this store is -/// preferred wherever a ConfigDb exists — including on a fused admin+driver node. +/// Reading per activation is what makes revocation prompt here, which is why this store is +/// preferred wherever a ConfigDb exists — including on a fused admin+driver node. ⚠️ It reaches a +/// new session activation only: an already-open session re-checks its key against the +/// in-memory artifact snapshot on every operation, not against this store, so clearing +/// Enabled here does not stop an open session until the next deployment. See +/// docs/security.md. +/// +/// +/// ⚠️ In production this store is WRAPPED by (arch-review +/// 03/S18), because these queries run inline in the SDK's impersonation callback under a +/// SessionManager-wide lock. A call may therefore be cancelled by the wrapper's deadline even +/// though the caller passed — the resulting +/// is a denial, not a bug. /// /// /// ⚠️ The ConfigDb holds the WHOLE fleet's ACL rows, not just this node's cluster's. That is diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs index 7aa19673..1548c3ea 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs @@ -7,6 +7,7 @@ using Xunit; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; using ZB.MOM.WW.OtOpcUa.Host.Configuration; using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; @@ -40,6 +41,41 @@ public sealed class ApiKeyStoreRegistrationTests authorizer.Clock.ShouldBeSameAs(registered); } + /// + /// An admin (or fused admin+driver) node's key store is the + /// decorator, wrapping the ConfigDb store. + /// + /// + /// The decorator lives in ONE branch of a role conditional, which is the "built but never wired" + /// shape OVERALL theme 3 names. Unwired, every unit test in BoundedApiKeyStoreTests still + /// passes and the deadline protects nothing. + /// + [Fact] + public void An_admin_node_wraps_the_ConfigDb_store_in_the_bound() + { + using var sp = BuildNode(hasAdmin: true, hasDriver: true); + + var store = sp.GetRequiredService().ShouldBeOfType(); + store.Inner.ShouldBeOfType(); + store.Clock.ShouldBeSameAs(sp.GetRequiredService()); + } + + /// + /// A driver-only node's key store is the bare artifact store — deliberately NOT wrapped. + /// + /// + /// It is two dictionary probes on an in-memory snapshot, so it can neither hang nor be worth + /// caching; wrapping it would add a deadline to a call that cannot miss one and a second copy of + /// data the snapshot already holds. + /// + [Fact] + public void A_driver_only_node_gets_the_bare_artifact_store() + { + using var sp = BuildNode(hasAdmin: false, hasDriver: true); + + sp.GetRequiredService().ShouldBeOfType(); + } + /// Builds the data-plane authentication wiring for one node role combination. /// Whether the node carries the admin role. /// Whether the node carries the driver role. From b05ebd635353be1d3b2880170e70c6f65ed081fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:46:30 -0400 Subject: [PATCH 12/45] fix(host): mesh gRPC endpoints opt out of the AdminUI cookie FallbackPolicy (archreview 03/S14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .AllowAnonymous() on DeploymentArtifactGrpcService, CentralCommandGrpcService, TelemetryStreamGrpcService and NodeCommandGrpcService — the four MapGrpcService maps that are chainable. On a login-enabled admin node the global cookie FallbackPolicy (RequireAuthenticatedUser) applies to every endpoint without explicit authorization metadata, gRPC included; the PSK bearer interceptors run INSIDE the endpoint invocation, strictly after that middleware, so a correctly-keyed node call was challenged before the interceptor's verdict could matter. Safe because all four interceptors fail closed on an unset key. The census now asserts IAllowAnonymous on all four (red before this change, green after) plus a scope guard proving the opt-out never leaks onto an unrelated endpoint. MapZbLocalDbSync (T3) is not chainable the same way and is asserted absent here on purpose, so this test cannot start passing for the wrong reason once T3 lands. --- .../Configuration/MeshGrpcEndpoints.cs | 33 ++++++-- .../MeshGrpcAnonymousMetadataTests.cs | 76 +++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs index c99c1194..c12e77f3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs @@ -10,11 +10,23 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// /// /// +/// Why AllowAnonymous (archreview 03/S14). On an admin-role node +/// AddOtOpcUaAuth installs a global FallbackPolicy = RequireAuthenticatedUser +/// under the cookie scheme, which the authorization middleware applies to EVERY endpoint +/// without explicit authorization metadata — gRPC included. These endpoints authenticate +/// with a per-cluster PSK bearer inside a gRPC interceptor, which runs INSIDE the endpoint +/// invocation, i.e. after that middleware. Without this opt-out a correctly-keyed node call +/// is challenged by the cookie handler before the interceptor's verdict matters: artifact +/// fetches fail, deploy acks are dropped, deployments seal TimedOut. Anonymous is safe here +/// because all five interceptors fail CLOSED — an unset key rejects every call and the +/// presented token is compared with FixedTimeEquals. +/// +/// /// Do not "tidy" this into Program.cs. A standing census guard /// (MeshGrpcAnonymousMetadataTests) drives THIS method, so a map that moves back /// inline leaves the guard asserting a shape production no longer has — which is exactly /// how the AdminUI cookie FallbackPolicy silently blocked every mesh gRPC call on a -/// login-enabled admin node for seven phases of live gates (archreview 03/S14). +/// login-enabled admin node for seven phases of live gates. /// /// public static class MeshGrpcEndpoints @@ -63,7 +75,11 @@ public static class MeshGrpcEndpoints // with no ConfigServe:ApiKey. if (hasAdmin && configServeGrpcPort > 0) { - app.MapGrpcService(); + // AllowAnonymous (03/S14): this is every FetchAndCache node's artifact fetch. Without the + // opt-out the AdminUI cookie FallbackPolicy 401s/redirects it before + // ConfigServeAuthInterceptor's verdict matters — a fleet-wide deploy-plane outage on any + // login-enabled admin node. + app.MapGrpcService().AllowAnonymous(); // Central command ingress (mesh gRPC command plane): the node→central deployment ack. It // rides the SAME listener rather than claiming a port of its own — a second cleartext h2c @@ -73,8 +89,9 @@ public static class MeshGrpcEndpoints // mode on a running fleet requires a central restart before any node can ack — the same // reasoning that registers CentralCommunicationActor in both modes. // CentralCommandAuthInterceptor (added at AddGrpc in Program.cs) is the ONLY inbound auth - // and fail-closes per cluster. - app.MapGrpcService(); + // and fail-closes per cluster. AllowAnonymous (03/S14): this is every node's deploy ack, + // from every cluster — same reasoning as the artifact fetch above. + app.MapGrpcService().AllowAnonymous(); } // Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by @@ -90,8 +107,12 @@ public static class MeshGrpcEndpoints // closed endpoint, not an open one. if (hasDriver && telemetryListenPort > 0) { - app.MapGrpcService(); - app.MapGrpcService(); + // AllowAnonymous (03/S14) on both: on a fused admin+driver node (the docker-dev central + // pair, and any production fused central) these sit behind the SAME cookie FallbackPolicy + // as the ConfigServe endpoints above and would be 401/redirected identically without the + // opt-out — central's own telemetry dialer and node-command dispatch to its own pair. + app.MapGrpcService().AllowAnonymous(); + app.MapGrpcService().AllowAnonymous(); } } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs index 1c734779..bdaf71d7 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; @@ -112,6 +113,74 @@ public sealed class MeshGrpcAnonymousMetadataTests endpoints.ShouldBeEmpty(); } + // --------------------------------------------------------------------------------------- + // 03/S14 — every mesh endpoint carries IAllowAnonymous, so the AdminUI cookie FallbackPolicy + // (RequireAuthenticatedUser) never reaches it; the PSK interceptors are the ONLY gate. T3 + // (LocalDbSync's empty-prefix group) is asserted separately below. + // --------------------------------------------------------------------------------------- + + [Fact] + public void AdminOnly_BothConfigServeEndpoints_CarryAllowAnonymous() + { + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: true, hasDriver: false, + syncListenPort: 0, configServeGrpcPort: 4055, telemetryListenPort: 0)); + + AnonymousPatternsOf(endpoints).ShouldBe([ArtifactFetch, CentralReportApply], ignoreOrder: true); + } + + [Fact] + public void DriverOnly_BothTelemetryAndNodeCommandEndpoints_CarryAllowAnonymous() + { + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: false, hasDriver: true, + syncListenPort: 0, configServeGrpcPort: 0, telemetryListenPort: 4056)); + + AnonymousPatternsOf(endpoints).ShouldBe([TelemetrySubscribe, NodeExecute], ignoreOrder: true); + } + + [Fact] + public void Fused_AllFourGrpcServiceMapsCarryAllowAnonymous_LocalDbSyncIsNotYetOwnedHere() + { + // T3 owns the LocalDbSync map (MapZbLocalDbSync returns IEndpointRouteBuilder, not + // IEndpointConventionBuilder, so .AllowAnonymous() cannot be chained onto it the same way). + // Asserting its ABSENCE here — rather than skipping it — is what makes this test go red the + // moment T3's fix lands with the wrong scope (e.g. anonymising nothing, or the group leaking + // beyond LocalDbSync) instead of silently starting to pass for the wrong reason. + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: true, hasDriver: true, + syncListenPort: 4054, configServeGrpcPort: 4055, telemetryListenPort: 4056)); + + AnonymousPatternsOf(endpoints).ShouldBe( + [ArtifactFetch, CentralReportApply, TelemetrySubscribe, NodeExecute], ignoreOrder: true); + } + + [Fact] + public void AllowAnonymous_NeverLeaksBeyondTheFiveKnownMeshEndpoints() + { + // Scope guard (theme 2): an endpoint mapped ALONGSIDE the mesh services, through no seam this + // extension touches, must never come out anonymous. If it did, some future edit widened the + // AllowAnonymous convention beyond the five known prefixes — the exact failure class the + // fallback-policy opt-out exists to bound. + var builder = WebApplication.CreateBuilder(); + builder.Environment.ApplicationName = typeof(MeshGrpcAnonymousMetadataTests).Assembly.GetName().Name!; + builder.Services.AddGrpc(); + var app = builder.Build(); + + app.MapOtOpcUaMeshGrpcServices( + hasAdmin: true, hasDriver: true, + syncListenPort: 4054, configServeGrpcPort: 4055, telemetryListenPort: 4056); + var sentinel = app.MapGet("/sentinel", () => "ok"); + + var sentinelEndpoint = ((IEndpointRouteBuilder)app).DataSources + .SelectMany(d => d.Endpoints) + .OfType() + .Single(e => e.RoutePattern.RawText == "/sentinel"); + + sentinelEndpoint.Metadata.GetMetadata().ShouldBeNull(); + _ = sentinel; // the builder handle itself is not the assertion target + } + // --------------------------------------------------------------------------------------- // harness // --------------------------------------------------------------------------------------- @@ -147,4 +216,11 @@ public sealed class MeshGrpcAnonymousMetadataTests .ToList(); private static string RoutePatternOf(RouteEndpoint e) => e.RoutePattern.RawText!; + + /// Route patterns of the endpoints in that carry . + private static List AnonymousPatternsOf(IEnumerable endpoints) => + endpoints + .Where(e => e.Metadata.GetMetadata() is not null) + .Select(RoutePatternOf) + .ToList(); } From 2f90156408c3f32c8831a8f0067e20478ca0f81c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:46:40 -0400 Subject: [PATCH 13/45] docs(plans): mark R3-01 T2 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 91648629..7e7d1e5d 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -11,7 +11,7 @@ { "id": "T2", "subject": "Census asserts IAllowAnonymous + scope/count guard-of-the-guard; .AllowAnonymous() on the four MapGrpcService maps", - "status": "pending", + "status": "done", "blockedBy": [ "T1" ] From bd80ff96ca70ce7fa75b019be4ee3a9f40e9fb27 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:46:56 -0400 Subject: [PATCH 14/45] =?UTF-8?q?feat(security):=20AclRevalidationTimer=20?= =?UTF-8?q?=E2=80=94=20clock-driven=20monitored-item=20revalidation=20(arc?= =?UTF-8?q?hreview=2003/S19)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 12 ++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../OpcUa/AclRevalidationTimer.cs | 99 ++++++++++++ .../OpcUa/AclRevalidationTimerTests.cs | 151 ++++++++++++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 8 +- 5 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/AclRevalidationTimer.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/AclRevalidationTimerTests.cs diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 1220328e..d6f82684 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -657,3 +657,15 @@ for the orchestrator. remark is now qualified by node role and points at the bound, and `ConfigDbApiKeyStore` records both that it is wrapped in production and that "revocation on the next request" is **activation**-scoped. - Host.Tests: 123 green (was 106 at base). + +### T6 — `AclRevalidationTimer` + +- **RED observed:** `error CS0246: The type or namespace name 'AclRevalidationTimer' could not be found`. +- **DEVIATION (visibility):** the plan declares the class `internal sealed`. `ZB.MOM.WW.OtOpcUa.Host` has + **no `InternalsVisibleTo`**, so the tests could not reach it. Made `public sealed`, matching + `BoundedApiKeyStore` and every other type in `Host/OpcUa/`. +- Added beyond the plan's list: `A_tick_runs_the_pass` — the control for the two "nothing happened" cases + (a `Tick()` that silently did nothing would satisfy the throwing-pass and dispose cases on its own). +- 5 tests, green. Also corrected the T4 `MeterRecorder` remark: the assembly declares + `CollectionBehavior(DisableTestParallelization = true)`, which is a stronger and simpler reason the + name-only filter is safe than the one first written. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index 1928adba..db16b8fb 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -43,7 +43,7 @@ { "id": "T6", "subject": "AclRevalidationTimer (pure, TimeProvider-driven, non-throwing tick) + unit tests (03/S19)", - "status": "pending", + "status": "done", "blockedBy": [ "T1" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/AclRevalidationTimer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/AclRevalidationTimer.cs new file mode 100644 index 00000000..233c9ddb --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/AclRevalidationTimer.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// Runs the node manager's monitored-item revalidation pass on a low-frequency tick, so a key that +/// expires by the CLOCK reaches a live subscription without waiting for a deployment (arch-review +/// 03/S19). +/// +/// +/// +/// Every request-shaped operation re-checks the calling key's usability per call and so honours +/// ExpiresUtc the moment it passes. A subscription is authorized once and then streams, and +/// streaming is how machine consumers actually read this server — so before this the only thing +/// that could stand a subscription down was a deployment. +/// +/// +/// ⚠️ The tick must never throw. An unhandled exception on a timer callback thread takes +/// the process down. Every tick is wrapped; a faulting pass is logged and the timer keeps running, +/// because the next tick is the recovery. +/// +/// +/// ⚠️ Each tick gets a fresh memo, and that is the pass's own constraint, not this class's. +/// RevalidateMonitoredItems allocates its verdict dictionary per call — see the remarks on +/// OtOpcUaNodeManager.OnAuthorityChanged. Nothing here may hoist it across ticks, or an +/// expiry landing between two ticks would be served the earlier tick's allow. +/// +/// +/// It adds no decision logic: the verdict comes from the authorizer's own +/// , and the stand-down (queue BadUserAccessDenied, then detach) +/// and its restore are the pass's, unchanged. The pass early-returns unless ACL enforcement is on, +/// so this is inert on the default configuration. +/// +/// +public sealed class AclRevalidationTimer : IDisposable +{ + private readonly Action _revalidate; + private readonly ILogger _log; + private readonly ITimer? _timer; + private volatile bool _disposed; + + /// Initializes a new instance of the class. + /// The revalidation pass to run on each tick. + /// + /// How often to run it. Non-positive arms no timer at all — the deploy-triggered-only behaviour + /// that predates 03/S19. + /// + /// The shared clock; its CreateTimer is what arms the tick. + /// The logger. + public AclRevalidationTimer(Action revalidate, TimeSpan interval, TimeProvider time, ILogger log) + { + ArgumentNullException.ThrowIfNull(revalidate); + ArgumentNullException.ThrowIfNull(time); + ArgumentNullException.ThrowIfNull(log); + + _revalidate = revalidate; + _log = log; + + if (interval <= TimeSpan.Zero) + { + // Say so once. A disabled guard that logs nothing is indistinguishable from a guard that is + // running and finding nothing, which is precisely the confusion 03/S19 is about. + _log.LogInformation( + "AclRevalidationTimer: disabled — {Section}:RevalidationIntervalSeconds is not positive, " + + "so live monitored items are re-authorized only when a deployment lands", + AclOptions.SectionName); + return; + } + + _timer = time.CreateTimer(_ => Tick(), state: null, interval, interval); + _log.LogInformation( + "AclRevalidationTimer: live monitored items will be re-authorized every {Interval}", interval); + } + + /// Runs one revalidation pass, absorbing anything it throws. + /// Exposed so a test can drive the callback body deterministically rather than by waiting. + public void Tick() + { + if (_disposed) return; + + try + { + _revalidate(); + } + catch (Exception ex) + { + _log.LogError(ex, "AclRevalidationTimer: the monitored-item revalidation pass threw; " + + "the timer keeps running and the next tick is the recovery"); + } + } + + /// + public void Dispose() + { + _disposed = true; + _timer?.Dispose(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/AclRevalidationTimerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/AclRevalidationTimerTests.cs new file mode 100644 index 00000000..fbe57857 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/AclRevalidationTimerTests.cs @@ -0,0 +1,151 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// Covers — the low-frequency trigger that makes key expiry reach +/// a live subscription without waiting for a deployment (arch-review 03/S19). +/// +public sealed class AclRevalidationTimerTests +{ + /// A positive interval arms one timer whose due time and period are both that interval. + [Fact] + public void A_positive_interval_creates_a_timer_with_that_period() + { + var clock = new RecordingTimeProvider(); + + using var _ = new AclRevalidationTimer( + () => { }, TimeSpan.FromSeconds(30), clock, NullLogger.Instance); + + clock.Created.Count.ShouldBe(1); + clock.Created[0].DueTime.ShouldBe(TimeSpan.FromSeconds(30)); + clock.Created[0].Period.ShouldBe(TimeSpan.FromSeconds(30)); + } + + /// + /// A non-positive interval arms nothing at all, and says so once — a disabled guard should be + /// visible in the log rather than inferred from an absence of activity. + /// + [Fact] + public void A_zero_interval_creates_no_timer() + { + var clock = new RecordingTimeProvider(); + var log = new CapturingLogger(); + + using var _ = new AclRevalidationTimer(() => { }, TimeSpan.Zero, clock, log); + + clock.Created.ShouldBeEmpty(); + log.Lines.ShouldContain(l => l.Contains("RevalidationIntervalSeconds", StringComparison.Ordinal)); + } + + /// + /// A pass that throws must not escape the tick. + /// + /// + /// An unhandled exception on a timer callback thread takes the process down. The next tick is the + /// recovery, so the fault is logged and the timer keeps running. + /// + [Fact] + public void A_throwing_pass_does_not_escape_the_tick() + { + var log = new CapturingLogger(); + using var timer = new AclRevalidationTimer( + () => throw new InvalidOperationException("pass blew up"), + TimeSpan.FromSeconds(30), + new RecordingTimeProvider(), + log); + + Should.NotThrow(timer.Tick); + log.Lines.ShouldContain(l => l.Contains("revalidation", StringComparison.OrdinalIgnoreCase)); + } + + /// A tick runs the pass — the control for the two "nothing happened" cases above. + [Fact] + public void A_tick_runs_the_pass() + { + var runs = 0; + using var timer = new AclRevalidationTimer( + () => runs++, TimeSpan.FromSeconds(30), new RecordingTimeProvider(), NullLogger.Instance); + + timer.Tick(); + timer.Tick(); + + runs.ShouldBe(2); + } + + /// Disposing disposes the underlying timer and stops further ticks. + [Fact] + public void Dispose_stops_further_ticks() + { + var clock = new RecordingTimeProvider(); + var runs = 0; + var timer = new AclRevalidationTimer( + () => runs++, TimeSpan.FromSeconds(30), clock, NullLogger.Instance); + + timer.Dispose(); + + clock.Created[0].Disposed.ShouldBeTrue(); + timer.Tick(); + runs.ShouldBe(0, "a disposed timer must not run the pass again"); + } + + // ---------- helpers ---------- + + /// + /// A that records the timers asked of it instead of arming real ones. + /// + private sealed class RecordingTimeProvider : TimeProvider + { + /// Every timer this provider was asked to create, in order. + public List Created { get; } = []; + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new FakeTimer(dueTime, period); + Created.Add(timer); + return timer; + } + + /// A timer that does nothing but remember how it was asked for, and whether it was disposed. + internal sealed class FakeTimer(TimeSpan dueTime, TimeSpan period) : ITimer + { + public TimeSpan DueTime { get; } = dueTime; + + public TimeSpan Period { get; } = period; + + public bool Disposed { get; private set; } + + public bool Change(TimeSpan due, TimeSpan p) => true; + + public void Dispose() => Disposed = true; + + public ValueTask DisposeAsync() + { + Disposed = true; + return ValueTask.CompletedTask; + } + } + } + + /// Collects rendered log messages so a "said so once" claim can be asserted. + private sealed class CapturingLogger : ILogger + { + public List Lines { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => Lines.Add(formatter(state, exception)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index 74210583..fd521810 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -421,11 +421,11 @@ public sealed class BoundedApiKeyStoreTests /// Collects otopcua.opcua.apikey_store.bounded measurements by their outcome tag. /// /// ⚠️ Filtered by instrument name only, which is normally the metrics-test leak (a listener sees - /// a PARALLEL test's measurements too). It is safe HERE for a checkable reason: + /// a PARALLEL test's measurements too). It is safe HERE for two checkable reasons: /// is the only thing in the product that writes this instrument, - /// and this class is the only one in the assembly that constructs one — and xUnit never runs two - /// tests of the same class in parallel. Adding a class that exercises the decorator means adding - /// a discriminating tag here. + /// and this assembly declares CollectionBehavior(DisableTestParallelization = true) in + /// AssemblyInfo.cs, so there is no parallel test to leak from. Removing that attribute + /// means adding a discriminating tag here. /// private sealed class MeterRecorder : IDisposable { From b09f78396f6b1f0edc8a7a98779928c45f94721f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:50:39 -0400 Subject: [PATCH 15/45] test(opcua): pin clock-driven expiry in the revalidation pass; correct the only-trigger remark --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 18 +++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../OtOpcUaNodeManager.cs | 28 ++-- .../NodeManagerStage6EnforcementTests.cs | 148 ++++++++++++++++++ 4 files changed, 186 insertions(+), 10 deletions(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index d6f82684..81c1d238 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -669,3 +669,21 @@ for the orchestrator. - 5 tests, green. Also corrected the T4 `MeterRecorder` remark: the assembly declares `CollectionBehavior(DisableTestParallelization = true)`, which is a stronger and simpler reason the name-only filter is safe than the one first written. + +### T7 — node-manager expiry tests + xmldoc correction + +- **RED: the three new cases PASSED on first run — as the plan predicted** ("the expiry case may already + PASS … that is the point: it pins the property the timer will rely on before the timer exists"). The + pass already reads the authorizer's `TimeProvider`, so no production change was needed. +- **Falsification run (recorded, since "it passed immediately" proves nothing on its own):** shortening the + clock advance in `An_expired_key_stands_down_a_live_item_with_no_deploy` from 20 min to 5 min (i.e. + short of `ExpiresUtc`) turned it **`[FAIL]`**, then restored. So the assertion tracks the clock rather + than a blanket deny. +- `A_renewed_key_re_attaches_the_item_on_the_next_pass` observes the restore **indirectly and + falsifiably**: the pass skips an already-suppressed item, so standing the item down a second time can + only succeed if it was genuinely re-attached in between. (`NodeHandle.MonitoredNode` is null on a + hand-tracked stub item, so attach/detach is not directly observable at this seam — it is proven over the + wire in T9.) +- Two stale remarks corrected, not one: `OnAuthorityChanged`'s "this is the only trigger" **and** + `IsSubscriptionStillPermitted`'s "a subscription is re-checked at a deployment, never on a timer". +- OpcUaServer.Tests: 493 passed / 1 skipped. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index db16b8fb..b2a67fb8 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -51,7 +51,7 @@ { "id": "T7", "subject": "Node-manager expiry tests (FakeTimeProvider advanced past ExpiresUtc, no deploy) + correct the ':316-328' only-trigger xmldoc", - "status": "pending", + "status": "done", "blockedBy": [ "T1" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index b12a03bc..8a1c3d1a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -317,12 +317,21 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 /// Re-authorizes live subscriptions when a deployment installs a new authorization generation. /// /// - /// ⚠️ This is the only trigger, and the pass's memo depends on that. - /// memoizes key revocation and expiry for the span of one - /// pass — safe only while revalidation is deploy-triggered, because a pass is then a momentary - /// event whose verdicts cannot go stale within it. A future periodic trigger must give each tick a - /// fresh memo (it already does — the dictionary is pass-local) and must not be tempted to reuse one - /// across ticks, or an expiry landing between ticks would be served the previous tick's allow. + /// + /// There are TWO triggers (arch-review 03/S19): this one, the deployment axis; and the + /// host's low-frequency AclRevalidationTimer, the clock axis — a key's + /// ExpiresUtc passes with the clock, not with a deployment, so a deploy-only trigger + /// left an expired key streaming to a live subscription indefinitely on a fleet that had been + /// deployed once and left running. + /// + /// + /// ⚠️ The second trigger is safe only because the memo is pass-local. + /// memoizes key revocation and expiry for the span + /// of one pass, which is sound because a pass is a momentary event whose verdicts cannot go + /// stale within it — its dictionary is constructed per call. Hoisting it across ticks, to save + /// the walk, would serve an expiry that landed between two ticks the earlier tick's allow, and + /// re-open exactly the gap the timer closes. + /// /// /// The newly published snapshot; unused — the current state is re-read per item. private void OnAuthorityChanged(DataPlaneAuthoritySnapshot snapshot) => RevalidateMonitoredItems(); @@ -3311,9 +3320,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 /// /// /// This memo does cover the key expiry check that the authorizer's own cache excludes, but only - /// for the duration of one pass — and a subscription is re-checked at a deployment, never on a - /// timer, so an expiry that lands inside a single pass was already going to be caught by the - /// next deployment rather than by this one. + /// for the duration of one pass. Since arch-review 03/S19 a subscription is re-checked on the + /// host's low-frequency timer as well as at a deployment, so an expiry that lands inside a + /// single pass is caught by the next TICK — one interval later, not one deployment later. What + /// makes that hold is that the memo is constructed per call and never reused across passes. /// /// private bool IsSubscriptionStillPermitted( diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerStage6EnforcementTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerStage6EnforcementTests.cs index 6d74b601..e4d0b926 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerStage6EnforcementTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerStage6EnforcementTests.cs @@ -615,8 +615,156 @@ public sealed class NodeManagerStage6EnforcementTests : IDisposable finally { await host.DisposeAsync(); } } + /// + /// A key that expires by the CLOCK stands its live monitored item down, with no deployment + /// anywhere in the test (arch-review 03/S19). + /// + /// + /// + /// The first pass is the falsification control, and it is not optional: without it a build + /// whose pass detached everything it saw — unconditionally, or because the authorizer denied + /// for some unrelated reason — would pass the second half and prove nothing about expiry. + /// + /// + /// DataPlaneAuthorityState.Apply is called exactly once here, in setup. Nothing + /// advances the generation between the two passes, so the only thing that changed is the + /// clock — which is the whole claim. + /// + /// + [Fact] + public async Task An_expired_key_stands_down_a_live_item_with_no_deploy() + { + var (host, server) = await BootAsync(); + try + { + var nm = server.NodeManager!; + var clock = new FakeTimeProvider(new DateTime(2026, 8, 20, 12, 0, 0, DateTimeKind.Utc)); + nm.Authorizer = KeyedSubscribeAuthorizer( + clock, out _, expiresUtc: new DateTime(2026, 8, 20, 12, 10, 0, DateTimeKind.Utc)); + + var item = TrackItem(nm, Variable(nm, "Plant/Modbus/dev1/Speed"), KeyedIdentity()); + + nm.RevalidateMonitoredItems(); + item.Denials.ShouldBe(0, "the key is still valid, so nothing may stand down"); + + clock.Advance(TimeSpan.FromMinutes(20)); + nm.RevalidateMonitoredItems(); + + item.Denials.ShouldBe(1); + } + finally { await host.DisposeAsync(); } + } + + /// + /// A key renewed with a later expiry re-attaches its item on the next pass — the clock axis + /// inherits the deploy axis's reversibility. + /// + /// + /// The restore is asserted by standing the item down a SECOND time: the pass skips an item that is + /// already suppressed, so a second denial can only be queued if the item was genuinely restored + /// in between. That is why this pins the design decision not to close the session — a client that + /// renews its key recovers with no action of its own. + /// + [Fact] + public async Task A_renewed_key_re_attaches_the_item_on_the_next_pass() + { + var (host, server) = await BootAsync(); + try + { + var nm = server.NodeManager!; + var clock = new FakeTimeProvider(new DateTime(2026, 8, 20, 12, 0, 0, DateTimeKind.Utc)); + nm.Authorizer = KeyedSubscribeAuthorizer( + clock, out var state, expiresUtc: new DateTime(2026, 8, 20, 12, 10, 0, DateTimeKind.Utc)); + + var item = TrackItem(nm, Variable(nm, "Plant/Modbus/dev1/Speed"), KeyedIdentity()); + + clock.Advance(TimeSpan.FromMinutes(20)); + nm.RevalidateMonitoredItems(); + item.Denials.ShouldBe(1, "non-vacuity: the item really was stood down before the renewal"); + + // Renew: same key id, a later expiry. + state.Apply( + Cluster, "rev-2", + [Key(new DateTime(2026, 8, 20, 14, 0, 0, DateTimeKind.Utc))], + [GrantOn(NodePermissions.Subscribe)], + Scopes()); + nm.RevalidateMonitoredItems(); + + // Expire it again. A still-suppressed item is skipped, so a second denial proves the restore. + clock.Advance(TimeSpan.FromHours(3)); + nm.RevalidateMonitoredItems(); + + item.Denials.ShouldBe(2); + } + finally { await host.DisposeAsync(); } + } + + /// A clock that moves but does not reach the expiry stands nothing down. + [Fact] + public async Task An_unexpired_key_is_untouched_by_a_clock_only_pass() + { + var (host, server) = await BootAsync(); + try + { + var nm = server.NodeManager!; + var clock = new FakeTimeProvider(new DateTime(2026, 8, 20, 12, 0, 0, DateTimeKind.Utc)); + nm.Authorizer = KeyedSubscribeAuthorizer( + clock, out _, expiresUtc: new DateTime(2026, 8, 20, 13, 0, 0, DateTimeKind.Utc)); + + var item = TrackItem(nm, Variable(nm, "Plant/Modbus/dev1/Speed"), KeyedIdentity()); + + clock.Advance(TimeSpan.FromMinutes(30)); + nm.RevalidateMonitoredItems(); + + item.Denials.ShouldBe(0); + } + finally { await host.DisposeAsync(); } + } + // ───────────────────────────── helpers ───────────────────────────── + /// The public id of the key the clock-driven expiry cases authenticate with. + private const string ExpiringKeyId = "k-expiry"; + + /// + /// A cluster-wide Subscribe grant PLUS a key that expires at , so the + /// verdict moves with the clock rather than with a deployment. + /// + /// The authorizer's clock — the one Evaluate checks expiry against. + /// The authority state, so a test can renew the key without a second helper. + /// When the key stops being usable. + /// The gate, enforcing. + private static DataPlaneAuthorizer KeyedSubscribeAuthorizer( + TimeProvider clock, out DataPlaneAuthorityState state, DateTime expiresUtc) + { + state = new DataPlaneAuthorityState(); + state.Apply(Cluster, "rev-1", [Key(expiresUtc)], [GrantOn(NodePermissions.Subscribe)], Scopes()); + return new DataPlaneAuthorizer( + state, new TriePermissionEvaluator(state.Tries), enabled: true, NullLogger.Instance, clock); + } + + /// The expiring key, at whatever expiry a case needs. + /// When the key stops being usable. + /// The key row. + private static ApiKey Key(DateTime expiresUtc) => new() + { + ApiKeyId = ExpiringKeyId, + Name = ExpiringKeyId, + SecretHash = "hash", + Principal = Principal, + Enabled = true, + CreatedUtc = new DateTime(2026, 8, 20, 0, 0, 0, DateTimeKind.Utc), + ExpiresUtc = expiresUtc, + }; + + /// + /// An identity carrying the key id, which is what makes the per-call expiry re-check apply — an + /// identity with no ApiKeyId skips it entirely. + /// + /// The identity. + private static RoleCarryingUserIdentity KeyedIdentity() => + new(new UserNameIdentityToken { UserName = "apikey" }, [], Principal, ExpiringKeyId); + /// A cluster-wide Subscribe grant — what CreateMonitoredItems resolves to. private static DataPlaneAuthorizer SubscribeAuthorizer(TimeProvider clock) { From e3cf363703a2de2dd7ff94498535db3153bb7cf9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:51:33 -0400 Subject: [PATCH 16/45] fix(host): LocalDb sync endpoint opts out of the cookie fallback via an empty-prefix anon group MapZbLocalDbSync returns IEndpointRouteBuilder, not an IEndpointConventionBuilder, so .AllowAnonymous() cannot be chained onto it the way the other four mesh maps are (T2). app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync() carries the same convention while leaving the gRPC route pattern byte-identical - verified directly (LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical). The census's RealMethodEndpoints filter had to move off a "route pattern starts with /" heuristic to an exact-route-pattern-set match: inside the empty-prefix group, grpc-dotnet's own "Unimplemented service"/"Unimplemented method" fallback endpoints also gain a leading "/", so the heuristic let them back in and over-counted (caught by DriverOnly_WithASyncPort_AlsoMapsTheLocalDbSyncEndpoint and LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical going red against the naive filter before this fix). All five mesh gRPC endpoints now carry IAllowAnonymous (archreview 03/S14, T3). --- .../Configuration/MeshGrpcEndpoints.cs | 9 +++- .../MeshGrpcAnonymousMetadataTests.cs | 47 ++++++++++++------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs index c12e77f3..68a51ed8 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs @@ -64,9 +64,16 @@ public static class MeshGrpcEndpoints // default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be // safe even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey // configured — but not mapping it at all is the stronger default. + // + // AllowAnonymous (03/S14) via an empty-prefix MapGroup: MapZbLocalDbSync returns + // IEndpointRouteBuilder, NOT an IEndpointConventionBuilder, so .AllowAnonymous() cannot be + // chained onto it the way the other four maps below are. An empty-prefix group carries the + // AllowAnonymous convention to whatever it maps while leaving the gRPC route pattern + // byte-identical to the ungrouped map (verified by + // LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical). if (hasDriver && syncListenPort > 0) { - app.MapZbLocalDbSync(); + app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync(); } // Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs index bdaf71d7..3cc91692 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAnonymousMetadataTests.cs @@ -26,11 +26,13 @@ namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// Filters out grpc-dotnet's own catch-all infrastructure. Every MapGrpcService /// call also registers a per-service "Unimplemented method for X" fallback, and the FIRST -/// such call in an app additionally registers one shared "Unimplemented service" catch-all — -/// neither carries a route pattern beginning with /, and neither ever reaches real -/// logic (they always answer Unimplemented regardless of caller identity), so their -/// authorization metadata carries no security meaning. -/// keeps only the five real, callable RPC methods. +/// such call in an app additionally registers one shared "Unimplemented service" catch-all. +/// Neither ever reaches real logic (they always answer Unimplemented regardless of +/// caller identity), so their authorization metadata carries no security meaning. +/// keeps only the five real, callable RPC methods, matched +/// against the exact known route-pattern set rather than a route-pattern-shape heuristic — +/// see that method's remarks for why a shape-based filter is NOT safe once LocalDbSync's +/// empty-prefix MapGroup (T3) is in play. /// /// /// Guard-of-the-guard. Every assertion below pins an EXACT expected count and the @@ -140,19 +142,30 @@ public sealed class MeshGrpcAnonymousMetadataTests } [Fact] - public void Fused_AllFourGrpcServiceMapsCarryAllowAnonymous_LocalDbSyncIsNotYetOwnedHere() + public void Fused_AllFiveMeshEndpoints_CarryAllowAnonymous() { - // T3 owns the LocalDbSync map (MapZbLocalDbSync returns IEndpointRouteBuilder, not - // IEndpointConventionBuilder, so .AllowAnonymous() cannot be chained onto it the same way). - // Asserting its ABSENCE here — rather than skipping it — is what makes this test go red the - // moment T3's fix lands with the wrong scope (e.g. anonymising nothing, or the group leaking - // beyond LocalDbSync) instead of silently starting to pass for the wrong reason. + // Includes LocalDbSync (T3): MapZbLocalDbSync returns IEndpointRouteBuilder, not an + // IEndpointConventionBuilder, so .AllowAnonymous() cannot be chained onto it the way the + // other four are — it needs the empty-prefix MapGroup wrapper instead. var endpoints = RealMethodEndpoints(BuildApp( hasAdmin: true, hasDriver: true, syncListenPort: 4054, configServeGrpcPort: 4055, telemetryListenPort: 4056)); - AnonymousPatternsOf(endpoints).ShouldBe( - [ArtifactFetch, CentralReportApply, TelemetrySubscribe, NodeExecute], ignoreOrder: true); + AnonymousPatternsOf(endpoints).ShouldBe(AllFivePrefixes, ignoreOrder: true); + } + + [Fact] + public void LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical() + { + // The empty-prefix MapGroup wrapper (T3) must be pure convention-carrying — it must not + // change the gRPC route pattern LocalDbSyncAuthInterceptor and the sync client both key on. + var endpoints = RealMethodEndpoints(BuildApp( + hasAdmin: false, hasDriver: true, + syncListenPort: 4054, configServeGrpcPort: 0, telemetryListenPort: 0)); + + var sync = endpoints.ShouldHaveSingleItem(); + sync.RoutePattern.RawText.ShouldBe(LocalDbSyncSync); + sync.Metadata.GetMetadata().ShouldNotBeNull(); } [Fact] @@ -205,14 +218,16 @@ public sealed class MeshGrpcAnonymousMetadataTests /// /// The real, callable RPC method endpoints — excludes grpc-dotnet's per-app "Unimplemented /// service" catch-all and per-service "Unimplemented method for X" fallback (see class - /// remarks). Both carry a route pattern that never starts with /, which is what this - /// filters on. + /// remarks). Filtered against the exact set of known method route patterns rather than a + /// "starts with /" heuristic: inside the empty-prefix MapGroup the LocalDbSync map + /// (T3) uses, grpc-dotnet's own fallback patterns ALSO gain a leading /, so a + /// prefix-shaped filter would let them back in. /// private static List RealMethodEndpoints(WebApplication app) => ((IEndpointRouteBuilder)app).DataSources .SelectMany(d => d.Endpoints) .OfType() - .Where(e => e.RoutePattern.RawText?.StartsWith('/') == true) + .Where(e => AllFivePrefixes.Contains(e.RoutePattern.RawText)) .ToList(); private static string RoutePatternOf(RouteEndpoint e) => e.RoutePattern.RawText!; From 1759f6047613ed714a2279c6f20054e6ef055b7c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:51:43 -0400 Subject: [PATCH 17/45] docs(plans): mark R3-01 T3 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 7e7d1e5d..95fe4701 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -19,7 +19,7 @@ { "id": "T3", "subject": "LocalDb sync endpoint anonymised via an empty-prefix MapGroup (MapZbLocalDbSync returns IEndpointRouteBuilder, not chainable)", - "status": "pending", + "status": "done", "blockedBy": [ "T2" ] From d97e4419794e1643b9f9502b8a3b290edc487051 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:53:38 -0400 Subject: [PATCH 18/45] test(runtime): operator restart re-subscribes the respawned child (archreview 03/S16, U6) Seen RED against a T3-reverted build: all 4 failed, Created[1].SubscribeCount == 0 on the headline case. 20x isolation run green after the fix was restored. --- ...R3-03-restart-resubscribe-watchdog-plan.md | 58 ++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverHostActor.cs | 20 +- .../Drivers/DriverHostActorRestartTests.cs | 287 ++++++++++++++++++ 4 files changed, 360 insertions(+), 7 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index 9d0de32d..4a0d7ace 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -537,3 +537,61 @@ added: `IReadOnlyDictionary? tagConfigByTag = null`. The seeder alarm-bearing raw tag that `Operator_restart_re_subscribes_alarm_refs_too` (a test the plan itself specifies) requires. It is trailing and optional, so it is the same source-compatible shape as `enabled`. + +### T5 — `DriverHostActorRestartTests` (operator-restart falsification, 03/U6's first half) + +**RED observed against a T3-reverted build** (the `SendDesiredSubscriptions(child, …)` call in +`SpawnChild` commented out, everything else on the branch): **all 4 FAILED, 0 passed**, each at the +bar the plan predicts — + +``` +Operator_restart_re_subscribes_the_respawned_child [FAIL] + factory.Created[1].SubscribeCount should be greater than or equal to 1 but was 0 +Operator_restart_re_subscribes_alarm_refs_too [FAIL] + factory.Created[1].SubscribeAlarmsCount should be greater than or equal to 1 but was 0 +Restart_of_a_driver_with_an_empty_value_set_still_sends_the_set [FAIL] + respawned.SubscribeAlarmsCount should be greater than or equal to 1 but was 0 +A_child_spawned_by_a_fresh_deploy_ends_with_the_NEW_refs [FAIL] + factory.Created[2].LastSubscribedRefs should be ["Plant/Modbus/dev1/temp"] but was null +``` + +The call was then restored; **4/4 pass**, and the class was run **20× in isolation, 20/20 green** (#500 +discipline for this assembly). + +**Two deviations, both found by running the tests rather than by reading:** + +1. ⚠️ **The plan's claim that the transient spawn-time set "is never acted on" is FALSE, and the + production comment that repeated it has been corrected.** The plan reasons that the child is still + `Connecting` when the stale set arrives, so it is only stored. But the child runs **concurrently** + with the rest of `HandleArtifactForApply` — `Context.ActorOf` returns and the child starts + processing while the host is still building `refsByDriver` — so a driver whose `InitializeAsync` + completes instantly reaches Connected and subscribes the stale set *before* the bulk push arrives. + Measured directly: `SubscribeCount == 2` on the respawned child, first with revision A's ref and + then with B's. What Akka's per-pair ordering does guarantee is **convergence**, and + `HandleSubscribeAsync` drops the prior subscription before establishing the new one, so the cost is + one extra subscribe round-trip on a *respawning* deploy — a real device driver, whose initialise + opens a socket, rarely wins that race. Accepted rather than "fixed": withholding the spawn-time send + is the 03/S16 defect, and the operator-Restart and supervision paths have no bulk push coming at + all. The `SpawnChild` comment now states this instead of the plan's claim, and the test asserts + convergence (the plan's own stated acceptance — "the child's live subscription carries the NEW + refs") rather than pinning the race. +2. **`A_child_spawned_by_a_fresh_deploy_ends_with_the_NEW_refs` had to be re-shaped.** The plan + describes revision B as one "which respawns nothing but re-pushes". It respawns: the raw tags are + folded into the merged `DriverConfig`, so a tag-set edit changes the `DriverInstanceSpec` and trips + the #516 config respawn. That is the *stronger* arrangement for this test's subject (a brand-new + child spawned while the authority still holds A), so the test keeps the same name and asserts the + respawned child converges on B — plus a subsequent operator Restart, which pins that the authority + is repopulated **whole** per apply rather than accumulated. +3. **`Restart_of_a_driver_with_no_authored_tags_sends_an_empty_set_not_nothing` was renamed + `Restart_of_a_driver_with_an_empty_value_set_still_sends_the_set`, with a real positive control.** + The seeder derives `DriverInstances` from the raw-tag rows, so a driver with *zero* tags is not + expressible, and — more importantly — an all-empty set has **no driver-side observable at all** + (the child stores it and, having no handle, does nothing), which would have made the test vacuous: + "it did not subscribe" passes equally against a build that sends the respawned child nothing, i.e. + against the defect itself. The subject is therefore a driver whose only raw tag is alarm-bearing: + its value half is genuinely empty while its alarm half is not, and **both halves ride in one + message** — so `SubscribeAlarmsCount >= 1` proves the message ARRIVED and `SubscribeCount == 0` + then proves the empty value half was honoured. The falsification run above confirms the control + works: with T3 reverted this test fails on the *positive control*, not on the absence. + **Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm + (a running child absent from BOTH ref maps) is not reachable through this seeder and is untested. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 42819372..7023cf45 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -31,7 +31,7 @@ { "id": "T5", "subject": "DriverHostActorRestartTests: operator restart re-subscribes the respawned child (falsifiable; 03/S16, U6)", - "status": "pending", + "status": "done", "blockedBy": [ "T3", "T4" diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 84bf70b6..9b58ed51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -2825,12 +2825,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // that connected and then read nothing until the next deployment — the #488 periodic reconcile cannot // help, because it early-returns on an empty desired set. // - // ORDERING, deliberately: ApplyDriverSpecs (which spawns) runs BEFORE ApplyDesiredSubscriptions (which - // rebuilds _desiredByDriver), so a newly-spawned child is handed the PREVIOUS apply's set — or nothing, - // on the first deploy. That transient is superseded milliseconds later by the bulk push on the same - // sender→receiver pair (Akka guarantees per-pair ordering), and the child is still Connecting, where - // SetDesiredSubscriptions only STORES; the subscribe happens at Connected entry. So the stale set is - // never acted on. Sent after InitializeRequested above, which keeps init-before-subscribe ordering. + // ORDERING, and its one accepted cost: ApplyDriverSpecs (which spawns) runs BEFORE + // ApplyDesiredSubscriptions (which rebuilds _desiredByDriver), so on a deploy that RESPAWNS a child + // (#516 — the merged DriverConfig folds in the raw tags, so a tag-set edit changes the spec) the new + // child is handed the PREVIOUS apply's set, and nothing at all on the first deploy. That transient is + // superseded a moment later by the bulk push, on the same sender→receiver pair, so Akka's per-pair + // ordering guarantees the child CONVERGES on the new set — but it is NOT guaranteed to be inert on + // the way there: the child runs concurrently with the rest of this handler, so a driver whose + // InitializeAsync completes quickly can reach Connected and subscribe the stale set before the bulk + // push arrives, then re-subscribe. The cost is one extra subscribe/unsubscribe round-trip on a + // respawning deploy (HandleSubscribeAsync drops the prior subscription before establishing the new + // one, so it converges); a real device driver, whose initialise opens a socket, rarely wins that race. + // Deliberately not "fixed" by withholding the send: a child that never gets a set is the 03/S16 + // defect, and the operator-Restart and supervision paths have no bulk push coming at all. + // Sent after InitializeRequested above, which keeps init-before-subscribe ordering. SendDesiredSubscriptions(child, spec.DriverInstanceId); // v3 (WP7): if the (real, non-stub) driver consumes other tags' live values, spawn a mux-adapter diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs new file mode 100644 index 00000000..621213c1 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs @@ -0,0 +1,287 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// The repo's first driver-child restart suite (archreview 03/S16, 03/U6): what a child obtained +/// by each of the three incarnation routes is actually handed. +/// +/// +/// +/// The falsification bar, and why this file exists at all. An actor-restart test must assert a +/// FUNCTION the new incarnation performs — it subscribed, it re-initialised — never that the +/// incarnation EXISTS. Existence is exactly what +/// DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision +/// asserts (no name collision, factory created a second driver, one child remains), and that test +/// passes green against the 03/S16 defect: a respawned child that connects and then reads nothing +/// forever. It is kept, because its own subject — the InvalidActorNameException — is real; it +/// is simply the weaker half of a pair, and this file is the other half. +/// +/// +/// The three incarnation routes, and what re-provisions each. (1) Deploy reconcile — +/// ApplyDriverSpecs → SpawnChild, then the apply's bulk desired-subscription push. +/// (2) Operator Restart — HandleRestartDriver stops the child and calls +/// SpawnChild with the cached spec; before 03/S16 that was the whole story, and the new child +/// started with an EMPTY desired set that nothing re-sent until the next deployment (the #488 +/// periodic reconcile early-returns on an empty set, so it is structurally inert here). +/// (3) Supervision restart — Akka's default decider recreates the actor object around the +/// SAME driver instance without going through SpawnChild at all, so nothing re-sent +/// InitializeRequested either and the child sat in Connecting forever around a driver +/// its own PostStop had already shut down. A fourth route, Force Reconnect, re-uses the +/// live actor instance and needs nothing — a verified negative, pinned here so a future edit to the +/// restart path cannot break the path that was already correct. +/// +/// +/// The supervision crash seam, and why the obvious ones do not work. The seam used is the +/// driver's own OnDataChange carrying a NULL Snapshot +/// (): DataChangeEventArgs is +/// a positional record with no null validation, and DriverInstanceActor.OnDataChangeForward +/// dereferences msg.Snapshot unguarded on the ACTOR thread, so it NREs inside the child and +/// the default decider Restarts it. Rejected alternatives, each for a specific reason: a throwing +/// IDriverHealthPublisher (PublishHealthSnapshot wraps the publish in a +/// catch (Exception), so it never escapes); a throwing SubscribeAsync (also swallowed — +/// the stub already models it via SubscribeFailuresBeforeSuccess); Kill.Instance (the +/// default decider Stops on ActorKilledException, a different directive and therefore a +/// different code path); and an internal "throw this" test message (a production surface added +/// purely for a test). The fault is arbitrary by design — what is under test is the RECOVERY. If +/// DataChangeEventArgs ever gains null validation, the fallback is an explicit +/// OneForOneStrategy(Directive.Restart) on the host plus an internal test-only throw message. +/// +/// +public sealed class DriverHostActorRestartTests : RuntimeActorTestBase +{ + private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64)); + private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64)); + private static readonly NodeId TestNode = NodeId.Parse("node-1:4053"); + + private const string DriverId = "DI-1"; + private const string SpeedPath = "Plant/Modbus/dev1/speed"; + private const string TempPath = "Plant/Modbus/dev1/temp"; + + /// Seeds + applies a v3 deployment whose single Modbus driver is ENABLED, so a real child is + /// spawned around a , and waits until that child has connected + /// and established its first subscription. + private (IActorRef Host, RecordingSubscribableDriverFactory Factory) DeployOneDriver( + IDbContextFactory db, + (string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags, + IReadOnlyDictionary? tagConfigByTag = null) + { + var factory = new RecordingSubscribableDriverFactory("Modbus"); + var deploymentId = V3ArtifactSeeder.SeedV3Deployment( + db, RevA, rawTags, Array.Empty<(string, string, string, string, string?)>(), + enabled: true, tagConfigByTag: tagConfigByTag); + var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory); + return (host, factory); + } + + /// + /// The 03/S16 falsification. An operator's Restart is supposed to recover a sick driver; before + /// the fix it silently killed it — the respawned child came up with _desiredRefs empty, and the + /// only SetDesiredSubscriptions sender in the repo was the apply's bulk push, so the driver + /// subscribed to nothing until the next deployment dispatch. + /// + /// Deliberately NOT the pass criterion: that a second driver was created, or that one child + /// remains. Both are true against the defect. + /// + /// + [Fact] + public void Operator_restart_re_subscribes_the_respawned_child() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeCount.ShouldBe(1); + factory.Created[0].LastSubscribedRefs.ShouldBe(new[] { SpeedPath }); + }, + duration: PresenceBudget); + + host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid())); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(2, "the restart did not respawn the child around a new driver"); + factory.Created[1].SubscribeCount.ShouldBeGreaterThanOrEqualTo( + 1, + "the RESPAWNED child never subscribed — it is Connected and reading nothing, which is the " + + "03/S16 defect (an operator pressed Restart to recover a driver and silently killed it)"); + factory.Created[1].LastSubscribedRefs.ShouldBe(new[] { SpeedPath }); + }, + duration: PresenceBudget); + } + + /// + /// The value refs and the alarm refs ride in ONE SetDesiredSubscriptions but are applied by two + /// different child code paths (the value set self-Tells Subscribe; the alarm set runs + /// SubscribeDesiredAlarms against the IAlarmSource feed, which stays gated shut until it + /// is called). Sending the message is therefore not evidence that both halves landed, so the alarm + /// half is pinned separately. + /// + [Fact] + public void Operator_restart_re_subscribes_alarm_refs_too() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver( + db, + new[] + { + (DriverId, "Modbus", "dev1", "speed", "Float", false), + (DriverId, "Modbus", "dev1", "temp", "Float", false), + }, + // An alarm-bearing raw tag is EXCLUDED from the value set and carried in the alarm set instead. + new Dictionary + { + ["temp"] = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}""", + }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(1); + factory.Created[0].LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath }); + }, + duration: PresenceBudget); + + host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid())); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(2); + factory.Created[1].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo( + 1, + "the respawned child never re-subscribed its alarm refs — the IAlarmSource feed stays " + + "gated shut, so no native alarm transition can ever reach the host again"); + factory.Created[1].LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath }); + // The value half must survive the alarm half being present. + factory.Created[1].LastSubscribedRefs.ShouldBe(new[] { SpeedPath }); + }, + duration: PresenceBudget); + } + + /// + /// The ordering guard for the spawn-time send. ApplyDriverSpecs (which spawns) runs BEFORE + /// ApplyDesiredSubscriptions (which rebuilds the authority), so a child spawned by a fresh + /// deploy is momentarily handed the PREVIOUS apply's set. That transient must never be acted on: it + /// is superseded by the bulk push on the same sender→receiver pair (Akka guarantees per-pair + /// ordering) while the child is still Connecting, where SetDesiredSubscriptions only + /// STORES; the subscribe happens on the Connected entry, from whatever was stored last. + /// + /// Revision B swaps the driver's tag, so A's and B's sets are distinguishable. Note the raw tags + /// are folded into the merged DriverConfig, so a tag-set change DOES change the driver + /// spec and therefore respawns the child (#516) — which is precisely the arrangement under test: + /// a brand-new child, spawned while the authority still holds A, must end up subscribed to B. + /// + /// + /// What is asserted is CONVERGENCE, not inertness, and the difference was measured rather + /// than assumed. The child runs concurrently with the rest of the apply handler, so a driver + /// whose InitializeAsync completes instantly (this stub does) can reach Connected and + /// subscribe the transient stale set BEFORE the bulk push arrives — observed here as + /// SubscribeCount == 2. Akka's per-pair ordering still guarantees the final state is B's, + /// and HandleSubscribeAsync drops the prior subscription before establishing the new one, + /// so the cost is one extra round-trip on a respawning deploy. Pinning the count would pin a + /// race; the final refs are the invariant. + /// + /// + [Fact] + public void A_child_spawned_by_a_fresh_deploy_ends_with_the_NEW_refs() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }); + + AwaitAssert( + () => factory.Created[0].LastSubscribedRefs.ShouldBe(new[] { SpeedPath }), + duration: PresenceBudget); + + var revB = V3ArtifactSeeder.SeedV3Deployment( + db, RevB, + new[] { (DriverId, "Modbus", "dev1", "temp", "Float", false) }, + Array.Empty<(string, string, string, string, string?)>(), + enabled: true); + host.Tell(new Commons.Messages.Deploy.DispatchDeployment(revB, RevB, CorrelationId.NewId())); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(2, "the tag-set change re-spawns the child (#516 config respawn)"); + factory.Created[1].LastSubscribedRefs.ShouldBe( + new[] { TempPath }, + "the newly-spawned child is on the PREVIOUS revision's refs — the spawn-time set was not superseded"); + }, + duration: PresenceBudget); + + // A Restart on top of that must hand the new child B's set, not A's — i.e. the authority really is + // repopulated WHOLE on every apply rather than accumulated. + host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid())); + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(3); + factory.Created[2].LastSubscribedRefs.ShouldBe(new[] { TempPath }); + }, + duration: PresenceBudget); + } + + /// + /// The empty-set semantics are load-bearing and must not degrade into "skip the send": an EMPTY value + /// set is what tells the child to Unsubscribe and drop a stale handle, so it must be SENT. + /// + /// The subject is a driver whose only raw tag is alarm-bearing: alarm tags are excluded from the + /// value set and carried in the alarm one, so its value half is genuinely empty while its alarm + /// half is not — and both halves ride in ONE SetDesiredSubscriptions. That is what makes + /// the absence assertion non-vacuous: SubscribeAlarmsCount >= 1 proves the message + /// ARRIVED, and SubscribeCount == 0 then proves the empty value half was honoured rather + /// than the whole send being skipped. Asserting only "it did not subscribe" would pass equally + /// against a build that sent the respawned child nothing at all — the 03/S16 defect itself. + /// + /// + [Fact] + public void Restart_of_a_driver_with_an_empty_value_set_still_sends_the_set() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver( + db, + new[] { (DriverId, "Modbus", "dev1", "temp", "Float", false) }, + new Dictionary + { + ["temp"] = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}""", + }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(1); + }, + duration: PresenceBudget); + + host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid())); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(2); + var respawned = factory.Created[1]; + respawned.InitializeCount.ShouldBe(1, "the respawned child was never initialised"); + // Positive control: the SetDesiredSubscriptions genuinely reached the respawned child. + respawned.SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo( + 1, "the respawned child received no desired set at all — the absence below would be vacuous"); + respawned.LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath }); + respawned.SubscribeCount.ShouldBe( + 0, "an EMPTY value set must route to Unsubscribe / no-op, never to a spurious subscribe"); + }, + duration: PresenceBudget); + } +} From a4f12bccef77008fca94a8283bdd6a065c6e494d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:55:01 -0400 Subject: [PATCH 19/45] fix(host): run monitored-item revalidation on a low-frequency timer (archreview 03/S19) --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 23 +++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../OpcUa/OtOpcUaServerHostedService.cs | 53 +++++++ .../ZB.MOM.WW.OtOpcUa.Host.csproj | 7 + ...cUaServerHostedServiceRevalidationTests.cs | 144 ++++++++++++++++++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceRevalidationTests.cs diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 81c1d238..57f5c808 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -687,3 +687,26 @@ for the orchestrator. - Two stale remarks corrected, not one: `OnAuthorityChanged`'s "this is the only trigger" **and** `IsSubscriptionStillPermitted`'s "a subscription is re-checked at a deployment, never on a timer". - OpcUaServer.Tests: 493 passed / 1 skipped. + +### T8 — wire the timer into `OtOpcUaServerHostedService` + +- **RED observed:** `error CS1061: 'OtOpcUaServerHostedService' does not contain a definition for + 'StartRevalidationTimer'` and `error CS1729: … does not contain a constructor that takes 13 arguments` + (the two new ctor parameters). +- **DEVIATION (test seam, and the reason for it):** the plan says "extend the hosted-service tests". **There + are none — `grep -rln OtOpcUaServerHostedService tests/` matched nothing**, because `StartAsync` boots + the real OPC UA SDK (PKI store, listening socket). A new suite, + `OtOpcUaServerHostedServiceRevalidationTests`, drives the timer's lifecycle through an **`internal`** + `StartRevalidationTimer()` (new `InternalsVisibleTo` on the Host csproj, the convention six driver + projects already use). Everything the timer's lifecycle touches — the options, the clock, the disposal — + is exercised exactly as production reaches it; only the SDK boot is skipped. +- **DEVIATION (assertion not made):** the plan asks the test to assert disposal happens **before** + `SetDataPlaneAuthorizer(null)`. That is **not observable**: both detaches go through `_server?.…`, and + `OtOpcUaSdkServer.NodeManager` is created only by the SDK's own `CreateMasterNodeManager` during a real + boot, so with no server both calls are no-ops. The ordering is instead guaranteed structurally — the + dispose is the **first statement** of `StopAsync`, with a comment saying why — and what IS asserted is + that `StopAsync` (and `DisposeAsync`) dispose the timer, with a non-vacuity check that it was live + beforehand. **Reviewer: this is the one plan assertion downgraded rather than met.** +- The node manager is resolved **per tick** (`() => _server?.NodeManager?.Revalidate…`) rather than + captured, so a restart cannot leave the tick walking a dead manager's item map. +- Host.Tests: 132 green. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index b2a67fb8..bbc19b89 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -59,7 +59,7 @@ { "id": "T8", "subject": "Wire the revalidation timer into OtOpcUaServerHostedService (create after StartAsync, dispose FIRST in StopAsync) + ordering test", - "status": "pending", + "status": "done", "blockedBy": [ "T6", "T7" diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index d05d5abf..33a0bb70 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -38,6 +38,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDataPlaneAuthoritySource _dataPlaneAuthority; private readonly DataPlaneAuthorizer? _dataPlaneAuthorizer; private readonly ServerHistorianOptions _serverHistorianOptions; + private readonly AclOptions _aclOptions; + private readonly TimeProvider _time; private readonly Func _actorSystemAccessor; private readonly ActorRegistry _actorRegistry; private readonly ILoggerFactory _loggerFactory; @@ -45,6 +47,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; + private AclRevalidationTimer? _revalidationTimer; /// /// Initializes a new instance of the OtOpcUaServerHostedService class. @@ -77,6 +80,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// #520 stage 5 — the node-scoped ACL gate. Nullable, and null means /// "not wired" rather than "deny": an unregistered gate must not lock a node out of writes to protect a /// feature that is off by default. Once wired AND enabled, every unresolvable case inside it denies. + /// #520 / arch-review 03/S19 — the Security:Acl options, read for + /// . Required, not optional: the built-in DI + /// container does not fall back to a default for an unregistered service, the same trap the + /// IOpcUaUserAuthenticator and IDataPlaneAuthoritySource parameters carry comments + /// about. + /// The shared clock — the one instance the authorizer and the API-key store bound + /// also read. Its CreateTimer arms the revalidation tick. /// The logger factory for creating loggers. public OtOpcUaServerHostedService( IOptions options, @@ -89,9 +99,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl Func actorSystemAccessor, ActorRegistry actorRegistry, DataPlaneAuthorizer? dataPlaneAuthorizer, + IOptions aclOptions, + TimeProvider time, ILoggerFactory loggerFactory) { + ArgumentNullException.ThrowIfNull(aclOptions); + ArgumentNullException.ThrowIfNull(time); + _dataPlaneAuthorizer = dataPlaneAuthorizer; + _aclOptions = aclOptions.Value; + _time = time; _options = options.Value; _deferredSink = deferredSink; _deferredServiceLevel = deferredServiceLevel; @@ -269,9 +286,34 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _loggerFactory.CreateLogger())); } + // arch-review 03/S19 — LAST, after the node manager is known non-null and after the ACL gate is + // attached, so the first tick cannot land on a half-wired server. + StartRevalidationTimer(); + _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Arms the clock-driven monitored-item revalidation tick (arch-review 03/S19). + /// + /// + /// Created unconditionally: the pass it drives early-returns unless the ACL gate is wired AND + /// Security:Acl:Enabled is on, so it stays inert on the default configuration, and + /// RevalidationIntervalSeconds = 0 arms no timer at all. + /// + /// The node manager is resolved per tick rather than captured: is + /// replaced across a restart, and a captured reference would go on revalidating a dead + /// manager's item map — the same detach hazard the authority-source setter documents. + /// + /// Separate from so the timer's lifecycle is testable without booting the SDK. + /// + internal void StartRevalidationTimer() => + _revalidationTimer = new AclRevalidationTimer( + () => _server?.NodeManager?.RevalidateMonitoredItems(), + TimeSpan.FromSeconds(_aclOptions.RevalidationIntervalSeconds), + _time, + _loggerFactory.CreateLogger()); + /// /// Stops the OPC UA server asynchronously. /// @@ -279,6 +321,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// A task that represents the asynchronous operation. public Task StopAsync(CancellationToken cancellationToken) { + // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose + // authority and gate have already been cleared is the shape this whole teardown block exists to + // prevent — a stopping node sheds the trigger before it sheds what the trigger drives. + _revalidationTimer?.Dispose(); + _revalidationTimer = null; + // Revert to Null adapters so any in-flight writes from a poison-pilled actor don't hit a // half-disposed NodeManager. _deferredSink.SetSink(null); @@ -304,6 +352,11 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { + // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must + // not leave a tick armed against a server that is about to go away. + _revalidationTimer?.Dispose(); + _revalidationTimer = null; + if (_appHost is not null) await _appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj index 1819d602..cc80ef36 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj @@ -13,6 +13,13 @@ $(NoWarn);NU1608 + + + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceRevalidationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceRevalidationTests.cs new file mode 100644 index 00000000..96c989f8 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceRevalidationTests.cs @@ -0,0 +1,144 @@ +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// Pins the ACL revalidation timer's lifecycle on the hosted service (arch-review 03/S19) — that it is +/// armed at the configured interval, that 0 arms nothing, and that stopping disposes it. +/// +/// +/// The timer is armed through rather +/// than by running StartAsync, because StartAsync boots the real OPC UA SDK (certificate +/// store, listening socket) and none of that is what this is about. Everything the timer's lifecycle +/// touches — the options, the clock, the disposal — is exercised exactly as production reaches it. +/// +public sealed class OtOpcUaServerHostedServiceRevalidationTests +{ + /// The configured interval is what the timer is armed at. + [Fact] + public void A_configured_interval_arms_one_timer_at_that_period() + { + var clock = new RecordingTimeProvider(); + var sut = Service(clock, intervalSeconds: 30); + + sut.StartRevalidationTimer(); + + clock.Periods.ShouldBe([TimeSpan.FromSeconds(30)]); + } + + /// + /// RevalidationIntervalSeconds = 0 arms nothing — the documented rollback to + /// deploy-triggered-only revalidation. + /// + [Fact] + public void A_zero_interval_arms_nothing() + { + var clock = new RecordingTimeProvider(); + var sut = Service(clock, intervalSeconds: 0); + + sut.StartRevalidationTimer(); + + clock.Periods.ShouldBeEmpty(); + } + + /// + /// StopAsync disposes the timer, so no tick can land on a node manager that is being torn + /// down. + /// + /// + /// The disposal is the FIRST thing StopAsync does, ahead of + /// SetDataPlaneAuthority(null) / SetDataPlaneAuthorizer(null) — a stopping node sheds + /// the trigger before it sheds the thing the trigger drives. + /// + [Fact] + public async Task StopAsync_disposes_the_timer() + { + var clock = new RecordingTimeProvider(); + var sut = Service(clock, intervalSeconds: 30); + sut.StartRevalidationTimer(); + + clock.Timers[0].Disposed.ShouldBeFalse("non-vacuity: the timer is live before the stop"); + + await sut.StopAsync(CancellationToken.None); + + clock.Timers[0].Disposed.ShouldBeTrue(); + } + + /// Disposing the service disposes the timer too, for a host that never called StopAsync. + [Fact] + public async Task DisposeAsync_disposes_the_timer() + { + var clock = new RecordingTimeProvider(); + var sut = Service(clock, intervalSeconds: 30); + sut.StartRevalidationTimer(); + + await sut.DisposeAsync(); + + clock.Timers[0].Disposed.ShouldBeTrue(); + } + + // ---------- helpers ---------- + + private static OtOpcUaServerHostedService Service(TimeProvider clock, int intervalSeconds) => new( + Options.Create(new OpcUaApplicationHostOptions()), + new DeferredAddressSpaceSink(), + new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, + NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, + new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no ActorSystem is needed for the timer lifecycle"), + new ActorRegistry(), + dataPlaneAuthorizer: null, + Options.Create(new AclOptions { RevalidationIntervalSeconds = intervalSeconds }), + clock, + NullLoggerFactory.Instance); + + /// Records the timers asked of it instead of arming real ones. + private sealed class RecordingTimeProvider : TimeProvider + { + /// Every timer this provider was asked to create, in order. + public List Timers { get; } = []; + + /// The period each of those timers was armed at. + public IReadOnlyList Periods => [.. Timers.Select(t => t.Period)]; + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new FakeTimer(period); + Timers.Add(timer); + return timer; + } + + /// A timer that remembers its period and whether it was disposed. + internal sealed class FakeTimer(TimeSpan period) : ITimer + { + public TimeSpan Period { get; } = period; + + public bool Disposed { get; private set; } + + public bool Change(TimeSpan dueTime, TimeSpan p) => true; + + public void Dispose() => Disposed = true; + + public ValueTask DisposeAsync() + { + Disposed = true; + return ValueTask.CompletedTask; + } + } + } +} From 2b9a397b2d4c7d1c471be8ceff409c91ea992413 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 09:56:05 -0400 Subject: [PATCH 20/45] fix(runtime): re-provision a driver child after a supervision restart (archreview 03/S6, S16) --- ...R3-03-restart-resubscribe-watchdog-plan.md | 10 +++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverHostActor.cs | 37 +++++++++++++++++++ .../Drivers/DriverInstanceActor.cs | 33 +++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index 4a0d7ace..6ab0d180 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -595,3 +595,13 @@ discipline for this assembly). works: with T3 reverted this test fails on the *positive control*, not on the absence. **Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm (a running child absent from BOTH ref maps) is not reachable through this seeder and is untested. + +### T6 — supervision reprovision hook + +`DriverInstanceActor.PostRestart` (calling `base.PostRestart(reason)` first, so `PreStart` re-attaches +the rediscovery / host-status sources and restarts the health timer) → `ReprovisionRequested` → +`DriverHostActor.HandleReprovisionRequested`, registered in **all three** `Receive` blocks that already +accept `RestartDriver`. The falsification is T7, below. + +Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **183/183 passed** (179 before this task +plus T5's four), no test moved. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 7023cf45..0e1d6c9f 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -40,7 +40,7 @@ { "id": "T6", "subject": "Supervision reprovision hook: DriverInstanceActor.PostRestart -> ReprovisionRequested -> host re-init + re-subscribe (03/S6, S16)", - "status": "pending", + "status": "done", "blockedBy": [ "T3" ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 9b58ed51..00352e15 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -1088,6 +1088,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleDeltaApplied); Receive(HandleApplyResult); Receive(HandleRestartDriver); + Receive(HandleReprovisionRequested); Receive(HandleReconnectDriver); Receive(HandleRouteNodeWrite); Receive(HandleRouteNativeAlarmAck); @@ -1143,6 +1144,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleDeltaApplied); Receive(HandleApplyResult); Receive(HandleRestartDriver); + Receive(HandleReprovisionRequested); Receive(HandleReconnectDriver); Receive(HandleRouteNodeWrite); Receive(HandleRouteNativeAlarmAck); @@ -1725,6 +1727,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleGetDiagnostics); Receive(_ => TryRecoverFromStale()); Receive(HandleRestartDriver); + Receive(HandleReprovisionRequested); Receive(HandleReconnectDriver); Receive(OnRedundancyStateChanged); // The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes). @@ -2991,6 +2994,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS SpawnChild(entry.Spec); } + /// + /// Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6, + /// 03/S16) — the third and least visible of the three incarnation routes. + /// + /// + /// A restart recreates the actor object around the SAME driver instance (the Props closure captured + /// it) without going through , so neither the InitializeRequested nor + /// the desired-subscription send that a spawn performs ever happens — and the child's + /// is unchanged, so _children needs no maintenance and the host has no + /// other notification that a new incarnation exists. Its PreRestart ran PostStop, which + /// called ShutdownAsync on the driver, so re-initialising is not optional: without it the + /// child sits in Connecting forever around a shut-down driver. + /// + /// The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the + /// instance actor restarting, so it is deliberately not touched here. + /// + /// + /// The restarted child's self-announcement. + private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) + { + // Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision. + if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return; + + // A crash that Akka silently restarted is otherwise invisible to operators; name it. + _log.Warning( + "DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning", + _localNode, msg.DriverInstanceId, msg.Reason); + + // A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested + // anyway), but it still stores a desired set, so only the initialise is conditional. + if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig)); + SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId); + } + private void HandleReconnectDriver(ReconnectDriver msg) { // DPS broadcast — only act if this node hosts the requested instance. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs index 81852667..b53a1418 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs @@ -42,6 +42,25 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers /// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm /// events. Fire-and-forget, mirroring . public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected); + + /// + /// Sent to the parent from so the host re-provisions THIS incarnation + /// (archreview 03/S6, 03/S16). + /// + /// + /// A supervision restart recreates the actor object in place: the desired subscription sets are gone + /// (they are instance fields), and PreRestart has already run — which + /// called on the driver object the Props closure re-uses. Nothing + /// else re-sends : its only sender is + /// DriverHostActor.SpawnChild, and a restart does NOT respawn (the child's + /// is unchanged, so the host is not otherwise notified that a new incarnation + /// exists at all). Without this the restarted child sits in Connecting forever around a + /// shut-down driver, with nothing subscribed — silently, since Akka logs the fault but the recovery + /// is invisible. + /// + /// The driver instance that was restarted. + /// The type name of the exception that tripped supervision, for the host's log. + public sealed record ReprovisionRequested(string DriverInstanceId, string Reason); public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation); public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation); /// @@ -375,6 +394,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval); } + /// + /// + /// Ask the host to re-provision this incarnation — see for why a + /// supervision restart otherwise leaves a dead child. base.PostRestart runs first because it + /// runs , which re-attaches the rediscovery / host-status sources and restarts + /// the health timer; the reprovision is the part cannot do, since only the + /// host knows this driver's config and desired set. + /// + protected override void PostRestart(Exception reason) + { + base.PostRestart(reason); + Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name)); + } + private void Stubbed() { // Stubbed drivers accept the standard message contracts but return deterministic From dfab7f614d2ae0e5ec2c0ea617696824d3509fd4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:00:52 -0400 Subject: [PATCH 21/45] test(integration): key expiry stops a live subscription over the wire, with no deploy --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 20 + ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- .../AclKeyExpiryEnforcementTests.cs | 370 ++++++++++++++++++ ...tOpcUa.OpcUaServer.IntegrationTests.csproj | 4 + 4 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AclKeyExpiryEnforcementTests.cs diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 57f5c808..88487801 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -710,3 +710,23 @@ for the orchestrator. - The node manager is resolved **per tick** (`() => _server?.NodeManager?.Revalidate…`) rather than captured, so a restart cannot leave the tick walking a dead manager's item map. - Host.Tests: 132 green. + +### T9 — over-the-wire expiry test (key-carrying session) + +- **RED observed (build):** `error CS1503: Argument 2: cannot convert from 'string' to 'byte[]'` — the + 1.5.378 `UserNameIdentityToken.DecryptedPassword` is raw bytes, not a string (the host already decodes + UTF-8; the token had to be built the same way). +- **FALSIFICATION (the plan's step 2, run and recorded):** deleting the single `rig.TickRevalidation()` + call from `An_expired_key_stops_a_live_subscription_with_no_deploy` makes it **`[FAIL]`** — the expired + key goes on streaming. So the denial is produced by the T8 tick and by nothing else. Restored, both + cases green. +- **DEVIATION (fake choice):** the plan calls for "an `ApiKeyOpcUaUserAuthenticator` over a fake + `IApiKeyStore`". The rig instead uses a 6-line `KeyPresentingAuthenticator` returning + `OpcUaUserAuthResult.AllowKey(principal, [], keyId)`. The credential-verification path (parse, peppered + HMAC, store lookup, deny shapes) is already covered in the Host suite and changes nothing about what + this file asks — **whether the SDK hands a monitored item back an identity that still carries the key + id**. Grants come from the deployed snapshot, so the authenticator returns no roles. +- The tick is driven through the **real `AclRevalidationTimer`**, which required a new `ProjectReference` + from the integration test project to `ZB.MOM.WW.OtOpcUa.Host` (the timer lives there). Without it the + test would have called the node manager's pass directly and could not have falsified T8. +- OpcUaServer.IntegrationTests: 20 green (was 18). diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index bbc19b89..1b459d39 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -68,7 +68,7 @@ { "id": "T9", "subject": "Over-the-wire AclKeyExpiryEnforcementTests: key-carrying session, clock advanced past expiry, item goes BadUserAccessDenied with no deploy (+ negative control)", - "status": "pending", + "status": "done", "blockedBy": [ "T8" ] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AclKeyExpiryEnforcementTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AclKeyExpiryEnforcementTests.cs new file mode 100644 index 00000000..cb97261d --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AclKeyExpiryEnforcementTests.cs @@ -0,0 +1,370 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Opc.Ua; +using Opc.Ua.Client; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +/// +/// arch-review 03/S19 over the wire — a key that expires by the CLOCK stops a live subscription, with +/// no deployment anywhere in the run. +/// +/// +/// +/// This cannot be reasoned into existence for the same reason +/// AclSubscriptionEnforcementTests gives: whether a monitored item can recover the identity +/// of the session that created it is an assumption about SDK behaviour. Here it must recover an +/// identity carrying an API key id — the field the per-call expiry re-check keys on — and +/// the existing rig has never exercised one, because it connects anonymously. +/// +/// +/// The tick is driven through the real rather than by calling +/// the node manager's pass directly, so what is asserted here is the whole T8 path: revert the +/// host's timer and the denial below cannot be produced by anything except a deployment. +/// +/// +public sealed class AclKeyExpiryEnforcementTests +{ + private const string Cluster = "SITE-A"; + private const string Device = "Plant/Modbus/dev1"; + private const string Tag = "Plant/Modbus/dev1/Speed"; + private const string Principal = "expiry-probe"; + private const string KeyId = "k-expiry"; + + private static readonly DateTime Start = new(2026, 8, 20, 12, 0, 0, DateTimeKind.Utc); + + /// + /// THE test this file exists for: the key's ExpiresUtc passes, a revalidation tick runs, and + /// the client is told BadUserAccessDenied — with DataPlaneAuthorityState.Apply called + /// exactly once, in setup. + /// + [Fact] + public async Task An_expired_key_stops_a_live_subscription_with_no_deploy() + { + var ct = TestContext.Current.CancellationToken; + var rig = await Rig.StartAsync("AclKeyExpiry", ct); + try + { + rig.Enforce(expiresUtc: Start.AddMinutes(10)); + rig.Materialise(); + rig.Write(1); + + using var session = await rig.ConnectAsync(ct); + var (subscription, received) = await rig.SubscribeAsync(session, ct); + + // Baseline: the key-carrying session's feed is live. + await WaitUntilAsync(() => received.Any(v => Equals(v.Value, 1f)), TimeSpan.FromSeconds(10)); + received.Clear(); + + // Only the clock moves. No deployment, no new generation, no grant change. + rig.Advance(TimeSpan.FromMinutes(20)); + rig.TickRevalidation(); + + await WaitUntilAsync( + () => received.Any(v => v.StatusCode.Code == StatusCodes.BadUserAccessDenied), + TimeSpan.FromSeconds(10)); + + // And the data really has stopped. + received.Clear(); + rig.Write(2); + await Task.Delay(TimeSpan.FromSeconds(2), ct); + received.ShouldNotContain(v => Equals(v.Value, 2f)); + + await subscription.DeleteAsync(true, ct); + } + finally { await rig.DisposeAsync(); } + } + + /// + /// The negative control: identical run, clock advanced but not past ExpiresUtc ⇒ values keep + /// arriving. + /// + /// + /// Without it the test above also passes on a build whose tick detaches every item it sees, which + /// would silence every subscription in the fleet once a minute. + /// + [Fact] + public async Task An_unexpired_key_keeps_streaming_across_a_tick() + { + var ct = TestContext.Current.CancellationToken; + var rig = await Rig.StartAsync("AclKeyLive", ct); + try + { + rig.Enforce(expiresUtc: Start.AddHours(4)); + rig.Materialise(); + rig.Write(1); + + using var session = await rig.ConnectAsync(ct); + var (subscription, received) = await rig.SubscribeAsync(session, ct); + await WaitUntilAsync(() => received.Any(v => Equals(v.Value, 1f)), TimeSpan.FromSeconds(10)); + + received.Clear(); + rig.Advance(TimeSpan.FromMinutes(20)); + rig.TickRevalidation(); + + rig.Write(2); + await WaitUntilAsync(() => received.Any(v => Equals(v.Value, 2f)), TimeSpan.FromSeconds(10)); + received.ShouldNotContain(v => v.StatusCode.Code == StatusCodes.BadUserAccessDenied); + + await subscription.DeleteAsync(true, ct); + } + finally { await rig.DisposeAsync(); } + } + + // ───────────────────────────── helpers ───────────────────────────── + + private static readonly IReadOnlyDictionary> NoScopes = + new Dictionary>(); + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) return; + await Task.Delay(50); + } + condition().ShouldBeTrue("condition not met within timeout"); + } + + /// A booted server, a key-carrying client session, and a hand-driven revalidation tick. + private sealed class Rig : IAsyncDisposable + { + private readonly string _pkiRoot; + private readonly FakeClock _clock = new(Start); + private OpcUaApplicationHost _host = null!; + private OtOpcUaSdkServer _server = null!; + private AclRevalidationTimer? _timer; + private int _port; + + private Rig(string pkiRoot) => _pkiRoot = pkiRoot; + + private OtOpcUaNodeManager Nm => _server.NodeManager!; + + public static async Task StartAsync(string name, CancellationToken ct) + { + var rig = new Rig(Path.Combine(Path.GetTempPath(), $"otopcua-{name}-{Guid.NewGuid():N}")); + rig._port = AllocateFreePort(); + rig._server = new OtOpcUaSdkServer(); + rig._host = new OpcUaApplicationHost( + new OpcUaApplicationHostOptions + { + ApplicationName = $"OtOpcUa.{name}", + ApplicationUri = $"urn:OtOpcUa.{name}.{Guid.NewGuid():N}", + OpcUaPort = rig._port, + PublicHostname = "127.0.0.1", + PkiStoreRoot = rig._pkiRoot, + EnabledSecurityProfiles = [OpcUaSecurityProfile.None], + AutoAcceptUntrustedClientCertificates = true, + }, + NullLogger.Instance, + new KeyPresentingAuthenticator()); + + await rig._host.StartAsync(rig._server, ct); + return rig; + } + + /// + /// Switches enforcement on with one cluster-wide grant and one key expiring at + /// , and arms the revalidation timer over the same clock. + /// + /// When the key stops being usable. + public void Enforce(DateTime expiresUtc) + { + var key = new ApiKey + { + ApiKeyId = KeyId, + Name = KeyId, + SecretHash = "hash", + Principal = Principal, + Enabled = true, + CreatedUtc = Start.AddDays(-1), + ExpiresUtc = expiresUtc, + }; + var grant = new NodeAcl + { + NodeAclId = "acl-cluster", + ClusterId = Cluster, + Principal = Principal, + ScopeKind = NodeAclScopeKind.Cluster, + PermissionFlags = NodePermissions.Read | NodePermissions.Browse | NodePermissions.Subscribe, + }; + + var state = new DataPlaneAuthorityState(); + + // The ONLY Apply in the whole test. Everything after this moves the clock. + state.Apply(Cluster, "rev-1", [key], [grant], NoScopes); + + Nm.DataPlaneAuthority = state; + Nm.Authorizer = new DataPlaneAuthorizer( + state, new TriePermissionEvaluator(state.Tries), enabled: true, NullLogger.Instance, _clock); + + // A long interval: the tick is driven by hand, so a real timer firing underneath would only + // make the test non-deterministic. What matters is that the pass is reached through the same + // component the hosted service arms. + _timer = new AclRevalidationTimer( + () => _server.NodeManager?.RevalidateMonitoredItems(), + TimeSpan.FromHours(1), + TimeProvider.System, + NullLogger.Instance); + } + + /// Moves the authorizer's clock forward — the only thing that changes mid-test. + /// How far forward. + public void Advance(TimeSpan by) => _clock.Advance(by); + + /// Runs one revalidation tick through the host's own timer component. + public void TickRevalidation() => _timer!.Tick(); + + public void Materialise() + { + Nm.EnsureFolder("Plant", null, "Plant", AddressSpaceRealm.Raw); + Nm.EnsureFolder("Plant/Modbus", "Plant", "Modbus", AddressSpaceRealm.Raw); + Nm.EnsureFolder(Device, "Plant/Modbus", "dev1", AddressSpaceRealm.Raw); + Nm.EnsureVariable(Tag, Device, "Speed", "Float", writable: false, AddressSpaceRealm.Raw); + } + + public void Write(float value) => + Nm.WriteValue(Tag, value, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); + + public ushort RawNamespace(ISession session) + { + var ns = (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri); + ns.ShouldBeGreaterThan((ushort)0); + return ns; + } + + public NodeId TagNodeId(ISession session) => new(Tag, RawNamespace(session)); + + public async Task<(Subscription Subscription, List Received)> SubscribeAsync( + ISession session, CancellationToken ct) + { + var received = new List(); + var gate = new object(); + + var subscription = new Subscription(session.DefaultSubscription) { PublishingInterval = 100 }; + session.AddSubscription(subscription); + await subscription.CreateAsync(ct); + + var item = new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = TagNodeId(session), + AttributeId = Attributes.Value, + SamplingInterval = 50, + QueueSize = 10, + }; + item.Notification += (mi, _) => + { + foreach (var v in mi.DequeueValues()) lock (gate) received.Add(v); + }; + subscription.AddItem(item); + await subscription.ApplyChangesAsync(ct); + + item.Status.Error.ShouldBeNull(); + return (subscription, received); + } + + /// Connects presenting the fixed apikey username with an otua_ credential. + /// The cancellation token. + /// The connected session. + public async Task ConnectAsync(CancellationToken ct) + { + var appConfig = new ApplicationConfiguration + { + ApplicationName = "OtOpcUa.AclKeyClient", + ApplicationUri = $"urn:OtOpcUa.AclKeyClient.{Guid.NewGuid():N}", + ApplicationType = ApplicationType.Client, + SecurityConfiguration = TestClientSecurity.Build(TestClientSecurity.AllocatePkiRoot()), + ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 60_000 }, + }; + await appConfig.ValidateAsync(ApplicationType.Client, ct); + appConfig.CertificateValidator.CertificateValidation += (_, e) => e.Accept = true; + + var endpoint = await CoreClientUtils.SelectEndpointAsync( + appConfig, $"opc.tcp://127.0.0.1:{_port}/OtOpcUa", false, DefaultTelemetry.Create(_ => { }), ct); + + return await new DefaultSessionFactory(DefaultTelemetry.Create(_ => { })).CreateAsync( + appConfig, + new ConfiguredEndpoint(null, endpoint, EndpointConfiguration.Create(appConfig)), + updateBeforeConnect: false, + checkDomain: false, + sessionName: "AclKeyExpiryEnforcementTests", + sessionTimeout: 60_000, + identity: new UserIdentity(new UserNameIdentityToken + { + UserName = "apikey", + // 1.5.378 carries the UserName token password as raw bytes; the host decodes UTF-8. + DecryptedPassword = Encoding.UTF8.GetBytes($"otua_{KeyId}_secret"), + }), + preferredLocales: null, + ct: ct); + } + + public async ValueTask DisposeAsync() + { + _timer?.Dispose(); + await _host.DisposeAsync(); + if (Directory.Exists(_pkiRoot)) + { + try { Directory.Delete(_pkiRoot, recursive: true); } + catch (IOException) { /* best effort */ } + } + } + + private static int AllocateFreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + /// + /// Admits the fixed apikey username as , carrying + /// . + /// + /// + /// Deliberately not the production ApiKeyOpcUaUserAuthenticator over a fake store: the + /// credential-verification path (parse, peppered HMAC, store lookup, deny shapes) is covered by + /// ApiKeyAuthenticationTests and BoundedApiKeyStoreTests in the Host suite, and + /// none of it changes what this file is about — whether the SDK hands a monitored item back an + /// identity that still carries the key id. Grants come from the deployed snapshot, so this + /// returns no roles. + /// + private sealed class KeyPresentingAuthenticator : IOpcUaUserAuthenticator + { + /// + public Task AuthenticateUserNameAsync( + string username, string password, CancellationToken ct) => + Task.FromResult( + string.Equals(username, "apikey", StringComparison.Ordinal) + && password.StartsWith($"otua_{KeyId}_", StringComparison.Ordinal) + ? OpcUaUserAuthResult.AllowKey(Principal, [], KeyId) + : OpcUaUserAuthResult.Deny("Invalid API key.")); + } + + /// A hand-advanced clock; the authorizer reads it on every decision. + private sealed class FakeClock(DateTime utcNow) : TimeProvider + { + private DateTimeOffset _now = new(utcNow, TimeSpan.Zero); + + /// + public override DateTimeOffset GetUtcNow() => _now; + + /// Moves the clock forward. + /// How far forward. + public void Advance(TimeSpan by) => _now = _now.Add(by); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.csproj b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.csproj index 65b05bdd..1ea5dbac 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.csproj +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.csproj @@ -25,6 +25,10 @@ + + From a9b592aeb8b32a530a1a4076aff6fb2b44b149c9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:04:03 -0400 Subject: [PATCH 22/45] docs(security): correct revocation latency (activation vs open session) + S18/S19 knobs --- CLAUDE.md | 32 ++++++- .../plans/R3-05-acl-hot-path-expiry-plan.md | 19 ++++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- docs/Configuration.md | 3 + docs/security.md | 94 +++++++++++++++---- 5 files changed, 129 insertions(+), 21 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ff22131d..76034579 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,7 +814,12 @@ Two things to know before relying on any of this: item is queued `BadUserAccessDenied` and then **detached from its `MonitoredNode2` fan-out — NOT set `MonitoringMode.Disabled`**: disabling discards the item's queued notifications, so the client is silenced without ever being told why (caught only by the over-the-wire test). Detaching is also - reversible, so restoring the grant resumes the feed with no client action. + reversible, so restoring the grant resumes the feed with no client action. ⚠️ **`Changed` is no longer + the only trigger (arch-review 03/S19):** `AclRevalidationTimer`, armed by `OtOpcUaServerHostedService` + at `Security:Acl:RevalidationIntervalSeconds` (default 60 s, `0` disables, inert while + `Security:Acl:Enabled` is false), runs the same pass on the clock — because `ExpiresUtc` passes with + the clock, not with a deployment. The pass's verdict memo is **pass-local and must stay that way**; + hoisting it across ticks would serve an expiry that landed between two ticks the earlier tick's allow. - **Anonymous is a normal grantable principal**, not a bypass. `OpcUa:AllowAnonymous` (default `true`, preserving today's behaviour) controls *admission*; when admitted the session authenticates as the reserved principal `anonymous` and gets roles from `NodeAcl` rows naming it, arriving at the gates as @@ -826,9 +831,28 @@ Two things to know before relying on any of this: deliberately *not* in the artifact — and `DeploymentArtifact.ParseDataPlaneAuthority` decodes them node-side alongside the cluster's `NodeAcl` rows. `OpcUaPublishActor` installs both on `IDataPlaneAuthoritySource` on every apply, **before** its `plan.IsEmpty` early-return, so a deployment - that only revokes a key still lands. A node with a ConfigDb keeps `ConfigDbApiKeyStore` (per-request - read); a driver-only node gets `ArtifactApiKeyStore`. ⚠️ **The consequence is asymmetric revocation: - immediate on a ConfigDb node, next-deployment on a driver-only one.** + that only revokes a key still lands. A node with a ConfigDb keeps `ConfigDbApiKeyStore` (per-activation + read, wrapped — see below); a driver-only node gets `ArtifactApiKeyStore`. ⚠️ **The asymmetry is + ACTIVATION-scoped, not session-scoped — this line used to say "immediate on a ConfigDb node" and that was + wrong.** Clearing `Enabled` in the ConfigDb stops the **next session activation** on an admin/fused node + (bounded by `Security:ApiKeys:StoreCacheSeconds`, default 5 s) and takes until the next deployment on a + driver-only one — but an **already-open** session re-checks its key against the in-memory *artifact + snapshot* (`DataPlaneAuthorizer.Evaluate` → `snapshot.FindUsableKey`), never against `IApiKeyStore`, so on + **both** node kinds an `Enabled` revocation reaches an open session only at the next deployment. + **Expiry is the exception**: `FindUsableKey` compares `ExpiresUtc` against the authorizer's clock on every + call, so it is immediate everywhere — and since arch-review 03/S19 it reaches live subscriptions too, on + the host's `Security:Acl:RevalidationIntervalSeconds` timer (default 60 s; the session is **silenced with + `BadUserAccessDenied`, not closed**, and recovers on re-activation). ⚠️ Event/alarm monitored items are + revalidated by neither trigger — pre-existing, and a named residual. +- ⚠️ **The activation lookup is BOUNDED (arch-review 03/S18).** The SDK raises impersonation synchronously + under a `SessionManager`-wide lock, so a ConfigDb that is *slow* rather than *refusing* stalled **every** + session activation on the server, Anonymous and X.509 included. `BoundedApiKeyStore` wraps the ConfigDb + store — and only it, never `ArtifactApiKeyStore`, which is two in-memory dictionary probes — with + `Security:ApiKeys:StoreTimeoutMs` (2000; exceeding it **denies**, via the authenticator's existing + catch-all) plus a `StoreCacheSeconds` (5) read-through cache. Faults and timeouts are never cached, a + cached hit whose `ExpiresUtc` has passed is re-read rather than served, and a timeout logs a distinct + Warning + `otopcua.opcua.apikey_store.bounded{outcome=timeout}`. Both knobs at `0` restore the old + behaviour exactly. - ⚠️ **The auth generation is a node-local COUNTER driven by the `RevisionHash`, not derived from its bits.** `PermissionTrieCache` treats the highest `GenerationId` as current, so a generation taken from hash bytes would install a *lower* number about half the time and the cache would go on serving the diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 88487801..420f325c 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -730,3 +730,22 @@ for the orchestrator. from the integration test project to `ZB.MOM.WW.OtOpcUa.Host` (the timer lives there). Without it the test would have called the node manager's pass directly and could not have falsified T8. - OpcUaServer.IntegrationTests: 20 green (was 18). + +### T10 — docs + +- `docs/security.md` store table: the "Revocation takes effect" column is now **two** columns — new + activation vs already-open session — with an explicit ⚠️ paragraph saying the old single column was + wrong and why (an open session re-checks the artifact snapshot, never `IApiKeyStore`), and that expiry + is the exception on both node kinds. +- `docs/security.md` new subsection **"Bounding the key store"** — the lock hazard, both knobs, what a + timeout looks like in the log, the counter to watch, why a timeout denies, and why faults are never + cached. +- `docs/security.md` revocation bullets: split into the **deployment axis** and the **clock axis**; the + live-subscription bullet now names **two triggers**, the timer's key and default, the silenced-not-closed + semantics with re-activation recovery, and the event-monitored-item residual. +- `CLAUDE.md`: the "asymmetric revocation" sentence corrected to activation-scoped (naming the old wording + as wrong), a new bullet for the S18 bound, and the stage-6 "Revocation reaches live subscriptions" bullet + extended with the second trigger and the pass-local-memo constraint. +- **Beyond the plan's file list:** `docs/Configuration.md` gained rows for all three new keys. It carries + the field-by-field tables for `Security:ApiKeys` and `Security:Acl`, so shipping the knobs without them + would have been the exact doc drift this plan's cross-cutting requirement is about. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index 1b459d39..46077638 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -76,7 +76,7 @@ { "id": "T10", "subject": "Docs: docs/security.md store table + revocation bullets, and the CLAUDE.md 'asymmetric revocation' sentence (activation vs already-open session)", - "status": "pending", + "status": "done", "blockedBy": [ "T5", "T8" diff --git a/docs/Configuration.md b/docs/Configuration.md index 2bccc843..5858bccb 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -79,12 +79,15 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only |---|---|---|---| | `Username` | string | `apikey` | The fixed username a key-bearing client sends in its OPC UA UserName token. Carries no information — the key identifies the principal — but is checked rather than ignored so a typo is an explicit denial. | | `Pepper` | string | `""` | Server-side pepper for the stored key hashes. **Supply via `Security__ApiKeys__Pepper` or `${secret:}` — never commit it.** No default on purpose: an empty pepper still hashes "successfully", so `ApiKeySecret.ComputeHash` throws on one rather than producing a deployment with none of the protection it believes it has. Rotating it invalidates every existing key. | +| `StoreTimeoutMs` | int | `2000` | Hard per-call deadline on the API-key store lookup that runs inline on session activation (arch-review 03/S18). Exceeding it **denies** the activation, logs a distinct Warning and meters `otopcua.opcua.apikey_store.bounded{outcome=timeout}`. `≤ 0` disables the deadline — do not, on a node with a ConfigDb: the SDK holds a `SessionManager`-wide lock across that lookup, so an unanswered query stalls **every** session activation on the server. Applies only to `ConfigDbApiKeyStore`; the in-memory artifact store on a driver-only node is not wrapped. | +| `StoreCacheSeconds` | int | `5` | TTL of the read-through cache in front of that store, so a burst of activations costs one round trip. `0` disables it (a store read per activation). Bounds how long an `Enabled` revocation authored in the ConfigDb takes to reach a **new** activation; it has no effect on already-open sessions (those re-check the artifact snapshot) and cannot delay an `ExpiresUtc` — a cached hit that has since expired is re-read rather than served. | **`Security:Acl`** key fields (`AclOptions`): | Key | Type | Default | Meaning | |---|---|---|---| | `Enabled` | bool | `false` | `true` makes scoped `NodeAcl` grants decide **write**, **alarm-command** and **HistoryRead** authorization, *replacing* the server-wide `WriteOperate` / `AlarmAck` role checks. `false` keeps those role checks and leaves HistoryRead ungated — byte-for-byte the pre-stage-5 behaviour. Read, Browse and Subscribe are ungated either way. | +| `RevalidationIntervalSeconds` | int | `60` | How often live monitored items are re-authorized against the clock, in addition to on every deployment (arch-review 03/S19). A key's `ExpiresUtc` passes with the clock, so without this an expired key kept feeding a live subscription until a deployment happened to land — indefinitely on a fleet deployed once and left running. `0` disables the timer (deploy-triggered only). **Inert while `Enabled` is false**, since the pass early-returns when enforcement is off. An affected session is silenced with `BadUserAccessDenied`, **not closed**, and recovers on re-activation. | > **Flipping `Security:Acl:Enabled` is a config change, not a redeploy** — both paths are wired in both modes (the node builds its permission trie on every deploy regardless), the same dark-switch discipline as [`MeshTransport`](#meshtransport), [`ConfigSource`](#configsource) and [`Telemetry`](#telemetry). It defaults off because the first deployment carrying the code would otherwise start denying every client that has no key, **including ScadaBridge's Data Connection Layer, which consumes this address space from another repo.** Enforcement fails closed on every unresolvable case, so a wiring or config mistake presents as *everything denied* — check the `OpcUaPublish: data-plane authority generation N installed` line for a non-zero grant count before debugging an individual rule. See [`security.md` § Node ACL enforcement](security.md#node-acl-enforcement-securityaclenabled). diff --git a/docs/security.md b/docs/security.md index 6872fbaf..512016c6 100644 --- a/docs/security.md +++ b/docs/security.md @@ -177,16 +177,59 @@ When no authenticator is supplied, `OpcUaApplicationHost` falls back to `NullOpc the Host wires the real `ApiKeyOpcUaUserAuthenticator` as a singleton in `Program.cs`. Which **`IApiKeyStore`** backs it depends on the node's role: -| Node | Store | Revocation takes effect | -|---|---|---| -| `admin` (including fused `admin,driver`) | `ConfigDbApiKeyStore` — reads the config database per request | On the **next request** | -| `driver`-only | `ArtifactApiKeyStore` — reads the keys carried by the deployment artifact, held in memory | On the **next deployment** | -| neither role | `NullOpcUaUserAuthenticator` — denies every UserName token | n/a | +| Node | Store | An `Enabled = 0` revocation reaches a **new activation** | …and an **already-open session** | +|---|---|---|---| +| `admin` (including fused `admin,driver`) | `ConfigDbApiKeyStore` — reads the config database per activation, wrapped in `BoundedApiKeyStore` | On the **next activation**, bounded by `Security:ApiKeys:StoreCacheSeconds` (default 5 s) | On the **next deployment** | +| `driver`-only | `ArtifactApiKeyStore` — reads the keys carried by the deployment artifact, held in memory | On the **next deployment** | On the **next deployment** | +| neither role | `NullOpcUaUserAuthenticator` — denies every UserName token | n/a | n/a | + +⚠️ **The last column is the correction, and it used to be missing.** This table once said a ConfigDb node +revokes "on the next request", which is true only of a **new session activation**. An operation on a +session that is *already open* does not re-read `IApiKeyStore` at all — `DataPlaneAuthorizer.Evaluate` +re-checks the key against the in-memory **artifact snapshot** (`DataPlaneAuthority.FindUsableKey`), which +`OpcUaPublishActor` installs on every apply. So clearing `Enabled` in the ConfigDb stops the *next* +connect on an admin node and nothing else until a deployment lands, on **both** node kinds. + +**Expiry is the exception, and it is immediate on both.** `FindUsableKey` compares `ExpiresUtc` against +the authorizer's clock on every call, so an `ExpiresUtc` that has already passed denies at once on an +open session — and since arch-review 03/S19 it reaches a live **subscription** too, on a timer (see +[Data-Plane Authorization](#data-plane-authorization)). The deny-all is registered **explicitly**, not left absent: `OtOpcUaServerHostedService` takes `IOpcUaUserAuthenticator` as a required constructor parameter, so an unregistered seam fails DI and the node does not boot at all. +### Bounding the key store (`StoreTimeoutMs` / `StoreCacheSeconds`) + +The SDK raises impersonation **synchronously, holding a `SessionManager`-wide event lock**, so the key +lookup does not stall one activation — it serializes **every** session activation on the server, +Anonymous and X.509 included. On a driver-only node that is harmless (the artifact store is two +dictionary probes). On an admin or fused node it is two EF round trips to central SQL, so a ConfigDb that +is *slow or hung* rather than *refusing* could hold that lock indefinitely. `BoundedApiKeyStore` +(`src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs`) wraps the ConfigDb store — and only it — +with two `Security:ApiKeys` knobs: + +| Key | Default | Meaning | +|---|---|---| +| `StoreTimeoutMs` | `2000` | Hard per-call deadline on the store. Exceeding it **denies** the activation. `≤ 0` disables the deadline (the pre-R3-05 unbounded behaviour — do not do this on an admin node). | +| `StoreCacheSeconds` | `5` | TTL of a read-through cache in front of the store, so a burst of activations costs one round trip. `0` disables it (a read per activation). | + +- **A timeout denies; it never admits.** The deadline surfaces as an `OperationCanceledException` into + `ApiKeyOpcUaUserAuthenticator`'s existing catch-all, which returns the same opaque denial as a bad + credential. There is no second denial path to get wrong, and the decorator can only ever return what + the inner store returned or throw. +- **What a timeout looks like:** a distinct Warning — *"ApiKeyStore: the key store did not answer within + N ms — denying this activation. Every OPC UA session activation on this server is serialized behind + this call; check ConfigDb health"* — plus + `otopcua.opcua.apikey_store.bounded{outcome=timeout}`. It is deliberately **not** the ordinary + *"no enabled, unexpired key with id X"* line, which would send an operator hunting for a key problem + that does not exist. +- **What to watch:** `otopcua.opcua.apikey_store.bounded{outcome=cache_hit}`. A busy fused central with no + `cache_hit` stream means `StoreCacheSeconds` is 0. +- **Faults and timeouts are never cached** — caching a deny would extend an outage past the outage. Only + a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has + since passed is re-read rather than served, so the TTL can never resurrect an expired key. + ### LDAP bind flow (`OtOpcUaLdapAuthService`) LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)). @@ -411,17 +454,36 @@ Behaviours worth knowing before enabling it: - **A deploy takes effect on sessions that are already open.** The gate stamps each decision with the current authority generation rather than the one the session activated under, so an ACL edit lands without bouncing clients — in both directions, widening and narrowing. -- **Revocation also reaches an open session.** Every decision re-checks the authenticating key against - the current snapshot, so a key disabled or expired by a later deployment stops working immediately - rather than at the client's next reconnect. On a driver-only node the artifact is the only revocation - channel there is, so without this a revoked credential could keep its authority indefinitely. -- **Revocation reaches a live subscription too** (#520 stage 6). Every other operation re-asks on each - request; a subscription is authorized once and then streams, so it needed a push. A deployment - re-authorizes every live monitored item: one whose grant has gone is queued a `BadUserAccessDenied` - and then detached from its node's value fan-out, and a later deployment that restores the grant - re-attaches it. This matters more than it sounds — subscriptions are how machine consumers actually - read this server, so without it revoking a grant would close the operations nobody uses and leave the - feed running. +- **Revocation also reaches an open session — but read the two axes separately.** Every decision + re-checks the authenticating key against the current snapshot, so an open session is re-authorized on + every operation. What that check can *see* differs by axis: + - **The deployment axis** (a key's `Enabled` cleared, a grant edited or removed). The snapshot only + changes when an artifact is applied, so this lands **on the next deployment**, on both node kinds — + including on a fused admin node, whose ConfigDb store is consulted at *activation* only. On a + driver-only node the artifact is the only revocation channel there is. + - **The clock axis** (a key's `ExpiresUtc` passing). `FindUsableKey` compares it against the + authorizer's clock on every call, so this lands **immediately**, with no deployment. +- **Revocation reaches a live subscription too** (#520 stage 6, extended by arch-review 03/S19). Every + other operation re-asks on each request; a subscription is authorized once and then streams, so it + needed a push. A denied item is queued a `BadUserAccessDenied` and then detached from its node's value + fan-out; restoring its authority re-attaches it. There are **two triggers**: + - a **deployment**, which re-authorizes every live monitored item (the deployment axis); + - the host's **low-frequency timer**, `Security:Acl:RevalidationIntervalSeconds` (default 60 s, `0` + disables it, inert while `Security:Acl:Enabled` is false). Without it a key that expired by the + clock kept streaming until a deployment happened to land — **indefinitely** on a fleet that has been + deployed once and left running, which is exactly the shape a production fleet settles into. + + ⚠️ **An expired-key session is silenced, not disconnected.** Its items go `BadUserAccessDenied` and the + session stays open, deliberately: a clock-driven denial must look to a client exactly like a + deploy-driven one, OPC UA has no honest "expire this session" verb (deleting it server-side gives the + client a transport failure rather than an authorization status), and every other operation on that + session already denies. **A client that re-activates with a renewed key recovers on the next tick with + no reconnect** — closing the session instead would force a full reconnect-and-resubscribe on every + machine consumer in the fleet at once when a shared key expired. + + ⚠️ **Event/alarm monitored items are revalidated by neither trigger.** The live-item map is fed only + from data-change items, so an alarm subscription is gated at creation and never again. Pre-existing — + the deployment trigger has the same hole — and tracked as a residual of arch-review 03/S19. - **An ancestor of a granted node is browsable**, or no narrow grant would be reachable: a client arrives at a deep node by browsing its ancestors, and the trie otherwise inherits downward only. Note what this necessarily discloses — the *names* of the folders on the path to a granted node are visible From 5c7e5725a5baabe9bf969d88b3ba80f9014cd604 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:05:45 -0400 Subject: [PATCH 23/45] =?UTF-8?q?test(runtime):=20first=20supervision-rest?= =?UTF-8?q?art=20test=20=E2=80=94=20child=20re-initialises=20+=20re-subscr?= =?UTF-8?q?ibes=20(archreview=2003/U6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crash seam: the planned null-Snapshot OnDataChange (no SupervisorStrategy fallback needed). Seen RED with T6 reverted: InitializeCount 1, not 2 — with a preceding ShutdownCount>=1 assertion proving the restart really happened, so the red is not ambiguous with 'never crashed'. --- ...R3-03-restart-resubscribe-watchdog-plan.md | 35 ++++++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- .../Drivers/DriverHostActorRestartTests.cs | 107 ++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index 6ab0d180..4b50050c 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -605,3 +605,38 @@ accept `RestartDriver`. The falsification is T7, below. Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **183/183 passed** (179 before this task plus T5's four), no test moved. + +### T7 — first supervision-restart test + the force-reconnect verified negative (03/U6) + +**Crash seam that landed: the planned one** — `RecordingSubscribableDriver.FireMalformedDataChange` +raises `OnDataChange` with a null `Snapshot`; `DriverInstanceActor.OnDataChangeForward` dereferences +`msg.Snapshot.SourceTimestampUtc` on the actor thread and NREs; Akka's default decider Restarts. No +fallback (`OneForOneStrategy` + an internal test message) was needed, so no production surface was +added for the test. + +**RED observed against a T6-reverted build** (the `Context.Parent.Tell(new ReprovisionRequested(…))` +in `PostRestart` commented out): + +``` +Supervision_restart_reinitialises_and_re_subscribes_the_child [FAIL] + driver.InitializeCount should be 2 but was 1 + "the restarted child was never re-initialised — it is sitting in Connecting around a driver + its own PostStop already shut down (03/S6, the worse half of 03/S16)" +``` + +**Deviation, and the reason for it — the first red run was AMBIGUOUS and the test was strengthened.** +`InitializeCount == 1` is equally consistent with "the child restarted and was not re-provisioned" +and with "the child never crashed at all", so a seam that silently stopped working would read as a +passing regression test forever. A preceding `AwaitAssert(ShutdownCount >= 1)` was added — `PreRestart` +runs `PostStop`, which calls `ShutdownAsync`, so that is the evidence a restart genuinely happened. +On the re-run of the falsification that assertion **passed** and the failure landed squarely on the +re-initialise, which is what makes the red observation above meaningful. + +`Force_reconnect_keeps_the_desired_set_and_re_subscribes` passed in **both** the red and green runs — +correct, and the point: it is the verified negative (`ForceReconnect` reaches the same live actor, so +`_desiredRefs` survive and the reconnect's Connected entry re-applies them). It carries a 30 s budget +rather than `PresenceBudget`, because `SpawnChild` gives the child the production +`DefaultReconnectInterval` (10 s) and the retry is a periodic timer, so the reconnect cannot land +sooner; a budget is only spent on failure. + +After restoring the hook: **6/6 pass**, and the class was run **20× in isolation, 20/20 green**. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 0e1d6c9f..625327b6 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -48,7 +48,7 @@ { "id": "T7", "subject": "First real supervision-restart test + force-reconnect verified-negative regression (03/U6)", - "status": "pending", + "status": "done", "blockedBy": [ "T4", "T6" diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs index 621213c1..e8abff55 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs @@ -284,4 +284,111 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase }, duration: PresenceBudget); } + + /// + /// The repo's first genuine Akka SUPERVISION-restart test (03/U6). Not a stop, not a stub: the + /// driver raises a malformed OnDataChange, the actor NREs on its own thread, and Akka's default + /// decider Restarts the child in place. + /// + /// A restart is strictly worse than the operator-Restart case 03/S16 names. It recreates the actor + /// object around the SAME driver instance without going through SpawnChild, so nothing + /// re-sends InitializeRequested — while PreRestart has already run PostStop, + /// which called ShutdownAsync on that very driver. The child therefore sat in + /// Connecting forever around a shut-down driver, with its desired set gone too, and + /// silently: Akka logs the fault, nothing logs the non-recovery. + /// + /// + /// The assertions are FUNCTIONS of the new incarnation — it re-initialised, it re-subscribed the + /// deployed refs — never that it exists. Created.Count == 1 is what distinguishes this from + /// the operator path: a restart must NOT produce a second driver object. + /// + /// + [Fact] + public void Supervision_restart_reinitialises_and_re_subscribes_the_child() + { + var db = NewInMemoryDbFactory(); + var (_, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeCount.ShouldBe(1); + }, + duration: PresenceBudget); + + var driver = factory.Created[0]; + driver.InitializeCount.ShouldBe(1); + + // The crash. Subscribing is what attaches the actor's OnDataChange handler, so this only reaches the + // actor thread because the arrangement above got as far as a live subscription. + driver.FireMalformedDataChange(SpeedPath); + + // FIRST: prove the crash seam actually tripped supervision, before asserting anything about the + // recovery. PreRestart runs PostStop, which calls ShutdownAsync on the driver — so a raised + // ShutdownCount is the evidence that a restart happened at all. Without this the assertions below + // would be ambiguous: "the child was not re-provisioned" and "the child never crashed" look + // identical from the outside, and a seam that silently stopped working would read as a passing + // regression test forever. + AwaitAssert( + () => driver.ShutdownCount.ShouldBeGreaterThanOrEqualTo( + 1, "the crash seam did not trip supervision — nothing below is testing a restart"), + duration: PresenceBudget); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe( + 1, "a supervision restart must re-use the same driver object, not respawn the child"); + driver.InitializeCount.ShouldBe( + 2, + "the restarted child was never re-initialised — it is sitting in Connecting around a " + + "driver its own PostStop already shut down (03/S6, the worse half of 03/S16)"); + driver.SubscribeCount.ShouldBeGreaterThanOrEqualTo( + 2, "the restarted child never re-subscribed — its desired set died with the old instance"); + driver.LastSubscribedRefs.ShouldBe(new[] { SpeedPath }); + }, + duration: PresenceBudget); + } + + /// + /// The verified negative, pinned so this plan's edits to the restart paths cannot break the one that + /// was already correct. Force Reconnect (ReconnectDriver) Tells ForceReconnect to the + /// SAME child actor: the instance survives, so _desiredRefs/_desiredAlarmRefs are intact + /// and the reconnect's Connected entry re-applies them by itself. Nothing needs re-provisioning, and + /// no new driver object is created. + /// + /// + /// The budget here is deliberately wider than : + /// SpawnChild gives the child the production DefaultReconnectInterval (10 s) and the + /// retry is a periodic timer, so the reconnect cannot land sooner than that. A budget is spent only + /// when the assertion fails, so widening it costs nothing on the happy path. + /// + [Fact] + public void Force_reconnect_keeps_the_desired_set_and_re_subscribes() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeCount.ShouldBe(1); + }, + duration: PresenceBudget); + + var driver = factory.Created[0]; + host.Tell(new ReconnectDriver("c1", DriverId, "tester", Guid.NewGuid())); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1, "Force Reconnect must NOT respawn the child"); + driver.SubscribeCount.ShouldBeGreaterThanOrEqualTo( + 2, "the reconnected driver never re-subscribed its (surviving) desired set"); + driver.LastSubscribedRefs.ShouldBe(new[] { SpeedPath }); + }, + duration: TimeSpan.FromSeconds(30)); + } } From 14094e3a187612667380e648cd0a428532552315 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:06:00 -0400 Subject: [PATCH 24/45] docs(archreview): file R3-05 residuals R1/R2 (Gitea #562/#563); execution notes --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 32 +++++++++++++++++++ ...-05-acl-hot-path-expiry-plan.md.tasks.json | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 420f325c..6352f8f7 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -749,3 +749,35 @@ for the orchestrator. - **Beyond the plan's file list:** `docs/Configuration.md` gained rows for all three new keys. It carries the field-by-field tables for `Security:ApiKeys` and `Security:Acl`, so shipping the knobs without them would have been the exact doc drift this plan's cross-cutting requirement is about. + +### T12 — residuals filed + +- **R1** → Gitea **#562** — *ACL: event/alarm monitored items are revalidated by neither trigger* + (`https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/562`). +- **R2** → Gitea **#563** — *ACL: an expired API key keeps its session's roles while + `Security:Acl:Enabled` is false* (`https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/563`). +- **R3** (activation-time revocation now bounded by `StoreCacheSeconds`) needs no issue — it is documented + behaviour after T10, in both `docs/security.md` and `docs/Configuration.md`. + +### T11 — LIVE GATE + +**Left `pending` for the orchestrator**, per the execution brief. Nothing in this branch has been run +against the docker-dev rig; the nine-step recipe in Finding 2 → Live gate is unchanged and still applies. +Two notes for whoever runs it: + +- `Security:Acl:RevalidationIntervalSeconds` defaults to 60, so step 6's "within one revalidation + interval" is ≤ 60 s unless the rig overrides it. Setting it to `15` shortens the wall-clock wait without + changing what is proven. +- Step 8's cache-hit leg reads `otopcua.opcua.apikey_store.bounded{outcome=cache_hit}`; the instrument only + exists on an **admin/fused** node, since the artifact store is not wrapped. + +### Summary of what a reviewer should look at + +1. **`Program.cs` → `DataPlaneAuthRegistration`** (T1): a verbatim move of the authorizer + key-store + branches out of top-level statements so they could be guarded. Confirm no branch changed. +2. **`OtOpcUaServerHostedServiceRevalidationTests`** (T8): the plan's dispose-**before**-`SetDataPlaneAuthorizer(null)` + ordering is guaranteed structurally but **not asserted** — see T8's notes for why it is unobservable. +3. **`BoundedApiKeyStore.IsExpired`** (T3): a two-line fail-closed addition beyond the plan, so the TTL + cannot resurrect a clock-expired key. +4. New public members added purely so wiring could be pinned by reference: `DataPlaneAuthorizer.Clock`, + `BoundedApiKeyStore.Inner` / `.Clock`. diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json index 46077638..82045b5b 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md.tasks.json @@ -94,7 +94,7 @@ { "id": "T12", "subject": "File residuals R1 (event monitored items revalidated by nothing) and R2 (expired key retains roles while Security:Acl:Enabled=false)", - "status": "pending", + "status": "done", "blockedBy": [] } ] From 458c84c34f0c8136f4d24a55aff5f80ed98c4d46 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:06:47 -0400 Subject: [PATCH 25/45] test(host): mesh gRPC boundary under the production auth stack (FallbackPolicy live) MeshGrpcAuthStackTests - the CI-shaped guard 03/S14 asked for. A real AddOtOpcUaAuth(config) + UseAuthentication/UseAuthorization pipeline (cookie scheme, DisableLogin=false, EnvironmentName=Production) with the production interceptor set, driving DeploymentArtifactService.Fetch, CentralCommandService.ReportApply and NodeCommandService.Execute through MapOtOpcUaMeshGrpcServices. Verified red-then-green by hand: with the T1 (pre-AllowAnonymous) mapper swapped in, all three positive calls fail with "HTTP/2 server reset the stream, PROTOCOL_ERROR" (RpcException/Internal) - the cookie challenge breaking prior-knowledge h2c, not a clean auth denial. With the current mapper (T2/T3) all three succeed. That asymmetry is the only thing distinguishing the two builds; every other assertion here (cookie handler identity, the falsifiability control) is unchanged by the fix and proves nothing on its own. Falsifiability control: LocalDbSyncService is mapped directly (bypassing MapOtOpcUaMeshGrpcServices) with no .AllowAnonymous(), and a correctly-shaped call to it is refused - proving the fixture's auth stack is genuinely live, not accidentally permissive. --- .../MeshGrpcAuthStackTests.cs | 306 ++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs new file mode 100644 index 00000000..d55a27d3 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs @@ -0,0 +1,306 @@ +using System.Diagnostics.CodeAnalysis; +using System.Net; +using Akka.Actor; +using Akka.Hosting; +using Grpc.Core; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shouldly; +using Xunit; +using ZB.MOM.WW.LocalDb.Replication; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Runtime; +using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry; +using ZB.MOM.WW.OtOpcUa.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// R3-01 T4 — the CI-shaped guard that would have caught 03/S14: mesh gRPC calls through the +/// REAL AddOtOpcUaAuth + UseAuthentication/UseAuthorization stack, not a +/// hand-built pipeline that never installs the middleware that broke production. +/// +/// +/// +/// The load-bearing assertion is the ALLOW. A blocked call and a refused call both +/// "fail"; only a correctly-keyed call SUCCEEDING through the real cookie +/// FallbackPolicy proves the .AllowAnonymous() opt-outs (T2/T3) actually work in +/// a pipeline that carries the policy at all. Every positive test here failed before T2/T3 — +/// confirmed by running this class against the T1 commit, where every one of them was +/// challenged by the cookie handler instead of reaching the interceptor. +/// +/// +/// Negative control (anti-vacuity). is mapped directly +/// here — via a bare MapGrpcService, deliberately NOT through +/// MapOtOpcUaMeshGrpcServices — with no .AllowAnonymous(). A correctly-shaped +/// call to it must be refused by the cookie challenge. If it were not, the fixture's auth +/// stack would be inert and every positive assertion above would be proving nothing. +/// +/// +public sealed class MeshGrpcAuthStackTests : IAsyncLifetime +{ + private const string ConfigServeKey = "test-configserve-key"; + private const string NodeCommandKey = "test-node-psk"; + private const string CentralCommandKey = "test-central-psk"; + private const string TelemetryKey = "test-telemetry-key"; + private const string ClusterId = "MAIN"; + + private WebApplication _app = null!; + private ActorSystem _system = null!; + private int _port; + private TaskCompletionSource _centralInbox = null!; + private TaskCompletionSource _nodeInbox = null!; + + /// + public async ValueTask InitializeAsync() + { + var builder = WebApplication.CreateBuilder(); + builder.Environment.ApplicationName = typeof(MeshGrpcAuthStackTests).Assembly.GetName().Name!; + // Production, not the WebApplicationBuilder default: forces ValidateOnBuild/ValidateScopes OFF + // (they engage only under Development), so services never resolved without a login attempt + // (ILdapAuthService's graph) don't need to be fully stood up for this fixture to build. + builder.Environment.EnvironmentName = Environments.Production; + builder.Logging.ClearProviders(); + + // DataProtection's PersistKeysToDbContext (inside AddOtOpcUaAuth) + // resolves a SCOPED OtOpcUaConfigDbContext via CreateScope() + GetRequiredService — it has no + // factory support. Mirrors AddOtOpcUaConfigDb's own factory+scoped-bridge shape, over InMemory. + var dbName = Guid.NewGuid().ToString("N"); + builder.Services.AddDbContextFactory(o => o.UseInMemoryDatabase(dbName)); + builder.Services.AddScoped(sp => + sp.GetRequiredService>().CreateDbContext()); + + builder.Configuration.AddInMemoryCollection(new Dictionary + { + // The whole point of this fixture: exercise the REAL cookie handler, not + // AutoLoginAuthenticationHandler. See CookieSchemeIsTheRealHandler_NotAutoLogin. + ["Security:Auth:DisableLogin"] = "false", + // This fixture serves cleartext h2c; a HTTPS-only cookie policy is irrelevant to what is + // under test (the FallbackPolicy applies identically either way) and would otherwise log + // a startup Warning for no reason. + ["Security:Cookie:RequireHttpsCookie"] = "false", + }); + + builder.Services.AddOtOpcUaAuth(builder.Configuration); + + // Interceptor options bound directly (NOT via AddOtOpcUaCluster, which also wires + // AkkaClusterOptions/ConfigSourceOptions/TelemetryDialOptions validators with ValidateOnStart — + // heavy machinery this fixture has no use for and no seed-node config to satisfy). + builder.Services.Configure(o => o.ApiKey = ConfigServeKey); + builder.Services.Configure(o => o.GrpcPsk = NodeCommandKey); + builder.Services.Configure(o => o.ApiKey = TelemetryKey); + builder.Services.AddSingleton(new FixedPskProvider(ClusterId, CentralCommandKey)); + builder.Services.AddSingleton(); + + _system = ActorSystem.Create("mesh-auth-stack-" + Guid.NewGuid().ToString("N")); + _centralInbox = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _nodeInbox = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var registry = new ActorRegistry(); + registry.Register(_system.ActorOf(CaptureActor.Props(_centralInbox))); + registry.Register(_system.ActorOf(CaptureActor.Props(_nodeInbox))); + builder.Services.AddSingleton(registry); + + // The production interceptor set — the ONLY inbound auth on the gRPC side. Deliberately does + // NOT include LocalDbSyncAuthInterceptor: LocalDbSyncService is mapped below purely as the + // negative control and is never given a matching interceptor here. + builder.Services.AddGrpc(o => + { + o.Interceptors.Add(); + o.Interceptors.Add(); + o.Interceptors.Add(); + o.Interceptors.Add(); + }); + + _port = GetFreePort(); + builder.WebHost.ConfigureKestrel(k => k.ListenAnyIP(_port, o => o.Protocols = HttpProtocols.Http2)); + + _app = builder.Build(); + _app.UseAuthentication(); + _app.UseAuthorization(); + + // The production mapper — the four chainable maps carry AllowAnonymous (T2); LocalDbSync is + // NOT mapped here (syncListenPort: 0) so it stays free for the negative control below without + // a route collision. + _app.MapOtOpcUaMeshGrpcServices( + hasAdmin: true, hasDriver: true, + syncListenPort: 0, configServeGrpcPort: 1, telemetryListenPort: 1); + + // Negative control (T4): mapped directly, NOT through MapOtOpcUaMeshGrpcServices, and + // deliberately with no .AllowAnonymous(). + _app.MapGrpcService(); + + await _app.StartAsync(TestContext.Current.CancellationToken); + } + + /// + public async ValueTask DisposeAsync() + { + await _app.StopAsync(TimeSpan.FromSeconds(10)); + await _app.DisposeAsync(); + await _system.Terminate().WaitAsync(TimeSpan.FromSeconds(15)); + } + + [Fact] + public async Task CookieSchemeIsTheRealHandler_NotAutoLogin() + { + // If this resolves AutoLoginAuthenticationHandler, every assertion below is meaningless — the + // fallback policy would pass every request regardless of AllowAnonymous. + var provider = _app.Services.GetRequiredService(); + var scheme = await provider.GetSchemeAsync(CookieAuthenticationDefaults.AuthenticationScheme); + + scheme.ShouldNotBeNull(); + scheme.HandlerType.ShouldBe(typeof(CookieAuthenticationHandler)); + } + + [Fact] + public async Task CorrectlyKeyed_ArtifactFetch_SucceedsThroughTheRealAuthStack() + { + var deploymentId = Guid.NewGuid(); + var blob = "R3-01-T4-artifact-bytes"u8.ToArray(); + var factory = _app.Services.GetRequiredService>(); + await using (var db = await factory.CreateDbContextAsync(TestContext.Current.CancellationToken)) + { + db.Deployments.Add(new Deployment + { + DeploymentId = deploymentId, + RevisionHash = new string('a', 64), + CreatedBy = "r3-01-t4", + ArtifactBlob = blob, + }); + await db.SaveChangesAsync(TestContext.Current.CancellationToken); + } + + var client = new DeploymentArtifactService.DeploymentArtifactServiceClient(Channel()); + var headers = new Metadata { { "authorization", $"Bearer {ConfigServeKey}" } }; + + using var call = client.Fetch( + new FetchRequest { DeploymentId = deploymentId.ToString() }, + headers, + cancellationToken: TestContext.Current.CancellationToken); + + using var buffer = new MemoryStream(); + await foreach (var chunk in call.ResponseStream.ReadAllAsync(TestContext.Current.CancellationToken)) + chunk.Data.WriteTo(buffer); + + buffer.ToArray().ShouldBe(blob); + } + + [Fact] + public async Task CorrectlyKeyed_CentralCommandReportApply_SucceedsThroughTheRealAuthStack() + { + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("node-a"), + ApplyAckOutcome.Applied, + FailureReason: null, + CorrelationId.NewId()); + + var client = new CentralCommandService.CentralCommandServiceClient(Channel()); + var headers = new Metadata + { + { "authorization", $"Bearer {CentralCommandKey}" }, + { "x-otopcua-cluster", ClusterId }, + }; + + await client.ReportApplyAsync( + MeshTransportDtoMapper.ToProto(ack), headers, cancellationToken: TestContext.Current.CancellationToken); + + var received = await _centralInbox.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + received.ShouldBeOfType().DeploymentId.ShouldBe(ack.DeploymentId); + } + + [Fact] + public async Task CorrectlyKeyed_NodeCommandExecute_SucceedsThroughTheRealAuthStack() + { + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + var client = new NodeCommandService.NodeCommandServiceClient(Channel()); + var headers = new Metadata { { "authorization", $"Bearer {NodeCommandKey}" } }; + + await client.ExecuteAsync( + MeshTransportDtoMapper.ToEnvelope(command), headers, cancellationToken: TestContext.Current.CancellationToken); + + var received = await _nodeInbox.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + received.ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1"); + } + + [Fact] + public async Task FalsifiabilityControl_AnUnAnonymisedServiceMapped_RefusesEvenACorrectlyShapedCall() + { + // LocalDbSyncService carries no AllowAnonymous in this fixture (see InitializeAsync). If this + // call succeeds, the fixture's auth stack is not actually challenging unmarked endpoints and + // every "succeeds through the real auth stack" assertion above proves nothing. + var client = new ZB.MOM.WW.LocalDb.Contracts.LocalDbSync.LocalDbSyncClient(Channel()); + var headers = new Metadata { { "authorization", "Bearer irrelevant-to-this-endpoint" } }; + + using var call = client.Sync(headers, cancellationToken: TestContext.Current.CancellationToken); + await call.RequestStream.CompleteAsync(); + + await Should.ThrowAsync( + () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); + } + + // --------------------------------------------------------------------------------------- + // harness + // --------------------------------------------------------------------------------------- + + private GrpcChannel Channel() => GrpcChannel.ForAddress($"http://localhost:{_port}"); + + private static int GetFreePort() + { + using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + /// + /// Hands out the one cluster key expects. Stands + /// in for the real MeshPskProvider (secret-store lookups, caching, the known-cluster + /// gate) — out of scope here, where the subject is the HTTP-layer auth stack, not the PSK + /// resolution chain (that is MeshPskProviderTests' job). + /// + private sealed class FixedPskProvider(string clusterId, string psk) : IMeshPskProvider + { + public bool TryGetPsk(string clusterId2, [NotNullWhen(true)] out string? psk2) + { + psk2 = string.Equals(clusterId2, clusterId, StringComparison.Ordinal) ? psk : null; + return psk2 is not null; + } + + public bool Matches(string clusterId2, string presented) => + string.Equals(clusterId2, clusterId, StringComparison.Ordinal) + && string.Equals(presented, psk, StringComparison.Ordinal); + + public void SetKnownClusters(IReadOnlyCollection clusterIds) { } + + public void Invalidate(string clusterId2) { } + + public void InvalidateAll() { } + } + + /// Completes an inbox with the first message it receives. + private sealed class CaptureActor : ReceiveActor + { + public static Props Props(TaskCompletionSource sink) => + Akka.Actor.Props.Create(() => new CaptureActor(sink)); + + public CaptureActor(TaskCompletionSource sink) => ReceiveAny(m => sink.TrySetResult(m)); + } +} From af9d3a69ac9d78632a0651abd08d8f34e55dcfd0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:06:55 -0400 Subject: [PATCH 26/45] docs(plans): mark R3-01 T4 done --- .../R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 95fe4701..b4413b6c 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -27,7 +27,7 @@ { "id": "T4", "subject": "MeshGrpcAuthStackTests: PSK-bearing calls succeed through the REAL AddOtOpcUaAuth + UseAuthentication/UseAuthorization stack, with an un-anonymised negative control", - "status": "pending", + "status": "done", "blockedBy": [ "T3" ] @@ -42,7 +42,7 @@ }, { "id": "T6", - "subject": "MeshGrpcDriverOnlyPipelineTests: driver-only nodes carry no authorization middleware — pins the 03/S14 exemption", + "subject": "MeshGrpcDriverOnlyPipelineTests: driver-only nodes carry no authorization middleware \u2014 pins the 03/S14 exemption", "status": "pending", "blockedBy": [ "T3" @@ -50,7 +50,7 @@ }, { "id": "T7", - "subject": "MeshGrpcMapConventionTests: Program.cs source scan — zero direct MapGrpcService/MapZbLocalDbSync sites, exactly one mapper call", + "subject": "MeshGrpcMapConventionTests: Program.cs source scan \u2014 zero direct MapGrpcService/MapZbLocalDbSync sites, exactly one mapper call", "status": "pending", "blockedBy": [ "T1" @@ -70,7 +70,7 @@ }, { "id": "T10", - "subject": "LIVE GATE: docker-dev with DisableLogin=false — step-0 falsification control MUST fail on the pre-fix build (Deployment.Status=4), then the fixed build seals Applied on all six nodes; restore DisableLogin=true", + "subject": "LIVE GATE: docker-dev with DisableLogin=false \u2014 step-0 falsification control MUST fail on the pre-fix build (Deployment.Status=4), then the fixed build seals Applied on all six nodes; restore DisableLogin=true", "status": "pending", "blockedBy": [ "T3", From 12c402847411d7a709fdf16986cd9e90467742bf Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:07:20 -0400 Subject: [PATCH 27/45] docs: restart re-provisioning + per-connection watchdog grace (R3-03) --- ...R3-03-restart-resubscribe-watchdog-plan.md | 13 +++ ...rt-resubscribe-watchdog-plan.md.tasks.json | 2 +- docs/DriverLifecycle.md | 79 ++++++++++++++++++- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index 4b50050c..f8043aa9 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -640,3 +640,16 @@ rather than `PresenceBudget`, because `SpawnChild` gives the child the productio sooner; a budget is only spent on failure. After restoring the hook: **6/6 pass**, and the class was run **20× in isolation, 20/20 green**. + +### T8 — docs + +`docs/DriverLifecycle.md`: + +- New §"What a (re)spawn re-provisions" (placed before §IDriverHealthPublisher): `_desiredByDriver` as + the single authority, a table of the three incarnation routes plus the verified-negative Force + Reconnect, the supervision route's invisibility, why replacing the default decider is out of scope, + the respawning-deploy ordering **with the measured caveat that the transient is not inert**, and the + assert-a-function-not-existence rule naming the weaker existing test. +- §"The comms-loss watchdog": conjunct 2 restated as the max, plus a new paragraph on the + per-connection grace — why `??` was wrong, that it **paces** rather than ends the half-broken-device + cycle, that the watchdog is not disabled by the change, and the two tests that pin each half. diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json index 625327b6..5e1d2dab 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md.tasks.json @@ -57,7 +57,7 @@ { "id": "T8", "subject": "docs/DriverLifecycle.md: what a (re)spawn re-provisions + per-connection watchdog grace", - "status": "pending", + "status": "done", "blockedBy": [] }, { diff --git a/docs/DriverLifecycle.md b/docs/DriverLifecycle.md index 3e4519db..c09fafce 100644 --- a/docs/DriverLifecycle.md +++ b/docs/DriverLifecycle.md @@ -164,6 +164,62 @@ mostly reusable — what was always missing is the trigger. --- +## What a (re)spawn re-provisions + +A driver child can come into existence by **three** routes, and until 2026-08-20 only the first of them +handed the child everything it needs to be a working driver. That is archreview **03/S16** (and its worse +half, **03/S6**): a child that exists without its desired subscription set is a *dead* driver — it +connects, reports `Connected`, publishes nothing and raises no alarms, and the #488 periodic reconcile +cannot rescue it (it early-returns on an empty desired set). The fix makes the desired set **part of what +it means to have a child** rather than a later step in the apply pipeline. + +**`DriverHostActor._desiredByDriver` is the single authority.** It is a per-driver +`(ValueRefs, AlarmRefs)` pair, cleared and repopulated **whole** on every apply in the same pass that +rebuilds the routing maps — keyed over the union of the value-ref map, the alarm-ref map and the running +`_children`, so a driver with no authored tags gets an explicit **empty** set rather than being absent. +Everything that hands a child a `SetDesiredSubscriptions` goes through the one helper +`SendDesiredSubscriptions(actor, driverId)`. (An empty set is meaningful, not a no-op to skip: the child +routes it to `Unsubscribe`, dropping a stale handle.) + +| Route | Trigger | What re-provisions it | +|---|---|---| +| **Deploy reconcile** | `ApplyDriverSpecs` → `SpawnChild` | `SpawnChild` sends `InitializeRequested` then the desired set; the apply's bulk push then sends the (freshly rebuilt) set to every child. | +| **Operator Restart** | `/hosts` → Restart → `HandleRestartDriver` | Stops the child, respawns from the cached spec — and `SpawnChild` sends the desired set, because it always does. **This is the 03/S16 fix**: there is no bulk push on this path, and before it the respawned child subscribed to nothing until the next deployment. | +| **Supervision restart** | a child throws ⇒ Akka's **default decider** ⇒ `Restart` | `DriverInstanceActor.PostRestart` Tells the host `ReprovisionRequested`; `HandleReprovisionRequested` logs a Warning naming the crash type, re-sends `InitializeRequested` (non-stub children) and the desired set. | +| **Force Reconnect** | `/hosts` → Force Reconnect → `HandleReconnectDriver` | **Nothing — verified.** It Tells `ForceReconnect` to the *same* child actor, so the instance and its `_desiredRefs`/`_desiredAlarmRefs` survive and the reconnect's Connected entry re-applies them itself. Pinned by `Force_reconnect_keeps_the_desired_set_and_re_subscribes` so a future edit to the restart paths cannot break the one that was already correct. | + +Things to know before touching any of this: + +- ⚠️ **The supervision route is the invisible one, and it was the worst.** There is **no** + `SupervisorStrategy` anywhere under `src/Server`, so a child that throws gets Akka's default `Restart`. + A restart recreates the actor object around the **same** `IDriver` — the Props closure captured it — + and its `PreRestart` runs `PostStop`, which calls `IDriver.ShutdownAsync`. It does **not** run + `SpawnChild`, and the child's `IActorRef` is unchanged, so the host had no notification a new + incarnation existed at all. Net effect before the hook: the child sat in `Connecting` forever around a + shut-down driver, with its desired set gone, and nothing logged the non-recovery. +- **Replacing the default decider is deliberately out of scope.** A `Stop` directive would silently + remove a driver; a bounded retry count would silently stop one after N crashes. Both change failure + semantics for every driver in the fleet and deserve their own plan and live gate. The reprovision hook + makes the *current* infinite-restart behaviour correct instead of dead. +- **Ordering on a respawning deploy, and its one accepted cost.** `ApplyDriverSpecs` (which spawns) runs + before the pass that rebuilds `_desiredByDriver`, so a child respawned by a deploy — raw tags are folded + into the merged `DriverConfig`, so a tag-set edit changes the spec and trips the #516 config respawn — + is first handed the **previous** apply's set, then the new one by the bulk push. Akka's per-pair + ordering guarantees it **converges**; it does not guarantee the transient is inert, because the child + runs concurrently with the rest of the apply handler and a fast-initialising driver can reach + `Connected` and subscribe the stale set first. The cost is one extra subscribe round-trip + (`HandleSubscribeAsync` drops the prior subscription before establishing the new one); a real device + driver, whose initialise opens a socket, rarely wins that race. Withholding the spawn-time send to avoid + it would simply restore 03/S16, since the Restart and supervision routes have no bulk push coming. +- **Restart tests must assert a FUNCTION, never existence.** `RestartDriver_respawns_the_child_without_an_actor_name_collision` + asserts only that a child exists after a restart, and it passed green against the dead driver 03/S16 + describes. It is kept — its own subject, the `InvalidActorNameException`, is real — but the load-bearing + suite is `DriverHostActorRestartTests`, which asserts that the new incarnation *subscribed* and + *re-initialised*. Its file remarks also record the supervision crash seam and why the obvious + alternatives (a throwing health publisher, a throwing `SubscribeAsync`, `Kill.Instance`) do not work. + +--- + ## IDriverHealthPublisher — health pub/sub sink [`Core.Abstractions/IDriverHealthPublisher.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/IDriverHealthPublisher.cs) @@ -236,7 +292,8 @@ The `Connected`-state health tick now also evaluates a watchdog, and self-sends 1. the driver reports a state other than `Healthy`, **and** 2. `DriverHealth.LastSuccessfulRead` has not advanced for `DefaultCommsLossTimeout` (90 s = three ticks), - measured from the Connected entry when the driver has never reported a successful read. + measured from **the later of that read stamp and the current Connected entry** (see "the grace is per + connection" below). **The conjunction is the whole design, and "non-Healthy for N polls" is the wrong rule.** `Degraded` means *connected but some data is bad* — the enum says so. Several drivers publish health **per tag inside their @@ -250,6 +307,26 @@ still needed as the second conjunct, because a `Healthy` driver with nothing sub reads and must not be declared dead for being idle. Both halves are pinned by their own test in `DriverInstanceActorCommsLossWatchdogTests`, each of which goes red when the *other* conjunct is deleted. +**The grace is per CONNECTION — the baseline is `max(LastSuccessfulRead, connectedSince)`, not +`LastSuccessfulRead ?? connectedSince`** (archreview 03/S17, fixed 2026-08-20). The two spellings differ +for exactly the case that matters, and the `??` one collapsed the grace to a single health tick. Because +`LastSuccessfulRead` is *preserved* across `Degraded`/`Faulted` writes — that preservation is the design +described above — a device that accepts a TCP connection but never serves data (a half-broken PLC, a wrong +unit id, a firewalled read path) reconnects carrying a read stamp frozen from before the outage, already +far older than the threshold. Measuring from that stamp alone re-trips on the **first** tick of every new +connection, so the reconnect cycle ran at health-tick cadence (~30 s in production) rather than at the +designed 90 s. Taking the *later* of the two gives each connection its own full window, which is what +re-stamping `connectedSince` on every Connected entry was always for; `??` only ever helped a driver that +had never read at all, and the max still covers that case. + +What this does **not** change: the conjunction, the disable-at-zero hatch, and a driver that stops reading +*within* a stable connection (its read stamp is the later term, so it still trips on schedule measured +from that read). And it **paces** the half-broken-device cycle rather than ending it — expect at most one +reconnect attempt per threshold per connection, not silence. Pinned by +`A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again` (the delay) and +`A_half_broken_device_is_still_declared_lost_once_the_grace_elapses` (the anti-vacuity half — without it, +a "fix" that simply disabled the watchdog after the first connection would look correct). + Two consequences worth knowing: - **`CalculationDriver` can never trip it, correctly.** It reports `Healthy` whenever it is running and has From 815423bd8853e8c37765ef81df846ef0353e5c08 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:07:20 -0400 Subject: [PATCH 28/45] docs(plans): restore literal UTF-8 em-dashes in R3-01 tasks.json (cosmetic) --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index b4413b6c..134df557 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -42,7 +42,7 @@ }, { "id": "T6", - "subject": "MeshGrpcDriverOnlyPipelineTests: driver-only nodes carry no authorization middleware \u2014 pins the 03/S14 exemption", + "subject": "MeshGrpcDriverOnlyPipelineTests: driver-only nodes carry no authorization middleware — pins the 03/S14 exemption", "status": "pending", "blockedBy": [ "T3" @@ -50,7 +50,7 @@ }, { "id": "T7", - "subject": "MeshGrpcMapConventionTests: Program.cs source scan \u2014 zero direct MapGrpcService/MapZbLocalDbSync sites, exactly one mapper call", + "subject": "MeshGrpcMapConventionTests: Program.cs source scan — zero direct MapGrpcService/MapZbLocalDbSync sites, exactly one mapper call", "status": "pending", "blockedBy": [ "T1" @@ -70,7 +70,7 @@ }, { "id": "T10", - "subject": "LIVE GATE: docker-dev with DisableLogin=false \u2014 step-0 falsification control MUST fail on the pre-fix build (Deployment.Status=4), then the fixed build seals Applied on all six nodes; restore DisableLogin=true", + "subject": "LIVE GATE: docker-dev with DisableLogin=false — step-0 falsification control MUST fail on the pre-fix build (Deployment.Status=4), then the fixed build seals Applied on all six nodes; restore DisableLogin=true", "status": "pending", "blockedBy": [ "T3", From c1ad9e24d2498a78892ee6a552abcec0e45119b9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:11:16 -0400 Subject: [PATCH 29/45] =?UTF-8?q?docs(archreview):=20R3-03=20execution=20n?= =?UTF-8?q?otes=20=E2=80=94=20final=20verification=20+=20deviation=20summa?= =?UTF-8?q?ry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...R3-03-restart-resubscribe-watchdog-plan.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index f8043aa9..c1ac048d 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -653,3 +653,35 @@ After restoring the hook: **6/6 pass**, and the class was run **20× in isolatio - §"The comms-loss watchdog": conjunct 2 restated as the max, plus a new paragraph on the per-connection grace — why `??` was wrong, that it **paces** rather than ends the half-broken-device cycle, that the watchdog is not disabled by the change, and the two tests that pin each half. + +### Final verification + +- `dotnet build ZB.MOM.WW.OtOpcUa.slnx` → **0 errors** (904 warnings, all pre-existing analyzer + `OTOPCUA0001` noise in unrelated driver test projects). +- **Full `Runtime.Tests`: 645 passed, 3 skipped, 0 failed** (1 m 9 s). Baseline on the branch point was + 637 + 3 skipped; the +8 are this plan's new tests (T1 ×2, T5 ×4, T7 ×2). The 3 skips are pre-existing + `DeploymentArtifact*` parity tests. +- The two other test projects that reference `ZB.MOM.WW.OtOpcUa.Runtime` were run as well, since T3 + changes what every driver child receives at spawn: `ControlPlane.Tests` **262/262**, `Host.Tests` + **106/106**. +- T9 (live gate) deliberately left **pending** — run by the orchestrator. + +### Summary of deviations (for the reviewer) + +1. **The plan's "the transient spawn-time set is never acted on" is false** — measured, corrected in + both the production comment and the docs, and accepted with its cost stated (T5, deviation 1). This + is the one behavioural claim in the plan that did not survive contact with the code. +2. `A_child_spawned_by_a_fresh_deploy_ends_with_the_NEW_refs` re-shaped — revision B *does* respawn the + child (raw tags fold into the merged `DriverConfig`) (T5, deviation 2). +3. `Restart_of_a_driver_with_no_authored_tags_sends_an_empty_set_not_nothing` renamed + + re-based on an alarm-only driver, because an all-empty set has no driver-side observable and the + planned test would have been vacuous; the `_children.Keys` union arm remains untested (T5, + deviation 3). +4. Seeder gained a second optional parameter (`tagConfigByTag`) — the only way to seed an alarm-bearing + raw tag, which one of the plan's own tests needs (T4). +5. The supervision test gained a preceding `ShutdownCount >= 1` assertion, because the plan's red + observation was ambiguous with "the child never crashed" (T7). +6. `Force_reconnect_keeps_the_desired_set_and_re_subscribes` uses a 30 s budget, not `PresenceBudget` — + `SpawnChild` gives the child the production 10 s `DefaultReconnectInterval` (T7). +7. `_lastComposition` and `_desiredByDriver` coexist for one commit (T2) so T2 stays compilable; the + deletion lands in T3, as T3's own step text specifies. From 8471db4b1dbd7c4e67d206f7c1d519a96d05c2ac Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:14:55 -0400 Subject: [PATCH 30/45] test(host): no-PSK mesh calls report the interceptor's verdict, not a cookie 401/redirect Extends MeshGrpcAuthStackTests (T5): a no-bearer or wrong-bearer call to DeploymentArtifactService.Fetch, TelemetryStreamService.Subscribe and NodeCommandService.Execute fails with the INTERCEPTOR's detail string ("Config-serve authentication failed.", "Telemetry stream authentication failed.", "Node command authentication failed."), asserted alongside the status code rather than in place of it - the vacuity trap the plan calls out (ConfigServeAuthInterceptor throws Unauthenticated, and Grpc.Net.Client can map an HTTP 401 to the same status, so status alone would not discriminate a broken build). Verified red-then-green against the T1 (pre-AllowAnonymous) mapper: all 6 new theory cases fail there with StatusCode.Internal ("HTTP/2 server reset the stream, PROTOCOL_ERROR") instead of Unauthenticated/PermissionDenied - the cookie challenge breaking the h2c stream before any gRPC trailers are written, never a clean auth denial. That is the exact signature the live gate (T10) should grep for; recorded in the class remarks. --- .../MeshGrpcAuthStackTests.cs | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs index d55a27d3..33c27bfa 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcAuthStackTests.cs @@ -19,6 +19,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Commons.Protos.DeploymentArtifact.V1; using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; @@ -255,6 +256,88 @@ public sealed class MeshGrpcAuthStackTests : IAsyncLifetime () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); } + // --------------------------------------------------------------------------------------- + // T5 - status fidelity: a no-PSK / wrong-PSK call reports the INTERCEPTOR's verdict, never a + // cookie challenge. This is the vacuity trap the plan calls out: ConfigServeAuthInterceptor + // throws StatusCode.Unauthenticated, and Grpc.Net.Client also maps an HTTP 401 to + // Unauthenticated - so "the call failed with Unauthenticated" is true on a broken build AND a + // fixed one. The interceptor's DETAIL string is the only thing that discriminates them. + // + // Observed cookie-challenge signature (captured red-then-green against this same fixture, see + // MeshGrpcAuthStackTests' positive tests): the challenge does not surface as a clean + // Unauthenticated/PermissionDenied at all. Grpc.Net.Client reports + // RpcException(StatusCode.Internal, "Error starting gRPC call. HttpRequestException: The HTTP/2 + // server reset the stream. HTTP/2 error code 'PROTOCOL_ERROR' (0x1)..."), because the cookie + // handler's redirect/401 breaks the prior-knowledge h2c stream before any gRPC trailers are + // written. So a status-only assertion here would ALSO be vacuous in the other direction: a + // regressed build fails with StatusCode.Internal, never Unauthenticated/PermissionDenied, and a + // caller checking only "did it throw" cannot tell a blocked call from a refused one. Recorded + // here per the plan (T5 step 2) - this is the exact string the live gate (T10) greps for. + // --------------------------------------------------------------------------------------- + + [Theory] + [InlineData(null)] + [InlineData("Bearer the-wrong-key")] + public async Task NoOrWrongPsk_ArtifactFetch_FailsWithTheInterceptorsDetail(string? authHeader) + { + var client = new DeploymentArtifactService.DeploymentArtifactServiceClient(Channel()); + var headers = new Metadata(); + if (authHeader is not null) + headers.Add("authorization", authHeader); + + using var call = client.Fetch( + new FetchRequest { DeploymentId = Guid.NewGuid().ToString() }, + headers, + cancellationToken: TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); + + ex.StatusCode.ShouldBe(StatusCode.Unauthenticated); + ex.Status.Detail.ShouldBe("Config-serve authentication failed."); + } + + [Theory] + [InlineData(null)] + [InlineData("Bearer the-wrong-key")] + public async Task NoOrWrongPsk_TelemetrySubscribe_FailsWithTheInterceptorsDetail(string? authHeader) + { + var client = new TelemetryStreamService.TelemetryStreamServiceClient(Channel()); + var headers = new Metadata(); + if (authHeader is not null) + headers.Add("authorization", authHeader); + + using var call = client.Subscribe( + new TelemetryStreamRequest { CorrelationId = "r3-01-t5" }, + headers, + cancellationToken: TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); + + ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); + ex.Status.Detail.ShouldBe("Telemetry stream authentication failed."); + } + + [Theory] + [InlineData(null)] + [InlineData("Bearer the-wrong-key")] + public async Task NoOrWrongPsk_NodeCommandExecute_FailsWithTheInterceptorsDetail(string? authHeader) + { + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + var client = new NodeCommandService.NodeCommandServiceClient(Channel()); + var headers = new Metadata(); + if (authHeader is not null) + headers.Add("authorization", authHeader); + + var ex = await Should.ThrowAsync(() => client.ExecuteAsync( + MeshTransportDtoMapper.ToEnvelope(command), headers, + cancellationToken: TestContext.Current.CancellationToken).ResponseAsync); + + ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); + ex.Status.Detail.ShouldBe("Node command authentication failed."); + } + // --------------------------------------------------------------------------------------- // harness // --------------------------------------------------------------------------------------- From 353ed6bb5893bc63633af2549c1c2f5dd659c304 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:15:03 -0400 Subject: [PATCH 31/45] docs(plans): mark R3-01 T5 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 134df557..3758cda5 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -35,7 +35,7 @@ { "id": "T5", "subject": "Status fidelity: a no-PSK call reports the interceptor's DETAIL string, not a cookie 401/302 (ConfigServe Unauthenticated collision makes status-only assertions vacuous)", - "status": "pending", + "status": "done", "blockedBy": [ "T4" ] From 62515f3279abc02316c863679d049eb39c8d8aee Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:18:52 -0400 Subject: [PATCH 32/45] fix(security): pin the cached-key expiry guard, meter cache_hit after it, and close two wiring gaps (review fixes) --- .../plans/R3-05-acl-hot-path-expiry-plan.md | 36 +++++++++++++++++ .../OpcUa/BoundedApiKeyStore.cs | 16 ++++++-- .../OpcUa/OtOpcUaServerHostedService.cs | 9 ++++- .../OpcUa/ApiKeyStoreRegistrationTests.cs | 20 ++++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 40 ++++++++++++++++++- 5 files changed, 116 insertions(+), 5 deletions(-) diff --git a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md index 6352f8f7..12d4c14b 100644 --- a/archreview/plans/R3-05-acl-hot-path-expiry-plan.md +++ b/archreview/plans/R3-05-acl-hot-path-expiry-plan.md @@ -781,3 +781,39 @@ Two notes for whoever runs it: cannot resurrect a clock-expired key. 4. New public members added purely so wiring could be pinned by reference: `DataPlaneAuthorizer.Clock`, `BoundedApiKeyStore.Inner` / `.Clock`. + +--- + +## Review outcome — FIX-FIRST, applied 2026-08-20 + +Coordinator verdict on the branch was **FIX-FIRST** with four items. All four applied; Host.Tests and +OpcUaServer.Tests rerun green. + +1. **[Important] `IsExpired` had no test — the review was right, and the gap was mine.** T3's execution + note called the guard a "fail-closed refinement" and shipped it with *no case that fails without it*: + deleting `&& !IsExpired(cached, utcNow)` left all 14 `BoundedApiKeyStoreTests` green. That is the + unchanged-observable trap — the guard's whole effect is invisible unless a cached entry outlives its + key's `ExpiresUtc` **inside** the TTL, which no existing case constructed. + Added **`A_cached_key_that_expires_by_the_clock_is_not_served`**: key expiring at `start + 2 s`, TTL + 60 s, first `FindAsync` non-null (and now cached); flip the inner store to `Revoked`; advance the fake + clock 5 s — past `ExpiresUtc`, far inside the TTL; assert `null` and `inner.Calls == 2`. The + `Revoked` flip is deliberate: it makes "served from cache" and "re-read" differ in the **answer**, not + only in a call count, so a build that re-read and ignored the result cannot pass either. + **RED confirmed by falsification:** with the guard removed, exactly **1 failed / 14 passed** — only the + new case — then restored, 15/15 green. +2. **[Minor] `cache_hit` was metered before the expiry guard could reject.** `TryServe` counted a hit for + an entry that then fell through to a store read, so the counter over-reported. The `Add` moved to the + two call sites, **after** the guard, behind a `MeterCacheHit()` helper; `TryServe` is now purely a + lookup. This matters operationally — `cache_hit` is the ratio §"Bounding the key store" tells an + operator to watch. +3. **[Minor] Added `A_node_with_neither_role_registers_the_deny_all_authenticator`** — asserts the + `IOpcUaUserAuthenticator` singleton **instance** is `NullOpcUaUserAuthenticator.Instance` and that no + `IApiKeyStore` is registered. It closes the third branch of the role conditional, which the guard suite + had left unpinned even though that branch is the one whose absence stops a node booting. +4. **[Minor] `StartRevalidationTimer` now disposes before overwriting.** `StartAsync` can run again when a + host restarts the service around the same instance, and the overwritten field would have left the + previous timer armed with nothing left to dispose it — the dispose-before-overwrite rule the driver + reconnect paths already carry. + +**Verification after the fixes:** solution build 0 errors · Host.Tests **134 passed / 0 failed** (was 132) +· OpcUaServer.Tests **493 passed / 1 skipped**. T11 remains `pending` for the orchestrator. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index cbf4cf76..bc63006e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -108,6 +108,10 @@ public sealed class BoundedApiKeyStore : IApiKeyStore && TryServe(_keys, apiKeyId, ttl, out var cached) && !IsExpired(cached, utcNow)) { + // Metered HERE rather than inside TryServe: an entry that is inside the TTL but whose key has + // since expired is not a hit — it falls through to a store read — so counting it in TryServe + // would report cache hits that never happened. + MeterCacheHit(); return cached; } @@ -122,7 +126,11 @@ public sealed class BoundedApiKeyStore : IApiKeyStore public async Task> FindGrantsAsync(string principal, CancellationToken ct) { var ttl = Ttl; - if (ttl > TimeSpan.Zero && TryServe(_grants, principal, ttl, out var cached)) return cached; + if (ttl > TimeSpan.Zero && TryServe(_grants, principal, ttl, out var cached)) + { + MeterCacheHit(); + return cached; + } var fresh = await BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct) .ConfigureAwait(false); @@ -140,6 +148,10 @@ public sealed class BoundedApiKeyStore : IApiKeyStore private static bool IsExpired(ApiKeyLookup? cached, DateTime utcNow) => cached?.Key.ExpiresUtc is { } expires && expires <= utcNow; + /// Counts one lookup the cache actually answered. + private static void MeterCacheHit() => + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "cache_hit")); + /// Serves an entry if one is present and still inside the TTL. /// The cached value type. /// The cache to read. @@ -152,8 +164,6 @@ public sealed class BoundedApiKeyStore : IApiKeyStore { if (cache.TryGetValue(key, out var entry) && _time.GetUtcNow() - entry.StampedUtc < ttl) { - OtOpcUaTelemetry.ApiKeyStoreBounded.Add( - 1, new KeyValuePair("outcome", "cache_hit")); value = entry.Value; return true; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 33a0bb70..adea5467 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -307,12 +307,19 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// /// Separate from so the timer's lifecycle is testable without booting the SDK. /// - internal void StartRevalidationTimer() => + internal void StartRevalidationTimer() + { + // Dispose before overwriting. StartAsync runs again when a host restarts the service around the + // same instance, and an overwritten field would leave the previous timer armed with nothing left + // to dispose it — the same dispose-before-overwrite rule the driver reconnect paths carry. + _revalidationTimer?.Dispose(); + _revalidationTimer = new AclRevalidationTimer( () => _server?.NodeManager?.RevalidateMonitoredItems(), TimeSpan.FromSeconds(_aclOptions.RevalidationIntervalSeconds), _time, _loggerFactory.CreateLogger()); + } /// /// Stops the OPC UA server asynchronously. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs index 1548c3ea..e425a12e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyStoreRegistrationTests.cs @@ -76,6 +76,26 @@ public sealed class ApiKeyStoreRegistrationTests sp.GetRequiredService().ShouldBeOfType(); } + /// + /// A node carrying neither role registers the deny-all authenticator explicitly, and no key + /// store at all. + /// + /// + /// Explicitly, not by absence: OtOpcUaServerHostedService takes + /// as a required constructor parameter, and the built-in + /// container does not fall back to a default for an unregistered service — it throws, and the node + /// does not start. The assertion is on the singleton INSTANCE, so a future registration that + /// happened to resolve some other deny-all type still surfaces here. + /// + [Fact] + public void A_node_with_neither_role_registers_the_deny_all_authenticator() + { + using var sp = BuildNode(hasAdmin: false, hasDriver: false); + + sp.GetRequiredService().ShouldBeSameAs(NullOpcUaUserAuthenticator.Instance); + sp.GetService().ShouldBeNull("a node that serves no data plane has no keys to read"); + } + /// Builds the data-plane authentication wiring for one node role combination. /// Whether the node carries the admin role. /// Whether the node carries the driver role. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index fd521810..9a4357dc 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -272,6 +272,43 @@ public sealed class BoundedApiKeyStoreTests inner.Calls.ShouldBe(callsAfterFlood + 1); } + /// + /// A cached key whose ExpiresUtc passes inside the TTL is not served: the store is + /// re-read instead. + /// + /// + /// + /// Without this the TTL would resurrect an expired key for up to StoreCacheSeconds — + /// the exact axis 03/S19 exists to close, re-opened by 03/S18's own cache. The cache key is + /// apiKeyId alone (utcNow changes every call and would defeat the cache), so the + /// clock check has to happen on the way OUT of the cache. + /// + /// + /// The inner store is flipped to Revoked first, so "served from cache" and "re-read" + /// give different ANSWERS rather than only different call counts — a build that re-read and + /// then ignored the result could not pass. + /// + /// + [Fact] + public async Task A_cached_key_that_expires_by_the_clock_is_not_served() + { + var start = new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero); + var (_, inner) = SetUp(expiresUtc: start.UtcDateTime.AddSeconds(2)); + var clock = new FakeClock(start); + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 60, clock); + + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)) + .ShouldNotBeNull("non-vacuity: the key was usable and IS now cached"); + + inner.Revoked = true; + + // Past ExpiresUtc (+2 s) but far inside the 60 s TTL — the entry is still "fresh" to the cache. + clock.Advance(TimeSpan.FromSeconds(5)); + + (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.Calls.ShouldBe(2, "the expired entry must be re-read, not served"); + } + // ---------- metering ---------- /// A deadline that fires meters outcome=timeout — the alarm signal for a sick ConfigDb. @@ -347,7 +384,7 @@ public sealed class BoundedApiKeyStoreTests Options.Create(new ApiKeyOptions { Username = "apikey", Pepper = Pepper }), NullLogger.Instance); - private static (string Presented, GatedApiKeyStore Store) SetUp() + private static (string Presented, GatedApiKeyStore Store) SetUp(DateTime? expiresUtc = null) { var (presented, apiKeyId, hash) = ApiKeySecret.Generate(Pepper); var key = new ApiKey @@ -358,6 +395,7 @@ public sealed class BoundedApiKeyStoreTests Principal = Principal, Enabled = true, CreatedUtc = DateTime.UtcNow, + ExpiresUtc = expiresUtc, }; return (presented, new GatedApiKeyStore(key)); } From 306ea1aecfc3eb87003ee5da8063b7b0e7445e0b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:20:56 -0400 Subject: [PATCH 33/45] test(host): driver-only nodes carry no authorization middleware (03/S14 exemption pinned) MeshGrpcDriverOnlyPipelineTests - the driver-role shape (no AddOtOpcUaAuth, no UseAuthentication/UseAuthorization at all) behaves identically to the fused shape's T4 positive/negative tests for NodeCommandService and TelemetryStreamService: a correctly-keyed call succeeds, a no-key call gets the interceptor's PermissionDenied + detail string. Also asserts the STRUCTURAL reason it is unaffected - no IAuthorizationPolicyProvider registered at all - so a future "register auth everywhere for consistency" change fails this test before it ever reaches a live site outage. Confirmed (by temporarily swapping in the pre-T2/T3 mapper) that this class passes identically on both builds: driver-only nodes were never exposed to 03/S14 in the first place, with or without .AllowAnonymous(). --- .../MeshGrpcDriverOnlyPipelineTests.cs | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcDriverOnlyPipelineTests.cs diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcDriverOnlyPipelineTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcDriverOnlyPipelineTests.cs new file mode 100644 index 00000000..3d394c43 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcDriverOnlyPipelineTests.cs @@ -0,0 +1,202 @@ +using System.Diagnostics.CodeAnalysis; +using System.Net; +using Akka.Actor; +using Akka.Hosting; +using Grpc.Core; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Runtime; +using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// R3-01 T6 — pins the report's "driver-only nodes are structurally unaffected" finding +/// (archreview 03/S14): a node with no admin role never registers +/// AddOtOpcUaAuth/UseAuthentication/UseAuthorization at all, so it has no +/// cookie FallbackPolicy to opt out of in the first place — the PSK interceptors were +/// always the ONLY gate on this shape, with or without .AllowAnonymous(). +/// +/// +/// A future "register auth everywhere for consistency" change would silently reintroduce 03/S14 +/// for every site node in the fleet. This class is what turns that into a failing test instead of +/// a live outage: it asserts BOTH that the driver-only mesh endpoints behave identically to the +/// fused shape's (T4) AND that the reason they do — no +/// registered at all — actually holds, so a change that adds auth services here fails on the +/// second assertion even if it somehow left the first passing. +/// +public sealed class MeshGrpcDriverOnlyPipelineTests : IAsyncLifetime +{ + private const string NodeCommandKey = "test-driveronly-node-psk"; + private const string TelemetryKey = "test-driveronly-telemetry-key"; + + private WebApplication _app = null!; + private ActorSystem _system = null!; + private int _port; + private ITelemetryLocalHub _hub = null!; + private TaskCompletionSource _nodeInbox = null!; + + /// + public async ValueTask InitializeAsync() + { + var builder = WebApplication.CreateBuilder(); + builder.Environment.ApplicationName = typeof(MeshGrpcDriverOnlyPipelineTests).Assembly.GetName().Name!; + builder.Logging.ClearProviders(); + + // Deliberately NO AddOtOpcUaAuth, NO UseAuthentication, NO UseAuthorization — this IS the + // driver-only shape (Program.cs gates all three on hasAdmin). Nothing here should need them. + builder.Services.Configure(o => o.GrpcPsk = NodeCommandKey); + builder.Services.Configure(o => o.ApiKey = TelemetryKey); + builder.Services.AddSingleton(); + + _system = ActorSystem.Create("mesh-driveronly-" + Guid.NewGuid().ToString("N")); + _nodeInbox = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var registry = new ActorRegistry(); + registry.Register(_system.ActorOf(CaptureActor.Props(_nodeInbox))); + builder.Services.AddSingleton(registry); + + // The driver-role interceptor subset only (matches Program.cs's `if (hasDriver)` branch); + // no ConfigServeAuthInterceptor/CentralCommandAuthInterceptor — those are admin-role. + builder.Services.AddGrpc(o => + { + o.Interceptors.Add(); + o.Interceptors.Add(); + }); + + _port = GetFreePort(); + builder.WebHost.ConfigureKestrel(k => k.ListenAnyIP(_port, o => o.Protocols = HttpProtocols.Http2)); + + _app = builder.Build(); + _hub = _app.Services.GetRequiredService(); + + // hasAdmin: false — the driver-only shape. syncListenPort: 0 so LocalDbSync (admin-agnostic + // but out of scope here) is not mapped; configServeGrpcPort: 0 because a driver-only node + // never carries ConfigServe. + _app.MapOtOpcUaMeshGrpcServices( + hasAdmin: false, hasDriver: true, + syncListenPort: 0, configServeGrpcPort: 0, telemetryListenPort: 1); + + await _app.StartAsync(TestContext.Current.CancellationToken); + } + + /// + public async ValueTask DisposeAsync() + { + await _app.StopAsync(TimeSpan.FromSeconds(10)); + await _app.DisposeAsync(); + await _system.Terminate().WaitAsync(TimeSpan.FromSeconds(15)); + } + + [Fact] + public void NoAuthorizationPolicyProviderIsRegistered() + { + // THE structural reason this shape is unaffected: with AddOtOpcUaAuth never called, there is + // no FallbackPolicy to opt out of. If a future change registers authorization services on a + // driver-only node, this goes red before any endpoint's behaviour does. + _app.Services.GetService().ShouldBeNull(); + } + + [Fact] + public async Task CorrectlyKeyed_NodeCommandExecute_Succeeds() + { + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + var client = new NodeCommandService.NodeCommandServiceClient(Channel()); + var headers = new Metadata { { "authorization", $"Bearer {NodeCommandKey}" } }; + + await client.ExecuteAsync( + MeshTransportDtoMapper.ToEnvelope(command), headers, + cancellationToken: TestContext.Current.CancellationToken); + + var received = await _nodeInbox.Task.WaitAsync( + TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + received.ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1"); + } + + [Fact] + public async Task NoKey_NodeCommandExecute_FailsWithTheInterceptorsDetail() + { + var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); + var client = new NodeCommandService.NodeCommandServiceClient(Channel()); + + var ex = await Should.ThrowAsync(() => client.ExecuteAsync( + MeshTransportDtoMapper.ToEnvelope(command), new Metadata(), + cancellationToken: TestContext.Current.CancellationToken).ResponseAsync); + + ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); + ex.Status.Detail.ShouldBe("Node command authentication failed."); + } + + [Fact] + public async Task CorrectlyKeyed_TelemetrySubscribe_Succeeds() + { + // Health is snapshot-style (ITelemetryLocalHub.Emit / class remarks): emitting BEFORE the + // Subscribe call means the new subscriber's reader is primed with it immediately, with no + // race against a live delta. + _hub.Emit(new TelemetryItem.Health(new DriverHealthChanged( + "C1", "driver-1", "Healthy", DateTime.UtcNow, null, 0, DateTime.UtcNow))); + + var client = new TelemetryStreamService.TelemetryStreamServiceClient(Channel()); + var headers = new Metadata { { "authorization", $"Bearer {TelemetryKey}" } }; + + using var call = client.Subscribe( + new TelemetryStreamRequest { CorrelationId = "r3-01-t6" }, + headers, + cancellationToken: TestContext.Current.CancellationToken); + + (await call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)).ShouldBeTrue(); + call.ResponseStream.Current.DriverHealth.DriverInstanceId.ShouldBe("driver-1"); + } + + [Fact] + public async Task NoKey_TelemetrySubscribe_FailsWithTheInterceptorsDetail() + { + var client = new TelemetryStreamService.TelemetryStreamServiceClient(Channel()); + + using var call = client.Subscribe( + new TelemetryStreamRequest { CorrelationId = "r3-01-t6-nokey" }, + cancellationToken: TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); + + ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); + ex.Status.Detail.ShouldBe("Telemetry stream authentication failed."); + } + + // --------------------------------------------------------------------------------------- + // harness + // --------------------------------------------------------------------------------------- + + private GrpcChannel Channel() => GrpcChannel.ForAddress($"http://localhost:{_port}"); + + private static int GetFreePort() + { + using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + /// Completes an inbox with the first message it receives. + private sealed class CaptureActor : ReceiveActor + { + public static Props Props(TaskCompletionSource sink) => + Akka.Actor.Props.Create(() => new CaptureActor(sink)); + + public CaptureActor(TaskCompletionSource sink) => ReceiveAny(m => sink.TrySetResult(m)); + } +} From 9bd3366b28041a9a6a1a8aae333ecdd26391e85b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:21:01 -0400 Subject: [PATCH 34/45] docs(plans): mark R3-01 T6 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 3758cda5..9d1cc850 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -43,7 +43,7 @@ { "id": "T6", "subject": "MeshGrpcDriverOnlyPipelineTests: driver-only nodes carry no authorization middleware — pins the 03/S14 exemption", - "status": "pending", + "status": "done", "blockedBy": [ "T3" ] From 82ad98e1a9e8cd4754a472c129c49816fa8e591b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:24:49 -0400 Subject: [PATCH 35/45] chore(docker-dev): parameterise Security__Auth__DisableLogin on the central pair Security__Auth__DisableLogin: "${OTOPCUA_DISABLE_LOGIN:-true}" on central-1 and central-2 - byte-identical default behaviour, verified via `docker compose config` both unset (resolves "true") and with OTOPCUA_DISABLE_LOGIN=false (resolves "false"). DisableLogin's always-succeeding handler, registered under the cookie scheme name, masks the AdminUI cookie FallbackPolicy entirely - which is exactly how 03/S14 (the FallbackPolicy blocking every mesh gRPC call on a login-enabled admin node) survived seven phases of prior live gates that all ran with the default. The production-faithful mode is now a one-variable flip instead of an edit-and-remember-to-revert; header comment names the flip and the LDAP users it needs (archreview 03/S14, R3-01 T8). --- docker-dev/docker-compose.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index 97abc07c..73aec571 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -90,6 +90,21 @@ # prior dark-switch shape). The ClusterNode.GrpcPort column (seeded by # docker-dev/seed/seed-clusters.sql) points central's dial supervisor at each node's :4056. # +# Production-faithful auth (R3-01, archreview 03/S14): central-1/central-2's +# Security__Auth__DisableLogin defaults to "true" here (byte-identical to the previous hardcoded +# value) but is now `${OTOPCUA_DISABLE_LOGIN:-true}` — a one-variable flip to the REAL cookie +# login flow instead of an edit-and-remember-to-revert. DisableLogin registers an always-succeeding +# handler UNDER the cookie scheme name, so every live gate run with the default masks the AdminUI +# cookie FallbackPolicy entirely — which is exactly how 03/S14 (the FallbackPolicy blocking every +# mesh gRPC call on a login-enabled admin node) survived seven phases of prior live gates. To drive +# this rig production-faithfully: +# OTOPCUA_DISABLE_LOGIN=false docker compose -f docker-dev/docker-compose.yml up -d central-1 central-2 +# Sign in as `alice` / `password` (or `bob`) against the shared GLAuth (10.100.0.35:3893, +# dc=zb,dc=local) — the same LDAP users every other node in this file already authenticates OPC UA +# sessions against. Restore the default with `docker compose ... up -d central-1 central-2` (no env +# var). See docs/security.md "Mesh gRPC ingress is AllowAnonymous by design" for what this actually +# exercises. +# # Usage: # docker compose -f docker-dev/docker-compose.yml up -d --build # open http://localhost:9200 # central Blazor admin UI @@ -382,7 +397,7 @@ services: Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" Security__Ldap__ServiceAccountPassword: "serviceaccount123" Security__DeployApiKey: "docker-dev-deploy-key" - Security__Auth__DisableLogin: "true" + Security__Auth__DisableLogin: "${OTOPCUA_DISABLE_LOGIN:-true}" # Map the shared-GLAuth data-plane groups (ReadOnly/WriteOperate/...) to data-plane role # strings the OPC UA session ACL gates read. The DB LdapGroupRoleMapping is AdminRole-typed # (AdminUI roles); this free-form GroupToRole baseline is the path for data-plane roles. @@ -508,7 +523,7 @@ services: Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" Security__Ldap__ServiceAccountPassword: "serviceaccount123" Security__DeployApiKey: "docker-dev-deploy-key" - Security__Auth__DisableLogin: "true" + Security__Auth__DisableLogin: "${OTOPCUA_DISABLE_LOGIN:-true}" # Data-plane group→role baseline (see central-1). Security__Ldap__GroupToRole__ReadOnly: "ReadOnly" Security__Ldap__GroupToRole__WriteOperate: "WriteOperate" From edd01fcc341e489f88edcc0b2ac176d115571415 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:24:52 -0400 Subject: [PATCH 36/45] fix(runtime): a changed alarm set re-subscribes instead of latching (review F1) The alarm subscribe gated on 'a handle exists', which is correct for an idempotent re-push and wrong for a deploy that widens/narrows the set. Combined with the spawn-time desired set, a respawning deploy could latch the PREVIOUS revision's alarm refs for the life of the connection; sourceNodeIds-filtering drivers (OpcUaClient/AbCip/Focas) would never deliver the new alarm. Seen RED: LastSubscribedAlarmRefs stayed [alarmX] after a deploy widening to [alarmX, alarmY]. --- docs/DriverLifecycle.md | 13 ++++ .../Drivers/DriverInstanceActor.cs | 60 ++++++++++++++-- .../Drivers/DriverHostActorRestartTests.cs | 68 +++++++++++++++++++ 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/docs/DriverLifecycle.md b/docs/DriverLifecycle.md index c09fafce..4fb2fb6f 100644 --- a/docs/DriverLifecycle.md +++ b/docs/DriverLifecycle.md @@ -211,6 +211,19 @@ Things to know before touching any of this: (`HandleSubscribeAsync` drops the prior subscription before establishing the new one); a real device driver, whose initialise opens a socket, rarely wins that race. Withholding the spawn-time send to avoid it would simply restore 03/S16, since the Restart and supervision routes have no bulk push coming. +- ⚠️ **Convergence is not free on the ALARM side, and "a subscription already exists" is the wrong guard + there.** The value path converges by construction — `StoreDesiredSubscriptions` is a full replace and + `HandleSubscribeAsync` drops the prior subscription before establishing the new one — but + `SubscribeDesiredAlarms` / `HandleSubscribeAlarmsAsync` used to gate purely on + `_alarmSubscriptionHandle is not null`, so once a handle existed the alarm set **latched for the life of + the connection**. Combined with the transient above, a deploy widening a driver's alarm refs `{X}` → + `{X, Y}` could leave a fast-initialising driver subscribed to `{X}` forever, and the drivers that filter + their feed on the subscribed `sourceNodeIds` (OPC UA Client, AB CIP, FOCAS) would simply never deliver + the new alarm — silently. The guard is now "the desired refs differ from the ones the live handle was + established with" (`_subscribedAlarmRefs`), and a changed set **releases the old subscription before + re-subscribing** so a filtering driver does not accumulate two overlapping ones. There is still + deliberately no unsubscribe-on-**empty**: a removed alarm tag's condition node is torn down on the + address-space rebuild and `ForwardNativeAlarm` drops any transition whose `ConditionId` no longer maps. - **Restart tests must assert a FUNCTION, never existence.** `RestartDriver_respawns_the_child_without_an_actor_name_collision` asserts only that a child exists after a restart, and it passed green against the dead driver 03/S16 describes. It is kept — its own subject, the `InvalidActorNameException`, is real — but the load-bearing diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs index b53a1418..cdcde90c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs @@ -271,6 +271,15 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers /// idempotent across repeated pushes. private IAlarmSubscriptionHandle? _alarmSubscriptionHandle; + /// The alarm references the live was actually established + /// with, so a CHANGED desired set can be told apart from a repeat of the same one. + /// Without this the subscription latched: both and + /// gate on "a handle already exists", which is the right guard + /// for an idempotent re-push and the WRONG one for a deploy that widens or narrows the alarm set — + /// drivers that filter on sourceNodeIds (OPC UA Client, AB CIP, FOCAS) would then never deliver a + /// newly-added alarm for the life of the connection. Empty whenever no handle is held. + private IReadOnlyList _subscribedAlarmRefs = Array.Empty(); + /// /// Gets or sets the timer scheduler for scheduling reconnection attempts. /// @@ -521,10 +530,14 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers // Native-alarm analogue: un-gate the IAlarmSource feed when alarm tags are (now) present. The // common live path — a deploy delivers SetDesiredSubscriptions while the driver is already // Connected — flows through HERE, so the alarm subscribe must happen on this message, not only - // on Connected entry. Unlike the value path above, there is deliberately no unsubscribe-on-empty: - // a removed alarm tag's condition node is torn down on the address-space rebuild and - // DriverHostActor.ForwardNativeAlarm drops any transition whose ConditionId no longer maps, so a - // lingering session-less alarm subscription can never surface a removed alarm. + // on Connected entry. A CHANGED set re-subscribes (release-then-subscribe inside + // HandleSubscribeAlarmsAsync): unlike the value path, "a subscription already exists" is NOT + // sufficient, because several drivers filter their feed on the subscribed sourceNodeIds, so a + // widened set that never re-subscribed would silently never deliver the new alarm. + // Still deliberately no unsubscribe-on-EMPTY: a removed alarm tag's condition node is torn down + // on the address-space rebuild and DriverHostActor.ForwardNativeAlarm drops any transition whose + // ConditionId no longer maps, so a lingering session-less alarm subscription can never surface a + // removed alarm. SubscribeDesiredAlarms(); }); ReceiveAsync(HandleSubscribeAlarmsAsync); @@ -1167,6 +1180,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers // (GalaxyDriver.SubscribeAlarmsAsync clears the set before adding), so handles no longer accumulate // across reconnects. The gate (Count > 0) and the one-shot fan-out are unchanged. _alarmSubscriptionHandle = null; + _subscribedAlarmRefs = Array.Empty(); } /// Establish the native-alarm subscription that un-gates an driver's @@ -1176,10 +1190,17 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers /// desired, and no subscription is yet established. private void SubscribeDesiredAlarms() { - if (_driver is IAlarmSource && _desiredAlarmRefs.Count > 0 && _alarmSubscriptionHandle is null) + if (_driver is IAlarmSource && _desiredAlarmRefs.Count > 0 && AlarmSubscriptionIsStale()) Self.Tell(new SubscribeAlarms()); } + /// True when the desired alarm refs are not the ones the live handle was established with — + /// either because there is no handle yet, or because a deploy changed the set. The second case is the + /// one a bare "handle is null" check misses; see . + private bool AlarmSubscriptionIsStale() => + _alarmSubscriptionHandle is null || + !_subscribedAlarmRefs.SequenceEqual(_desiredAlarmRefs, StringComparer.Ordinal); + /// Calls the driver's (bounded) to register the /// alarm subscription that un-gates its native feed, caching the returned handle. Re-checks the guard /// (the desired set may have cleared, or another SubscribeAlarms may have already established a handle, @@ -1187,10 +1208,36 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers /// stays gated until then. private async Task HandleSubscribeAlarmsAsync(SubscribeAlarms _) { - if (_driver is not IAlarmSource src || _desiredAlarmRefs.Count == 0 || _alarmSubscriptionHandle is not null) + if (_driver is not IAlarmSource src || _desiredAlarmRefs.Count == 0 || !AlarmSubscriptionIsStale()) return; var refs = _desiredAlarmRefs; using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // A handle held against a DIFFERENT set must be released before re-subscribing, or a + // sourceNodeIds-filtering driver accumulates two overlapping subscriptions. Best-effort and + // deliberately non-fatal: the re-subscribe below is what actually matters, and a driver whose + // unsubscribe throws must not be left on the stale set because of it. Cleared FIRST so a throw + // cannot leave a handle that the staleness check would keep treating as live. + if (_alarmSubscriptionHandle is { } stale) + { + _alarmSubscriptionHandle = null; + _subscribedAlarmRefs = Array.Empty(); + try + { + await _invoker.ExecuteAsync( + DriverCapability.AlarmSubscribe, + _driverInstanceId, + async ct => await src.UnsubscribeAlarmsAsync(stale, ct), + cts.Token); + } + catch (Exception ex) + { + _log.Warning(ex, + "DriverInstance {Id}: releasing the previous native-alarm subscription ({Diag}) threw; re-subscribing anyway", + _driverInstanceId, stale.DiagnosticId); + } + } + try { _alarmSubscriptionHandle = await _invoker.ExecuteAsync( @@ -1198,6 +1245,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers _driverInstanceId, async ct => await src.SubscribeAlarmsAsync(refs, ct), cts.Token); + _subscribedAlarmRefs = refs; _log.Info("DriverInstance {Id}: native-alarm subscription established for {Count} alarm ref(s) ({Diag})", _driverInstanceId, refs.Count, _alarmSubscriptionHandle.DiagnosticId); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs index e8abff55..020e36d4 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs @@ -64,6 +64,11 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase private const string DriverId = "DI-1"; private const string SpeedPath = "Plant/Modbus/dev1/speed"; private const string TempPath = "Plant/Modbus/dev1/temp"; + private const string AlarmXPath = "Plant/Modbus/dev1/alarmX"; + private const string AlarmYPath = "Plant/Modbus/dev1/alarmY"; + + /// TagConfig marking a raw tag as alarm-bearing (parsed by TagConfigIntent). + private const string AlarmConfig = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}"""; /// Seeds + applies a v3 deployment whose single Modbus driver is ENABLED, so a real child is /// spawned around a , and waits until that child has connected @@ -285,6 +290,69 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase duration: PresenceBudget); } + /// + /// Review finding F1 — the alarm subscription must FOLLOW a changed desired set, not latch. + /// The value side converges by construction (HandleSubscribeAsync drops the prior subscription + /// before establishing the new one, and StoreDesiredSubscriptions is a full replace), but the + /// alarm side is guarded by _alarmSubscriptionHandle is not null in BOTH + /// SubscribeDesiredAlarms and HandleSubscribeAlarmsAsync — so once a handle exists it + /// was never re-established for the life of the connection. + /// + /// That became reachable when every spawn started carrying a desired set: a tag-set edit forces a + /// respawn (raw tags fold into the merged DriverConfig, #516), and a fast-initialising + /// driver connects and subscribes the transient spawn-time set — the PREVIOUS revision's alarm + /// refs — before the bulk push lands. The bulk push then early-returns, latching the old set for + /// the connection's life. Drivers that FILTER on sourceNodeIds (OPC UA Client, AB CIP, + /// FOCAS) would silently never deliver the newly-added alarm. + /// + /// + [Fact] + public void A_deploy_that_widens_the_alarm_set_re_subscribes_the_new_refs() + { + var db = NewInMemoryDbFactory(); + var (host, factory) = DeployOneDriver( + db, + new[] { (DriverId, "Modbus", "dev1", "alarmX", "Float", false) }, + new Dictionary { ["alarmX"] = AlarmConfig }); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].LastSubscribedAlarmRefs.ShouldBe(new[] { AlarmXPath }, ignoreOrder: true); + }, + duration: PresenceBudget); + + // Revision B WIDENS the alarm set: X stays, Y is added. + var revB = V3ArtifactSeeder.SeedV3Deployment( + db, RevB, + new[] + { + (DriverId, "Modbus", "dev1", "alarmX", "Float", false), + (DriverId, "Modbus", "dev1", "alarmY", "Float", false), + }, + Array.Empty<(string, string, string, string, string?)>(), + enabled: true, + tagConfigByTag: new Dictionary + { + ["alarmX"] = AlarmConfig, + ["alarmY"] = AlarmConfig, + }); + host.Tell(new Commons.Messages.Deploy.DispatchDeployment(revB, RevB, CorrelationId.NewId())); + + AwaitAssert( + () => + { + var current = factory.Created[^1]; + current.LastSubscribedAlarmRefs.ShouldBe( + new[] { AlarmXPath, AlarmYPath }, + ignoreOrder: true, + "the driver is still subscribed to the PREVIOUS revision's alarm refs — the alarm " + + "subscription latched on the transient spawn-time set and never followed the new one"); + }, + duration: PresenceBudget); + } + /// /// The repo's first genuine Akka SUPERVISION-restart test (03/U6). Not a stop, not a stub: the /// driver raises a malformed OnDataChange, the actor NREs on its own thread, and Akka's default From f3832aa48c7364f0f466689f0bfe177541a2e283 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:25:02 -0400 Subject: [PATCH 37/45] test(host): a mesh gRPC map added outside MeshGrpcEndpoints fails the build MeshGrpcMapConventionTests - source-scan of Program.cs (solution-root walk, following FlushLoggingConventionTests.cs's idiom): zero direct MapGrpcService +/// R3-01 T7 — a source-scan guard, following the solution-root-walk idiom +/// ZB.MOM.WW.OtOpcUa.Driver.S7.Cli.Tests.FlushLoggingConventionTests uses. Complements the +/// endpoint-metadata census (MeshGrpcAnonymousMetadataTests): the census catches a map +/// added to MeshGrpcEndpoints without the AllowAnonymous convention, but it cannot +/// catch a map added to Program.cs that bypasses the helper entirely — that shape never +/// goes through MapOtOpcUaMeshGrpcServices at all, so the census would simply never see it. +/// +/// +/// Would have FAILED at HEAD before T1: Program.cs used to carry the five map sites +/// inline (MapZbLocalDbSync(), four MapGrpcService<T>() calls) with no +/// MapOtOpcUaMeshGrpcServices call at all. +/// +public sealed class MeshGrpcMapConventionTests +{ + private static readonly string ProgramCsPath = LocateProgramCs(); + + [Fact] + public void ProgramCs_ContainsNoDirectMeshGrpcMapSites_AndExactlyOneMapperCall() + { + var source = File.ReadAllText(ProgramCsPath); + + // Guard-of-the-guard: a bad path (an empty file, or one that never had the mapper in the + // first place) must not pass this test by finding nothing to count. + source.ShouldNotBeNullOrWhiteSpace(); + source.ShouldContain("MapOtOpcUaMeshGrpcServices("); + + CountOccurrences(source, "MapGrpcService<").ShouldBe(0, + "every MapGrpcService() call must live inside MeshGrpcEndpoints.MapOtOpcUaMeshGrpcServices, " + + "not inline in Program.cs (03/S14 — a map added outside the helper is invisible to the " + + "endpoint-metadata census)."); + CountOccurrences(source, "MapZbLocalDbSync(").ShouldBe(0, + "the LocalDb sync map must live inside MeshGrpcEndpoints.MapOtOpcUaMeshGrpcServices, not inline " + + "in Program.cs, for the same reason as MapGrpcService() above."); + CountOccurrences(source, "MapOtOpcUaMeshGrpcServices(").ShouldBe(1, + "Program.cs must call the mapper exactly once; more than one call would map the mesh gRPC " + + "services twice (a route conflict) and zero calls would drop them entirely."); + } + + private static int CountOccurrences(string source, string needle) + { + var count = 0; + var index = 0; + while ((index = source.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private static string LocateProgramCs() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.OtOpcUa.slnx"))) + dir = dir.Parent; + dir.ShouldNotBeNull("Could not find solution root (ZB.MOM.WW.OtOpcUa.slnx)."); + return Path.Combine(dir!.FullName, "src", "Server", "ZB.MOM.WW.OtOpcUa.Host", "Program.cs"); + } +} From 4dde5cb8bf8356a61c0167e9418159d6f11dc84a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:25:12 -0400 Subject: [PATCH 38/45] docs(plans): mark R3-01 T7 and T8 done --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 9d1cc850..799e0fa9 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -51,7 +51,7 @@ { "id": "T7", "subject": "MeshGrpcMapConventionTests: Program.cs source scan — zero direct MapGrpcService/MapZbLocalDbSync sites, exactly one mapper call", - "status": "pending", + "status": "done", "blockedBy": [ "T1" ] @@ -59,7 +59,7 @@ { "id": "T8", "subject": "docker-dev: Security__Auth__DisableLogin parameterised as ${OTOPCUA_DISABLE_LOGIN:-true} on central-1/central-2 + header docs", - "status": "pending", + "status": "done", "blockedBy": [] }, { From 2b7100cf19ae1a7cb33d3a4b757a3a38ae826870 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:27:02 -0400 Subject: [PATCH 39/45] fix(runtime): route a reprovision by incarnation, not driver id (review F2) A ReprovisionRequested in flight from an old incarnation could reach the NEW child after HandleRestartDriver swapped it in. In Connecting that child DOES handle InitializeRequested, so it started a second concurrent InitializeAsync, racing the #527 dispose-before-overwrite. Seen RED: InitializeCount == 2 on a child held in Connecting by a gated init (an already-connected child swallows InitializeRequested and passes vacuously). --- .../Drivers/DriverHostActor.cs | 14 +++++ .../Drivers/DriverHostActorRestartTests.cs | 59 +++++++++++++++++++ .../Drivers/StubDrivers.cs | 25 +++++++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 00352e15..588e40ee 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -3017,6 +3017,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision. if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return; + // Route by INCARNATION, not just by id. This message is delivered asynchronously, so one can still + // be in flight from an OLD incarnation after HandleRestartDriver stopped that child and spawned a + // replacement under the same DriverInstanceId. Delivered to the new child — which is in Connecting, + // where InitializeRequested IS handled — it would start a second, concurrent InitializeAsync, + // racing the dispose-before-overwrite that #527's teardown exists to serialise + // (ModbusDriver._transport, MqttDriver._connection each leak a transport on that race). + if (!Equals(Sender, entry.Actor)) + { + _log.Debug( + "DriverHost {Node}: dropping a reprovision for {Id} from a superseded incarnation ({Stale})", + _localNode, msg.DriverInstanceId, Sender); + return; + } + // A crash that Akka silently restarted is otherwise invisible to operators; name it. _log.Warning( "DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning", diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs index 020e36d4..53652f9f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs @@ -5,6 +5,7 @@ using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; @@ -419,6 +420,64 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase duration: PresenceBudget); } + /// + /// Review finding F2 — a reprovision must be routed by INCARNATION, not just by driver id. + /// ReprovisionRequested is delivered asynchronously, so one can still be in flight from an old + /// incarnation when HandleRestartDriver has already stopped that child and swapped in a new + /// one under the same DriverInstanceId. Routing on the id alone delivers it to the NEW child, + /// which is in Connecting around a healthy driver: it triggers a second, concurrent + /// InitializeAsync that races the very dispose-before-overwrite the #527 teardown exists to + /// serialise (ModbusDriver._transport, MqttDriver._connection). + /// + /// A stale sender is simulated directly — a probe standing in for the dead incarnation — because + /// provoking the real race would be timing-dependent. The assertion is an ABSENCE, so it carries + /// a positive control: the current child is proven alive and provisioned first, and its + /// InitializeCount must then not move. + /// + /// + [Fact] + public void A_reprovision_from_a_stale_incarnation_is_ignored() + { + var db = NewInMemoryDbFactory(); + // The child must be in CONNECTING for this defect to bite: InitializeRequested is unhandled in the + // Connected behaviour, so an already-connected child would swallow the stale reprovision for the + // wrong reason and the test would pass vacuously. The gated init holds it in Connecting. + var factory = new RecordingSubscribableDriverFactory("Modbus", gateInitialize: true); + var deploymentId = V3ArtifactSeeder.SeedV3Deployment( + db, RevA, + new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }, + Array.Empty<(string, string, string, string, string?)>(), + enabled: true); + var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory); + + // Positive control: the current child is alive and has begun its one and only initialise. + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].InitializeCount.ShouldBe(1); + }, + duration: PresenceBudget); + var driver = factory.Created[0]; + + // A ReprovisionRequested that did NOT come from the child the host currently holds. + var staleIncarnation = CreateTestProbe(); + host.Tell( + new DriverInstanceActor.ReprovisionRequested(DriverId, "NullReferenceException"), + staleIncarnation.Ref); + + // Round-trip a message the host must answer, so the reprovision above is provably processed before + // the absence is asserted — otherwise "nothing happened yet" would pass for "it was ignored". + var coordinator = CreateTestProbe(); + host.Tell(new Commons.Messages.Fleet.GetDiagnostics(CorrelationId.NewId()), coordinator.Ref); + coordinator.ExpectMsg(PresenceBudget); + + driver.InitializeCount.ShouldBe( + 1, + "a reprovision from a dead incarnation reached the CURRENT child and started a second, " + + "concurrent InitializeAsync on a driver that was already connected"); + } + /// /// The verified negative, pinned so this plan's edits to the restart paths cannot break the one that /// was already correct. Force Reconnect (ReconnectDriver) Tells ForceReconnect to the diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs index 30a51113..3c4783d0 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/StubDrivers.cs @@ -261,11 +261,19 @@ internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAla get { lock (_gate) return _lastSubscribedAlarmRefs; } } + /// + /// When set, awaits it instead of completing inline — so the actor + /// stays in Connecting for as long as the test wants. Needed because several defects only + /// bite a child that has NOT yet connected (InitializeRequested is unhandled in + /// Connected, so a test built on an already-connected child passes vacuously). + /// + public TaskCompletionSource? InitializeGate { get; set; } + /// - public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) + public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { Interlocked.Increment(ref _initializeCount); - return Task.CompletedTask; + if (InitializeGate is { } gate) await gate.Task.ConfigureAwait(false); } /// @@ -359,9 +367,18 @@ internal sealed class RecordingSubscribableDriverFactory : IDriverFactory private readonly Lock _gate = new(); private readonly List _created = new(); + private readonly bool _gateInitialize; + /// Initializes a factory that materialises exactly one driver type. /// The driver type this factory answers for. - public RecordingSubscribableDriverFactory(string supportedType) => _supportedType = supportedType; + /// When true, every created driver gets an un-completed + /// , so its child actor stays in + /// Connecting until the test completes it. + public RecordingSubscribableDriverFactory(string supportedType, bool gateInitialize = false) + { + _supportedType = supportedType; + _gateInitialize = gateInitialize; + } /// Every driver this factory created, in creation order. public IReadOnlyList Created @@ -374,6 +391,8 @@ internal sealed class RecordingSubscribableDriverFactory : IDriverFactory { if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null; var driver = new RecordingSubscribableDriver(driverInstanceId, driverType); + if (_gateInitialize) + driver.InitializeGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); lock (_gate) _created.Add(driver); return driver; } From a222421374a251e7bbf5265b3328f3821877ca59 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:27:40 -0400 Subject: [PATCH 40/45] docs: mesh gRPC ingress is AllowAnonymous; the PSK interceptors are the gate (R3-01) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New docs/security.md section "Mesh gRPC ingress is AllowAnonymous by design" (archreview 03/S14): what the cookie FallbackPolicy covers, why it can block a correctly-PSK'd call before the interceptor's verdict matters, why the AllowAnonymous opt-out is safe (all five interceptors fail-closed), the affected-endpoint table (fused-vs-driver-only distinction), and the two standing guards (MeshGrpcAnonymousMetadataTests, MeshGrpcMapConventionTests) plus the auth-stack/driver-only boundary tests. One cross-reference line each in docs/Configuration.md (§MeshTransport, §ConfigSource/ConfigServe) and docs/Telemetry.md, so an operator reading about the listener ports meets the constraint where they'd look for it. --- docs/Configuration.md | 4 +++ docs/Telemetry.md | 8 +++++ docs/security.md | 73 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/docs/Configuration.md b/docs/Configuration.md index 2bccc843..2adcc05f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -190,6 +190,8 @@ On the docker-dev rig every node carries these keys already; the rig's compose d See the [ClusterClient→gRPC migration design](plans/2026-08-06-clusterclient-to-grpc-design.md), [implementation plan](plans/2026-08-06-clusterclient-to-grpc.md), [tracking doc](plans/2026-08-06-clusterclient-to-grpc-tracking.md) and [live gate](plans/2026-08-06-clusterclient-to-grpc-live-gate.md) for the migration's phase-by-phase history — those are historical records of the cutover and are not updated further by this sweep. +**On an admin-role node, `NodeCommandService` and `CentralCommandService` are `AllowAnonymous` by design** — the AdminUI cookie `FallbackPolicy` would otherwise 401/redirect a correctly-PSK'd call before the interceptor above ever ran (archreview 03/S14). See [`security.md` § Mesh gRPC ingress is `AllowAnonymous` by design](security.md#mesh-grpc-ingress-is-allowanonymous-by-design). + ### `ConfigSource` / `ConfigServe` (config fetch-and-cache) - **Purpose:** [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) **Phase 3**. `ConfigSource:Mode` selects where a **driver** node reads its deployed configuration: `Direct` (read `Deployment.ArtifactBlob` from central SQL, today's behaviour) or `FetchAndCache` (fetch the artifact bytes from central over a gRPC stream, verify SHA-256, cache in LocalDb, and read **only** the cache — no central-SQL config read). `ConfigServe` is the **central** (admin-role) serve side that a `FetchAndCache` node dials. @@ -217,6 +219,8 @@ See the [ClusterClient→gRPC migration design](plans/2026-08-06-clusterclient-t **`RevisionHash == SHA-256(artifact bytes)` is load-bearing.** `ConfigComposer` computes the deployment's revision hash as the lowercase-hex SHA-256 of the exact artifact bytes, so the node verifies a fetch against the `RevisionHash` the dispatch already carries — no hash rides in the proto. If a future change makes the revision hash anything else, every fetch fails verification. +**`DeploymentArtifactService` is `AllowAnonymous` by design** — same reasoning as `NodeCommandService`/`CentralCommandService` above: the AdminUI cookie `FallbackPolicy` on an admin-role node would otherwise block a correctly-keyed fetch before `ConfigServeAuthInterceptor`'s verdict matters. See [`security.md` § Mesh gRPC ingress is `AllowAnonymous` by design](security.md#mesh-grpc-ingress-is-allowanonymous-by-design). + #### Artifact compression (2026-08-15) The artifact blob is **gzip-compressed, inside the hashed bytes**. `ConfigComposer.Pack` gzips the diff --git a/docs/Telemetry.md b/docs/Telemetry.md index d1364b46..5caae91a 100644 --- a/docs/Telemetry.md +++ b/docs/Telemetry.md @@ -190,6 +190,14 @@ transports. ScadaBridge itself has since closed that gap with this identical interceptor pattern, so Phase 5 ships authenticated from the start rather than deferring auth to a later hardening pass. See the design doc's superseded note for the full history. +**On a fused `admin,driver` node, `TelemetryStreamService` and `NodeCommandService` are `AllowAnonymous` +by design** — a *different* gate from the interceptor above. `TelemetryStreamAuthInterceptor` is the PSK +check; on an admin-role node the AdminUI cookie `FallbackPolicy` sits in front of it and would 401/redirect +a correctly-keyed call before the interceptor's verdict matters, because it runs in the authorization +middleware, strictly *before* the interceptor executes (archreview 03/S14). A driver-only node never +registers that middleware at all, so it is structurally unaffected either way. See +[`security.md` § Mesh gRPC ingress is `AllowAnonymous` by design](security.md#mesh-grpc-ingress-is-allowanonymous-by-design). + ## Upgrading an existing deployment: populate `ClusterNode.GrpcPort` first (#493) `GrpcPort` is a **nullable column added by Phase 5**, and nothing backfills it onto rows that diff --git a/docs/security.md b/docs/security.md index 6872fbaf..bd44294b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -846,6 +846,79 @@ Responses: `202 Accepted` (`{ outcome, deploymentId, revisionHash }`) when a dep --- +## Mesh gRPC ingress is `AllowAnonymous` by design + +Archreview 03/S14 (Critical, closed R3-01). On an admin-role node `AddOtOpcUaAuth` installs a global +`FallbackPolicy = RequireAuthenticatedUser` bound to the cookie scheme (the same fallback that protects +every AdminUI page above). ASP.NET Core's authorization middleware applies that fallback to **every +endpoint that carries no authorization metadata of its own** — gRPC endpoints included. The five mesh +gRPC endpoints (config-serve artifact fetch, central command ack, telemetry stream, node command, LocalDb +sync) authenticate with a per-cluster/per-node **PSK bearer inside a gRPC interceptor**, and that +interceptor runs *inside* the endpoint invocation — i.e. strictly *after* the authorization middleware. +Without an explicit `.AllowAnonymous()` opt-out, a correctly-keyed node call is challenged by the cookie +handler before the interceptor's verdict ever matters: on a login-enabled admin node this is a fleet-wide +deploy-plane outage (artifact fetches fail, deploy acks are dropped, deployments seal `TimedOut`), and it +went unnoticed for seven phases of prior live gates because the docker-dev rig always ran +`Security__Auth__DisableLogin=true` (an always-succeeding handler registered *under the cookie scheme +name*, so the fallback trivially passes) and every prior mesh-gRPC test built its own auth-free pipeline +by hand. + +**Why `AllowAnonymous` is the correct fix, not a hole.** All five interceptors are fail-closed: an unset +key rejects every call, and the presented token is compared with `CryptographicOperations.FixedTimeEquals`. +`Program.cs:574` states the design contract directly — "the interceptor is the ONLY inbound auth on any +endpoint (the libraries verify nothing)" — which was already true of the gRPC pipeline; the missing opt-out +made it false of the HTTP pipeline in front of it. The pattern itself predates this fix: static assets +(`MapStaticAssets().AllowAnonymous()`), the metrics endpoint (`MapZbMetrics().AllowAnonymous()`), and the +headless deploy API above are all `AllowAnonymous` for the identical reason — each is gated by something +other than the cookie, and the fallback would otherwise 401/redirect it regardless. + +| Endpoint | Role gate | Affected without `AllowAnonymous`, on a login-enabled node | +|---|---|---| +| `DeploymentArtifactService.Fetch` | `hasAdmin && ConfigServe:GrpcListenPort > 0` | Always — every `FetchAndCache` node's artifact fetch. | +| `CentralCommandService.ReportApply` | same | Always — every node's deploy ack, from every cluster. | +| `TelemetryStreamService.Subscribe` | `hasDriver && Telemetry:GrpcListenPort > 0` | Only on a **fused** `admin,driver` node (e.g. docker-dev's central pair) — a driver-only node never registers the cookie stack at all, so it is structurally unaffected either way. | +| `NodeCommandService.Execute` | same | Only on a fused node, same reasoning. | +| `localdb_sync.v1.LocalDbSync/Sync` | `hasDriver && LocalDb:SyncListenPort > 0` | Only on a fused node that also sets a sync port — no node in the fleet currently has that shape, but the map is anonymised anyway. | + +**A fused central pair is affected on its OWN dispatch, not only inbound from sites.** `CentralCommunicationActor` dials every enabled `ClusterNode` row, its own pair included, and central's telemetry dialer dials its own pair too — so a login-enabled fused central breaks its own command/telemetry plane, not just the sites'. + +**Mapping mechanics.** All five maps live in one place, `MeshGrpcEndpoints.MapOtOpcUaMeshGrpcServices` +(`src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs`) — `Program.cs` calls it once and +carries no map sites of its own. Four maps chain `.AllowAnonymous()` directly onto `MapGrpcService()`. +The fifth, `MapZbLocalDbSync()`, returns `IEndpointRouteBuilder` rather than an +`IEndpointConventionBuilder`, so it cannot take the same chain; it is wrapped in an empty-prefix +`app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync()`, which carries the convention without +changing the wire route pattern. + +**Two standing guards**, both in `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/`, so a future map +cannot silently reopen this: + +- `MeshGrpcAnonymousMetadataTests` — an endpoint-metadata census that calls the SAME + `MapOtOpcUaMeshGrpcServices` production helper across all three role shapes (admin-only, driver-only, + fused) and asserts every mesh endpoint carries `IAllowAnonymous`, with an exact expected count per shape + and a scope check that the opt-out never leaks onto an unrelated endpoint. +- `MeshGrpcMapConventionTests` — a `Program.cs` source scan asserting zero direct `MapGrpcService Date: Thu, 20 Aug 2026 10:27:45 -0400 Subject: [PATCH 41/45] docs(plans): mark R3-01 T9 done; T10 remains pending (live gate, orchestrator-run) --- .../plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json index 799e0fa9..9464b7e2 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md.tasks.json @@ -65,7 +65,7 @@ { "id": "T9", "subject": "docs/security.md \"Mesh gRPC ingress is AllowAnonymous by design\" + cross-refs in docs/Configuration.md and docs/Telemetry.md", - "status": "pending", + "status": "done", "blockedBy": [] }, { From b725788954d8beaa26491b9581fbc7aee755a495 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:28:55 -0400 Subject: [PATCH 42/45] docs(plans): R3-01 execution notes (T1-T9) - filter gotcha, T4/T5 fixture scope, observed cookie-challenge signature --- .../R3-01-mesh-grpc-allowanonymous-plan.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md index a0400ede..4f552be4 100644 --- a/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md +++ b/archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md @@ -352,3 +352,87 @@ T10 last — and its step-0 control runs on master BEFORE the branch is merged ``` Overall effort: **~half a dev-day code+tests, plus the two-pass rig session.** The Critical itself is five lines; everything else in this plan exists because the five lines were invisible to a seven-phase live-gate program. + +--- + +## Execution notes (T1–T9, 2026-08-20) + +Branch `fix/r3-01-mesh-grpc-allowanonymous`, one commit per task as specified. T1–T9 all landed; +T10 (live gate) left `pending` for the orchestrator per the run brief. Every code-touching task +(T1–T7) was verified red-before-fix by hand — for T2–T5 by asserting against the un-fixed +`MeshGrpcEndpoints.cs`/`Program.cs` (via a temporary `git show :` swap, +rebuild, run, then restore — confirmed byte-identical restore via `git diff --stat` each time) +and observing the expected failure before reapplying the fix. Two things surfaced during that +process that are worth recording: + +1. **The census's endpoint filter could not stay a "route pattern starts with `/`" heuristic once + T3's empty-prefix `MapGroup` landed.** `RealMethodEndpoints` in `MeshGrpcAnonymousMetadataTests` + originally filtered on `RoutePattern.RawText?.StartsWith('/')` to exclude grpc-dotnet's own + per-app "Unimplemented service" catch-all and per-service "Unimplemented method for X" + fallback (neither carries a route pattern starting with `/` in the ungrouped case). Once + LocalDbSync moved behind `app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync()`, + those SAME fallback patterns *also* gained a leading `/` inside the group, so the heuristic + started over-counting (`DriverOnly_WithASyncPort_AlsoMapsTheLocalDbSyncEndpoint` and + `LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical` both went red for the + wrong reason — 3 endpoints found where 1 was expected). Fixed by matching against the exact + known set of five method route-pattern literals instead of a shape heuristic. Not a plan defect + — the plan didn't specify the filter mechanism — but a genuine gotcha for anyone extending this + census: **grpc-dotnet's route-pattern-shape for its own fallback endpoints is not stable across + a plain `MapGrpcService` vs. a grouped one**, so don't filter on shape once any mesh map moves + behind a group. + +2. **T4's fixture does not map or wire `LocalDbSyncService`'s real dependencies.** The plan's + fixture description says "maps the mesh services through `MapOtOpcUaMeshGrpcServices`"; T4's + `MeshGrpcAuthStackTests` calls it with `syncListenPort: 0` (LocalDbSync NOT mapped by the + helper) and instead hand-maps the library's `LocalDbSyncService` directly, with no + `.AllowAnonymous()`, purely as the negative control. This is a minimal, deliberate narrowing: + no assertion in T4 or T5 exercises LocalDbSync's real RPC (the plan's own positive/negative + lists never name it), and standing up its DI graph (`AddZbLocalDb`/`AddZbLocalDbReplication`, + a temp-file SQLite `ILocalDb`) would have added fixture weight with nothing to verify — the + authorization middleware refuses an unauthorized call before the service is ever constructed, + so the DI graph doesn't need to exist for the negative control to be meaningful. Recorded here + because it's a case where the fixture is deliberately narrower than a literal reading of the + plan's prose, even though every named assertion is still covered. + +3. **The positive/negative service lists T4 and T5 assert are asymmetric, and that asymmetry is + the plan's own, not introduced here.** T4's positive list is {`DeploymentArtifactService.Fetch`, + `CentralCommandService.ReportApply`, `NodeCommandService.Execute`} — Telemetry is not asserted + positively there. T5's no-PSK detail-string list is {Config-serve, Telemetry stream, Node + command} — `CentralCommandService`'s own detail string + (`"Central command authentication failed."`) is never asserted anywhere in T4/T5. Implemented + exactly as specified (no scope widening per the run brief); flagging the gap in case a + follow-up wants a `NoOrWrongPsk_CentralCommandReportApply_FailsWithTheInterceptorsDetail` case + for full symmetry — trivial to add given the fixture already maps and keys that service. + +4. **Observed cookie-challenge signature (T5 step 2), captured directly against this repo's + fixture stack (macOS, Kestrel h2c, `Grpc.Net.Client` over `HttpClient`/`SocketsHttpHandler`):** + every blocked positive-test call in the red state failed as + `RpcException(StatusCode.Internal, "Error starting gRPC call. HttpRequestException: The HTTP/2 + server reset the stream. HTTP/2 error code 'PROTOCOL_ERROR' (0x1). (HttpProtocolError) + HttpProtocolException: The HTTP/2 server reset the stream. HTTP/2 error code 'PROTOCOL_ERROR' + (0x1). (HttpProtocolError)")` — never `Unauthenticated`/`PermissionDenied`. The cookie handler's + redirect/401 breaks the prior-knowledge h2c stream before any gRPC trailers are written, so the + client sees a transport-level reset rather than a clean gRPC status. **The live gate (T10) + should grep node/central logs and browser network traces for `PROTOCOL_ERROR`, `/login`, `302` + and `401` together** — not for `401` alone, and not for a gRPC `Unauthenticated`/ + `PermissionDenied` status, which this environment never actually produces for the blocked case. + (A real reverse-proxy in front of a production deployment could still surface a clean HTTP 401 + instead of a stream reset, depending on how it terminates HTTP/2 — the plan's own multi-signal + grep advice already covers that; this note is the empirical confirmation of one concrete shape.) + +5. **T6 passes identically on both the pre-fix and post-fix mapper (confirmed by the same + temporary-swap procedure).** Driver-only nodes never register `AddOtOpcUaAuth` at all, so + `.AllowAnonymous()` presence or absence on the mesh maps has zero effect on that shape — T6 is + a structural pin/regression-control test, not a red-then-green TDD test for a behaviour T1–T3 + changed. This matches the plan's own "standard (test-only)" classification; recorded because + the run brief asked for red-before-fix confirmation to be checked (and reported) for every task, + and this is the one task where "red" never existed to begin with. + +6. **T8's `docker compose config` verification ran locally on macOS (OrbStack-backed Docker + Compose v5.1.2)** — no Windows-VM `lmxopcua-fix` helper needed, consistent with the Mac-path + guidance in CLAUDE.md. Confirmed both branches: unset `OTOPCUA_DISABLE_LOGIN` resolves + `Security__Auth__DisableLogin: "true"` on both central-1/central-2; `OTOPCUA_DISABLE_LOGIN=false` + resolves `"false"` on both. + +No other deviations from the plan's design or task breakdown. T10 was not run — left `pending` +per the run brief, for the orchestrator. From 0793b0cba837c31a15314edee8e74339a107b7d8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:39:13 -0400 Subject: [PATCH 43/45] =?UTF-8?q?fix(runtime):=20rate-limit=20the=20superv?= =?UTF-8?q?ision=20reprovision=20=E2=80=94=20deferred,=20not=20dropped=20(?= =?UTF-8?q?review=20F3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default decider restarts a faulting child immediately and without limit, and the hook makes every restart do ShutdownAsync+InitializeAsync+SubscribeAsync, so a driver faulting on every value became a reconnect storm. Per-driver last-reprovision stamp + DefaultReprovisionBackoff (10s, matched to DefaultReconnectInterval); a held request is re-delivered by a single per-driver timer carrying the original incarnation ref, so the F2 identity check still applies. Seen RED with the backoff disabled: InitializeCount == 3 (second applied immediately) where the test requires 2-then-3. Also: the _children.Keys union-arm test (seeder gained taglessDrivers) and the _desiredByDriver xmldoc note that it lingers on last-known-good after a failed apply. --- docs/DriverLifecycle.md | 18 +++ .../Drivers/DriverHostActor.cs | 74 ++++++++++-- .../Drivers/DriverHostActorRestartTests.cs | 109 ++++++++++++++++++ .../Harness/V3ArtifactSeeder.cs | 24 +++- 4 files changed, 212 insertions(+), 13 deletions(-) diff --git a/docs/DriverLifecycle.md b/docs/DriverLifecycle.md index 4fb2fb6f..a157a5d7 100644 --- a/docs/DriverLifecycle.md +++ b/docs/DriverLifecycle.md @@ -201,6 +201,24 @@ Things to know before touching any of this: remove a driver; a bounded retry count would silently stop one after N crashes. Both change failure semantics for every driver in the fleet and deserve their own plan and live gate. The reprovision hook makes the *current* infinite-restart behaviour correct instead of dead. +- ⚠️ **…which is exactly why the reprovision is RATE-LIMITED.** The default decider restarts a faulting + child **immediately and without limit**, and the hook now makes every one of those restarts do + `ShutdownAsync` + `InitializeAsync` + `SubscribeAsync` — so a driver that faults on every value would + hammer its device as fast as it can crash. `DriverHostActor` keeps a per-driver last-reprovision stamp + and holds anything inside `DefaultReprovisionBackoff` (10 s, matched to + `DriverInstanceActor.DefaultReconnectInterval` because it drives the same connect work). The held + request is **deferred, not dropped** — one `StartSingleTimer` per driver, so a burst collapses to a + single pending reprovision — because dropping it would leave a child whose *final* crash fell inside + the window un-provisioned forever, which is the 03/S16 failure this hook exists to close. Each + deferral logs a Warning naming the crash type, so a crash loop is visible rather than merely quiet. +- **A reprovision is routed by INCARNATION, not by driver id.** `ReprovisionRequested` is delivered + asynchronously, so one can still be in flight from an old incarnation after `HandleRestartDriver` + stopped that child and spawned a replacement under the same id. Delivered to the new child — which is + in `Connecting`, where `InitializeRequested` *is* handled — it would start a second, concurrent + `InitializeAsync`, racing the dispose-before-overwrite that #527's teardown exists to serialise + (`ModbusDriver._transport` / `MqttDriver._connection` each leak a transport on that race). The handler + therefore checks `Sender` against the child it currently holds and drops anything else; the deferred + path carries the original ref so the same check still applies when the timer re-delivers it. - **Ordering on a respawning deploy, and its one accepted cost.** `ApplyDriverSpecs` (which spawns) runs before the pass that rebuilds `_desiredByDriver`, so a child respawned by a deploy — raw tags are folded into the merged `DriverConfig`, so a tag-set edit changes the spec and trips the #516 config respawn — diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 588e40ee..8b5f5c2c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -82,6 +82,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5); + /// Minimum interval between two supervision reprovisions of the SAME driver child (archreview + /// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the + /// reprovision hook makes every restart do ShutdownAsync + InitializeAsync + SubscribeAsync — so an + /// un-limited hook turns a driver that faults on every value into a reconnect storm against its device. + /// Matched to : the reprovision drives the same + /// connect work, so it should not run more often than an ordinary reconnect attempt. + public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10); + /// /// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped /// deployments). A literal rather than an empty string so the row is visibly deliberate when @@ -357,7 +365,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// push, and the supervision-reprovision handler all send from here, so a /// child obtained by ANY route carries the same set. Cleared and repopulated whole on every apply, /// alongside the routing maps, so renames/removals can never leave a stale entry. Empty until the - /// first apply — a child spawned before any apply is handed nothing, which is what it should have. + /// first apply — a child spawned before any apply is handed nothing, which is what it should have. + /// It is only ever rebuilt by a SUCCESSFUL apply, so after a failed one it deliberately lingers on + /// the last-good composition — matching the address space, drivers and subscriptions that also keep + /// serving through a failed apply (#485/#486). A respawn or reprovision in that window therefore hands + /// the child the last-known-good set, not an empty one. private readonly Dictionary _desiredByDriver = new(StringComparer.Ordinal); /// @@ -413,6 +425,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// ). private readonly TimeSpan _applyWedgeTimeout; + /// Per-driver rate limit for the supervision-reprovision hook (see + /// ). + private readonly TimeSpan _reprovisionBackoff; + + /// When each driver was last reprovisioned after a supervision restart, so a crash loop is + /// paced rather than amplified. Bounded by the number of driver children; an entry for a driver that is + /// later removed simply stops being consulted (the handler resolves _children first). + private readonly Dictionary _lastReprovisionUtc = new(StringComparer.Ordinal); + /// /// Host of this node's LocalDb replication partner — the only node that holds a copy of this /// node's alarm queue, and therefore the only node it may ever stand down in favour of. @@ -554,7 +575,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS IDriverPrimaryRoleSource? primaryRoleSource = null, IHistorizationRoleView? historizationRoleView = null, Func? sharesHistorizationScopeWithLocal = null, - TimeSpan? applyWedgeTimeout = null) => + TimeSpan? applyWedgeTimeout = null, + TimeSpan? reprovisionBackoff = null) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -566,7 +588,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider, deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher, alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView, - sharesHistorizationScopeWithLocal, applyWedgeTimeout)); + sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -652,9 +674,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS IDriverPrimaryRoleSource? primaryRoleSource = null, IHistorizationRoleView? historizationRoleView = null, Func? sharesHistorizationScopeWithLocal = null, - TimeSpan? applyWedgeTimeout = null) + TimeSpan? applyWedgeTimeout = null, + TimeSpan? reprovisionBackoff = null) { _applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout; + _reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff; _historizationRoleView = historizationRoleView; _sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal; _telemetryHub = telemetryHub; @@ -1089,6 +1113,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleApplyResult); Receive(HandleRestartDriver); Receive(HandleReprovisionRequested); + Receive(m => ReprovisionChild(m.Request, m.Child)); Receive(HandleReconnectDriver); Receive(HandleRouteNodeWrite); Receive(HandleRouteNativeAlarmAck); @@ -1145,6 +1170,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleApplyResult); Receive(HandleRestartDriver); Receive(HandleReprovisionRequested); + Receive(m => ReprovisionChild(m.Request, m.Child)); Receive(HandleReconnectDriver); Receive(HandleRouteNodeWrite); Receive(HandleRouteNativeAlarmAck); @@ -1728,6 +1754,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(_ => TryRecoverFromStale()); Receive(HandleRestartDriver); Receive(HandleReprovisionRequested); + Receive(m => ReprovisionChild(m.Request, m.Child)); Receive(HandleReconnectDriver); Receive(OnRedundancyStateChanged); // The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes). @@ -3012,7 +3039,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// /// /// The restarted child's self-announcement. - private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) + private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) => + ReprovisionChild(msg, Sender); + + /// A reprovision held back by , carrying the incarnation it + /// was requested by so the identity check still applies when it is re-delivered by the timer. + /// The original request. + /// The incarnation that asked; re-validated against _children on delivery, + /// so a driver restarted or removed in the meantime discards it. + private sealed record ReprovisionDeferred( + DriverInstanceActor.ReprovisionRequested Request, IActorRef Child); + + private static string ReprovisionTimerKey(string driverInstanceId) => "reprovision-" + driverInstanceId; + + private void ReprovisionChild(DriverInstanceActor.ReprovisionRequested msg, IActorRef child) { // Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision. if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return; @@ -3023,14 +3063,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // where InitializeRequested IS handled — it would start a second, concurrent InitializeAsync, // racing the dispose-before-overwrite that #527's teardown exists to serialise // (ModbusDriver._transport, MqttDriver._connection each leak a transport on that race). - if (!Equals(Sender, entry.Actor)) + if (!Equals(child, entry.Actor)) { _log.Debug( "DriverHost {Node}: dropping a reprovision for {Id} from a superseded incarnation ({Stale})", - _localNode, msg.DriverInstanceId, Sender); + _localNode, msg.DriverInstanceId, child); return; } + // Rate-limit per driver (review F3). DEFERRED, never dropped: the last request must still be + // honoured, or a child whose final crash fell inside the window is left un-provisioned forever — + // exactly the 03/S16 failure this hook exists to close. One timer key per driver, so a burst + // collapses to a single pending reprovision rather than a queue of them. + var now = DateTime.UtcNow; + if (_lastReprovisionUtc.TryGetValue(msg.DriverInstanceId, out var last) && + now - last < _reprovisionBackoff) + { + var delay = _reprovisionBackoff - (now - last); + _log.Warning( + "DriverHost {Node}: driver child {Id} was restarted by supervision again after {Reason} " + + "within {Backoff} — deferring the re-provision by {Delay} (crash loop?)", + _localNode, msg.DriverInstanceId, msg.Reason, _reprovisionBackoff, delay); + Timers.StartSingleTimer( + ReprovisionTimerKey(msg.DriverInstanceId), new ReprovisionDeferred(msg, child), delay); + return; + } + + _lastReprovisionUtc[msg.DriverInstanceId] = now; + // A crash that Akka silently restarted is otherwise invisible to operators; name it. _log.Warning( "DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning", diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs index 53652f9f..414932b0 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs @@ -478,6 +478,115 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase "concurrent InitializeAsync on a driver that was already connected"); } + /// + /// Review finding F3 - a crash-looping driver must not become a reconnect storm. There is no + /// SupervisorStrategy anywhere under src/Server, so the default decider restarts a + /// faulting child immediately and without limit. That was survivable while a restart did + /// nothing; the reprovision hook now makes every one of them do + /// ShutdownAsync + InitializeAsync + SubscribeAsync, so a driver that faults on + /// every value would hammer its device as fast as it can crash. + /// + /// The reprovision is therefore rate-limited per driver. Crucially it is deferred, not + /// dropped - the last request must still be honoured, or a child whose final crash fell + /// inside the window would be left un-provisioned forever, which is the very failure this whole + /// plan exists to close. Two genuine back-to-back supervision restarts: the first reprovision + /// applies immediately, the second is held and then lands. + /// + /// + [Fact] + public void A_second_reprovision_inside_the_backoff_window_is_deferred_not_dropped() + { + var db = NewInMemoryDbFactory(); + // Generous relative to the in-memory step time below, so "inside the window" is a decision rather + // than a race, while still far under the presence budget the deferred landing waits on. + var backoff = TimeSpan.FromSeconds(3); + var factory = new RecordingSubscribableDriverFactory("Modbus"); + var deploymentId = V3ArtifactSeeder.SeedV3Deployment( + db, RevA, + new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }, + Array.Empty<(string, string, string, string, string?)>(), + enabled: true); + V3ArtifactSeeder.SpawnHostAndApply( + this, db, TestNode, deploymentId, RevA, factory, reprovisionBackoff: backoff); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(1); + factory.Created[0].SubscribeCount.ShouldBe(1); + }, + duration: PresenceBudget); + var driver = factory.Created[0]; + + // Crash 1: the first reprovision has no prior stamp, so it applies immediately. + driver.FireMalformedDataChange(SpeedPath); + AwaitAssert( + () => + { + driver.InitializeCount.ShouldBe(2); + driver.SubscribeCount.ShouldBe(2, "the re-subscribe must land before the second crash is armed"); + }, + duration: PresenceBudget); + + // Crash 2, immediately after - well inside the window. + driver.FireMalformedDataChange(SpeedPath); + AwaitAssert( + () => driver.ShutdownCount.ShouldBeGreaterThanOrEqualTo( + 2, "the second crash seam did not trip supervision - nothing below is testing a backoff"), + duration: PresenceBudget); + + driver.InitializeCount.ShouldBe( + 2, "the second reprovision was applied immediately - the backoff is not rate-limiting anything"); + + // ...but it must still land once the window elapses. + AwaitAssert( + () => driver.InitializeCount.ShouldBe( + 3, "the deferred reprovision was DROPPED, not deferred"), + duration: PresenceBudget); + } + + /// + /// The _children.Keys arm of the desired-set authority's union. A driver with NO authored tags + /// is absent from both the value- and alarm-ref maps, so keying the authority off those maps alone + /// would leave it with no entry at all — and "absent" must not be allowed to differ from "empty", + /// because an empty set is what tells a child to Unsubscribe and drop a stale handle. + /// + /// Asserted the only way it is observable: the tagless child is initialised (so it is provably + /// alive and was provisioned) and never subscribes, while its tag-owning sibling in the same + /// deployment does — the sibling is the positive control proving the apply's push pass actually + /// ran, so "it did not subscribe" cannot pass for "nothing happened". + /// + /// + [Fact] + public void A_running_driver_with_no_authored_tags_is_handed_an_empty_set() + { + var db = NewInMemoryDbFactory(); + var factory = new RecordingSubscribableDriverFactory("Modbus"); + var deploymentId = V3ArtifactSeeder.SeedV3Deployment( + db, RevA, + new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) }, + Array.Empty<(string, string, string, string, string?)>(), + enabled: true, + taglessDrivers: new[] { "DI-TAGLESS" }); + V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory); + + AwaitAssert( + () => + { + factory.Created.Count.ShouldBe(2); + // Positive control: the sibling that DOES have a tag subscribed, so the push pass ran. + factory.Created.Single(d => d.DriverInstanceId == DriverId) + .SubscribeCount.ShouldBe(1); + factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS") + .InitializeCount.ShouldBe(1); + }, + duration: PresenceBudget); + + factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS") + .SubscribeCount.ShouldBe( + 0, "a driver with no authored tags must be handed an EMPTY set, never a spurious subscribe"); + } + /// /// The verified negative, pinned so this plan's edits to the restart paths cannot break the one that /// was already correct. Force Reconnect (ReconnectDriver) Tells ForceReconnect to the diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs index 4831fdd2..11160db8 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs @@ -47,18 +47,25 @@ public static class V3ArtifactSeeder /// is parsed out of TagConfig.alarm by TagConfigIntent), which the alarm-ref half of the /// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried /// in the alarm one instead, so the two travel through different child code paths. + /// Extra DriverInstance ids carrying NO devices and NO tags. The + /// only way to express a driver that is a running child yet absent from both the value- and alarm-ref + /// maps — the _children.Keys arm of ApplyDesiredSubscriptions's union, which must still + /// hand that child an explicit EMPTY set rather than skipping it. /// The seeded deployment's id. public static DeploymentId SeedV3Deployment( IDbContextFactory db, RevisionHash rev, (string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags, (string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs, bool enabled = false, - IReadOnlyDictionary? tagConfigByTag = null) + IReadOnlyDictionary? tagConfigByTag = null, + string[]? taglessDrivers = null) { var drivers = rawTags .Select(t => (t.Driver, t.DriverName)) .Distinct() .Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }) + .Concat((taglessDrivers ?? Array.Empty()) + .Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })) .ToArray(); var devices = rawTags .Select(t => (t.Driver, t.Device)) @@ -130,14 +137,18 @@ public static class V3ArtifactSeeder /// Optional driver factory forwarded to DriverHostActor.Props. Null (the /// default, preserving every existing call site) leaves the host on its own default, which materialises no /// real drivers. Supply one — with enabled: true on the seed — to actually spawn driver children. + /// Optional per-driver supervision-reprovision rate limit forwarded to + /// DriverHostActor.Props; null keeps the production default. Exists so a test can observe the + /// deferral without waiting the production window. /// The host actor plus the publish and mux probes wired into it. public static (IActorRef Actor, TestProbe Publish, TestProbe Mux) SpawnHostAndApply( TestKitBase kit, IDbContextFactory db, NodeId localNode, DeploymentId deploymentId, RevisionHash rev, - IDriverFactory? driverFactory = null) + IDriverFactory? driverFactory = null, + TimeSpan? reprovisionBackoff = null) { var publish = kit.CreateTestProbe(); - var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory); + var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff); Dispatch(actor, coordinator, deploymentId, rev); publish.ExpectMsg(RuntimeActorTestBase.PresenceBudget); @@ -157,14 +168,14 @@ public static class V3ArtifactSeeder TestKitBase kit, IDbContextFactory db, NodeId localNode, DeploymentId deploymentId, RevisionHash rev) { - var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null); + var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null, reprovisionBackoff: null); Dispatch(actor, coordinator, deploymentId, rev); return (actor, mux); } private static (IActorRef Actor, TestProbe Mux, TestProbe Coordinator) Spawn( TestKitBase kit, IDbContextFactory db, NodeId localNode, - IActorRef? opcUaPublishActor, IDriverFactory? driverFactory) + IActorRef? opcUaPublishActor, IDriverFactory? driverFactory, TimeSpan? reprovisionBackoff) { var coordinator = kit.CreateTestProbe(); var mux = kit.CreateTestProbe(); @@ -177,7 +188,8 @@ public static class V3ArtifactSeeder dependencyMux: mux.Ref, opcUaPublishActor: opcUaPublishActor, virtualTagEvaluator: NullVirtualTagEvaluator.Instance, - virtualTagHostOverride: vtHost.Ref)); + virtualTagHostOverride: vtHost.Ref, + reprovisionBackoff: reprovisionBackoff)); return (actor, mux, coordinator); } From 6d0002889a7ef214c882b6ae4488d3f793d5410b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:39:47 -0400 Subject: [PATCH 44/45] =?UTF-8?q?docs(archreview):=20R3-03=20review=20outc?= =?UTF-8?q?ome=20=E2=80=94=20F1/F2/F3=20fixed=20with=20red-before=20eviden?= =?UTF-8?q?ce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...R3-03-restart-resubscribe-watchdog-plan.md | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md index c1ac048d..10648423 100644 --- a/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md +++ b/archreview/plans/R3-03-restart-resubscribe-watchdog-plan.md @@ -685,3 +685,88 @@ After restoring the hook: **6/6 pass**, and the class was run **20× in isolatio `SpawnChild` gives the child the production 10 s `DefaultReconnectInterval` (T7). 7. `_lastComposition` and `_desiredByDriver` coexist for one commit (T2) so T2 stays compilable; the deletion lands in T3, as T3's own step text specifies. + +--- + +## Review outcome + +Verdict **FIX-FIRST**, 3 items. All three confirmed against the code, fixed TDD (test first, seen red +for the predicted reason, then fixed), one commit each. + +### F1 [HIGH regression] — a changed alarm set latched (`edd01fcc`) + +**Confirmed.** `SubscribeDesiredAlarms` and `HandleSubscribeAlarmsAsync` both gated on +`_alarmSubscriptionHandle is not null`, which is the right guard for an idempotent re-push and the wrong +one for a deploy that changes the set. The value side converges by construction +(`StoreDesiredSubscriptions` is a full replace; `HandleSubscribeAsync` drops the prior subscription +first) — the alarm side did not. **T3 is what made it reachable**: a tag-set edit forces a respawn, and +a fast-initialising driver subscribes the transient spawn-time set before the bulk push, so the +PREVIOUS revision's alarm refs latched for the connection's life. `sourceNodeIds`-filtering drivers +(OPC UA Client, AB CIP, FOCAS) would silently never deliver the newly-added alarm. + +**RED:** `A_deploy_that_widens_the_alarm_set_re_subscribes_the_new_refs` — after deploying +alarm `{alarmX}` then `{alarmX, alarmY}`, `LastSubscribedAlarmRefs` was still `["Plant/Modbus/dev1/alarmX"]`. + +**Fix:** new `_subscribedAlarmRefs` records what the live handle was established with; +`AlarmSubscriptionIsStale()` (no handle **or** the desired refs differ) replaces the bare null check in +both places; a stale handle is **released before** re-subscribing so a filtering driver cannot +accumulate two overlapping subscriptions. The release clears the handle *first*, so a throwing +unsubscribe cannot leave the staleness check treating a dead handle as live, and it is best-effort +(logged, non-fatal — the re-subscribe is what matters). Unsubscribe-on-**empty** is still deliberately +absent, unchanged. Two notes: the release had to be wrapped in `_invoker.ExecuteAsync` or the repo's +`OTOPCUA0001` analyzer fails the build; and the Connected-handler comment that stated the +value-only convergence claim was corrected, as was the matching bullet in `docs/DriverLifecycle.md`. + +### F2 [MEDIUM] — reprovision routed by id, not incarnation (`2b7100cf`) + +**Confirmed.** Fixed with `if (!Equals(Sender, entry.Actor)) return;` (plus a Debug line naming the +superseded ref). + +⚠️ **The first version of the test PASSED against the unfixed code — the arrangement mattered.** With +the child already **Connected**, `InitializeRequested` is unhandled, so the stale reprovision was +swallowed for the wrong reason and the test was vacuous. The defect only bites a child in +`Connecting`, which is exactly where `InitializeRequested` *is* handled. `RecordingSubscribableDriver` +therefore gained an `InitializeGate` (a `TaskCompletionSource` its `InitializeAsync` awaits) so the +test can hold the child in `Connecting`. **RED with that arrangement:** `InitializeCount == 2` — the +second concurrent `InitializeAsync` racing the #527 dispose-before-overwrite. The absence assertion +carries a positive control (the child is proven alive at `InitializeCount == 1` first) and a +`GetDiagnostics` round-trip so the dropped message is provably processed before the absence is read. + +### F3 [IMPORTANT] — no backoff on the reprovision (`0793b0cb`) + +**Confirmed.** Added `DefaultReprovisionBackoff` (10 s, matched to +`DriverInstanceActor.DefaultReconnectInterval` because the reprovision drives the same connect work), +a per-driver `_lastReprovisionUtc` stamp, and a `reprovisionBackoff` Props parameter following the +existing `applyWedgeTimeout` precedent so a test need not wait the production window. + +A held request is **deferred, not dropped**: one `StartSingleTimer` per driver re-delivers a +`ReprovisionDeferred` carrying the **original incarnation ref**, so F2's identity check still applies +on re-delivery, and a burst collapses to a single pending reprovision rather than a queue. Each +deferral logs a Warning naming the crash type, so a crash loop is visible rather than quiet. + +**RED** (backoff disabled): `InitializeCount == 3` where the test requires 2-then-3 — i.e. the second +reprovision applied immediately. The test drives **two genuine supervision restarts** (not synthetic +messages), waits for the re-subscribe before arming the second crash, and asserts `ShutdownCount >= 2` +so "the second crash tripped supervision" is proven before the backoff is read. + +### Optional adds (both done) + +- `A_running_driver_with_no_authored_tags_is_handed_an_empty_set` — the `_children.Keys` union arm, + previously untested and called out as a gap in the T5 notes. It needed a driver with no tags at all, + which the seeder could not express, so `SeedV3Deployment` gained `taglessDrivers`. The tagless child + is asserted initialised-but-never-subscribed, with its tag-owning sibling's subscribe as the positive + control that the push pass ran. +- `_desiredByDriver` xmldoc now records that it is rebuilt only by a **successful** apply, so it + lingers on the last-known-good composition after a failed one — consistent with the address space, + drivers and subscriptions that also keep serving (#485/#486). + +### Verification after the three fixes + +- `dotnet build ZB.MOM.WW.OtOpcUa.slnx` → **0 errors**. +- **Full `Runtime.Tests`: 649 passed, 3 skipped, 0 failed** (up 4 from 645 — F1, F2, F3 and the + union-arm test). +- **`ControlPlane.Tests`: 262/262.** +- `DriverHostActorRestartTests` (now 10 cases) run **10× in isolation, 10/10 green**. +- T9 (live gate) still **pending** — the orchestrator runs it. Worth noting for that session: F1 is + directly observable on the rig, by deploying a second alarm-bearing raw tag to a driver that already + has one and confirming a transition on the NEW tag reaches `/alerts`. From c8cf5c8032a61411fd8bf29401cf59f95ce8dbbe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:43:49 -0400 Subject: [PATCH 45/45] test(host): the R3-01 falsifiability control asserts the cookie-challenge signature (a bare Exception was satisfiable by an anonymised endpoint too); docs wording --- docs/security.md | 4 ++-- .../MeshGrpcAuthStackTests.cs | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/security.md b/docs/security.md index bd44294b..d313d65e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -890,7 +890,7 @@ The fifth, `MapZbLocalDbSync()`, returns `IEndpointRouteBuilder` rather than an `app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync()`, which carries the convention without changing the wire route pattern. -**Two standing guards**, both in `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/`, so a future map +**Three standing guards**, all in `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/`, so a future map cannot silently reopen this: - `MeshGrpcAnonymousMetadataTests` — an endpoint-metadata census that calls the SAME @@ -899,7 +899,7 @@ cannot silently reopen this: and a scope check that the opt-out never leaks onto an unrelated endpoint. - `MeshGrpcMapConventionTests` — a `Program.cs` source scan asserting zero direct `MapGrpcService( + // Review (R3-01): asserting a bare Exception was vacuous — an anonymised endpoint ALSO throws + // here (handler deps are not wired in this fixture => RpcException Unknown). Only the + // cookie-challenge signature discriminates: the FallbackPolicy's 302 to /login is a + // non-gRPC response, surfaced by grpc-dotnet as Internal + PROTOCOL_ERROR. + var ex = await Should.ThrowAsync( () => call.ResponseStream.MoveNext(TestContext.Current.CancellationToken)); + ex.StatusCode.ShouldBe(StatusCode.Internal); + ex.Status.Detail.ShouldContain("PROTOCOL_ERROR"); } // ---------------------------------------------------------------------------------------