From eccb15bdab964bf449ec7d4012916cec52f6c61c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 14 Jul 2026 15:13:43 -0400 Subject: [PATCH] docs(archreview): R2-04 T13/T15 live-passed 2026-07-14 Both R2-04 verification gates for the S4 primary-gate default-deny run: - T15 (docker-dev 2-node rig, rebuilt from master 365a90e6 + Modbus sim): ServiceLevel 250/240; boot-window write rejected 'not primary (role unknown)' (meter reason=role-unknown,site=write); steady-state secondary rejected 'not primary' + node reverted (read-back 1234 not 9999, meter reason=secondary); primary write reaches device; clean apply (no apply.failed). The behavior-affecting S4 slice is trusted. - T13 (in-process PrimaryGateFailoverTests): 1/1, ran without SQL; the real DPS-delivered snapshot drives the gate; negative control covered. STATUS.md + R2-04 tasks.json updated. --- ...2-04-failure-visibility-plan.md.tasks.json | 134 +++++++++++++++--- archreview/plans/STATUS.md | 51 ++++++- 2 files changed, 168 insertions(+), 17 deletions(-) diff --git a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json index 07d3a34a..9fe1c05b 100644 --- a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json +++ b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json @@ -2,20 +2,122 @@ "planPath": "archreview/plans/R2-04-failure-visibility-plan.md", "lastUpdated": "2026-07-12", "tasks": [ - { "id": "T1", "subject": "Extend AddressSpaceApplyOutcome with RebuildFailed/FailedNodes + SafeRebuild returns bool (01/S-1)", "status": "completed", "blockedBy": [] }, - { "id": "T2", "subject": "Safe* helpers return bool; Apply tallies removal-pass failures into FailedNodes", "status": "completed", "blockedBy": ["T1"] }, - { "id": "T3", "subject": "Materialise* passes return swallowed-failure counts (void -> int, five methods)", "status": "completed", "blockedBy": ["T2"] }, - { "id": "T4", "subject": "OpcUaApplyFailed counter + OpcUaPublishActor Error/meter surfacing of degraded applies", "status": "completed", "blockedBy": ["T3"] }, - { "id": "T5", "subject": "Galaxy internal IMxWriteOps seam under GatewayGalaxyDataWriter (sealed SDK session untestable) (06/S-1)", "status": "completed", "blockedBy": [] }, - { "id": "T6", "subject": "Galaxy fail-closed write on AdviseSupervisory failure -> BadCommunicationError, no raw write", "status": "completed", "blockedBy": ["T5"] }, - { "id": "T7", "subject": "Galaxy counters galaxy.writes.advise_failed + galaxy.writes.unconfirmed (empty-statuses Good metered)", "status": "completed", "blockedBy": ["T6"] }, - { "id": "T8", "subject": "Galaxy skip-gated live smoke: normal advised write still returns Good post-change", "status": "completed", "blockedBy": ["T6"] }, - { "id": "T9", "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", "status": "completed", "blockedBy": [] }, - { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] }, - { "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "completed", "blockedBy": ["T10"] }, - { "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "completed", "blockedBy": ["T9"] }, - { "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "deferred-live", "note": "rig/live; Host.IntegrationTests (~16GB/run leak + needs SQL/2-node rig) — implemented + build-verified, controller runs it in the serialized heavy pass", "blockedBy": ["T11"] }, - { "id": "T14", "subject": "docs/Redundancy.md: gate policy table, boot-window client experience, new meters, Galaxy fail-closed semantics", "status": "completed", "blockedBy": [] }, - { "id": "T15", "subject": "LIVE GATE: 2-node docker-dev rig verify (ServiceLevel 240/100, boot-window write rejected+reverted, single /alerts row)", "status": "deferred-live", "note": "rig/live; 2-node docker-dev rig session (rebuild both centrals) — merge-blocking gate the controller runs in the serialized heavy pass", "blockedBy": ["T10", "T11", "T12", "T13"] } + { + "id": "T1", + "subject": "Extend AddressSpaceApplyOutcome with RebuildFailed/FailedNodes + SafeRebuild returns bool (01/S-1)", + "status": "completed", + "blockedBy": [] + }, + { + "id": "T2", + "subject": "Safe* helpers return bool; Apply tallies removal-pass failures into FailedNodes", + "status": "completed", + "blockedBy": [ + "T1" + ] + }, + { + "id": "T3", + "subject": "Materialise* passes return swallowed-failure counts (void -> int, five methods)", + "status": "completed", + "blockedBy": [ + "T2" + ] + }, + { + "id": "T4", + "subject": "OpcUaApplyFailed counter + OpcUaPublishActor Error/meter surfacing of degraded applies", + "status": "completed", + "blockedBy": [ + "T3" + ] + }, + { + "id": "T5", + "subject": "Galaxy internal IMxWriteOps seam under GatewayGalaxyDataWriter (sealed SDK session untestable) (06/S-1)", + "status": "completed", + "blockedBy": [] + }, + { + "id": "T6", + "subject": "Galaxy fail-closed write on AdviseSupervisory failure -> BadCommunicationError, no raw write", + "status": "completed", + "blockedBy": [ + "T5" + ] + }, + { + "id": "T7", + "subject": "Galaxy counters galaxy.writes.advise_failed + galaxy.writes.unconfirmed (empty-statuses Good metered)", + "status": "completed", + "blockedBy": [ + "T6" + ] + }, + { + "id": "T8", + "subject": "Galaxy skip-gated live smoke: normal advised write still returns Good post-change", + "status": "completed", + "blockedBy": [ + "T6" + ] + }, + { + "id": "T9", + "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", + "status": "completed", + "blockedBy": [] + }, + { + "id": "T10", + "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", + "status": "completed", + "blockedBy": [ + "T9" + ] + }, + { + "id": "T11", + "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", + "status": "completed", + "blockedBy": [ + "T10" + ] + }, + { + "id": "T12", + "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", + "status": "completed", + "blockedBy": [ + "T9" + ] + }, + { + "id": "T13", + "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", + "status": "completed", + "note": "DONE 2026-07-14: PrimaryGateFailoverTests PASS 1/1 (3s), ran without SQL (gate test needs only the in-process 2-node cluster + DPS snapshot). Delivered snapshot promotes exactly one Primary; negative-control (broken subscribe) times out. See STATUS.md.", + "blockedBy": [ + "T11" + ] + }, + { + "id": "T14", + "subject": "docs/Redundancy.md: gate policy table, boot-window client experience, new meters, Galaxy fail-closed semantics", + "status": "completed", + "blockedBy": [] + }, + { + "id": "T15", + "subject": "LIVE GATE: 2-node docker-dev rig verify (ServiceLevel 240/100, boot-window write rejected+reverted, single /alerts row)", + "status": "completed", + "note": "DONE 2026-07-14: docker-dev 2-node rig rebuilt from master 365a90e6 + Modbus sim up. ServiceLevel 250/240; boot-window write rejected \"not primary (role unknown)\" (meter reason=role-unknown,site=write); steady-state secondary rejected \"not primary\" + node reverted (read-back 1234 not 9999, meter reason=secondary); primary write reaches device; clean apply (no apply.failed). Full record in STATUS.md.", + "blockedBy": [ + "T10", + "T11", + "T12", + "T13" + ] + } ] -} +} \ No newline at end of file diff --git a/archreview/plans/STATUS.md b/archreview/plans/STATUS.md index 1d9581c4..9cde706e 100644 --- a/archreview/plans/STATUS.md +++ b/archreview/plans/STATUS.md @@ -223,7 +223,7 @@ shared docker-dev rig / VPN infra isn't contended. To run one-at-a-time: - **Full integration-suite sweep** (10 `*.IntegrationTests` projects, one at a time): discharges R2-02 T18, R2-03 T11, R2-07 T5/T6, R2-08 T14, R2-11 T24. - **docker-dev `:9200` live-`/run`** (rebuild BOTH centrals): R2-02 T15 (ResilienceConfig warning banner / input lockout / unknown-key survival), R2-03 T12 (VT fail→Bad→recover + P7 no-recompile), R2-10 T10 (breaker-open on both centrals + recovery + FOLLOWUP-10 log line), R2-11 T22 (typed-editor Writable checkbox), **R2-07 T12 (surgical add/remove/mixed — F10b inertness risk, MUST run)**, R2-05 T15 (over-gating regression only — deny unobservable under `DisableLogin`). -- **2-node redundancy rig:** **R2-04 T13/T15 — behavior-affecting S4 primary-gate default-deny; MUST run before trusting the failover-role change.** +- **2-node redundancy rig:** ~~R2-04 T13/T15~~ **BOTH ✅ PASSED 2026-07-14** — T15 live docker-dev gate + T13 in-process delivery-path test (see below). - **Genuinely external / needs-user:** R2-06 T12 (VPN to a 0.2.0 HistorianGateway for the live EnsureTags retype probe), R2-08 live legs (real GLAuth outage/recovery cycle; `Category=LiveIntegration` gateway soak), R2-01 T11 (S7 SYN-blackhole outage on the `10.100.0.35` s7_1500 fixture). ### Deferred behavior-change follow-up (deliberately not shipped) @@ -280,6 +280,55 @@ gates (R2-06 VPN, R2-08 live LDAP, R2-01 S7 blackhole) still need their infra. --- +## R2-04 T15 — 2-node primary-gate LIVE GATE ✅ PASSED (2026-07-14) + +Ran the S4 primary-gate live recipe on the docker-dev 2-node central cluster **rebuilt from master `365a90e6`** +(`up -d --build central-1 central-2`). Brought up the **Modbus sim** (`otopcua-pymodbus-standard` on +`10.100.0.35:5020`) so `MAIN`'s Modbus equipment tags read **Good** (a data-less rig materialises every node +`Bad_WaitingForInitialData`, and the CLI write reads-first — so a live driver is required to have a writable-Good +node to drive the gate). Made `EQ-55297329838d/ModbusHR200` writable (`TagConfig.writable=true` via SQL + +`POST :9200/api/deployments`). Data-plane writes bound `multi-role`/`password` (carries the `WriteOperate` LDAP role). + +**All core assertions PASS:** +- **Step 1 — ServiceLevel split:** `Server_ServiceLevel` (`i=2267`) = **250 on `:4840` (primary) / 240 on `:4841` + (secondary)** — differentiated, no split-brain. (Plan said 240/100; the deployment's healthy values are 250/240 — + the invariant "primary > secondary" holds.) +- **Step 2 — boot-window default-DENY (the NEW S4 behavior):** restarting **both** centrals (to widen the + role-null-with-≥2-members window — a single-node restart's window is sub-second because cluster-join and the first + redundancy snapshot land near-simultaneously) caught, ~5 s into boot: + `WRN Operator write to EQ-55297329838d/ModbusHR200 rejected: not primary (role unknown)`. Meter + `otopcua_redundancy_primary_gate_denied_total{reason="role-unknown",site="write"} = 1`. +- **Step 3 — steady-state secondary DENY + #5 revert:** write `9999` to `:4841` → CLI reports optimistic + `Write successful` (synchronous, by design) but read-back = **1234 (the primary's value, reverted — NOT 9999)**; + central-2 logs `rejected: not primary`; meter `reason="secondary",site="write" = 1`. +- **Step 3b — primary write reaches the device:** write `1234` to `:4840` → `Write successful` and read-back on + `:4841` = `1234` (the Modbus driver wrote through). +- **T4 sanity — clean apply:** both centrals logged `AddressSpaceApplier: applied plan (kind=PureAdd … failed=0)` / + `materialised (… failed=0)`; **no `[ERR]` / `apply.failed` / degraded-apply line**; the `apply_failed` meter is + absent (never incremented). +- **Step 4 — `/alerts`:** endpoint 200, **0 rows** — no active alarm on the data-less rig, so trivially no + dual-primary double-emission (the double-row regression is only observable with a firing alarm, which needs live + driver data this rig lacks; the alerts-emit gate shares the same `PrimaryGatePolicy` and is unit-covered). + +**Net:** the R2-04 `PrimaryGatePolicy` membership-aware **default-DENY-on-unknown** is confirmed live end-to-end — +both denial reasons (`role-unknown`, `secondary`) fire the log + the new meter, the #5 optimistic-write revert +corrects the node, and the primary path is unaffected. The behavior-affecting S4 slice is trusted. + +**R2-04 T13 — in-process delivery-path test ✅ PASSED (2026-07-14).** +`dotnet test …Host.IntegrationTests --filter FullyQualifiedName~PrimaryGateFailoverTests` → **1/1, 3 s**. Ran +**without SQL** (`OTOPCUA_HARNESS_USE_SQL` unset) — the gate test needs only the in-process 2-node Akka cluster + +the DPS-delivered redundancy snapshot, not a ConfigDb deploy. The `TwoNodeClusterHarness` formed the cluster and +the **real DistributedPubSub-delivered** snapshot promoted exactly one node to Primary (Secondary's +`DriverHostActor.RouteNodeWrite` → `not primary`; Primary passed the gate, rejecting only with a mapping reason). +Its built-in negative control (a broken redundancy-topic subscribe leaves both stuck `role unknown` → 45 s timeout) +means the PASS proves genuine snapshot delivery drives the gate — the half the pure-TestKit unit guards can't cover. + +**Rig mutations (this pass):** `ModbusHR200.TagConfig.writable=true`; Modbus sim brought up on the shared docker host +(`10.100.0.35`). Reset the central rig with `docker compose -f docker-dev/docker-compose.yml down -v`; the sim with +`ssh dohertj2@10.100.0.35 'cd /opt/otopcua-modbus && docker compose --profile standard down'`. + +--- + ## Post-round-2 loose-end closure (2026-07-13) - **Pre-existing red unit test closed** (theme-#4 "CI to all suites" hygiene; predates round 2, not an R2 plan).