diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index acab60d8..d9e7f283 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -74,13 +74,14 @@ { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", - "status": "blocked-structural", + "status": "completed", "blockedBy": [ "T6", "T8", "T9" ], "openNote": "PARTIAL \u2014 the resilience PIPELINE is live-verified (Polly CircuitBreaker -> Retry -> Timeout -> CapabilityInvoker in central-1 stack traces), but breaker-OPEN state was never forced: R2-09's ConnectionBackoff throttles retries so failures don't accumulate fast enough to open the Polly breaker. The reliable trigger is a rapid idempotent write to a dead endpoint. Confirmed still open 2026-07-27 (deferment.md \u00a78.5).", + "passNote": "PASSED 2026-08-08 after the structural gap was FIXED (feat/wrap-driver-init-in-resilience): IDriver.InitializeAsync now routes through the invoker as DriverCapability.Initialize (no retry \u2014 the actor's RetryConnect timer is the retry loop; tier-default breaker threshold 2, because at the 10 s reconnect cadence at most three attempts fit the 30 s sampling window and a threshold of 5 could never open). Live on docker-dev (image df04c76ce06b, symbol-verified on both centrals): rtu-gate (DRV-3616122eb53d, endpoint 10.100.0.35:5021 with no listener) opened its Initialize breaker on BOTH centrals ('circuit-breaker OPENED \u2026 capability=Initialize \u2026 Connection refused'), the /hosts RESILIENCE panel showed the red 'BREAKER OPEN \u00b7 OPENED 19S AGO' chip, and after a socat 5021\u21925020 forwarder brought the endpoint alive both breakers closed within 1 s of each other ('circuit-breaker closed (recovered)' + 'reconnected') and the panel cleared to 'All targets healthy.' Control: healthy MAIN-modbus recorded zero breaker events. Two nuances: (1) 'both-replica chips' resolves as both nodes' TRACKERS tripping (proven in both logs) rendered through the single ACTIVE AdminUI \u2014 Traefik's /health/active check deliberately routes the UI to the active admin node only (standby returns 503 'Standby: active node for role admin is \u2026'), so there is never a second replica serving chips; (2) the staleness-dim leg was NOT exercised \u2014 both pair nodes publish under the same key, so staleness needs every publisher for a key to stop, which requires deliberate node-kill churn this gate did not do. Companion fix shipped in the same change: DriverCapabilityInvokerFactory.Create now Invalidates the instance's TRACKER snapshots alongside its pipelines, because a respawn during an outage discarded the only breaker whose OnClosed could clear the OPEN flag \u2014 a deploy mid-outage would have stranded a false OPEN chip forever (unreachable before this change made breaker-open reachable). Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md \u00a77.", "closureNote": "ATTEMPTED 2026-08-07, NOT ACHIEVABLE AS SPECIFIED \u2014 and the reason is structural, not environmental. The breaker only sees failures that propagate out of a CapabilityInvoker-wrapped call, and the dead-endpoint path produces none: (1) IDriver.InitializeAsync is NOT wrapped (DriverInstanceActor.cs:707 calls it directly; the invoker wraps Read/Write/Subscribe/Discover/AlarmAcknowledge only), so a driver that cannot connect contributes zero breaker input \u2014 refuting this task's own premise; (2) ModbusDriver swallows per-tag failures in BOTH directions \u2014 ReadAsync -> Degraded health (:365-375,:838-855), WriteAsync -> WriteResult status \u2014 so even the fallback trigger this plan suggests (rapid idempotent write-to-dead-endpoint) cannot work. Live-proven: MAIN-modbus pointed at a TCP blackhole went DEGRADED with every read timing out (0x80050000) while the tracker held ConsecutiveFailures=0 and the panel showed \"All targets healthy.\" The breaker is configured to open on 2 failures in a 30 s window at ratio 1.0, so the previously-recorded ConnectionBackoff-throttling explanation is not the operative one. VERIFIED LIVE regardless: panel renders on /hosts (rehomed by #546; this plan's \"8 driver pages\" placement is stale), RESILIENCE block present, and the publisher emits real tracker state every 5 s. To close: wrap the connect path, stop drivers swallowing capability failures, or use a driver whose wrapped call fails AFTER a successful init. Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md \u00a74." }, { diff --git a/archreview/plans/STATUS.md b/archreview/plans/STATUS.md index 3f34dc1e..f728d943 100644 --- a/archreview/plans/STATUS.md +++ b/archreview/plans/STATUS.md @@ -272,6 +272,21 @@ line + resilience chip were not triggered.)* > blackhole showed `ConsecutiveFailures = 0` and a panel reading *"All targets healthy."* > Full evidence: [`docs/plans/2026-08-07-adminui-followups-and-live-gates.md`](../../docs/plans/2026-08-07-adminui-followups-and-live-gates.md). +> **Update 2026-08-08 — R2-10 T10 ✅ PASSED, by fixing the structural gap the entry above diagnosed.** +> `IDriver.InitializeAsync` now routes through the invoker as **`DriverCapability.Initialize`** +> (branch `feat/wrap-driver-init-in-resilience`): no pipeline retry (the actor's reconnect timer is +> the retry loop, parser-enforced), breaker threshold **2** (at the 10 s reconnect cadence at most +> three attempts fit the 30 s sampling window — a Read-sized 5 could never open; pinned by its own +> test), and `DriverCapabilityInvokerFactory.Create` now invalidates the instance's **tracker** +> snapshots with its pipelines (else a deploy mid-outage strands a false OPEN chip forever). +> Live on docker-dev: `rtu-gate` vs. dead `:5021` → breaker OPEN on **both** centrals + the red +> "BREAKER OPEN" chip on `/hosts`; socat `5021→5020` up → both breakers closed within 1 s and the +> panel cleared to "All targets healthy." (healthy `MAIN-modbus` control: zero breaker events). +> This is also the **first live sighting of the `circuit-breaker OPENED` log line**, closing +> FOLLOWUP-10's behavioral gate. Two narrowings vs. this plan's T10 wording: the chips render on the +> single **active** AdminUI (Traefik `/health/active` 503s the standby by design — "both replicas" +> means both trackers), and the staleness-dim leg was not exercised. Evidence: the same doc, §7. + ### Residual follow-ups — tracked in Gitea (2026-07-15) The remaining non-blocking items are filed as issues (none gate the remediation, which is complete): diff --git a/deferment.md b/deferment.md index 2725e32e..bbe3711e 100644 --- a/deferment.md +++ b/deferment.md @@ -53,7 +53,7 @@ So everything below is **untracked work**. Nothing in it is a data-loss or outag | What | Nature | Where | |---|---|---| -| **4 live gates blocked on hardware/licence** — archreview R2-03 (VT Good→Bad on a data-fed rig), R2-10 (force breaker-OPEN), Galaxy `ScanStateProbeParityTests` (licence-gated), v2 GA exit criteria (~~FOCAS live-CNC smoke #54~~ *closed — live-validated on a real 31i-B 2026-06-25, see `docs/plans/2026-06-25-focas-pdu-v3-30i-b-support.md`*, OPC UA CTT pass, Ignition 8.3 cutover) | Cannot be closed from this box. Not decisions — inputs. | §5 | +| **4 live gates blocked on hardware/licence** — ~~archreview R2-03 (VT Good→Bad on a data-fed rig)~~ ✅ *PASSED 2026-08-07*, ~~R2-10 (force breaker-OPEN)~~ ✅ *PASSED 2026-08-08 by fixing the structural gap (`DriverCapability.Initialize`)*, Galaxy `ScanStateProbeParityTests` (licence-gated), v2 GA exit criteria (~~FOCAS live-CNC smoke #54~~ *closed — live-validated on a real 31i-B 2026-06-25, see `docs/plans/2026-06-25-focas-pdu-v3-30i-b-support.md`*, OPC UA CTT pass, Ignition 8.3 cutover) | Cannot be closed from this box. Not decisions — inputs. *(The two archreview gates turned out to be closable from this box after all — §5.)* | §5 | | ~~**1 deferred live-verify task** — `2026-06-14-driver-reconfigure-while-faulted` Task 3~~ ✅ **PASSED 2026-08-07.** A faulted driver ("initialize failed: Connection refused") adopted a corrected config and connected while faulted, with the node's `startedAt` byte-identical before and after and `RestartCount=0` — no restart. | §5 | | **G-7 residual** — Sql / MTConnect / Calculation / OpcUaClient have no Contracts-side `Inspect` | **The deploy gate is weaker than the authoring gate.** `TagConfigValidator` covers 11 of 12, so config arriving by **CSV import or API** — the paths that skip the UI — meets only the weaker gate. | §1.2 | | ~~**3 carried-open archreview findings** — the failure-visibility trio~~ **Update 2026-08-07 (truth sweep): mostly CLOSED.** 01/S-1 **FIXED** — `AddressSpaceApplier.cs:95-97` tallies swallowed removal-pass failures into `AddressSpaceApplyOutcome.FailedNodes` (comment cites 01/S-1). 03/S4 **FIXED** — `PrimaryGatePolicy.cs` default-denies on unknown role when a driver peer exists (doc-comment "Fixes archreview 03/S4"), live-proven R2-04 T15. 06/S-1 **PARTIAL** — fail-closed advise path + `galaxy.writes.unconfirmed` meter shipped in `GatewayGalaxyDataWriter.cs`; residual = gateway-side WriteComplete correlation (cross-repo, mxaccessgw). | Residual is only the 06/S-1 cross-repo half. | §6.3 | @@ -87,7 +87,7 @@ themes into tracked decisions. What remains: | **G-7** `EquipmentTagConfigInspector` coverage | ◐ **Narrowed + guarded** (2026-07-28): MQTT wired (it shipped an `Inspect` nobody called), and a reflection guard now fails the build if a reachable driver ships one and is not mapped. **Residual:** Sql / MTConnect / Calculation / OpcUaClient have no Contracts-side `Inspect` at all, so the deploy gate stays blind for them while the AdminUI validates them — see §1.2. | §1.2 | | ~~**13 `EquipmentTagsDarkBatch4` skipped tests**~~ | ✅ **Done** — 6 revived onto the raw/UNS shape, 3 folded into a widened live parity corpus, 4 deleted as retired. The reason constant is gone. **Skips now: `Runtime.Tests` 13 → 3, `OpcUaServer.Tests` 4 → 1** — all remaining are deliberate `EquipmentDeviceBindingRetired` tombstones. | §9 (2026-07-28, skips) | | 17 other tracker issues | Driver features + deferred work, unchanged by this remediation. | §2 | -| Open live gates | ~~#491 continuous historization, #468 browser tree~~ ✅ **both PASSED since** (#468 `c2c3c522`, #491 `e77ea4aa` + the #537/#538 gate `4bdce910`); archreview R2-03 / R2-10 remain. *(The un-gated drift detector's gate closed by deletion — see #523.)* | §5 | +| Open live gates | ~~#491 continuous historization, #468 browser tree~~ ✅ **both PASSED since** (#468 `c2c3c522`, #491 `e77ea4aa` + the #537/#538 gate `4bdce910`); ~~archreview R2-03 / R2-10~~ ✅ **both PASSED** (R2-03 2026-08-07; R2-10 2026-08-08, closed by fixing the structural gap). *(The un-gated drift detector's gate closed by deletion — see #523.)* | §5 | ✅ **The tracker is now clean of stale-open issues** (2026-07-30). **#516**, **#518**, **#507** and **#523** were closed, each with a resolution comment recording what was actually done — which differs @@ -411,7 +411,7 @@ mere env-var presence, so it cannot pass vacuously. | **Continuous-historization value capture** (#491) | ✅ **PASSED** `e77ea4aa` (2026-08-04) — and it found a **production-fatal defect the code-complete reading missed**: `ConfigureAwait(false)` in the Akka handler destroyed `Self`, the actor restarted, and the restart discarded the (empty-seeded) ref set. One value captured, then permanent silence. Two follow-ups filed and closed the same day: **#537** (capture had no Primary gate — both pair nodes wrote duplicates) and **#538** (a poison tag pinned the FIFO head, re-writing every healthy tag forever). | `docs/plans/2026-08-01-continuous-historization-live-gate.md`; `ad744734`, `e77ea4aa`, `e7fbe977`, `4bdce910` | | **Wave-0 universal discovery browser** full tree render (#468) | ✅ **PASSED** `c2c3c522` (2026-08-01) — unblocked by **building** the missing sim capability (an `ab_server` patch) rather than waiting for hardware. The gate then found a decoder defect **350 unit tests agreed with**. | tracking doc §Wave 0 | | **archreview R2-03** (T11 + T12) — VT Good→Bad on a data-fed rig | ✅ **PASSED 2026-08-07.** The rig is no longer data-less — the Modbus fixture is reachable from inside it — so the VT was staged Good and tripped **without a deployment** (a register was flipped on the fixture, so the VT child survived and could hold a last value): `42 → 0x80020000 → 42`, plus the P7 identical-redeploy spot-check (zero disturbance). ⚠️ **Found:** the Bad publish's "carrying the last-known value" is **unobservable by any client** — subscription *and* raw Read both return NULL. The actor is correct (new regression test pins it); the docs overstate what a client sees. | `docs/plans/2026-08-07-adminui-followups-and-live-gates.md` §2–3 | -| **archreview R2-10** — force breaker-OPEN live | ❌ **STILL OPEN — but the recorded cause was wrong, and the real one is structural.** Not ConnectionBackoff throttling (the breaker opens on 2 failures / 30 s): the dead-endpoint failure mode **never enters the resilience pipeline**. `IDriver.InitializeAsync` is not invoker-wrapped (`DriverInstanceActor.cs:707`), and `ModbusDriver` swallows per-tag failures in **both** directions, so even the plan's own fallback trigger (write-to-dead-endpoint) cannot work. Live-proven: a driver timing out every read against a blackhole reported `ConsecutiveFailures = 0` and *"All targets healthy."* — **the panel is permanently quiet for the failure operators most care about.** | `docs/plans/2026-08-07-adminui-followups-and-live-gates.md` §4 | +| **archreview R2-10** — force breaker-OPEN live | ✅ **PASSED 2026-08-08 — by FIXING the structural gap** (`feat/wrap-driver-init-in-resilience`), the day after the 2026-08-07 run re-diagnosed it. `IDriver.InitializeAsync` now routes through the invoker as **`DriverCapability.Initialize`** (no retry — the actor's timer is the retry loop; **breaker threshold 2**, because at the 10 s reconnect cadence at most three attempts fit the 30 s sampling window and a Read-sized 5 could literally never open). Live: `rtu-gate` against dead `10.100.0.35:5021` opened its breaker on **both** centrals, the `/hosts` panel showed the red **"BREAKER OPEN"** chip, and a socat 5021→5020 forwarder recovered it — both breakers closed within 1 s, panel back to *"All targets healthy."* Control: healthy `MAIN-modbus`, zero breaker events. Companion fix: a respawn now `Invalidate`s the instance's **tracker** snapshots with its pipelines, or a deploy mid-outage would strand a false OPEN chip forever. Caveats recorded: the UI is served by the **active admin node only** (Traefik `/health/active` 503s the standby — "both-replica chips" means both *trackers*, one UI), and the staleness-dim leg was not exercised. | `docs/plans/2026-08-07-adminui-followups-and-live-gates.md` §7 | | **Driver-reconfigure-while-faulted** Task 3 | ✅ **PASSED 2026-08-07** (the block indeed no longer held — docker-dev runs `DisableLogin`). A driver faulted on a bad device config adopted a corrected config **while faulted** and connected, with the node's `startedAt` byte-identical before/after and `RestartCount=0` — no restart, which is the whole point (pre-fix it retried the stale config forever). | `docs/plans/2026-08-07-adminui-followups-and-live-gates.md` §5 | | Galaxy `ScanStateProbeParityTests` | **OPEN** — licence-gated (one `$WinPlatform` on this box) | `docs/v2/Galaxy.ParityRig.md:180` | | **v2 GA exit criteria** — FOCAS live-CNC wire smoke (#54), OPC UA CTT/compliance pass, Ignition 8.3 redundancy cutover | **OPEN** — manual/hardware/external-tool | `docs/v2/v2-release-readiness.md:105-120` | @@ -421,8 +421,9 @@ mere env-var presence, so it cannot pass vacuously. **Note on the 14 archreview `deferred-live` tasks across 8 plans:** `archreview/plans/STATUS.md:242-305` records a 2026-07-13 live pass closing several (R2-02, R2-07, R2-11, R2-04, R2-05-partial) and -2026-07-15 closures for R2-01/R2-08/R2-06 — **but the `.tasks.json` files were never updated.** Only -R2-03 and R2-10 have no closure record anywhere. +2026-07-15 closures for R2-01/R2-08/R2-06 — **but the `.tasks.json` files were never updated.** ~~Only +R2-03 and R2-10 have no closure record anywhere.~~ *(Both now closed with full records: R2-03 +2026-08-07, R2-10 2026-08-08 — their `.tasks.json` files carry the closure notes.)* --- @@ -1279,6 +1280,31 @@ drivers), 3 carried-open archreview findings led by 01/S-1 (`AddressSpaceApplier --- +### 2026-08-08 — R2-10 closed by fixing the structural gap (`DriverCapability.Initialize`) + +The 2026-08-07 entry below left R2-10 T10 "still open — structural." The user chose the fix over the +other three options (gate through a different driver / drivers reporting out-of-band / retire the gate), +and the gate then **PASSED** against the fixed build. Branch `feat/wrap-driver-init-in-resilience`; +evidence in [`docs/plans/2026-08-07-adminui-followups-and-live-gates.md`](docs/plans/2026-08-07-adminui-followups-and-live-gates.md) §7. + +- **`IDriver.InitializeAsync` now routes through the invoker** as `DriverCapability.Initialize` — the + one capability call the pipeline never saw, which is why a dead endpoint produced a quiet panel. No + pipeline retry (the actor's `RetryConnect` timer is the retry loop; parser-enforced), and **breaker + threshold 2** — at the 10 s reconnect cadence at most three attempts fit the 30 s sampling window, + so the Read-sized 5 could literally never open. That threshold arithmetic is the load-bearing detail + and is pinned by its own test. +- **Live:** `rtu-gate` against dead `:5021` → breaker OPEN on both centrals + the red panel chip; + socat forwarder up → both breakers closed within 1 s, panel cleared. Healthy `MAIN-modbus`: zero + breaker events (control). Bonus: a config-invalid OpcUaClient driver also now trips — equally + invisible before. +- **Companion fix found by asking "what clears the flag":** a respawn (`Create`) now `Invalidate`s the + instance's tracker snapshots along with its pipelines — a deploy mid-outage discards the only + breaker whose `OnClosed` could clear the OPEN flag, so the old behaviour would have stranded a false + OPEN chip forever. Unreachable until this change made breaker-open reachable at all. +- **Two honest narrowings of the original T10 wording:** "both-replica chips" = both *trackers* (one + active AdminUI — Traefik's `/health/active` deliberately 503s the standby, discovered fighting the + sticky cookie); the staleness-dim leg was not exercised (needs every publisher for a key to stop). + ### 2026-08-07 — adminui-followups closed by audit; three of four runnable gates PASSED Took the two items §0.1 named as the remaining untracked engineering. Full evidence: diff --git a/docs/DriverLifecycle.md b/docs/DriverLifecycle.md index d92babec..95571dd5 100644 --- a/docs/DriverLifecycle.md +++ b/docs/DriverLifecycle.md @@ -270,6 +270,40 @@ re-entered this path; the watchdog makes it automatic and unbounded, so a device have leaked ~360 transports at the 10 s retry interval. The shutdown is best-effort: a driver that throws on it still gets its initialise attempt. +### The connect path feeds the circuit breaker (R2-10 T10 closure) + +`IDriver.InitializeAsync` is routed through the capability invoker as +**`DriverCapability.Initialize`**, keyed on the driver-instance id (init spans the whole driver, +like a bulk subscribe). Before this, the connect path was the ONE capability call the invoker never +saw: a driver that could not connect contributed **zero** breaker input, so the AdminUI resilience +panel read *"All targets healthy"* against a dead endpoint — live-proven during the R2-10 T10 gate, +and the reason that gate was structurally unsatisfiable for a year of attempts. Things to know +before touching it: + +- **The Initialize policy never retries** — this actor's `RetryConnect` timer *is* the retry loop, + and pipeline retries would stack inside each timer tick (and stretch one attempt past the timer + interval so attempts overlap). The parser forces a `retryCount` override back to 0, same funnel + as the write-shaped rule. +- ⚠️ **The breaker threshold is 2, and raising it silently re-breaks the feature.** The breaker + samples over 30 s and the reconnect timer fires every 10 s, so at most **three** init attempts fit + one sampling window — a Read-sized threshold of 5 can literally never open on connect failures, + which reads exactly like "the panel doesn't work" with nothing logged. Pinned by + `Initialize_BreakerThreshold_IsReachableAtTheReconnectCadence` and, behaviourally, by + `Initialize_BreakerOpens_AtTierDefaults_AndFastFailsTheNextAttempt`. +- **An open breaker cannot wedge the reconnect loop.** A fast-failed attempt + (`BrokenCircuitException`) lands in the same catch as a real connect failure → + `InitializeFailed` → the timer keeps ticking. The 15 s break window is shorter than two 10 s + ticks, so recovery costs at most one extra tick. The teardown `ShutdownAsync` stays deliberately + UNWRAPPED — best-effort cleanup of an already-dead transport must not count toward the breaker. +- **A respawn resets the instance's tracker snapshots** (`DriverResilienceStatusTracker.Invalidate`, + called by `DriverCapabilityInvokerFactory.Create` alongside the pipeline invalidation). The + pipelines carry the actual breaker state; once they are discarded, a retained "Breaker OPEN" + snapshot could never self-correct — only the discarded pipeline's `OnClosed` stamps the close — + so a deploy during an outage would otherwise strand a false OPEN chip forever. This was + unreachable while breakers never opened; the Initialize wrap made it reachable. +- `ConsecutiveFailures` stays 0 during a connect outage **by design** — the tracker counts retry + attempts, and Initialize has none. The breaker-OPEN chip is the operator signal. + --- ## IDriverConfigEditor — custom AdminUI config editor (plug-point) diff --git a/docs/plans/2026-08-07-adminui-followups-and-live-gates.md b/docs/plans/2026-08-07-adminui-followups-and-live-gates.md index 9b938b6a..e8b40abd 100644 --- a/docs/plans/2026-08-07-adminui-followups-and-live-gates.md +++ b/docs/plans/2026-08-07-adminui-followups-and-live-gates.md @@ -215,3 +215,67 @@ original one-line body, so it mirrors `MainPressure` again rather than the `×6` One anomaly recorded rather than explained: both centrals were **recreated** at 09:49:40/09:49:50 (new `Created`, `RestartCount=0`) by something this session did not issue. It postdates the observations above and the old container's logs went with it, so it is noted, not attributed. + +--- + +## 7. Addendum 2026-08-08 — R2-10 T10 ✅ PASSED after fixing the structural gap + +§4 concluded the gate was not achievable as specified because the connect path never entered the +resilience pipeline. The user chose to fix that ("do option 1"), and the gate then passed against +the fixed build. Branch `feat/wrap-driver-init-in-resilience`. + +### The fix + +- **`DriverCapability.Initialize`** — `IDriver.InitializeAsync` routes through the capability + invoker in `DriverInstanceActor.InitializeAsync` (both the initial connect and the + `Reconnecting` retry path; the best-effort teardown `ShutdownAsync` stays deliberately + unwrapped). Keyed on the driver-instance id, like a bulk subscribe. +- **No pipeline retry for Initialize** — the actor's `RetryConnect` timer is the retry loop. + Enforced at the parser funnel (`retryCount` override forced to 0, same shape as the + write-shaped rule) and in every tier default. +- **Breaker threshold 2, not the Read-sized 5** — the breaker samples 30 s and the reconnect + timer fires every 10 s, so at most three init attempts fit one window; threshold 5 could never + open. Pinned by `Initialize_BreakerThreshold_IsReachableAtTheReconnectCadence` + + `Initialize_BreakerOpens_AtTierDefaults_AndFastFailsTheNextAttempt`. +- **Companion fix — respawn resets tracker snapshots.** `DriverCapabilityInvokerFactory.Create` + now calls `DriverResilienceStatusTracker.Invalidate(driverInstanceId)` alongside the pipeline + invalidation. Without it, a deploy during an outage discards the only breaker whose `OnClosed` + could clear the OPEN flag, stranding a false "Breaker OPEN" chip forever. Unreachable before + this change made breaker-open reachable; found by asking what clears the flag, not by a test. +- Wiring pinned by `Initialize_routes_through_the_capability_invoker` + + `An_invoker_fast_fail_on_init_keeps_the_actor_retrying_until_recovery` (falsification-controlled: + both go red with the actor wrap stashed; `Create_resets_the_instances_tracker_state_with_its_pipelines` + goes red with the factory call stashed). + +### The gate run (docker-dev, image `df04c76ce06b`, symbol-verified on both centrals) + +The staged failure was free: `rtu-gate` (`DRV-3616122eb53d`) was already authored against +`10.100.0.35:5021`, where nothing listens — connection refused on every init. + +1. **Breaker OPEN, both replicas** — within ~20 s of boot, both centrals logged + `circuit-breaker OPENED: instance=DRV-3616122eb53d … capability=Initialize … Connection refused`, + re-opening each half-open probe cycle. Bonus coverage: `DRV-opcua1` (invalid config → init + throws) also tripped — a permanently-misconfigured driver was equally invisible before. +2. **Panel chip** — `/hosts` → rtu-gate → Controls showed the red **"BREAKER OPEN · OPENED 19S + AGO"** chip in the RESILIENCE section, row FAULTED with "Connection refused". +3. **Control** — healthy `MAIN-modbus` recorded zero breaker events throughout. +4. **Recovery clear** — a socat forwarder (`5021 → 5020` on the docker host) brought the endpoint + alive; both centrals logged `circuit-breaker closed (recovered)` + `reconnected` within 1 s of + each other, and the panel dropped to "All targets healthy." +5. **Teardown** — socat removed; 5021 dead again (as-found). rtu-gate returns to a truthfully-OPEN + breaker chip, which is the feature working. + +### Two nuances against the original T10 wording + +- **"Both-replica chips"** resolves as both nodes' *trackers* tripping (proven in both logs) + rendered through the single **active** AdminUI. Traefik's `/health/active` check deliberately + routes the UI to the active admin node only — the standby returns 503 + (`Standby: active node for role 'admin' is …`), so there is never a second replica serving + chips. Discovered while trying to force the sticky-cookie LB onto central-1. +- **The staleness-dim leg was not exercised.** Both pair nodes publish under the same + `(instance, host)` key, so staleness requires *every* publisher for a key to stop — deliberate + node-kill churn this gate did not do. The dim is a pure render predicate (`IsStale` → opacity). + +Also corrected while here: §4's "ConnectionBackoff throttles retries" explanation (the previously +recorded root cause) is confirmed wrong — the breaker config opens on 2 failures in 30 s; the +operative cause was always that no failure reached the pipeline at all. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverCapability.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverCapability.cs index b6f9ea0d..d9a38d49 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverCapability.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverCapability.cs @@ -39,4 +39,15 @@ public enum DriverCapability /// reads (Raw/Processed/AtTime/Events). Retries by default. HistoryRead, + + /// + /// IDriver.InitializeAsync — connection establishment (R2-10 T10 closure: without this, + /// a driver that cannot connect contributes zero breaker input and the resilience panel stays + /// quiet for the failure operators most care about). Does NOT retry — the + /// DriverInstanceActor reconnect timer is the retry loop, and pipeline-level retries + /// would stack inside each timer tick. The tier-default breaker threshold is deliberately low + /// (2): at the 10 s reconnect cadence at most three attempts fit the breaker's 30 s sampling + /// window, so a Read-sized threshold of 5 could never open on connect failures. + /// + Initialize, } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverCapabilityInvokerFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverCapabilityInvokerFactory.cs index ba014e52..e0a0584c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverCapabilityInvokerFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverCapabilityInvokerFactory.cs @@ -77,6 +77,10 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac // Cleanup (no longer the correctness mechanism): drop this instance's cached pipelines across all // generations so lingering old-generation entries don't accumulate. No-op on first spawn. _builder.Invalidate(driverInstanceId); + // The tracker's snapshots describe the breakers inside the pipelines just discarded, so they go + // too — a retained "Breaker OPEN" from a pre-respawn outage could never self-correct (only the + // discarded pipeline's OnClosed stamps the close) and would show a false OPEN chip forever. + _statusTracker.Invalidate(driverInstanceId); return new CapabilityInvoker( _builder, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptions.cs index 723cbb39..4481bb66 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptions.cs @@ -82,6 +82,11 @@ public sealed record DriverResilienceOptions [DriverCapability.AlarmSubscribe] = new(TimeoutSeconds: 5, RetryCount: 3, BreakerFailureThreshold: 5), [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 5, RetryCount: 0, BreakerFailureThreshold: 5), [DriverCapability.HistoryRead] = new(TimeoutSeconds: 30, RetryCount: 2, BreakerFailureThreshold: 5), + // Initialize: no retry (the DriverInstanceActor reconnect timer is the retry loop) and a + // threshold of 2, NOT the Read-sized 5 — the breaker samples over 30 s and the reconnect + // timer fires every 10 s, so at most three init attempts fit one window; a threshold of 5 + // could never open on connect failures (R2-10 T10). + [DriverCapability.Initialize] = new(TimeoutSeconds: 5, RetryCount: 0, BreakerFailureThreshold: 2), }, DriverTier.B => new Dictionary { @@ -93,6 +98,8 @@ public sealed record DriverResilienceOptions [DriverCapability.AlarmSubscribe] = new(TimeoutSeconds: 8, RetryCount: 3, BreakerFailureThreshold: 5), [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 8, RetryCount: 0, BreakerFailureThreshold: 5), [DriverCapability.HistoryRead] = new(TimeoutSeconds: 60, RetryCount: 2, BreakerFailureThreshold: 5), + // See the Tier A Initialize note: threshold 2 is cadence-reachability, not a tuning choice. + [DriverCapability.Initialize] = new(TimeoutSeconds: 8, RetryCount: 0, BreakerFailureThreshold: 2), }, DriverTier.C => new Dictionary { @@ -104,6 +111,8 @@ public sealed record DriverResilienceOptions [DriverCapability.AlarmSubscribe] = new(TimeoutSeconds: 15, RetryCount: 1, BreakerFailureThreshold: 0), [DriverCapability.AlarmAcknowledge] = new(TimeoutSeconds: 15, RetryCount: 0, BreakerFailureThreshold: 0), [DriverCapability.HistoryRead] = new(TimeoutSeconds: 120, RetryCount: 1, BreakerFailureThreshold: 0), + // Tier C stays breaker-less like every other Tier C capability (process-level supervision). + [DriverCapability.Initialize] = new(TimeoutSeconds: 15, RetryCount: 0, BreakerFailureThreshold: 0), }, _ => throw new ArgumentOutOfRangeException(nameof(tier), tier, $"No default policy table defined for tier {tier}."), }; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptionsParser.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptionsParser.cs index fc26669b..22a0ae05 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptionsParser.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceOptionsParser.cs @@ -158,6 +158,17 @@ public static class DriverResilienceOptionsParser retry = 0; } + // Initialize's retry loop is the DriverInstanceActor reconnect timer, not the pipeline — + // pipeline retries would stack inside each timer tick (and, worse, stretch a single init + // attempt past the timer interval so attempts overlap). Same funnel-enforced invariant + // shape as the write-shaped rule above, different rationale. + if (capability == DriverCapability.Initialize && retry > 0) + { + AppendDiagnostic(ref diagnostic, + $"{capability} never retries (the reconnect timer is the retry loop); retryCount override ignored."); + retry = 0; + } + var breaker = merged.BreakerFailureThreshold; if (breaker < ResiliencePolicyRanges.MinBreakerFailureThreshold) { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs index b1134d06..eb1296f7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs @@ -83,6 +83,27 @@ public sealed class DriverResilienceStatusTracker (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }); } + /// + /// Drop every snapshot for one driver instance. Called by + /// DriverCapabilityInvokerFactory.Create alongside + /// DriverResiliencePipelineBuilder.Invalidate: the pipelines carry the actual breaker + /// state, so once they are discarded on a respawn, a retained snapshot describes a breaker + /// that no longer exists — and a stale "Breaker OPEN" could never self-correct, because only + /// the discarded pipeline's OnClosed hook stamps the close. + /// + /// The driver instance whose snapshots should be dropped. + /// The number of snapshots removed. + public int Invalidate(string driverInstanceId) + { + var removed = 0; + foreach (var key in _status.Keys) + { + if (key.DriverInstanceId == driverInstanceId && _status.TryRemove(key, out _)) + removed++; + } + return removed; + } + /// Record a process recycle event (Tier C only). /// The driver instance identifier. /// The host name. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/ResilienceFormModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/ResilienceFormModel.cs index 58af2e4b..e0f64f0a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/ResilienceFormModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/ResilienceFormModel.cs @@ -21,7 +21,7 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers; public sealed class ResilienceFormModel { public static readonly string[] Capabilities = - ["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"]; + ["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead", "Initialize"]; // RecycleIntervalSeconds is GONE (Gitea #522) — the scheduled-recycle machinery it authored was // deleted, so the field configured nothing. It is deliberately NOT explicitly stripped from stored diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs index c96b70c0..4ab6af32 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs @@ -687,6 +687,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers var log = _log; var driverInstanceId = _driverInstanceId; var driver = _driver; + var invoker = _invoker; _ = Task.Run(async () => { try @@ -695,6 +696,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers { try { + // Deliberately UNWRAPPED: teardown is best-effort cleanup of an already-dead + // transport — a shutdown failure must not count toward the Initialize breaker. await driver.ShutdownAsync(CancellationToken.None); } catch (Exception ex) @@ -704,7 +707,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers "initialising anyway", driverInstanceId); } } - await driver.InitializeAsync(driverConfigJson, CancellationToken.None); + // R2-10 T10: the connect path was the ONE capability call the invoker never saw, so a + // driver that could not connect contributed zero breaker input and the resilience panel + // stayed quiet for a dead endpoint. Keyed on the driver-instance id (init spans the + // whole driver, like a bulk subscribe). The Initialize policy never retries — this + // actor's RetryConnect timer is the retry loop — and its pipeline timeout now bounds a + // hanging connect via the ct the call site must honour. An open breaker fast-fails the + // attempt with BrokenCircuitException, which lands in the catch below exactly like a + // real connect failure: InitializeFailed → keep retrying on the timer. The 15 s break + // window is shorter than two 10 s timer ticks, so recovery costs at most one extra tick. + await invoker.ExecuteAsync( + DriverCapability.Initialize, + driverInstanceId, + async ct => await driver.InitializeAsync(driverConfigJson, ct), + CancellationToken.None); self.Tell(new InitializeSucceeded(generation)); } catch (Exception ex) diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverCapabilityInvokerFactoryTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverCapabilityInvokerFactoryTests.cs index 329096b3..f4246677 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverCapabilityInvokerFactoryTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverCapabilityInvokerFactoryTests.cs @@ -22,6 +22,29 @@ public sealed class DriverCapabilityInvokerFactoryTests ILogger? logger = null) => new(builder, new DriverResilienceStatusTracker(), _ => tier, logger); + /// + /// A respawn (every Create) resets the instance's tracker state alongside its pipeline + /// cache. The pipelines carry the actual breaker state and are discarded by the same call, so a + /// snapshot recorded against them — a "Breaker OPEN" from a pre-deploy outage — refers to a + /// breaker that no longer exists; keeping it would show a false OPEN chip forever, because only + /// the discarded pipeline's OnClosed could have cleared it. + /// + [Fact] + public void Create_resets_the_instances_tracker_state_with_its_pipelines() + { + var tracker = new DriverResilienceStatusTracker(); + var factory = new DriverCapabilityInvokerFactory( + new DriverResiliencePipelineBuilder(), tracker, _ => DriverTier.A); + tracker.RecordBreakerOpen("d1", "d1", DateTime.UtcNow); + tracker.RecordBreakerOpen("d2", "d2", DateTime.UtcNow); + + factory.Create("d1", "Modbus"); + + tracker.TryGet("d1", "d1").ShouldBeNull( + "the respawn discarded the pipeline whose breaker this snapshot described"); + tracker.TryGet("d2", "d2")!.IsBreakerOpen.ShouldBeTrue("other instances must be untouched"); + } + /// /// End-to-end proof the per-instance override REACHES execution: a ResilienceConfig that sets /// Read's retryCount to 0 must suppress the tier-A Read retry — a failing Read is attempted once, diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsParserTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsParserTests.cs index 67895cd3..bdf0e519 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsParserTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsParserTests.cs @@ -331,6 +331,19 @@ public sealed class DriverResilienceOptionsParserTests diag.ShouldContain("AlarmAcknowledge never retries"); } + /// An Initialize retryCount override is forced to 0 with a diagnostic — the + /// DriverInstanceActor reconnect timer is the retry loop, so pipeline retries would stack. + [Fact] + public void InitializeRetryOverride_IsForcedToZero_WithDiagnostic() + { + var options = DriverResilienceOptionsParser.ParseOrDefaults( + DriverTier.A, "{\"capabilityPolicies\":{\"Initialize\":{\"retryCount\":3}}}", out var diag); + + options.Resolve(DriverCapability.Initialize).RetryCount.ShouldBe(0); + diag.ShouldNotBeNull(); + diag.ShouldContain("Initialize never retries"); + } + /// A Read (idempotent) retryCount override is still honored — the invariant is write-shaped only. [Fact] public void ReadRetryOverride_StillHonored() diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsTests.cs index b2a4a827..1bf1a56c 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceOptionsTests.cs @@ -78,6 +78,39 @@ public sealed class DriverResilienceOptionsTests defaults[DriverCapability.AlarmAcknowledge].RetryCount.ShouldBe(0); } + /// + /// Initialize never retries by default — the DriverInstanceActor reconnect timer IS the + /// retry loop; pipeline retries would stack inside each timer tick. + /// + /// The driver tier to test. + [Theory] + [InlineData(DriverTier.A)] + [InlineData(DriverTier.B)] + [InlineData(DriverTier.C)] + public void Initialize_NeverRetries_ByDefault(DriverTier tier) + { + var defaults = DriverResilienceOptions.GetTierDefaults(tier); + defaults[DriverCapability.Initialize].RetryCount.ShouldBe(0); + } + + /// + /// The Initialize breaker threshold must be reachable at the reconnect cadence (R2-10 T10): + /// the breaker samples over 30 s and the reconnect timer fires every 10 s, so at most THREE + /// init attempts fit one sampling window. A threshold above 3 can never open on connect + /// failures — the exact silent-inertness this capability exists to fix — so the default is + /// pinned at 2 (Polly's minimum enabled throughput). Tier C stays breaker-less, consistent + /// with every other Tier C capability (the supervisor's process-level breaker owns it). + /// + [Theory] + [InlineData(DriverTier.A, 2)] + [InlineData(DriverTier.B, 2)] + [InlineData(DriverTier.C, 0)] + public void Initialize_BreakerThreshold_IsReachableAtTheReconnectCadence(DriverTier tier, int expectedThreshold) + { + var defaults = DriverResilienceOptions.GetTierDefaults(tier); + defaults[DriverCapability.Initialize].BreakerFailureThreshold.ShouldBe(expectedThreshold); + } + /// Verifies that idempotent capabilities retry by default. /// The driver tier to test. /// The driver capability to test. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs index 44642c28..8fef9521 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs @@ -315,6 +315,47 @@ public sealed class DriverResiliencePipelineBuilderTests snap.IsBreakerOpen.ShouldBeFalse("the breaker must read closed after a successful recovery probe"); } + /// + /// R2-10 T10 closure: with the TIER DEFAULTS (no override), consecutive Initialize failures + /// open the breaker — and a subsequent attempt fast-fails WITHOUT reaching the call site. + /// The load-bearing half is the threshold: at the 10 s reconnect cadence at most three init + /// attempts fit the 30 s sampling window, so this test drives exactly the tier-default + /// threshold of failures and demands an open breaker — it goes red if anyone raises the + /// Initialize threshold back into never-opens territory (5), the silent inertness the + /// live gate proved. + /// + [Fact] + public async Task Initialize_BreakerOpens_AtTierDefaults_AndFastFailsTheNextAttempt() + { + var tracker = new DriverResilienceStatusTracker(); + var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker); + var pipeline = builder.GetOrCreate("drv-init", "drv-init", DriverCapability.Initialize, TierAOptions); + + var threshold = TierAOptions.Resolve(DriverCapability.Initialize).BreakerFailureThreshold; + for (var i = 0; i < threshold; i++) + { + await Should.ThrowAsync(async () => + await pipeline.ExecuteAsync(async _ => + { + await Task.Yield(); + throw new InvalidOperationException("connect refused"); + })); + } + + tracker.TryGet("drv-init", "drv-init")!.IsBreakerOpen.ShouldBeTrue( + "the tier-default threshold of consecutive connect failures must open the Initialize breaker"); + + // The open breaker fast-fails the next attempt without invoking the connect at all. + var reached = false; + await Should.ThrowAsync(async () => + await pipeline.ExecuteAsync(_ => + { + reached = true; + return ValueTask.CompletedTask; + })); + reached.ShouldBeFalse("an open breaker must not let the connect attempt through"); + } + /// Verifies that the tracker isolates counters per host. [Fact] public async Task Tracker_IsolatesCounters_PerHost() diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs index b38de154..ba901a7e 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs @@ -90,6 +90,27 @@ public sealed class DriverResilienceStatusTrackerTests tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); } + /// + /// Invalidate drops every entry for one instance and leaves other instances alone. Without + /// this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale + /// "Breaker OPEN" snapshot: the respawn discards the pipelines whose OnClosed is the + /// only thing that stamps the close, so nothing can ever clear the flag again. + /// + [Fact] + public void Invalidate_DropsTheInstance_LeavesOthers() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordBreakerOpen("drv", "host-a", Now); + tracker.RecordBreakerOpen("drv", "host-b", Now); + tracker.RecordBreakerOpen("other", "host-a", Now); + + tracker.Invalidate("drv"); + + tracker.TryGet("drv", "host-a").ShouldBeNull(); + tracker.TryGet("drv", "host-b").ShouldBeNull(); + tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue(); + } + /// Verifies that RecordRecycle populates the LastRecycleUtc timestamp. [Fact] public void RecordRecycle_Populates_LastRecycleUtc() diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorResilienceWiringTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorResilienceWiringTests.cs index 745f3f6b..6908e445 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorResilienceWiringTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorResilienceWiringTests.cs @@ -80,6 +80,100 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB invoker.WriteCalls.ShouldContain(c => c.Host == "plc-7" && c.IsIdempotent == false); } + /// + /// R2-10 T10 closure: IDriver.InitializeAsync routes through the invoker as + /// keyed on the driver-instance id (init spans the + /// whole driver, like a bulk subscribe). Before this wiring the connect path was the ONE + /// capability call the invoker never saw — a driver that could not connect contributed zero + /// breaker input, so the resilience panel stayed quiet for a dead endpoint. + /// + [Fact] + public void Initialize_routes_through_the_capability_invoker() + { + var invoker = new RecordingInvoker(); + var driver = new HostResolvingWritableDriver(); + var actor = Sys.ActorOf(DriverInstanceActor.Props(driver, invoker: invoker)); + + actor.Tell(new DriverInstanceActor.InitializeRequested("{}")); + + AwaitCondition( + () => invoker.Calls.Any(c => c.Capability == DriverCapability.Initialize), + TimeSpan.FromSeconds(3)); + invoker.Calls.ShouldContain(c => + c.Capability == DriverCapability.Initialize && c.Host == driver.DriverInstanceId); + } + + /// + /// The open-breaker shape end to end at the actor: when the INVOKER itself throws on the + /// Initialize route (exactly what an open circuit does — the call site is never reached), + /// the actor treats it as an ordinary init failure — lands in Reconnecting, keeps the retry + /// timer running, and recovers to Connected once the invoker lets an attempt through. This + /// pins two things: the reconnect-RETRY path is wrapped too (not just the initial connect), + /// and a breaker fast-fail can never wedge or crash the actor. + /// + [Fact] + public async Task An_invoker_fast_fail_on_init_keeps_the_actor_retrying_until_recovery() + { + var invoker = new FastFailingInitInvoker(failFirst: 2); + var driver = new HostResolvingWritableDriver(); + var actor = Sys.ActorOf(DriverInstanceActor.Props( + driver, invoker: invoker, reconnectInterval: TimeSpan.FromMilliseconds(100))); + + actor.Tell(new DriverInstanceActor.InitializeRequested("{}")); + + // The first two init attempts die inside the invoker (open breaker); recovery is proven by a + // write succeeding — WriteAttribute fast-fails synchronously in Connecting/Reconnecting, so a + // Success reply is only possible from Connected. + DriverInstanceActor.WriteAttributeResult? reply = null; + for (var i = 0; i < 120 && reply is not { Success: true }; i++) + { + reply = await actor.Ask( + new DriverInstanceActor.WriteAttribute("tag-1", 42), TimeSpan.FromSeconds(1)); + if (reply is not { Success: true }) await Task.Delay(25); + } + + reply.ShouldNotBeNull(); + reply!.Success.ShouldBeTrue("the actor must recover to Connected once the breaker lets an attempt through"); + invoker.InitializeAttempts.ShouldBeGreaterThanOrEqualTo(3, + "the reconnect-retry path must route through the invoker (initial attempt + at least the two fast-failed retries)"); + } + + /// Stands in for an OPEN circuit breaker on the Initialize route: the first + /// Initialize calls throw WITHOUT invoking the call site (the + /// driver's connect never runs — exactly Polly's BrokenCircuitException shape), then the + /// breaker "closes" and calls pass through. All other capabilities always pass through. + private sealed class FastFailingInitInvoker : IDriverCapabilityInvoker + { + private readonly int _failFirst; + private int _initializeAttempts; + + public FastFailingInitInvoker(int failFirst) => _failFirst = failFirst; + + public int InitializeAttempts => Volatile.Read(ref _initializeAttempts); + + public ValueTask ExecuteAsync( + DriverCapability capability, string hostName, + Func> callSite, CancellationToken cancellationToken) + => callSite(cancellationToken); + + public ValueTask ExecuteAsync( + DriverCapability capability, string hostName, + Func callSite, CancellationToken cancellationToken) + { + if (capability == DriverCapability.Initialize + && Interlocked.Increment(ref _initializeAttempts) <= _failFirst) + { + throw new InvalidOperationException("circuit open (simulated fast-fail)"); + } + return callSite(cancellationToken); + } + + public ValueTask ExecuteWriteAsync( + string hostName, bool isIdempotent, + Func> callSite, CancellationToken cancellationToken) + => callSite(cancellationToken); + } + /// Records every capability + resolved-host key routed through it, then delegates to the /// real call-site so the driver still runs. Stands in for CapabilityInvoker without pulling /// Polly / Core into the Runtime test.