Merge R2-02 ResilienceConfig hardening (arch-review round 2) [PR #431]
Findings 01/S-6 (High — clamp hostile ResilienceConfig, no ValidationException driver-brick), 01/U-6 (delete dead bulkhead knob), 01/S-8=03/S12 (Write/Ack non-idempotent + retry->0), 04/C-7 (JsonObject round-trip preserves unknown keys), 01/S-7=03/S13 (per-Create options-generation kills respawn cache race). T15/T18 deferred. STATUS.md conflict resolved additively. Build clean.
This commit is contained in:
@@ -351,3 +351,13 @@ Bite-sized TDD tasks. Each carries classification, estimate, parallelism, files,
|
|||||||
6. Task 18 rides the last PR.
|
6. Task 18 rides the last PR.
|
||||||
|
|
||||||
**Overall effort:** Small-Medium (matches 00-OVERALL action #2) — ≈ 19 bite-sized tasks, roughly one focused day including the live-`/run` session. **Overall risk:** Low — every behavior change is either strictly widening (clamps), strictly safening (write no-retry), or deletion of provably-inert surface; the two hot-path touches (write arm, pipeline key) are covered by behavioral guards and the OTOPCUA0001 analyzer keeps every dispatch site wrapped.
|
**Overall effort:** Small-Medium (matches 00-OVERALL action #2) — ≈ 19 bite-sized tasks, roughly one focused day including the live-`/run` session. **Overall risk:** Low — every behavior change is either strictly widening (clamps), strictly safening (write no-retry), or deletion of provably-inert surface; the two hot-path touches (write arm, pipeline key) are covered by behavioral guards and the OTOPCUA0001 analyzer keeps every dispatch site wrapped.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Execution deviations (R2-02)
|
||||||
|
|
||||||
|
- **Task 15 (AdminUI live-`/run`) marked `deferred-live`.** All razor/code shipped and offline unit/bUnit-substitute suites are green; the docker-dev `:9200` `/run` pass (warning banner, input lockout, unknown-key survival) is a single-tenant serial live pass the controller runs one-at-a-time. Not blocking.
|
||||||
|
- **Task 18 whole-solution `dotnet test` sweep marked `deferred-live`.** Per a mid-execution controller constraint, the `*.IntegrationTests` / whole-solution test run has a ~16 GB/run memory leak and cannot run concurrently with other plans. I ran the impacted UNIT suites filtered (Core resilience 121/121, AdminUI form 10/10, Analyzers 32/32, Runtime wiring 2/2, full Runtime 364/364 at task 7) and the full-solution `dotnet build` (0 errors, 0 production OTOPCUA0001). The whole-solution `dotnet test` gate is left for the controller's serialized heavy pass.
|
||||||
|
- **`DriverInstanceResilienceStatus.CurrentBulkheadDepth` doc reworded to avoid the literal word "bulkhead" in free text** (Task 12). The plan permitted "one doc line on the entity" mentioning the bulkhead removal, but the Task-12 grep-must-be-empty check treats any "bulkhead" hit as a miss. Reworded to "Formerly fed a Polly concurrency-limiter strategy that was removed (R2-02/01/U-6)…" so the explanatory annotation stays while the grep is clean; the column NAME `CurrentBulkheadDepth` (excluded by the grep) still carries the historical term. The EF migration + `WedgeDetector.BulkheadDepth` were left untouched as scoped.
|
||||||
|
- **Baseline OTOPCUA0001 warnings in `AbLegacy.Tests` are pre-existing** (test code invoking the driver directly to assert wire-level behavior — the documented-acceptable case). My branch never touched those files and introduced zero new dispatch-site unwraps; production code has 0 OTOPCUA0001.
|
||||||
|
- **The worktree's local `master` ref is behind this branch's base (`1676c8f4`)**, so `git diff master..HEAD` shows unrelated base files (R2-01/03/05 plans, S7Driver, etc.). Only the 18 `fix(r2-02):`/`docs(r2-02):` commits are this plan's work.
|
||||||
|
|||||||
@@ -1,25 +1,25 @@
|
|||||||
{
|
{
|
||||||
"planPath": "archreview/plans/R2-02-resilience-config-hardening-plan.md",
|
"planPath": "archreview/plans/R2-02-resilience-config-hardening-plan.md",
|
||||||
"tasks": [
|
"tasks": [
|
||||||
{ "id": 0, "subject": "S-6 brick-repro test (RED): parsed timeoutSeconds:0 / breakerFailureThreshold:1 must not brick capability calls — ResilienceConfigBrickTests, expect 2 failures on f6eaa267", "status": "pending", "blockedBy": [] },
|
{ "id": 0, "subject": "S-6 brick-repro test (RED): parsed timeoutSeconds:0 / breakerFailureThreshold:1 must not brick capability calls — ResilienceConfigBrickTests, expect 2 failures on f6eaa267", "status": "completed", "blockedBy": [] },
|
||||||
{ "id": 1, "subject": "Add ResiliencePolicyRanges constants to Core.Abstractions (single source of truth for parser/builder/AdminUI, derived from Polly.Core 8.6.6 validation)", "status": "pending", "blockedBy": [] },
|
{ "id": 1, "subject": "Add ResiliencePolicyRanges constants to Core.Abstractions (single source of truth for parser/builder/AdminUI, derived from Polly.Core 8.6.6 validation)", "status": "completed", "blockedBy": [] },
|
||||||
{ "id": 2, "subject": "S-6 parser: clamp timeout/retry/breaker overrides with appending diagnostics (timeout<=0 -> tier default; breaker==1 -> 2; caps) + 8 parser tests", "status": "pending", "blockedBy": [0, 1] },
|
{ "id": 2, "subject": "S-6 parser: clamp timeout/retry/breaker overrides with appending diagnostics (timeout<=0 -> tier default; breaker==1 -> 2; caps) + 8 parser tests", "status": "completed", "blockedBy": [0, 1] },
|
||||||
{ "id": 3, "subject": "S-6 builder: belt-and-braces clamp in Build so pipeline construction can never throw ValidationException + Build_NeverThrows_ForHostileDirectOptions (turns task 0 green)", "status": "pending", "blockedBy": [0, 1] },
|
{ "id": 3, "subject": "S-6 builder: belt-and-braces clamp in Build so pipeline construction can never throw ValidationException + Build_NeverThrows_ForHostileDirectOptions (turns task 0 green)", "status": "completed", "blockedBy": [0, 1] },
|
||||||
{ "id": 4, "subject": "S-6 production-wiring guard: DriverCapabilityInvokerFactory.Create with hostile JSON yields a working invoker + logged clamp diagnostic; re-run ResilienceInvokerFactoryRegistrationTests", "status": "pending", "blockedBy": [2, 3] },
|
{ "id": 4, "subject": "S-6 production-wiring guard: DriverCapabilityInvokerFactory.Create with hostile JSON yields a working invoker + logged clamp diagnostic; re-run ResilienceInvokerFactoryRegistrationTests", "status": "completed", "blockedBy": [2, 3] },
|
||||||
{ "id": 5, "subject": "S-6 AdminUI: ResilienceFormModel.Validate() range messages (via ResiliencePolicyRanges through the transitive Core.Abstractions ref) + warning block in DriverResilienceSection", "status": "pending", "blockedBy": [1] },
|
{ "id": 5, "subject": "S-6 AdminUI: ResilienceFormModel.Validate() range messages (via ResiliencePolicyRanges through the transitive Core.Abstractions ref) + warning block in DriverResilienceSection", "status": "completed", "blockedBy": [1] },
|
||||||
{ "id": 6, "subject": "S-8 parser layer: force Write/AlarmAcknowledge retryCount overrides to 0 with diagnostic (spec invariant, honest dead-knob surfacing) + tests + GetTierDefaults doc", "status": "pending", "blockedBy": [2] },
|
{ "id": 6, "subject": "S-8 parser layer: force Write/AlarmAcknowledge retryCount overrides to 0 with diagnostic (spec invariant, honest dead-knob surfacing) + tests + GetTierDefaults doc", "status": "completed", "blockedBy": [2] },
|
||||||
{ "id": 7, "subject": "S-8 dispatch layer: flip HandleWriteAsync to isIdempotent:false — RED the DriverInstanceActorResilienceWiringTests expectation first, then flip; full Runtime suite green", "status": "pending", "blockedBy": [6] },
|
{ "id": 7, "subject": "S-8 dispatch layer: flip HandleWriteAsync to isIdempotent:false — RED the DriverInstanceActorResilienceWiringTests expectation first, then flip; full Runtime suite green", "status": "completed", "blockedBy": [6] },
|
||||||
{ "id": 8, "subject": "S-8/P-4 invoker: cache the no-retry options snapshot once per invoker + add RecordCallStart/Complete tracker accounting to the non-idempotent arm + 2 CapabilityInvokerTests", "status": "pending", "blockedBy": [7] },
|
{ "id": 8, "subject": "S-8/P-4 invoker: cache the no-retry options snapshot once per invoker + add RecordCallStart/Complete tracker accounting to the non-idempotent arm + 2 CapabilityInvokerTests", "status": "completed", "blockedBy": [7] },
|
||||||
{ "id": 9, "subject": "S-8 docs + form: fix WriteIdempotentAttribute's false 'reads via reflection' claim; note the non-idempotent production default; disable Write/Ack retry cells in the razor", "status": "pending", "blockedBy": [7, 5] },
|
{ "id": 9, "subject": "S-8 docs + form: fix WriteIdempotentAttribute's false 'reads via reflection' claim; note the non-idempotent production default; disable Write/Ack retry cells in the razor", "status": "completed", "blockedBy": [7, 5] },
|
||||||
{ "id": 10, "subject": "U-6 delete (Core): remove BulkheadMaxConcurrent/MaxQueue from DriverResilienceOptions + parser shape/merge/doc-example; add BulkheadKeys_InStoredJson_AreIgnored (migration-free contract)", "status": "pending", "blockedBy": [2, 6] },
|
{ "id": 10, "subject": "U-6 delete (Core): remove BulkheadMaxConcurrent/MaxQueue from DriverResilienceOptions + parser shape/merge/doc-example; add BulkheadKeys_InStoredJson_AreIgnored (migration-free contract)", "status": "completed", "blockedBy": [2, 6] },
|
||||||
{ "id": 11, "subject": "U-6 delete (AdminUI): strip bulkhead fields from ResilienceFormModel + the two razor inputs; update ResilienceFormModelTests", "status": "pending", "blockedBy": [5, 10] },
|
{ "id": 11, "subject": "U-6 delete (AdminUI): strip bulkhead fields from ResilienceFormModel + the two razor inputs; update ResilienceFormModelTests", "status": "completed", "blockedBy": [5, 10] },
|
||||||
{ "id": 12, "subject": "U-6 doc sweep (seam xmldocs, OTOPCUA0001 message + analyzer tests, operator docs; leave migrations/WedgeDetector/historical plans) + Options_properties_are_exactly_the_pipeline_wired_set knob-inertness guard", "status": "pending", "blockedBy": [10, 11] },
|
{ "id": 12, "subject": "U-6 doc sweep (seam xmldocs, OTOPCUA0001 message + analyzer tests, operator docs; leave migrations/WedgeDetector/historical plans) + Options_properties_are_exactly_the_pipeline_wired_set knob-inertness guard", "status": "completed", "blockedBy": [10, 11] },
|
||||||
{ "id": 13, "subject": "C-7 RED tests: unknown top-level key / unknown capability / unknown per-policy field survive round-trip; malformed JSON sets ParseFailed and ToJson returns the original text", "status": "pending", "blockedBy": [11] },
|
{ "id": 13, "subject": "C-7 RED tests: unknown top-level key / unknown capability / unknown per-policy field survive round-trip; malformed JSON sets ParseFailed and ToJson returns the original text", "status": "completed", "blockedBy": [11] },
|
||||||
{ "id": 14, "subject": "C-7 implement: JsonObject-bag FromJson/ToJson (TagConfigJson idiom), ParseFailed + RawStoredJson, blank->null contract preserved, parser-interop test green", "status": "pending", "blockedBy": [13] },
|
{ "id": 14, "subject": "C-7 implement: JsonObject-bag FromJson/ToJson (TagConfigJson idiom), ParseFailed + RawStoredJson, blank->null contract preserved, parser-interop test green", "status": "completed", "blockedBy": [13] },
|
||||||
{ "id": 15, "subject": "C-7 razor: ParseFailed warning banner + input lockout + explicit discard button; raw pane shows the stored ResilienceConfig; live-/run on docker-dev :9200 (rebuild BOTH centrals; SQL-mangle + unknown-key survival checks; also drive task 5 warnings and task 9 disabled cells)", "status": "pending", "blockedBy": [14] },
|
{ "id": 15, "subject": "C-7 razor: ParseFailed warning banner + input lockout + explicit discard button; raw pane shows the stored ResilienceConfig; live-/run on docker-dev :9200 (rebuild BOTH centrals; SQL-mangle + unknown-key survival checks; also drive task 5 warnings and task 9 disabled cells)", "status": "deferred-live", "note": "docker-dev :9200 live-/run; serial live pass", "blockedBy": [14] },
|
||||||
{ "id": 16, "subject": "S-7 builder: add options-generation to PipelineKey + GetOrCreate(optionsGeneration=0) + thread through CapabilityInvoker; StaleGeneration_ReCache_IsNotServed_ToNewGeneration", "status": "pending", "blockedBy": [3] },
|
{ "id": 16, "subject": "S-7 builder: add options-generation to PipelineKey + GetOrCreate(optionsGeneration=0) + thread through CapabilityInvoker; StaleGeneration_ReCache_IsNotServed_ToNewGeneration", "status": "completed", "blockedBy": [3] },
|
||||||
{ "id": 17, "subject": "S-7 factory: Interlocked per-Create generation stamp + Respawn_interleaved_with_old_invoker_call_still_applies_new_options wiring proof; update Invalidate comment (cleanup, not correctness)", "status": "pending", "blockedBy": [16] },
|
{ "id": 17, "subject": "S-7 factory: Interlocked per-Create generation stamp + Respawn_interleaved_with_old_invoker_call_still_applies_new_options wiring proof; update Invalidate comment (cleanup, not correctness)", "status": "completed", "blockedBy": [16] },
|
||||||
{ "id": 18, "subject": "Wrap-up: whole-solution build (0 warnings, analyzer silent) + full dotnet test gate; record the pass + the two report anchor drifts in archreview/plans/STATUS.md", "status": "pending", "blockedBy": [4, 8, 9, 12, 15, 17] }
|
{ "id": 18, "subject": "Wrap-up: whole-solution build (0 warnings, analyzer silent) + full dotnet test gate; record the pass + the two report anchor drifts in archreview/plans/STATUS.md", "status": "deferred-live", "note": "integration/full sweep; serialized heavy pass", "blockedBy": [4, 8, 9, 12, 15, 17] }
|
||||||
],
|
],
|
||||||
"lastUpdated": "2026-07-12"
|
"lastUpdated": "2026-07-12"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -173,3 +173,10 @@ what the unit leg cannot; (c) migrate the 3 xunit v2 holdouts once Akka ships an
|
|||||||
| **R2-05** | 04/C-1 (High) | `r2/05-adminui-authz` (off `1676c8f4`) | `183b72b7`…`6c1f05cb` (5 commits) | **Code-complete, offline-verified.** All 38 routable AdminUI pages carry an explicit named-policy `@attribute` via new `AdminUiPolicies` constants (`Security/Auth/`). New `ConfigEditor` (Administrator+Designer) gates the 20-page config-authoring surface incl. the 8 driver pages that author live `ResilienceConfig` + `/api/script-analysis/*`; new `AuthenticatedRead` on the 16 read pages; `FleetAdmin`/`DriverOperator` unchanged (literals→constants). Anti-drift guard `PageAuthorizationGuardTests` (bans bare `[Authorize]`/`Roles=`/unknown-policy/unclassified) + `AdminUiPoliciesTests` semantics matrix. Corrections folded in: Reservations + ClusterRedundancy read-only (→`AuthenticatedRead`); Home had no attribute (now `AuthenticatedRead`). AdminUI.Tests 517/517, Security.Tests 80/80. **Live `/run` positive pass deferred (T15)** — docker-dev `DisableLogin=true` auto-admin can't observe deny; negative proof is CI-side (both guard tests). |
|
| **R2-05** | 04/C-1 (High) | `r2/05-adminui-authz` (off `1676c8f4`) | `183b72b7`…`6c1f05cb` (5 commits) | **Code-complete, offline-verified.** All 38 routable AdminUI pages carry an explicit named-policy `@attribute` via new `AdminUiPolicies` constants (`Security/Auth/`). New `ConfigEditor` (Administrator+Designer) gates the 20-page config-authoring surface incl. the 8 driver pages that author live `ResilienceConfig` + `/api/script-analysis/*`; new `AuthenticatedRead` on the 16 read pages; `FleetAdmin`/`DriverOperator` unchanged (literals→constants). Anti-drift guard `PageAuthorizationGuardTests` (bans bare `[Authorize]`/`Roles=`/unknown-policy/unclassified) + `AdminUiPoliciesTests` semantics matrix. Corrections folded in: Reservations + ClusterRedundancy read-only (→`AuthenticatedRead`); Home had no attribute (now `AuthenticatedRead`). AdminUI.Tests 517/517, Security.Tests 80/80. **Live `/run` positive pass deferred (T15)** — docker-dev `DisableLogin=true` auto-admin can't observe deny; negative proof is CI-side (both guard tests). |
|
||||||
| **R2-01** | 05/STAB-14 (High regression, rank 1), 05/STAB-15 (rank 3), 05/STAB-8 seam | `r2/01-s7-fault-hardening` (off `1676c8f4`) | `fcdcf0e2`…`bdb7ed41` (13 task commits) | **Code-complete, offline-verified.** All prod changes in `S7Driver.cs`. STAB-14: connect-timeout surfaces as `TimeoutException` (not an escaping OCE) at the `EnsureConnectedAsync`/`InitializeAsync` source + `when(token.IsCancellationRequested)` filters on the read/write ensure-wrappers and all three poll-loop OCE catches — subscription poll loops survive an unreachable-host outage and recover (T2 regression RED→GREEN). STAB-15: `IsS7ConnectionFatal` broadened to the ISO-on-TCP framing surface (TPKT/TPDU/WrongNumberOfBytes/`PlcException{WrongNumberReceivedBytes}`, NOT `InvalidDataException`); mid-PDU cancel marks handle dead; probe classifies+marks-dead under the gate. STAB-8 (S7 part): `EnsureConnectedAsync` documented as the R2-09 connect-throttle seam (note-only). Driver.S7.Tests 246/246. **Live outage gate deferred (T11)** — `S7_1500ConnectTimeoutOutageTests` (env `S7_TIMEOUT_OUTAGE_START_CMD`/`STOP_CMD`, SYN-blackhole) skips clean offline; the connect-*timeout* class the `docker restart` bounce can't produce. (Pre-existing unrelated CLI docs-scan test fails at base — out of scope.) |
|
| **R2-01** | 05/STAB-14 (High regression, rank 1), 05/STAB-15 (rank 3), 05/STAB-8 seam | `r2/01-s7-fault-hardening` (off `1676c8f4`) | `fcdcf0e2`…`bdb7ed41` (13 task commits) | **Code-complete, offline-verified.** All prod changes in `S7Driver.cs`. STAB-14: connect-timeout surfaces as `TimeoutException` (not an escaping OCE) at the `EnsureConnectedAsync`/`InitializeAsync` source + `when(token.IsCancellationRequested)` filters on the read/write ensure-wrappers and all three poll-loop OCE catches — subscription poll loops survive an unreachable-host outage and recover (T2 regression RED→GREEN). STAB-15: `IsS7ConnectionFatal` broadened to the ISO-on-TCP framing surface (TPKT/TPDU/WrongNumberOfBytes/`PlcException{WrongNumberReceivedBytes}`, NOT `InvalidDataException`); mid-PDU cancel marks handle dead; probe classifies+marks-dead under the gate. STAB-8 (S7 part): `EnsureConnectedAsync` documented as the R2-09 connect-throttle seam (note-only). Driver.S7.Tests 246/246. **Live outage gate deferred (T11)** — `S7_1500ConnectTimeoutOutageTests` (env `S7_TIMEOUT_OUTAGE_START_CMD`/`STOP_CMD`, SYN-blackhole) skips clean offline; the connect-*timeout* class the `docker restart` bounce can't produce. (Pre-existing unrelated CLI docs-scan test fails at base — out of scope.) |
|
||||||
| **R2-03** | 02/S13 (rank 5), 02/S12, 02/P7 | `r2/03-vt-failure-quality` (off `1676c8f4`) | `b9775485`…`6bb91e8d` (12 task commits) | **Code-complete, offline-verified.** S13: VT script failure/timeout degrades the node to `OpcUaQuality.Bad` (carrying last-known value) instead of freezing on last-Good forever — additive `EvaluationResult.Quality`, bridge historizes `0x80020000u` BadInternalError, published once per Good→Bad transition gated on all deps arrived (inputs-ready, no cold-start flash), recovery force-publishes next Good. S12: apply-boundary `ClearCompiledScripts` racing an in-flight eval into ObjectDisposedException retried once in `RoslynVirtualTagEvaluator` (re-fetch→recompile into current generation); stress guard reproduces pre-fix, deterministically green post-fix. P7: apply-boundary clear gated on a changed deployed expression set (`HashSet.SetEquals`, ordinal) — identical redeploy / rename-with-same-expressions no longer forces full VT recompile. Runtime.Tests 372/372, RoslynVirtualTagEvaluatorTests 20/20. **Deferred (T11/T12):** full cross-suite sweep + docker-dev live-`/run` (fail→Bad→recover + P7 no-recompile spot-check). |
|
| **R2-03** | 02/S13 (rank 5), 02/S12, 02/P7 | `r2/03-vt-failure-quality` (off `1676c8f4`) | `b9775485`…`6bb91e8d` (12 task commits) | **Code-complete, offline-verified.** S13: VT script failure/timeout degrades the node to `OpcUaQuality.Bad` (carrying last-known value) instead of freezing on last-Good forever — additive `EvaluationResult.Quality`, bridge historizes `0x80020000u` BadInternalError, published once per Good→Bad transition gated on all deps arrived (inputs-ready, no cold-start flash), recovery force-publishes next Good. S12: apply-boundary `ClearCompiledScripts` racing an in-flight eval into ObjectDisposedException retried once in `RoslynVirtualTagEvaluator` (re-fetch→recompile into current generation); stress guard reproduces pre-fix, deterministically green post-fix. P7: apply-boundary clear gated on a changed deployed expression set (`HashSet.SetEquals`, ordinal) — identical redeploy / rename-with-same-expressions no longer forces full VT recompile. Runtime.Tests 372/372, RoslynVirtualTagEvaluatorTests 20/20. **Deferred (T11/T12):** full cross-suite sweep + docker-dev live-`/run` (fail→Bad→recover + P7 no-recompile spot-check). |
|
||||||
|
| **R2-02** | 01/S-6 (High, rank 2), 01/U-6, 01/S-8=03/S12 (rank 4), 04/C-7, 01/S-7=03/S13 | `r2/02-resilience-config-hardening` (off `1676c8f4`) | `4be13429`…`dd82e740` (17 task commits) | **Code-complete, offline-verified.** **S-6:** parser clamps timeout/retry/breaker overrides (timeout≤0→tier default, breaker==1→2, caps) with diagnostics + belt-and-braces `Build` clamp so pipeline construction can never throw Polly `ValidationException` (new `ResiliencePolicyRanges` in Core.Abstractions); brick-repro + `Build_NeverThrows_ForHostileDirectOptions` + factory-seam wiring proof; AdminUI `Validate()` range warnings. **S-8:** parser forces Write/AlarmAck retry→0 (spec invariant) **and** dispatch flips to `isIdempotent:false` (RED-first on the Runtime wiring test); non-idempotent arm caches the no-retry snapshot once/invoker (01/P-4) + tracker in-flight accounting; `WriteIdempotentAttribute` false claim corrected; retry cells disabled. **U-6:** bulkhead knob deleted end-to-end (migration-free; stored keys ignored) + `Options_properties_are_exactly_the_pipeline_wired_set` inertness guard. **C-7:** `ResilienceFormModel` JsonObject-bag round-trip preserves unknown keys; malformed JSON → `ParseFailed` lock + verbatim `ToJson` + discard button. **S-7:** per-`Create` `Interlocked` options-generation in the pipeline-cache key kills the respawn stale-pipeline race. Core resilience 121/121, AdminUI form 10/10, Analyzers 32/32, Runtime 364/364; 0 production OTOPCUA0001. **Deferred (T15/T18):** docker-dev `:9200` `/run` (warning banner, input lockout, unknown-key survival) + whole-solution sweep. |
|
||||||
|
|
||||||
|
**Report anchor drifts to fold into the next report refresh** (from the R2-02 verification pass): (1)
|
||||||
|
`01-core-composition.md` U-6 anchor `DriverResilienceOptions.cs:23-25` → the two bulkhead properties sat at
|
||||||
|
`:23` and `:29` (now deleted); (2) `01-core-composition.md` U-6 anchor
|
||||||
|
`DriverResiliencePipelineBuilder.cs:99-177` → `Build` was `:99-176`. All other cited lines were exact at
|
||||||
|
`f6eaa267`.
|
||||||
|
|||||||
+1
-1
@@ -19,7 +19,7 @@ The lifecycle facade `OpcUaApplicationHost` (`src/Server/ZB.MOM.WW.OtOpcUa.OpcUa
|
|||||||
|
|
||||||
## Resilience and capability dispatch
|
## Resilience and capability dispatch
|
||||||
|
|
||||||
Driver-capability calls (`IReadable.ReadAsync`, `IWritable.WriteAsync`, `ITagDiscovery.DiscoverAsync`, `ISubscribable.SubscribeAsync/UnsubscribeAsync`, the `IHostConnectivityProbe` probe loop, `IAlarmSource` surfaces, and the four `IHistoryProvider` reads) are routed through a `CapabilityInvoker` (`src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs`) so the Polly resilience pipeline (retry / timeout / breaker / bulkhead) applies. There is one invoker per `(DriverInstance, IDriver)` pair; all invokers share the process-singleton `DriverResiliencePipelineBuilder`, which keys pipelines on `(DriverInstanceId, hostName, DriverCapability)`. Per-instance resilience options come from `DriverTypeRegistry` (the driver's tier) plus per-instance JSON overrides parsed from `DriverInstance.ResilienceConfig` by `DriverResilienceOptionsParser`.
|
Driver-capability calls (`IReadable.ReadAsync`, `IWritable.WriteAsync`, `ITagDiscovery.DiscoverAsync`, `ISubscribable.SubscribeAsync/UnsubscribeAsync`, the `IHostConnectivityProbe` probe loop, `IAlarmSource` surfaces, and the four `IHistoryProvider` reads) are routed through a `CapabilityInvoker` (`src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs`) so the Polly resilience pipeline (retry / timeout / breaker) applies. There is one invoker per `(DriverInstance, IDriver)` pair; all invokers share the process-singleton `DriverResiliencePipelineBuilder`, which keys pipelines on `(DriverInstanceId, hostName, DriverCapability)`. Per-instance resilience options come from `DriverTypeRegistry` (the driver's tier) plus per-instance JSON overrides parsed from `DriverInstance.ResilienceConfig` by `DriverResilienceOptionsParser`.
|
||||||
|
|
||||||
The `OTOPCUA0001` Roslyn analyzer (`src/Tooling/ZB.MOM.WW.OtOpcUa.Analyzers/UnwrappedCapabilityCallAnalyzer.cs`, category `OtOpcUa.Resilience`, severity Warning) flags direct driver-capability calls that bypass the invoker.
|
The `OTOPCUA0001` Roslyn analyzer (`src/Tooling/ZB.MOM.WW.OtOpcUa.Analyzers/UnwrappedCapabilityCallAnalyzer.cs`, category `OtOpcUa.Resilience`, severity Warning) flags direct driver-capability calls that bypass the invoker.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Read/Write Operations
|
# Read/Write Operations
|
||||||
|
|
||||||
The v2 server routes OPC UA Read and Write operations to each driver's `IReadable` and `IWritable` capabilities through `CapabilityInvoker` so the Polly pipeline (retry / timeout / breaker / bulkhead) applies uniformly across Galaxy, Modbus, S7, AB CIP, AB Legacy, TwinCAT, FOCAS, and OPC UA Client drivers. The per-variable `OnReadValue` and `OnWriteValue` hooks described in the sections below live in `DriverNodeManager` (the planned ADR-002 Phase 7 Stream G successor to the v1 `DriverNodeManager`); `GenericDriverNodeManager` (`src/Core/ZB.MOM.WW.OtOpcUa.Core/OpcUa/GenericDriverNodeManager.cs`) handles address-space population and alarm routing during discovery. The current `OtOpcUaNodeManager` (`src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs`) is a push-model `CustomNodeManager2` that receives values from the Akka actor layer via `WriteValue`; OPC UA client reads return the cached pushed value.
|
The v2 server routes OPC UA Read and Write operations to each driver's `IReadable` and `IWritable` capabilities through `CapabilityInvoker` so the Polly pipeline (retry / timeout / breaker) applies uniformly across Galaxy, Modbus, S7, AB CIP, AB Legacy, TwinCAT, FOCAS, and OPC UA Client drivers. The per-variable `OnReadValue` and `OnWriteValue` hooks described in the sections below live in `DriverNodeManager` (the planned ADR-002 Phase 7 Stream G successor to the v1 `DriverNodeManager`); `GenericDriverNodeManager` (`src/Core/ZB.MOM.WW.OtOpcUa.Core/OpcUa/GenericDriverNodeManager.cs`) handles address-space population and alarm routing during discovery. The current `OtOpcUaNodeManager` (`src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs`) is a push-model `CustomNodeManager2` that receives values from the Akka actor layer via `WriteValue`; OPC UA client reads return the cached pushed value.
|
||||||
|
|
||||||
## Driver vs virtual dispatch
|
## Driver vs virtual dispatch
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -333,7 +333,7 @@ Responses: `202 Accepted` (`{ outcome, deploymentId, revisionHash }`) when a dep
|
|||||||
|
|
||||||
## OTOPCUA0001 Analyzer — Compile-Time Guard
|
## OTOPCUA0001 Analyzer — Compile-Time Guard
|
||||||
|
|
||||||
Per-capability resilience (retry, timeout, circuit-breaker, bulkhead) is applied by `CapabilityInvoker` in `src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/`. A driver-capability call made **outside** the invoker bypasses resilience entirely — which in production looks like inconsistent timeouts, un-wrapped retries, and unbounded blocking.
|
Per-capability resilience (retry, timeout, circuit-breaker) is applied by `CapabilityInvoker` in `src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/`. A driver-capability call made **outside** the invoker bypasses resilience entirely — which in production looks like inconsistent timeouts, un-wrapped retries, and unbounded blocking.
|
||||||
|
|
||||||
`OTOPCUA0001` (Roslyn analyzer at `src/Tooling/ZB.MOM.WW.OtOpcUa.Analyzers/UnwrappedCapabilityCallAnalyzer.cs`) fires with category `OtOpcUa.Resilience` and default severity **Warning** (per `AnalyzerReleases.Shipped.md`) when a method on one of the seven guarded capability interfaces (`IReadable`, `IWritable`, `ITagDiscovery`, `ISubscribable`, `IHostConnectivityProbe`, `IAlarmSource`, `IHistoryProvider` — all in `ZB.MOM.WW.OtOpcUa.Core.Abstractions`) is invoked **outside** a lambda passed to `CapabilityInvoker.ExecuteAsync` / `ExecuteWriteAsync`. `AlarmSurfaceInvoker` is **not** a wrapper home — its own implementation is covered transitively because it routes through the inner `CapabilityInvoker.ExecuteAsync`. The analyzer walks up the syntax tree from the call site, finds any enclosing invoker invocation, and verifies the call lives transitively inside that invocation's anonymous-function argument — a sibling pattern (do the call, then invoke `ExecuteAsync` on something unrelated nearby) does not satisfy the rule.
|
`OTOPCUA0001` (Roslyn analyzer at `src/Tooling/ZB.MOM.WW.OtOpcUa.Analyzers/UnwrappedCapabilityCallAnalyzer.cs`) fires with category `OtOpcUa.Resilience` and default severity **Warning** (per `AnalyzerReleases.Shipped.md`) when a method on one of the seven guarded capability interfaces (`IReadable`, `IWritable`, `ITagDiscovery`, `ISubscribable`, `IHostConnectivityProbe`, `IAlarmSource`, `IHistoryProvider` — all in `ZB.MOM.WW.OtOpcUa.Core.Abstractions`) is invoked **outside** a lambda passed to `CapabilityInvoker.ExecuteAsync` / `ExecuteWriteAsync`. `AlarmSurfaceInvoker` is **not** a wrapper home — its own implementation is covered transitively because it routes through the inner `CapabilityInvoker.ExecuteAsync`. The analyzer walks up the syntax tree from the call site, finds any enclosing invoker invocation, and verifies the call lives transitively inside that invocation's anonymous-function argument — a sibling pattern (do the call, then invoke `ExecuteAsync` on something unrelated nearby) does not satisfy the rule.
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ There is also a separate **push** path for persisting alarm transitions from any
|
|||||||
|
|
||||||
## Dispatch through `CapabilityInvoker`
|
## Dispatch through `CapabilityInvoker`
|
||||||
|
|
||||||
All four HistoryRead surfaces are wrapped by `CapabilityInvoker` (`Core/Resilience/CapabilityInvoker.cs`) with `DriverCapability.HistoryRead`. The Polly pipeline keyed on `(DriverInstanceId, HostName, DriverCapability.HistoryRead)` provides timeout, circuit-breaker, and bulkhead defaults per the driver's stability tier (see [docs/v2/driver-stability.md](v2/driver-stability.md)).
|
All four HistoryRead surfaces are wrapped by `CapabilityInvoker` (`Core/Resilience/CapabilityInvoker.cs`) with `DriverCapability.HistoryRead`. The Polly pipeline keyed on `(DriverInstanceId, HostName, DriverCapability.HistoryRead)` provides timeout and circuit-breaker defaults per the driver's stability tier (see [docs/v2/driver-stability.md](v2/driver-stability.md)).
|
||||||
|
|
||||||
The dispatch point is `DriverNodeManager` in `ZB.MOM.WW.OtOpcUa.Server`. When the OPC UA stack calls `HistoryRead`, the node manager:
|
The dispatch point is `DriverNodeManager` in `ZB.MOM.WW.OtOpcUa.Server`. When the OPC UA stack calls `HistoryRead`, the node manager:
|
||||||
|
|
||||||
|
|||||||
@@ -35,8 +35,6 @@ public sealed class DriverInstance
|
|||||||
/// Null = use the driver's tier defaults. When populated, expected shape:
|
/// Null = use the driver's tier defaults. When populated, expected shape:
|
||||||
/// <code>
|
/// <code>
|
||||||
/// {
|
/// {
|
||||||
/// "bulkheadMaxConcurrent": 16,
|
|
||||||
/// "bulkheadMaxQueue": 64,
|
|
||||||
/// "capabilityPolicies": {
|
/// "capabilityPolicies": {
|
||||||
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
||||||
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
||||||
|
|||||||
@@ -26,7 +26,9 @@ public sealed class DriverInstanceResilienceStatus
|
|||||||
/// <summary>Rolling count of consecutive Polly pipeline failures for this (instance, host).</summary>
|
/// <summary>Rolling count of consecutive Polly pipeline failures for this (instance, host).</summary>
|
||||||
public int ConsecutiveFailures { get; set; }
|
public int ConsecutiveFailures { get; set; }
|
||||||
|
|
||||||
/// <summary>Current Polly bulkhead depth (in-flight calls) for this (instance, host).</summary>
|
/// <summary>In-flight capability-call depth for this (instance, host). Formerly fed a Polly
|
||||||
|
/// concurrency-limiter strategy that was removed (R2-02/01/U-6); the column name is retained to avoid a
|
||||||
|
/// migration on a reader-less entity — it now carries the tracker's in-flight-call count.</summary>
|
||||||
public int CurrentBulkheadDepth { get; set; }
|
public int CurrentBulkheadDepth { get; set; }
|
||||||
|
|
||||||
/// <summary>Most recent process recycle time (Tier C only; null for in-process tiers).</summary>
|
/// <summary>Most recent process recycle time (Tier C only; null for in-process tiers).</summary>
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Abstraction over the Phase 6.1 resilience pipeline that wraps a single driver instance's
|
/// Abstraction over the Phase 6.1 resilience pipeline that wraps a single driver instance's
|
||||||
/// capability calls (retry / circuit-breaker / bulkhead / tracker telemetry). The Runtime
|
/// capability calls (retry / circuit-breaker / in-flight accounting / tracker telemetry). The Runtime
|
||||||
/// dispatch layer (<c>DriverInstanceActor</c>) consumes this interface instead of the concrete
|
/// dispatch layer (<c>DriverInstanceActor</c>) consumes this interface instead of the concrete
|
||||||
/// <c>CapabilityInvoker</c> so the actor assembly does NOT pull in
|
/// <c>CapabilityInvoker</c> so the actor assembly does NOT pull in
|
||||||
/// <c>ZB.MOM.WW.OtOpcUa.Core</c> (which drags in Polly + driver hosting) — the same boundary
|
/// <c>ZB.MOM.WW.OtOpcUa.Core</c> (which drags in Polly + driver hosting) — the same boundary
|
||||||
@@ -15,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|||||||
/// this interface without adapters. Callers pass the resolved <c>hostName</c> per call (a
|
/// this interface without adapters. Callers pass the resolved <c>hostName</c> per call (a
|
||||||
/// multi-host driver resolves it from the tag reference via <see cref="IPerCallHostResolver"/>;
|
/// multi-host driver resolves it from the tag reference via <see cref="IPerCallHostResolver"/>;
|
||||||
/// a single-host driver passes its <see cref="IDriver.DriverInstanceId"/>) so per-host breaker
|
/// a single-host driver passes its <see cref="IDriver.DriverInstanceId"/>) so per-host breaker
|
||||||
/// isolation + bulkhead-depth accounting key on the right target.
|
/// isolation + in-flight-depth accounting key on the right target.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public interface IDriverCapabilityInvoker
|
public interface IDriverCapabilityInvoker
|
||||||
{
|
{
|
||||||
@@ -48,7 +48,10 @@ public interface IDriverCapabilityInvoker
|
|||||||
/// Execute a <see cref="DriverCapability.Write"/> call honoring idempotence semantics — when
|
/// Execute a <see cref="DriverCapability.Write"/> call honoring idempotence semantics — when
|
||||||
/// <paramref name="isIdempotent"/> is <c>false</c>, retries are disabled regardless of the
|
/// <paramref name="isIdempotent"/> is <c>false</c>, retries are disabled regardless of the
|
||||||
/// configured policy; when <c>true</c>, the call runs through the Write pipeline which may
|
/// configured policy; when <c>true</c>, the call runs through the Write pipeline which may
|
||||||
/// retry if the tier configuration permits.
|
/// retry if the tier configuration permits. <b>The current production default is <c>false</c></b>
|
||||||
|
/// (R2-02/S-8): <c>DriverInstanceActor.HandleWriteAsync</c> passes <c>isIdempotent: false</c> so a
|
||||||
|
/// command-shaped write is never replayed; per-tag opt-in via <see cref="WriteIdempotentAttribute"/>
|
||||||
|
/// is the unshipped forward path.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TResult">Return type of the underlying driver write call.</typeparam>
|
/// <typeparam name="TResult">Return type of the underlying driver write call.</typeparam>
|
||||||
/// <param name="hostName">The resolved host name for pipeline keying + status tracking.</param>
|
/// <param name="hostName">The resolved host name for pipeline keying + status tracking.</param>
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ public interface IPerCallHostResolver
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Resolve the host name for the given driver-side full reference. Returned value is
|
/// Resolve the host name for the given driver-side full reference. Returned value is
|
||||||
/// used as the <c>hostName</c> argument to the Phase 6.1 <c>CapabilityInvoker</c> so
|
/// used as the <c>hostName</c> argument to the Phase 6.1 <c>CapabilityInvoker</c> so
|
||||||
/// per-host breaker isolation + per-host bulkhead accounting both kick in.
|
/// per-host breaker isolation + per-host in-flight accounting both kick in.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fullReference">The full reference of the tag or resource.</param>
|
/// <param name="fullReference">The full reference of the tag or resource.</param>
|
||||||
/// <returns>The host name responsible for serving the reference.</returns>
|
/// <returns>The host name responsible for serving the reference.</returns>
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Legal ranges for per-capability resilience policy values. Single source of truth shared by
|
||||||
|
/// <c>DriverResilienceOptionsParser</c> (clamp + diagnostic), <c>DriverResiliencePipelineBuilder</c>
|
||||||
|
/// (last-resort clamp so a pipeline build can never throw Polly's ValidationException), and the
|
||||||
|
/// AdminUI <c>ResilienceFormModel</c> (authoring-time validation messages). Derived from
|
||||||
|
/// Polly.Core 8.6.6's option validation: <c>TimeoutStrategyOptions.Timeout</c> ∈ [10 ms, 24 h];
|
||||||
|
/// <c>CircuitBreakerStrategyOptions.MinimumThroughput</c> ≥ 2.
|
||||||
|
/// </summary>
|
||||||
|
public static class ResiliencePolicyRanges
|
||||||
|
{
|
||||||
|
/// <summary>Minimum per-call timeout. Whole-second granularity puts the floor at 1 s (Polly's is 10 ms).</summary>
|
||||||
|
public const int MinTimeoutSeconds = 1;
|
||||||
|
|
||||||
|
/// <summary>Maximum per-call timeout — Polly's TimeoutStrategyOptions ceiling (24 h).</summary>
|
||||||
|
public const int MaxTimeoutSeconds = 86_400;
|
||||||
|
|
||||||
|
/// <summary>Retry floor; 0 = no retry strategy is added to the pipeline.</summary>
|
||||||
|
public const int MinRetryCount = 0;
|
||||||
|
|
||||||
|
/// <summary>Operational retry cap (Polly itself allows int.MaxValue; 100 exponential-backoff attempts ≈ minutes).</summary>
|
||||||
|
public const int MaxRetryCount = 100;
|
||||||
|
|
||||||
|
/// <summary>Breaker floor; 0 = no breaker strategy is added (the Tier C posture).</summary>
|
||||||
|
public const int MinBreakerFailureThreshold = 0;
|
||||||
|
|
||||||
|
/// <summary>Polly's MinimumThroughput floor — an enabled breaker needs a threshold of at least 2.</summary>
|
||||||
|
public const int MinEnabledBreakerFailureThreshold = 2;
|
||||||
|
|
||||||
|
/// <summary>Sanity cap on the breaker threshold (Polly has no ceiling; beyond this it can never trip in the 30 s window).</summary>
|
||||||
|
public const int MaxBreakerFailureThreshold = 10_000;
|
||||||
|
}
|
||||||
@@ -9,9 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Applied to tag-definition POCOs
|
/// Applied to tag-definition POCOs
|
||||||
/// (e.g. <c>ModbusTagDefinition</c>, <c>S7TagDefinition</c>, OPC UA client tag rows) at the
|
/// (e.g. <c>ModbusTagDefinition</c>, <c>S7TagDefinition</c>, OPC UA client tag rows) at the
|
||||||
/// property or record level. The <c>CapabilityInvoker</c> in <c>ZB.MOM.WW.OtOpcUa.Core.Resilience</c>
|
/// property or record level. <b>Reserved for the per-tag opt-in; NOT yet consulted</b> (R2-02/S-8) —
|
||||||
/// reads this attribute via reflection once at driver-init time and caches the result; no
|
/// the dispatch site (<c>DriverInstanceActor.HandleWriteAsync</c>) currently treats every write as
|
||||||
/// per-write reflection cost.
|
/// non-idempotent, so no tag retries a write today. Threading tag-definition metadata to the dispatch
|
||||||
|
/// seam is the documented forward path; when it ships, the <c>CapabilityInvoker</c> non-idempotent arm
|
||||||
|
/// and the parser's Write/AlarmAcknowledge no-retry clamp are the two sites that relax.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = false, Inherited = true)]
|
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = false, Inherited = true)]
|
||||||
public sealed class WriteIdempotentAttribute : Attribute
|
public sealed class WriteIdempotentAttribute : Attribute
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ public sealed class AlarmSurfaceInvoker
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Subscribe to alarm events for a set of source node ids, fanning out by resolved host
|
/// Subscribe to alarm events for a set of source node ids, fanning out by resolved host
|
||||||
/// so per-host breakers / bulkheads apply. Returns one handle per host — callers that
|
/// so per-host breakers apply. Returns one handle per host — callers that
|
||||||
/// don't care about per-host separation may concatenate them. Each returned handle wraps
|
/// don't care about per-host separation may concatenate them. Each returned handle wraps
|
||||||
/// the driver's opaque handle together with its resolved host so <see cref="UnsubscribeAsync"/>
|
/// the driver's opaque handle together with its resolved host so <see cref="UnsubscribeAsync"/>
|
||||||
/// routes through the same host's pipeline that the subscription was created on.
|
/// routes through the same host's pipeline that the subscription was created on.
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
|||||||
private readonly string _driverType;
|
private readonly string _driverType;
|
||||||
private readonly Func<DriverResilienceOptions> _optionsAccessor;
|
private readonly Func<DriverResilienceOptions> _optionsAccessor;
|
||||||
private readonly DriverResilienceStatusTracker? _statusTracker;
|
private readonly DriverResilienceStatusTracker? _statusTracker;
|
||||||
|
private readonly long _optionsGeneration;
|
||||||
|
private DriverResilienceOptions? _noRetryWriteOptions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Construct an invoker for one driver instance.
|
/// Construct an invoker for one driver instance.
|
||||||
@@ -39,13 +41,19 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
|||||||
/// pipeline-invalidate can take effect without restarting the invoker.
|
/// pipeline-invalidate can take effect without restarting the invoker.
|
||||||
/// </param>
|
/// </param>
|
||||||
/// <param name="driverType">Driver type name for structured-log enrichment (e.g. <c>"Modbus"</c>).</param>
|
/// <param name="driverType">Driver type name for structured-log enrichment (e.g. <c>"Modbus"</c>).</param>
|
||||||
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the bulkhead-depth proxy.</param>
|
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the in-flight-call-depth proxy.</param>
|
||||||
|
/// <param name="optionsGeneration">
|
||||||
|
/// Options epoch for this invoker (01/S-7). Threaded into every pipeline-cache lookup so a lingering
|
||||||
|
/// old child's late re-cache can never poison a newer invoker's pipeline. Defaults to 0 so existing
|
||||||
|
/// callers/tests are unaffected.
|
||||||
|
/// </param>
|
||||||
public CapabilityInvoker(
|
public CapabilityInvoker(
|
||||||
DriverResiliencePipelineBuilder builder,
|
DriverResiliencePipelineBuilder builder,
|
||||||
string driverInstanceId,
|
string driverInstanceId,
|
||||||
Func<DriverResilienceOptions> optionsAccessor,
|
Func<DriverResilienceOptions> optionsAccessor,
|
||||||
string driverType = "Unknown",
|
string driverType = "Unknown",
|
||||||
DriverResilienceStatusTracker? statusTracker = null)
|
DriverResilienceStatusTracker? statusTracker = null,
|
||||||
|
long optionsGeneration = 0)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(builder);
|
ArgumentNullException.ThrowIfNull(builder);
|
||||||
ArgumentNullException.ThrowIfNull(optionsAccessor);
|
ArgumentNullException.ThrowIfNull(optionsAccessor);
|
||||||
@@ -55,6 +63,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
|||||||
_driverType = driverType;
|
_driverType = driverType;
|
||||||
_optionsAccessor = optionsAccessor;
|
_optionsAccessor = optionsAccessor;
|
||||||
_statusTracker = statusTracker;
|
_statusTracker = statusTracker;
|
||||||
|
_optionsGeneration = optionsGeneration;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
@@ -116,27 +125,44 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
|||||||
|
|
||||||
if (!isIdempotent)
|
if (!isIdempotent)
|
||||||
{
|
{
|
||||||
// Snapshot the options exactly once per call — invoking _optionsAccessor twice can
|
// Build the no-retry options snapshot ONCE per invoker lifetime (01/P-4). The options are
|
||||||
// (a) observe two different snapshots if an Admin edit lands between them and
|
// immutable for the invoker's lifetime — a ResilienceConfig change respawns the driver child,
|
||||||
// (b) wastes an allocation on the per-write hot path (Phase 6.1 1% pipeline budget).
|
// which constructs a fresh invoker (#13 respawn-on-change) — so lazily caching the snapshot is
|
||||||
var snapshot = _optionsAccessor();
|
// safe and removes the per-write allocation the previous once-per-call snapshot incurred. The
|
||||||
var noRetryOptions = snapshot with
|
// null-check-assign tolerates a benign first-call race (both threads build an equivalent record).
|
||||||
|
var noRetryOptions = _noRetryWriteOptions ??= BuildNoRetryWriteOptions();
|
||||||
|
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions, _optionsGeneration);
|
||||||
|
// Record in-flight accounting on the CALLER's host (the "::non-idempotent" suffix is a
|
||||||
|
// pipeline-cache key, not an operator-facing host) so Admin /hosts sees write depth too.
|
||||||
|
_statusTracker?.RecordCallStart(_driverInstanceId, hostName);
|
||||||
|
try
|
||||||
{
|
{
|
||||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
||||||
{
|
|
||||||
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
|
|
||||||
},
|
|
||||||
};
|
|
||||||
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions);
|
|
||||||
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
|
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
|
||||||
{
|
{
|
||||||
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
|
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_statusTracker?.RecordCallComplete(_driverInstanceId, hostName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return await ExecuteAsync(DriverCapability.Write, hostName, callSite, cancellationToken).ConfigureAwait(false);
|
return await ExecuteAsync(DriverCapability.Write, hostName, callSite, cancellationToken).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResiliencePipeline ResolvePipeline(DriverCapability capability, string hostName) =>
|
private DriverResilienceOptions BuildNoRetryWriteOptions()
|
||||||
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor());
|
{
|
||||||
|
var snapshot = _optionsAccessor();
|
||||||
|
return snapshot with
|
||||||
|
{
|
||||||
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||||
|
{
|
||||||
|
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResiliencePipeline ResolvePipeline(DriverCapability capability, string hostName) =>
|
||||||
|
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor(), _optionsGeneration);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,11 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|||||||
///
|
///
|
||||||
/// <para>Options are snapshotted once per <see cref="Create"/> (a driver actor's options are fixed
|
/// <para>Options are snapshotted once per <see cref="Create"/> (a driver actor's options are fixed
|
||||||
/// for its lifetime — a ResilienceConfig change respawns the child), so the invoker's per-call
|
/// for its lifetime — a ResilienceConfig change respawns the child), so the invoker's per-call
|
||||||
/// options accessor is allocation-free on the hot path. Because the pipeline builder caches
|
/// options accessor is allocation-free on the hot path. The pipeline builder caches pipelines keyed on
|
||||||
/// pipelines keyed on <c>(instance, host, capability)</c> and <b>ignores options on a cache hit</b>,
|
/// <c>(instance, host, capability, generation)</c> and ignores options on a cache hit; each
|
||||||
/// <see cref="Create"/> first <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s any pipelines
|
/// <see cref="Create"/> stamps a fresh monotonic <b>generation</b> (01/S-7) so the new invoker can only
|
||||||
/// cached for the instance so a respawn with changed options rebuilds them (rather than silently
|
/// read pipelines it built — a lingering old child's post-invalidate re-cache lands under its own older
|
||||||
/// reusing the stale pipeline). On a first spawn this removes nothing.</para>
|
/// generation and is unreachable. <see cref="Create"/> still
|
||||||
|
/// <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s the instance's pipelines as cleanup (bounds
|
||||||
|
/// the cache), but correctness now rests on the generation key, not the invalidate timing.</para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFactory
|
public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFactory
|
||||||
{
|
{
|
||||||
@@ -31,6 +33,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
private readonly DriverResilienceStatusTracker _statusTracker;
|
private readonly DriverResilienceStatusTracker _statusTracker;
|
||||||
private readonly Func<string, DriverTier> _tierResolver;
|
private readonly Func<string, DriverTier> _tierResolver;
|
||||||
private readonly ILogger? _logger;
|
private readonly ILogger? _logger;
|
||||||
|
private long _generation;
|
||||||
|
|
||||||
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
||||||
/// <param name="builder">Process-singleton Polly pipeline builder (shared pipeline cache).</param>
|
/// <param name="builder">Process-singleton Polly pipeline builder (shared pipeline cache).</param>
|
||||||
@@ -65,9 +68,14 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
driverInstanceId, driverType, diagnostic);
|
driverInstanceId, driverType, diagnostic);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop any pipelines cached for this instance under the PREVIOUS options — the builder keys on
|
// Stamp a fresh options generation for this invoker. The generation is part of the pipeline-cache
|
||||||
// (instance, host, capability) and reuses a cached pipeline regardless of options, so a respawn
|
// key (01/S-7), so this invoker can only ever read pipelines it built — a lingering old child's
|
||||||
// with a changed ResilienceConfig would otherwise keep serving the stale pipeline. No-op on first spawn.
|
// late re-cache lands under its OWN (older) generation and is unreachable here. Correctness rests on
|
||||||
|
// the epoch key, not on the timing of the invalidate below.
|
||||||
|
var generation = Interlocked.Increment(ref _generation);
|
||||||
|
|
||||||
|
// Cleanup (no longer the correctness mechanism): drop this instance's cached pipelines across all
|
||||||
|
// generations so lingering old-generation entries don't accumulate. No-op on first spawn.
|
||||||
_builder.Invalidate(driverInstanceId);
|
_builder.Invalidate(driverInstanceId);
|
||||||
|
|
||||||
return new CapabilityInvoker(
|
return new CapabilityInvoker(
|
||||||
@@ -75,6 +83,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
driverInstanceId,
|
driverInstanceId,
|
||||||
optionsAccessor: () => options,
|
optionsAccessor: () => options,
|
||||||
driverType: driverType,
|
driverType: driverType,
|
||||||
statusTracker: _statusTracker);
|
statusTracker: _statusTracker,
|
||||||
|
optionsGeneration: generation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,15 +19,6 @@ public sealed record DriverResilienceOptions
|
|||||||
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityPolicies { get; init; }
|
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityPolicies { get; init; }
|
||||||
= new Dictionary<DriverCapability, CapabilityPolicy>();
|
= new Dictionary<DriverCapability, CapabilityPolicy>();
|
||||||
|
|
||||||
/// <summary>Bulkhead (max concurrent in-flight calls) for every capability. Default 32.</summary>
|
|
||||||
public int BulkheadMaxConcurrent { get; init; } = 32;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Bulkhead queue depth. Zero = no queueing; overflow fails fast with
|
|
||||||
/// <c>BulkheadRejectedException</c>. Default 64.
|
|
||||||
/// </summary>
|
|
||||||
public int BulkheadMaxQueue { get; init; } = 64;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Periodic scheduled recycle interval for Tier C out-of-process hosts, in seconds.
|
/// Periodic scheduled recycle interval for Tier C out-of-process hosts, in seconds.
|
||||||
/// Null (the default) = no scheduled recycle; the driver's Host process keeps running
|
/// Null (the default) = no scheduled recycle; the driver's Host process keeps running
|
||||||
@@ -69,8 +60,10 @@ public sealed record DriverResilienceOptions
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Per-tier per-capability default policy table, per the Phase 6.1
|
/// Per-tier per-capability default policy table, per the Phase 6.1
|
||||||
/// Stream A.2 specification. Retries skipped on <see cref="DriverCapability.Write"/> and
|
/// Stream A.2 specification. The parser enforces no-retry on
|
||||||
/// <see cref="DriverCapability.AlarmAcknowledge"/> regardless of tier.
|
/// <see cref="DriverCapability.Write"/> and <see cref="DriverCapability.AlarmAcknowledge"/>
|
||||||
|
/// as an invariant (R2-02/S-8) — a JSON retryCount override on those capabilities is forced back
|
||||||
|
/// to 0; per-tag opt-in via <see cref="WriteIdempotentAttribute"/> is the future relaxation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="tier">The driver tier to get defaults for.</param>
|
/// <param name="tier">The driver tier to get defaults for.</param>
|
||||||
/// <returns>The default policy dictionary for the specified tier.</returns>
|
/// <returns>The default policy dictionary for the specified tier.</returns>
|
||||||
|
|||||||
@@ -13,8 +13,6 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|||||||
/// <para>Example JSON shape per Phase 6.1 Stream A.2:</para>
|
/// <para>Example JSON shape per Phase 6.1 Stream A.2:</para>
|
||||||
/// <code>
|
/// <code>
|
||||||
/// {
|
/// {
|
||||||
/// "bulkheadMaxConcurrent": 16,
|
|
||||||
/// "bulkheadMaxQueue": 64,
|
|
||||||
/// "capabilityPolicies": {
|
/// "capabilityPolicies": {
|
||||||
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
||||||
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
||||||
@@ -83,15 +81,16 @@ public static class DriverResilienceOptionsParser
|
|||||||
{
|
{
|
||||||
if (!Enum.TryParse<DriverCapability>(capName, ignoreCase: true, out var capability))
|
if (!Enum.TryParse<DriverCapability>(capName, ignoreCase: true, out var capability))
|
||||||
{
|
{
|
||||||
parseDiagnostic ??= $"Unknown capability '{capName}' in ResilienceConfig; skipped.";
|
AppendDiagnostic(ref parseDiagnostic, $"Unknown capability '{capName}' in ResilienceConfig; skipped.");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var basePolicy = merged[capability];
|
var basePolicy = merged[capability];
|
||||||
merged[capability] = new CapabilityPolicy(
|
var mergedPolicy = new CapabilityPolicy(
|
||||||
TimeoutSeconds: overridePolicy.TimeoutSeconds ?? basePolicy.TimeoutSeconds,
|
TimeoutSeconds: overridePolicy.TimeoutSeconds ?? basePolicy.TimeoutSeconds,
|
||||||
RetryCount: overridePolicy.RetryCount ?? basePolicy.RetryCount,
|
RetryCount: overridePolicy.RetryCount ?? basePolicy.RetryCount,
|
||||||
BreakerFailureThreshold: overridePolicy.BreakerFailureThreshold ?? basePolicy.BreakerFailureThreshold);
|
BreakerFailureThreshold: overridePolicy.BreakerFailureThreshold ?? basePolicy.BreakerFailureThreshold);
|
||||||
|
merged[capability] = ClampPolicy(capability, mergedPolicy, basePolicy, ref parseDiagnostic);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,9 +101,9 @@ public static class DriverResilienceOptionsParser
|
|||||||
if (shape.RecycleIntervalSeconds is int secs)
|
if (shape.RecycleIntervalSeconds is int secs)
|
||||||
{
|
{
|
||||||
if (secs <= 0)
|
if (secs <= 0)
|
||||||
parseDiagnostic ??= $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.";
|
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.");
|
||||||
else if (tier != DriverTier.C)
|
else if (tier != DriverTier.C)
|
||||||
parseDiagnostic ??= $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.";
|
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.");
|
||||||
else
|
else
|
||||||
recycleIntervalSeconds = secs;
|
recycleIntervalSeconds = secs;
|
||||||
}
|
}
|
||||||
@@ -113,18 +112,90 @@ public static class DriverResilienceOptionsParser
|
|||||||
{
|
{
|
||||||
Tier = tier,
|
Tier = tier,
|
||||||
CapabilityPolicies = merged,
|
CapabilityPolicies = merged,
|
||||||
BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent ?? baseOptions.BulkheadMaxConcurrent,
|
|
||||||
BulkheadMaxQueue = shape.BulkheadMaxQueue ?? baseOptions.BulkheadMaxQueue,
|
|
||||||
RecycleIntervalSeconds = recycleIntervalSeconds,
|
RecycleIntervalSeconds = recycleIntervalSeconds,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Clamp a merged per-capability override to the legal <see cref="ResiliencePolicyRanges"/> so a
|
||||||
|
/// downstream Polly pipeline build can never throw <c>ValidationException</c> (01/S-6). Each clamp
|
||||||
|
/// appends a diagnostic naming the field + what happened. A non-positive timeout snaps to the
|
||||||
|
/// capability's <paramref name="tierDefault"/> (a nonsense value, not merely an extreme — snapping
|
||||||
|
/// to 1 s could be a 30× surprise on a Discover policy); everything else clamps to the nearest legal
|
||||||
|
/// value. Preserves maximum operator intent while never bricking.
|
||||||
|
/// </summary>
|
||||||
|
private static CapabilityPolicy ClampPolicy(
|
||||||
|
DriverCapability capability,
|
||||||
|
CapabilityPolicy merged,
|
||||||
|
CapabilityPolicy tierDefault,
|
||||||
|
ref string? diagnostic)
|
||||||
|
{
|
||||||
|
var timeout = merged.TimeoutSeconds;
|
||||||
|
if (timeout <= 0)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic,
|
||||||
|
$"{capability}.timeoutSeconds {timeout} is non-positive; using the tier default ({tierDefault.TimeoutSeconds}s).");
|
||||||
|
timeout = tierDefault.TimeoutSeconds;
|
||||||
|
}
|
||||||
|
else if (timeout > ResiliencePolicyRanges.MaxTimeoutSeconds)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic,
|
||||||
|
$"{capability}.timeoutSeconds {timeout} exceeds the max ({ResiliencePolicyRanges.MaxTimeoutSeconds}s); clamped.");
|
||||||
|
timeout = ResiliencePolicyRanges.MaxTimeoutSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
var retry = merged.RetryCount;
|
||||||
|
if (retry < ResiliencePolicyRanges.MinRetryCount)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} is negative; clamped to {ResiliencePolicyRanges.MinRetryCount}.");
|
||||||
|
retry = ResiliencePolicyRanges.MinRetryCount;
|
||||||
|
}
|
||||||
|
else if (retry > ResiliencePolicyRanges.MaxRetryCount)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} exceeds the cap ({ResiliencePolicyRanges.MaxRetryCount}); clamped.");
|
||||||
|
retry = ResiliencePolicyRanges.MaxRetryCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spec invariant (01/S-8 = 03/S12): Write and AlarmAcknowledge are write-shaped — a duplicate is
|
||||||
|
// not harmless (pulse coils, counter increments, Galaxy supervisory writes, alarm acks). Retries are
|
||||||
|
// "skipped regardless of tier" per Phase 6.1; the tier defaults are already 0, so only an override can
|
||||||
|
// differ — force it back to 0 here so the JSON→options funnel enforces the invariant. When per-tag
|
||||||
|
// opt-in via WriteIdempotentAttribute ships, this is the one line to relax.
|
||||||
|
if ((capability == DriverCapability.Write || capability == DriverCapability.AlarmAcknowledge) && retry > 0)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic,
|
||||||
|
$"{capability} never retries (writes are treated as non-idempotent until per-tag opt-in ships); retryCount override ignored.");
|
||||||
|
retry = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var breaker = merged.BreakerFailureThreshold;
|
||||||
|
if (breaker < ResiliencePolicyRanges.MinBreakerFailureThreshold)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic, $"{capability}.breakerFailureThreshold {breaker} is negative; breaker disabled.");
|
||||||
|
breaker = ResiliencePolicyRanges.MinBreakerFailureThreshold;
|
||||||
|
}
|
||||||
|
else if (breaker == 1)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic,
|
||||||
|
$"{capability}.breakerFailureThreshold 1 is below Polly's minimum throughput; raised to {ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} (closest to open-on-first-failure intent).");
|
||||||
|
breaker = ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold;
|
||||||
|
}
|
||||||
|
else if (breaker > ResiliencePolicyRanges.MaxBreakerFailureThreshold)
|
||||||
|
{
|
||||||
|
AppendDiagnostic(ref diagnostic,
|
||||||
|
$"{capability}.breakerFailureThreshold {breaker} exceeds the cap ({ResiliencePolicyRanges.MaxBreakerFailureThreshold}); clamped.");
|
||||||
|
breaker = ResiliencePolicyRanges.MaxBreakerFailureThreshold;
|
||||||
|
}
|
||||||
|
|
||||||
|
return merged with { TimeoutSeconds = timeout, RetryCount = retry, BreakerFailureThreshold = breaker };
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Append a diagnostic to the running channel (joined with "; ") so multiple clamps all surface.</summary>
|
||||||
|
private static void AppendDiagnostic(ref string? diagnostic, string message)
|
||||||
|
=> diagnostic = diagnostic is null ? message : $"{diagnostic}; {message}";
|
||||||
|
|
||||||
private sealed class ResilienceConfigShape
|
private sealed class ResilienceConfigShape
|
||||||
{
|
{
|
||||||
/// <summary>Gets or sets the maximum concurrent bulkhead requests.</summary>
|
|
||||||
public int? BulkheadMaxConcurrent { get; set; }
|
|
||||||
/// <summary>Gets or sets the maximum bulkhead queue size.</summary>
|
|
||||||
public int? BulkheadMaxQueue { get; set; }
|
|
||||||
/// <summary>Gets or sets the scheduled recycle interval in seconds.</summary>
|
/// <summary>Gets or sets the scheduled recycle interval in seconds.</summary>
|
||||||
public int? RecycleIntervalSeconds { get; set; }
|
public int? RecycleIntervalSeconds { get; set; }
|
||||||
/// <summary>Gets or sets the per-capability resilience policies.</summary>
|
/// <summary>Gets or sets the per-capability resilience policies.</summary>
|
||||||
|
|||||||
@@ -15,11 +15,17 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Per-device isolation. Composition from outside-in:
|
/// Per-device isolation. Composition from outside-in:
|
||||||
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits) → Bulkhead</b>.
|
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits)</b>.
|
||||||
///
|
///
|
||||||
/// <para>Pipeline resolution is lock-free on the hot path: the inner
|
/// <para>Pipeline resolution is lock-free on the hot path: the inner
|
||||||
/// <see cref="ConcurrentDictionary{TKey,TValue}"/> caches a <see cref="ResiliencePipeline"/> per key;
|
/// <see cref="ConcurrentDictionary{TKey,TValue}"/> caches a <see cref="ResiliencePipeline"/> per key;
|
||||||
/// first-call cost is one <see cref="ResiliencePipelineBuilder"/>.Build. Thereafter reads are O(1).</para>
|
/// first-call cost is one <see cref="ResiliencePipelineBuilder"/>.Build. Thereafter reads are O(1).</para>
|
||||||
|
///
|
||||||
|
/// <para><b>Never-throws invariant (01/S-6):</b> <see cref="Build"/> clamps every policy value to
|
||||||
|
/// <see cref="ResiliencePolicyRanges"/> before constructing strategy options, so a hostile
|
||||||
|
/// <see cref="DriverResilienceOptions"/> (constructed directly, bypassing the parser) can never make
|
||||||
|
/// the memoizing <c>GetOrAdd</c> factory throw Polly's <c>ValidationException</c> — which would
|
||||||
|
/// otherwise brick every capability call for that key.</para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed class DriverResiliencePipelineBuilder
|
public sealed class DriverResiliencePipelineBuilder
|
||||||
{
|
{
|
||||||
@@ -33,7 +39,7 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
/// <param name="statusTracker">When non-null, every built pipeline wires Polly telemetry into
|
/// <param name="statusTracker">When non-null, every built pipeline wires Polly telemetry into
|
||||||
/// the tracker — retries increment <c>ConsecutiveFailures</c>, breaker-open stamps
|
/// the tracker — retries increment <c>ConsecutiveFailures</c>, breaker-open stamps
|
||||||
/// <c>LastBreakerOpenUtc</c>, breaker-close resets failures. Feeds Admin <c>/hosts</c> +
|
/// <c>LastBreakerOpenUtc</c>, breaker-close resets failures. Feeds Admin <c>/hosts</c> +
|
||||||
/// the Polly bulkhead-depth column. Absent tracker means no telemetry (unit tests +
|
/// the in-flight-call-depth column. Absent tracker means no telemetry (unit tests +
|
||||||
/// deployments that don't care about resilience observability).</param>
|
/// deployments that don't care about resilience observability).</param>
|
||||||
/// <param name="logger">When non-null, retry / circuit-breaker-open / breaker-close events are
|
/// <param name="logger">When non-null, retry / circuit-breaker-open / breaker-close events are
|
||||||
/// logged (instance + host + capability + attempt/exception). This is the operator-facing
|
/// logged (instance + host + capability + attempt/exception). This is the operator-facing
|
||||||
@@ -63,23 +69,35 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
/// </param>
|
/// </param>
|
||||||
/// <param name="capability">Which capability surface is being called.</param>
|
/// <param name="capability">Which capability surface is being called.</param>
|
||||||
/// <param name="options">Per-driver-instance options (tier + per-capability overrides).</param>
|
/// <param name="options">Per-driver-instance options (tier + per-capability overrides).</param>
|
||||||
|
/// <param name="optionsGeneration">
|
||||||
|
/// Monotonic options epoch stamped by <see cref="DriverCapabilityInvokerFactory"/> per invoker
|
||||||
|
/// (01/S-7). Part of the cache key so a lingering old child's post-invalidate re-cache lands under
|
||||||
|
/// its OWN (old) generation — a key the new invoker's generation never reads. Defaults to 0 so
|
||||||
|
/// every existing caller/test compiles unchanged.
|
||||||
|
/// </param>
|
||||||
/// <returns>The cached or newly built resilience pipeline for the key.</returns>
|
/// <returns>The cached or newly built resilience pipeline for the key.</returns>
|
||||||
public ResiliencePipeline GetOrCreate(
|
public ResiliencePipeline GetOrCreate(
|
||||||
string driverInstanceId,
|
string driverInstanceId,
|
||||||
string hostName,
|
string hostName,
|
||||||
DriverCapability capability,
|
DriverCapability capability,
|
||||||
DriverResilienceOptions options)
|
DriverResilienceOptions options,
|
||||||
|
long optionsGeneration = 0)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(options);
|
ArgumentNullException.ThrowIfNull(options);
|
||||||
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
|
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
|
||||||
|
|
||||||
var key = new PipelineKey(driverInstanceId, hostName, capability);
|
var key = new PipelineKey(driverInstanceId, hostName, capability, optionsGeneration);
|
||||||
return _pipelines.GetOrAdd(key, static (k, state) => Build(
|
return _pipelines.GetOrAdd(key, static (k, state) => Build(
|
||||||
k.DriverInstanceId, k.HostName, state.capability, state.options, state.timeProvider, state.tracker, state.logger),
|
k.DriverInstanceId, k.HostName, state.capability, state.options, state.timeProvider, state.tracker, state.logger),
|
||||||
(capability, options, timeProvider: _timeProvider, tracker: _statusTracker, logger: _logger));
|
(capability, options, timeProvider: _timeProvider, tracker: _statusTracker, logger: _logger));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change). Test + Admin-reload use.</summary>
|
/// <summary>
|
||||||
|
/// Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change), across ALL
|
||||||
|
/// option generations. With generation-keyed entries (01/S-7) correctness no longer depends on this
|
||||||
|
/// sweep — a stale-epoch re-cache is already unreachable by the new generation — but it bounds the
|
||||||
|
/// cache by clearing lingering old-generation entries on the next respawn. Test + Admin-reload use.
|
||||||
|
/// </summary>
|
||||||
/// <param name="driverInstanceId">The driver instance ID whose pipelines should be invalidated.</param>
|
/// <param name="driverInstanceId">The driver instance ID whose pipelines should be invalidated.</param>
|
||||||
/// <returns>The number of pipelines removed.</returns>
|
/// <returns>The number of pipelines removed.</returns>
|
||||||
public int Invalidate(string driverInstanceId)
|
public int Invalidate(string driverInstanceId)
|
||||||
@@ -108,16 +126,23 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
var policy = options.Resolve(capability);
|
var policy = options.Resolve(capability);
|
||||||
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
|
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
|
||||||
|
|
||||||
|
// Belt-and-braces: clamp the resolved policy to ResiliencePolicyRanges so building a pipeline can
|
||||||
|
// NEVER throw Polly's ValidationException (01/S-6). The parser already clamps every JSON→options
|
||||||
|
// funnel, but DriverResilienceOptions is a public record constructible by paths that never traverse
|
||||||
|
// the parser (tests today; any future caller), so this makes "a pipeline build never throws" a
|
||||||
|
// builder invariant. Three integer ops once per pipeline build (not per call).
|
||||||
|
var timeoutSeconds = Math.Clamp(
|
||||||
|
policy.TimeoutSeconds, ResiliencePolicyRanges.MinTimeoutSeconds, ResiliencePolicyRanges.MaxTimeoutSeconds);
|
||||||
builder.AddTimeout(new TimeoutStrategyOptions
|
builder.AddTimeout(new TimeoutStrategyOptions
|
||||||
{
|
{
|
||||||
Timeout = TimeSpan.FromSeconds(policy.TimeoutSeconds),
|
Timeout = TimeSpan.FromSeconds(timeoutSeconds),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (policy.RetryCount > 0)
|
if (policy.RetryCount > 0)
|
||||||
{
|
{
|
||||||
var retryOptions = new RetryStrategyOptions
|
var retryOptions = new RetryStrategyOptions
|
||||||
{
|
{
|
||||||
MaxRetryAttempts = policy.RetryCount,
|
MaxRetryAttempts = Math.Min(policy.RetryCount, ResiliencePolicyRanges.MaxRetryCount),
|
||||||
BackoffType = DelayBackoffType.Exponential,
|
BackoffType = DelayBackoffType.Exponential,
|
||||||
UseJitter = true,
|
UseJitter = true,
|
||||||
Delay = TimeSpan.FromMilliseconds(100),
|
Delay = TimeSpan.FromMilliseconds(100),
|
||||||
@@ -143,7 +168,7 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
var breakerOptions = new CircuitBreakerStrategyOptions
|
var breakerOptions = new CircuitBreakerStrategyOptions
|
||||||
{
|
{
|
||||||
FailureRatio = 1.0,
|
FailureRatio = 1.0,
|
||||||
MinimumThroughput = policy.BreakerFailureThreshold,
|
MinimumThroughput = Math.Max(ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold, policy.BreakerFailureThreshold),
|
||||||
SamplingDuration = TimeSpan.FromSeconds(30),
|
SamplingDuration = TimeSpan.FromSeconds(30),
|
||||||
BreakDuration = TimeSpan.FromSeconds(15),
|
BreakDuration = TimeSpan.FromSeconds(15),
|
||||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
||||||
@@ -175,5 +200,5 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
return builder.Build();
|
return builder.Build();
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability);
|
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability, long Generation);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ public sealed class DriverResilienceStatusTracker
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Record the entry of a capability call for this (instance, host). Increments the
|
/// Record the entry of a capability call for this (instance, host). Increments the
|
||||||
/// in-flight counter used as the <see cref="ResilienceStatusSnapshot.CurrentInFlight"/>
|
/// in-flight counter used as the <see cref="ResilienceStatusSnapshot.CurrentInFlight"/>
|
||||||
/// surface (a cheap stand-in for Polly bulkhead depth). Paired with
|
/// surface (in-flight capability-call depth). Paired with
|
||||||
/// <see cref="RecordCallComplete"/>; callers use try/finally.
|
/// <see cref="RecordCallComplete"/>; callers use try/finally.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
||||||
@@ -159,7 +159,7 @@ public sealed record ResilienceStatusSnapshot
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// In-flight capability calls against this (instance, host). Bumped on call entry +
|
/// In-flight capability calls against this (instance, host). Bumped on call entry +
|
||||||
/// decremented on completion. Feeds <c>DriverInstanceResilienceStatus.CurrentBulkheadDepth</c>
|
/// decremented on completion. Feeds <c>DriverInstanceResilienceStatus.CurrentBulkheadDepth</c>
|
||||||
/// for Admin <c>/hosts</c> — a cheap proxy for the Polly bulkhead depth until the full
|
/// for Admin <c>/hosts</c> — in-flight capability-call depth until the full
|
||||||
/// telemetry observer lands.
|
/// telemetry observer lands.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int CurrentInFlight { get; init; }
|
public int CurrentInFlight { get; init; }
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
|
|||||||
/// <para>Wire layer is libplctag 1.6.x. Per-device host addresses use
|
/// <para>Wire layer is libplctag 1.6.x. Per-device host addresses use
|
||||||
/// the <c>ab://gateway[:port]/cip-path</c> canonical form parsed via
|
/// the <c>ab://gateway[:port]/cip-path</c> canonical form parsed via
|
||||||
/// <see cref="AbCipHostAddress.TryParse"/>; those strings become the <c>hostName</c> key
|
/// <see cref="AbCipHostAddress.TryParse"/>; those strings become the <c>hostName</c> key
|
||||||
/// for Polly bulkhead + circuit-breaker isolation.</para>
|
/// for Polly per-host circuit-breaker isolation.</para>
|
||||||
///
|
///
|
||||||
/// <para>Tier A — in-process, shares server lifetime, no
|
/// <para>Tier A — in-process, shares server lifetime, no
|
||||||
/// sidecar. <see cref="ReinitializeAsync"/> is the Tier-B escape hatch for recovering
|
/// sidecar. <see cref="ReinitializeAsync"/> is the Tier-B escape hatch for recovering
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parsed <c>ab://gateway[:port]/cip-path</c> host-address string used by the AbCip driver
|
/// Parsed <c>ab://gateway[:port]/cip-path</c> host-address string used by the AbCip driver
|
||||||
/// as the <c>hostName</c> key across <see cref="Core.Abstractions.IHostConnectivityProbe"/>,
|
/// as the <c>hostName</c> key across <see cref="Core.Abstractions.IHostConnectivityProbe"/>,
|
||||||
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly bulkhead key
|
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly per-host pipeline key
|
||||||
/// <c>(DriverInstanceId, hostName)</c>.
|
/// <c>(DriverInstanceId, hostName)</c>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// FOCAS driver configuration. One instance supports N CNC devices. Each device gets its own
|
/// FOCAS driver configuration. One instance supports N CNC devices. Each device gets its own
|
||||||
/// <c>(DriverInstanceId, HostAddress)</c> bulkhead key at the Phase 6.1 resilience layer.
|
/// <c>(DriverInstanceId, HostAddress)</c> per-host pipeline key at the Phase 6.1 resilience layer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class FocasDriverOptions
|
public sealed class FocasDriverOptions
|
||||||
{
|
{
|
||||||
|
|||||||
+49
-9
@@ -6,13 +6,23 @@
|
|||||||
<p class="form-text mb-3">Blank fields use the driver type's stability-tier defaults
|
<p class="form-text mb-3">Blank fields use the driver type's stability-tier defaults
|
||||||
(see <span class="mono">docs/v2/driver-stability.md</span>). Set only what you need to override.</p>
|
(see <span class="mono">docs/v2/driver-stability.md</span>). Set only what you need to override.</p>
|
||||||
|
|
||||||
|
@if (_m.ParseFailed)
|
||||||
|
{
|
||||||
|
<div class="alert alert-danger" role="alert">
|
||||||
|
<strong>Stored resilience config could not be parsed.</strong>
|
||||||
|
Editing is locked so an accidental keystroke can't overwrite it. Fix the stored JSON directly,
|
||||||
|
or discard it and start from tier defaults.
|
||||||
|
<div class="mt-2">
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-danger" @onclick="DiscardAsync">
|
||||||
|
Discard stored JSON and start blank
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
|
||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-md-4"><label class="form-label">Bulkhead max concurrent</label>
|
|
||||||
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxConcurrent" @bind:after="EmitAsync" placeholder="tier default" /></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Bulkhead max queue</label>
|
|
||||||
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxQueue" @bind:after="EmitAsync" placeholder="tier default" /></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Recycle interval (s, Tier C only)</label>
|
<div class="col-md-4"><label class="form-label">Recycle interval (s, Tier C only)</label>
|
||||||
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" /></div>
|
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" disabled="@_m.ParseFailed" /></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="table-wrap mt-3">
|
<div class="table-wrap mt-3">
|
||||||
@@ -22,20 +32,37 @@
|
|||||||
@foreach (var cap in ResilienceFormModel.Capabilities)
|
@foreach (var cap in ResilienceFormModel.Capabilities)
|
||||||
{
|
{
|
||||||
var row = _m.Policies[cap];
|
var row = _m.Policies[cap];
|
||||||
|
var writeShaped = cap is "Write" or "AlarmAcknowledge";
|
||||||
<tr>
|
<tr>
|
||||||
<td class="mono">@cap</td>
|
<td class="mono">@cap</td>
|
||||||
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" /></td>
|
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
|
||||||
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="default" /></td>
|
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="@(writeShaped ? "never" : "default")" disabled="@(writeShaped || _m.ParseFailed)" title="@(writeShaped ? "Writes/acks never retry — per-tag opt-in not yet available" : null)" /></td>
|
||||||
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" /></td>
|
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
|
||||||
</tr>
|
</tr>
|
||||||
}
|
}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
@{ var _warnings = _m.Validate(); }
|
||||||
|
@if (_warnings.Count > 0)
|
||||||
|
{
|
||||||
|
<div class="alert alert-warning mt-3" role="alert">
|
||||||
|
<strong>Out-of-range values will be clamped to a safe value on deploy:</strong>
|
||||||
|
<ul class="mb-0">
|
||||||
|
@foreach (var w in _warnings)
|
||||||
|
{
|
||||||
|
<li>@w</li>
|
||||||
|
}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
|
||||||
<details class="mt-3">
|
<details class="mt-3">
|
||||||
<summary class="small text-muted">Raw JSON (advanced)</summary>
|
<summary class="small text-muted">Raw JSON (advanced)</summary>
|
||||||
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(_m.ToJson() ?? "(null — all tier defaults)")</pre>
|
@* Render the bound STORED value, not the re-serialized model — shows the truth in both the
|
||||||
|
healthy and the malformed case. *@
|
||||||
|
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(ResilienceConfig ?? "(null — all tier defaults)")</pre>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
@@ -59,9 +86,22 @@
|
|||||||
|
|
||||||
private async Task EmitAsync()
|
private async Task EmitAsync()
|
||||||
{
|
{
|
||||||
|
// Never emit while the stored config is unparseable — the section is locked, and an unrelated
|
||||||
|
// keystroke must never silently overwrite what we couldn't read (04/C-7).
|
||||||
|
if (_m.ParseFailed) return;
|
||||||
|
|
||||||
var json = _m.ToJson();
|
var json = _m.ToJson();
|
||||||
_lastParsed = json;
|
_lastParsed = json;
|
||||||
ResilienceConfig = json;
|
ResilienceConfig = json;
|
||||||
await ResilienceConfigChanged.InvokeAsync(json);
|
await ResilienceConfigChanged.InvokeAsync(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task DiscardAsync()
|
||||||
|
{
|
||||||
|
// Explicit, visible destruction — replace the unparseable blob with a blank (tier-default) config.
|
||||||
|
_m = new ResilienceFormModel();
|
||||||
|
_lastParsed = null;
|
||||||
|
ResilienceConfig = null;
|
||||||
|
await ResilienceConfigChanged.InvokeAsync(null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+158
-72
@@ -1,23 +1,28 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Nodes;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed
|
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed fields in
|
||||||
/// fields in DriverResilienceSection; null/blank = "use the driver's tier default", so a
|
/// DriverResilienceSection; null/blank = "use the driver's tier default", so a blank form serializes
|
||||||
/// blank form serializes back to null (preserving DriverInstance.ResilienceConfig = null).
|
/// back to null (preserving DriverInstance.ResilienceConfig = null). Emits / reads the exact override
|
||||||
/// Emits / reads the exact override JSON shape DriverResilienceOptionsParser consumes.
|
/// JSON shape DriverResilienceOptionsParser consumes.
|
||||||
|
/// <para>
|
||||||
|
/// Round-trip is <b>non-lossy</b> (04/C-7): the stored JSON is kept as a <see cref="JsonObject"/> bag,
|
||||||
|
/// and <see cref="ToJson"/> overlays only the known fields onto that bag — so unknown top-level keys,
|
||||||
|
/// unknown capability entries, and unknown per-policy fields all survive a load→save, mirroring the
|
||||||
|
/// driver tag editors' preserve-unknown discipline (<c>TagConfigJson</c>). A stored blob that could not
|
||||||
|
/// be parsed sets <see cref="ParseFailed"/> and is <b>never</b> overwritten — <see cref="ToJson"/>
|
||||||
|
/// returns the original text verbatim.
|
||||||
|
/// </para>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class ResilienceFormModel
|
public sealed class ResilienceFormModel
|
||||||
{
|
{
|
||||||
public static readonly string[] Capabilities =
|
public static readonly string[] Capabilities =
|
||||||
["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"];
|
["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"];
|
||||||
|
|
||||||
/// <summary>Gets or sets the bulkhead max-concurrency override; null = use the tier default.</summary>
|
|
||||||
public int? BulkheadMaxConcurrent { get; set; }
|
|
||||||
/// <summary>Gets or sets the bulkhead max-queue-length override; null = use the tier default.</summary>
|
|
||||||
public int? BulkheadMaxQueue { get; set; }
|
|
||||||
/// <summary>Gets or sets the driver recycle-interval override, in seconds; null = use the tier default.</summary>
|
/// <summary>Gets or sets the driver recycle-interval override, in seconds; null = use the tier default.</summary>
|
||||||
public int? RecycleIntervalSeconds { get; set; }
|
public int? RecycleIntervalSeconds { get; set; }
|
||||||
|
|
||||||
@@ -26,6 +31,18 @@ public sealed class ResilienceFormModel
|
|||||||
public Dictionary<string, CapabilityRow> Policies { get; set; } =
|
public Dictionary<string, CapabilityRow> Policies { get; set; } =
|
||||||
Capabilities.ToDictionary(c => c, _ => new CapabilityRow(), StringComparer.OrdinalIgnoreCase);
|
Capabilities.ToDictionary(c => c, _ => new CapabilityRow(), StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// True when the stored ResilienceConfig could not be parsed as JSON. The section locks editing and
|
||||||
|
/// <see cref="ToJson"/> returns <see cref="RawStoredJson"/> unchanged so an unrelated keystroke can
|
||||||
|
/// never silently overwrite an unreadable column.
|
||||||
|
/// </summary>
|
||||||
|
public bool ParseFailed { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>The original stored text when <see cref="ParseFailed"/> — returned verbatim by <see cref="ToJson"/>.</summary>
|
||||||
|
public string? RawStoredJson { get; private set; }
|
||||||
|
|
||||||
|
private JsonObject _bag = new();
|
||||||
|
|
||||||
/// <summary>Per-capability timeout/retry/breaker override row; null fields fall back to the tier default.</summary>
|
/// <summary>Per-capability timeout/retry/breaker override row; null fields fall back to the tier default.</summary>
|
||||||
public sealed class CapabilityRow
|
public sealed class CapabilityRow
|
||||||
{
|
{
|
||||||
@@ -39,88 +56,157 @@ public sealed class ResilienceFormModel
|
|||||||
public bool IsEmpty => TimeoutSeconds is null && RetryCount is null && BreakerFailureThreshold is null;
|
public bool IsEmpty => TimeoutSeconds is null && RetryCount is null && BreakerFailureThreshold is null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static readonly JsonSerializerOptions ReadOpts = new() { PropertyNameCaseInsensitive = true };
|
/// <summary>
|
||||||
private static readonly JsonSerializerOptions WriteOpts = new()
|
/// Range-validate the per-capability overrides against <see cref="ResiliencePolicyRanges"/> and return
|
||||||
|
/// one human-readable message per violation (naming capability + field + legal range). This is
|
||||||
|
/// authoring-time <b>feedback</b>, not the enforcement layer — the parser's clamp is the authoritative
|
||||||
|
/// guard, so the form still emits. Mirrors the driver tag editors' <c>Validate()</c> idiom (01/S-6).
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A list of range-violation messages; empty when every override is in range or blank.</returns>
|
||||||
|
public IReadOnlyList<string> Validate()
|
||||||
{
|
{
|
||||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
var msgs = new List<string>();
|
||||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
foreach (var (cap, row) in Policies)
|
||||||
};
|
{
|
||||||
|
if (row.TimeoutSeconds is { } t &&
|
||||||
|
(t < ResiliencePolicyRanges.MinTimeoutSeconds || t > ResiliencePolicyRanges.MaxTimeoutSeconds))
|
||||||
|
msgs.Add($"{cap} timeout must be between {ResiliencePolicyRanges.MinTimeoutSeconds} and " +
|
||||||
|
$"{ResiliencePolicyRanges.MaxTimeoutSeconds} seconds (got {t}).");
|
||||||
|
|
||||||
/// <summary>Parses the override JSON into a form model; malformed or blank JSON yields an empty (all-default) form.</summary>
|
if (row.RetryCount is { } r &&
|
||||||
|
(r < ResiliencePolicyRanges.MinRetryCount || r > ResiliencePolicyRanges.MaxRetryCount))
|
||||||
|
msgs.Add($"{cap} retries must be between {ResiliencePolicyRanges.MinRetryCount} and " +
|
||||||
|
$"{ResiliencePolicyRanges.MaxRetryCount} (got {r}).");
|
||||||
|
|
||||||
|
if (row.BreakerFailureThreshold is { } b &&
|
||||||
|
(b < ResiliencePolicyRanges.MinBreakerFailureThreshold || b == 1 ||
|
||||||
|
b > ResiliencePolicyRanges.MaxBreakerFailureThreshold))
|
||||||
|
msgs.Add($"{cap} breaker threshold must be 0 (off) or between " +
|
||||||
|
$"{ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} and " +
|
||||||
|
$"{ResiliencePolicyRanges.MaxBreakerFailureThreshold} (got {b}).");
|
||||||
|
}
|
||||||
|
return msgs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parse the override JSON into a form model, preserving every key in an internal bag. Malformed JSON
|
||||||
|
/// sets <see cref="ParseFailed"/> + <see cref="RawStoredJson"/> and yields an otherwise-empty model.
|
||||||
|
/// </summary>
|
||||||
/// <param name="json">The raw resilience-override JSON, or null/blank if there is no override.</param>
|
/// <param name="json">The raw resilience-override JSON, or null/blank if there is no override.</param>
|
||||||
/// <returns>A populated form model, or an all-default one when <paramref name="json"/> is blank or malformed.</returns>
|
/// <returns>A populated form model.</returns>
|
||||||
public static ResilienceFormModel FromJson(string? json)
|
public static ResilienceFormModel FromJson(string? json)
|
||||||
{
|
{
|
||||||
var model = new ResilienceFormModel();
|
var model = new ResilienceFormModel();
|
||||||
if (string.IsNullOrWhiteSpace(json)) return model;
|
if (string.IsNullOrWhiteSpace(json)) return model;
|
||||||
|
|
||||||
Shape? shape;
|
JsonObject bag;
|
||||||
try { shape = JsonSerializer.Deserialize<Shape>(json, ReadOpts); }
|
try
|
||||||
catch (JsonException) { return model; } // malformed -> empty form; raw view (next task) shows the text
|
|
||||||
if (shape is null) return model;
|
|
||||||
|
|
||||||
model.BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent;
|
|
||||||
model.BulkheadMaxQueue = shape.BulkheadMaxQueue;
|
|
||||||
model.RecycleIntervalSeconds = shape.RecycleIntervalSeconds;
|
|
||||||
if (shape.CapabilityPolicies is not null)
|
|
||||||
foreach (var (cap, p) in shape.CapabilityPolicies)
|
|
||||||
if (model.Policies.TryGetValue(cap, out var row))
|
|
||||||
{
|
{
|
||||||
row.TimeoutSeconds = p.TimeoutSeconds;
|
bag = JsonNode.Parse(json) as JsonObject ?? new JsonObject();
|
||||||
row.RetryCount = p.RetryCount;
|
}
|
||||||
row.BreakerFailureThreshold = p.BreakerFailureThreshold;
|
catch (JsonException)
|
||||||
|
{
|
||||||
|
model.ParseFailed = true;
|
||||||
|
model.RawStoredJson = json;
|
||||||
|
return model;
|
||||||
|
}
|
||||||
|
|
||||||
|
model._bag = bag;
|
||||||
|
model.RecycleIntervalSeconds = GetIntOrNull(bag, "recycleIntervalSeconds");
|
||||||
|
|
||||||
|
if (bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject cp)
|
||||||
|
{
|
||||||
|
foreach (var (capName, polNode) in cp)
|
||||||
|
{
|
||||||
|
// Known capability names populate rows (reading only the known per-policy fields — unknown
|
||||||
|
// per-policy fields stay untouched in the bag). Unknown capability entries stay in the bag.
|
||||||
|
if (polNode is JsonObject pol && model.Policies.TryGetValue(capName, out var row))
|
||||||
|
{
|
||||||
|
row.TimeoutSeconds = GetIntOrNull(pol, "timeoutSeconds");
|
||||||
|
row.RetryCount = GetIntOrNull(pol, "retryCount");
|
||||||
|
row.BreakerFailureThreshold = GetIntOrNull(pol, "breakerFailureThreshold");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return model;
|
return model;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Emit only the non-null overrides; returns null when nothing is overridden.</summary>
|
/// <summary>
|
||||||
/// <returns>The serialized override JSON, or null when no field in this form is overridden.</returns>
|
/// Overlay the model's non-null known values onto the preserved bag and serialize. Returns null when
|
||||||
|
/// the result is empty (blank form = tier defaults). When <see cref="ParseFailed"/>, returns the
|
||||||
|
/// original unparseable text unchanged.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The serialized override JSON, the original text when parse failed, or null when nothing is set.</returns>
|
||||||
public string? ToJson()
|
public string? ToJson()
|
||||||
{
|
{
|
||||||
var caps = Policies
|
if (ParseFailed) return RawStoredJson;
|
||||||
.Where(kv => !kv.Value.IsEmpty)
|
|
||||||
.ToDictionary(kv => kv.Key, kv => new PolicyShape
|
|
||||||
{
|
|
||||||
TimeoutSeconds = kv.Value.TimeoutSeconds,
|
|
||||||
RetryCount = kv.Value.RetryCount,
|
|
||||||
BreakerFailureThreshold = kv.Value.BreakerFailureThreshold,
|
|
||||||
});
|
|
||||||
|
|
||||||
var hasAny = BulkheadMaxConcurrent is not null || BulkheadMaxQueue is not null
|
SetOrRemoveInt(_bag, "recycleIntervalSeconds", RecycleIntervalSeconds);
|
||||||
|| RecycleIntervalSeconds is not null || caps.Count > 0;
|
|
||||||
if (!hasAny) return null;
|
|
||||||
|
|
||||||
var shape = new Shape
|
var cp = _bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject existing
|
||||||
{
|
? existing
|
||||||
BulkheadMaxConcurrent = BulkheadMaxConcurrent,
|
: null;
|
||||||
BulkheadMaxQueue = BulkheadMaxQueue,
|
|
||||||
RecycleIntervalSeconds = RecycleIntervalSeconds,
|
|
||||||
CapabilityPolicies = caps.Count > 0 ? caps : null,
|
|
||||||
};
|
|
||||||
return JsonSerializer.Serialize(shape, WriteOpts);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>Wire shape of the resilience-override JSON, as consumed by DriverResilienceOptionsParser.</summary>
|
foreach (var (cap, row) in Policies)
|
||||||
private sealed class Shape
|
|
||||||
{
|
{
|
||||||
/// <summary>Gets or sets the bulkhead max-concurrency override.</summary>
|
// Find this known capability's existing policy object (case-insensitive), else its canonical key.
|
||||||
public int? BulkheadMaxConcurrent { get; set; }
|
var polKey = cap;
|
||||||
/// <summary>Gets or sets the bulkhead max-queue-length override.</summary>
|
JsonObject? pol = null;
|
||||||
public int? BulkheadMaxQueue { get; set; }
|
if (cp is not null)
|
||||||
/// <summary>Gets or sets the driver recycle-interval override, in seconds.</summary>
|
|
||||||
public int? RecycleIntervalSeconds { get; set; }
|
|
||||||
/// <summary>Gets or sets the per-capability overrides, keyed by capability name.</summary>
|
|
||||||
public Dictionary<string, PolicyShape>? CapabilityPolicies { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>Wire shape of a single capability's timeout/retry/breaker override.</summary>
|
|
||||||
private sealed class PolicyShape
|
|
||||||
{
|
{
|
||||||
/// <summary>Gets or sets the timeout override, in seconds.</summary>
|
foreach (var (k, v) in cp)
|
||||||
public int? TimeoutSeconds { get; set; }
|
{
|
||||||
/// <summary>Gets or sets the retry-count override.</summary>
|
if (string.Equals(k, cap, StringComparison.OrdinalIgnoreCase) && v is JsonObject vo)
|
||||||
public int? RetryCount { get; set; }
|
{
|
||||||
/// <summary>Gets or sets the circuit-breaker failure-threshold override.</summary>
|
polKey = k;
|
||||||
public int? BreakerFailureThreshold { get; set; }
|
pol = vo;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (row.IsEmpty)
|
||||||
|
{
|
||||||
|
// Clear the known fields; drop the policy entry only if no unknown fields remain.
|
||||||
|
if (pol is not null)
|
||||||
|
{
|
||||||
|
pol.Remove("timeoutSeconds");
|
||||||
|
pol.Remove("retryCount");
|
||||||
|
pol.Remove("breakerFailureThreshold");
|
||||||
|
if (pol.Count == 0) cp!.Remove(polKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (cp is null)
|
||||||
|
{
|
||||||
|
cp = new JsonObject();
|
||||||
|
_bag["capabilityPolicies"] = cp;
|
||||||
|
}
|
||||||
|
if (pol is null)
|
||||||
|
{
|
||||||
|
pol = new JsonObject();
|
||||||
|
cp[polKey] = pol;
|
||||||
|
}
|
||||||
|
SetOrRemoveInt(pol, "timeoutSeconds", row.TimeoutSeconds);
|
||||||
|
SetOrRemoveInt(pol, "retryCount", row.RetryCount);
|
||||||
|
SetOrRemoveInt(pol, "breakerFailureThreshold", row.BreakerFailureThreshold);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cp is not null && cp.Count == 0) _bag.Remove("capabilityPolicies");
|
||||||
|
|
||||||
|
return _bag.Count == 0 ? null : _bag.ToJsonString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int? GetIntOrNull(JsonObject o, string name)
|
||||||
|
=> o.TryGetPropertyValue(name, out var n) && n is JsonValue v && v.TryGetValue<int>(out var i)
|
||||||
|
? i
|
||||||
|
: null;
|
||||||
|
|
||||||
|
private static void SetOrRemoveInt(JsonObject o, string name, int? value)
|
||||||
|
{
|
||||||
|
if (value is null) o.Remove(name);
|
||||||
|
else o[name] = JsonValue.Create(value.Value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1627,7 +1627,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
|
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
|
||||||
// route through the retry/breaker/bulkhead/telemetry pipeline. The tier defaults are layered
|
// route through the retry/breaker/telemetry pipeline. The tier defaults are layered
|
||||||
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
|
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
|
||||||
// no-op invoker on nodes without the real resilience factory bound.
|
// no-op invoker on nodes without the real resilience factory bound.
|
||||||
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
|
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
|||||||
private readonly IDriver _driver;
|
private readonly IDriver _driver;
|
||||||
|
|
||||||
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
|
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
|
||||||
/// routed through this invoker (retry / breaker / bulkhead / tracker telemetry). Defaults to
|
/// routed through this invoker (retry / breaker / in-flight accounting / tracker telemetry). Defaults to
|
||||||
/// <see cref="NullDriverCapabilityInvoker"/> (a genuine pass-through) when none is supplied — so
|
/// <see cref="NullDriverCapabilityInvoker"/> (a genuine pass-through) when none is supplied — so
|
||||||
/// unit tests + the F7 skeleton path behave exactly as a raw driver call. The fused Host builds a
|
/// unit tests + the F7 skeleton path behave exactly as a raw driver call. The fused Host builds a
|
||||||
/// real per-instance invoker via <see cref="IDriverCapabilityInvokerFactory"/> and injects it at
|
/// real per-instance invoker via <see cref="IDriverCapabilityInvokerFactory"/> and injects it at
|
||||||
@@ -589,14 +589,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
|||||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Route through the resilience invoker (Write pipeline: timeout, no retry by default,
|
// Route through the resilience invoker (Write pipeline: timeout, per-host breaker). Writes
|
||||||
// per-host breaker). An OPC UA attribute set is idempotent (writing value X twice == once),
|
// default to NON-idempotent (03/S12): a timed-out write may already have committed at the
|
||||||
// so a configured Write retry is safe to apply; the Write tier default is RetryCount=0, so
|
// device, and replaying a command-shaped write (pulse coil, counter increment, Galaxy
|
||||||
// this stays behavior-neutral until an operator opts in via ResilienceConfig. The outer cts
|
// supervisory write) can duplicate an irreversible field action. The invoker's no-retry arm is
|
||||||
// is retained as a hard backstop above the (typically tighter) tier timeout.
|
// therefore authoritative regardless of any configured Write retry. Per-tag opt-in to retries via
|
||||||
|
// WriteIdempotentAttribute is the unshipped forward path (see WriteIdempotentAttribute). The outer
|
||||||
|
// cts is retained as a hard backstop above the (typically tighter) tier timeout.
|
||||||
var results = await _invoker.ExecuteWriteAsync(
|
var results = await _invoker.ExecuteWriteAsync(
|
||||||
ResolveHost(msg.TagId),
|
ResolveHost(msg.TagId),
|
||||||
isIdempotent: true,
|
isIdempotent: false,
|
||||||
async ct => await writable.WriteAsync(request, ct),
|
async ct => await writable.WriteAsync(request, ct),
|
||||||
cts.Token);
|
cts.Token);
|
||||||
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
|
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.Analyzers;
|
|||||||
/// Diagnostic analyzer that flags direct invocations of Phase 6.1-wrapped driver-capability
|
/// Diagnostic analyzer that flags direct invocations of Phase 6.1-wrapped driver-capability
|
||||||
/// methods when the call is NOT already running inside a <c>CapabilityInvoker.ExecuteAsync</c>
|
/// methods when the call is NOT already running inside a <c>CapabilityInvoker.ExecuteAsync</c>
|
||||||
/// or <c>CapabilityInvoker.ExecuteWriteAsync</c> lambda. The wrapping is what gives us
|
/// or <c>CapabilityInvoker.ExecuteWriteAsync</c> lambda. The wrapping is what gives us
|
||||||
/// per-host breaker isolation, retry semantics, bulkhead-depth accounting, and alarm-ack
|
/// per-host breaker isolation, retry semantics, in-flight-depth accounting, and alarm-ack
|
||||||
/// idempotence guards — raw calls bypass all of that.
|
/// idempotence guards — raw calls bypass all of that.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
@@ -102,7 +102,7 @@ public sealed class UnwrappedCapabilityCallAnalyzer : DiagnosticAnalyzer
|
|||||||
private static readonly DiagnosticDescriptor Rule = new(
|
private static readonly DiagnosticDescriptor Rule = new(
|
||||||
id: DiagnosticId,
|
id: DiagnosticId,
|
||||||
title: "Driver capability call must be wrapped in CapabilityInvoker",
|
title: "Driver capability call must be wrapped in CapabilityInvoker",
|
||||||
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, bulkhead, tracker telemetry) is bypassed for this call.",
|
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, tracker telemetry) is bypassed for this call.",
|
||||||
category: "OtOpcUa.Resilience",
|
category: "OtOpcUa.Resilience",
|
||||||
defaultSeverity: DiagnosticSeverity.Warning,
|
defaultSeverity: DiagnosticSeverity.Warning,
|
||||||
isEnabledByDefault: true,
|
isEnabledByDefault: true,
|
||||||
|
|||||||
@@ -188,6 +188,53 @@ public sealed class CapabilityInvokerTests
|
|||||||
"ExecuteWriteAsync's non-idempotent branch must capture the options snapshot exactly once per call");
|
"ExecuteWriteAsync's non-idempotent branch must capture the options snapshot exactly once per call");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// S-8 residual: the non-idempotent write arm must record tracker in-flight accounting (start +
|
||||||
|
/// complete) on the caller's host, exactly like the idempotent arms — the previous arm skipped it.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task NonIdempotentWrite_RecordsTrackerStartAndComplete()
|
||||||
|
{
|
||||||
|
var tracker = new DriverResilienceStatusTracker();
|
||||||
|
var invoker = new CapabilityInvoker(
|
||||||
|
new DriverResiliencePipelineBuilder(),
|
||||||
|
"drv-test",
|
||||||
|
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
||||||
|
statusTracker: tracker);
|
||||||
|
|
||||||
|
await invoker.ExecuteWriteAsync(
|
||||||
|
"host-1",
|
||||||
|
isIdempotent: false,
|
||||||
|
_ => ValueTask.FromResult(0),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var snap = tracker.TryGet("drv-test", "host-1");
|
||||||
|
snap.ShouldNotBeNull("the non-idempotent write arm must record tracker accounting on the caller's host");
|
||||||
|
snap!.CurrentInFlight.ShouldBe(0, "in-flight must return to 0 after the write completes");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 01/P-4: the non-idempotent write arm must build the no-retry options snapshot ONCE per invoker
|
||||||
|
/// lifetime (not per call). Two writes to the same host reuse the cached snapshot — the options
|
||||||
|
/// accessor is invoked at most once and only one pipeline is cached.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task NonIdempotentWrite_ReusesCachedNoRetryOptions()
|
||||||
|
{
|
||||||
|
var builder = new DriverResiliencePipelineBuilder();
|
||||||
|
var accessorCalls = 0;
|
||||||
|
var invoker = new CapabilityInvoker(
|
||||||
|
builder,
|
||||||
|
"drv-test",
|
||||||
|
() => { Interlocked.Increment(ref accessorCalls); return new DriverResilienceOptions { Tier = DriverTier.A }; });
|
||||||
|
|
||||||
|
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
||||||
|
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
||||||
|
|
||||||
|
accessorCalls.ShouldBeLessThanOrEqualTo(1, "the no-retry snapshot must be cached across calls");
|
||||||
|
builder.CachedPipelineCount.ShouldBe(1, "two writes to the same host share one cached no-retry pipeline");
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Core-009 regression — companion consistency assertion: the non-idempotent branch must
|
/// Core-009 regression — companion consistency assertion: the non-idempotent branch must
|
||||||
/// not observe two different option snapshots even if the accessor's returned value changes
|
/// not observe two different option snapshots even if the accessor's returned value changes
|
||||||
|
|||||||
+67
@@ -136,6 +136,73 @@ public sealed class DriverCapabilityInvokerFactoryTests
|
|||||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Message.Contains("resilience config", StringComparison.OrdinalIgnoreCase));
|
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Message.Contains("resilience config", StringComparison.OrdinalIgnoreCase));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 01/S-6 production-wiring proof: the exact <c>DriverHostActor.SpawnChild</c> path
|
||||||
|
/// (<c>factory.Create(id, type, hostileJson)</c>) must yield a WORKING invoker whose capability
|
||||||
|
/// call succeeds, and must log the clamp diagnostic as a warning — a hostile ResilienceConfig can
|
||||||
|
/// never brick the spawn path.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Create_with_out_of_range_config_yields_working_invoker()
|
||||||
|
{
|
||||||
|
var logger = new CapturingLogger();
|
||||||
|
var factory = MakeFactory(new DriverResiliencePipelineBuilder(), logger: logger);
|
||||||
|
|
||||||
|
var invoker = factory.Create("i1", "Modbus",
|
||||||
|
resilienceConfigJson: "{\"capabilityPolicies\":{\"Subscribe\":{\"timeoutSeconds\":0,\"breakerFailureThreshold\":1}}}");
|
||||||
|
|
||||||
|
var result = await invoker.ExecuteAsync(
|
||||||
|
DriverCapability.Subscribe, "host-1", _ => ValueTask.FromResult(99), CancellationToken.None);
|
||||||
|
|
||||||
|
result.ShouldBe(99, "a hostile ResilienceConfig must not brick the Subscribe capability call");
|
||||||
|
logger.Entries.ShouldContain(e =>
|
||||||
|
e.Level == LogLevel.Warning &&
|
||||||
|
e.Message.Contains("resilience config", StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
(e.Message.Contains("timeoutSeconds") || e.Message.Contains("breakerFailureThreshold")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 01/S-7 production-wiring proof: a respawn interleaved with a lingering old-invoker call must still
|
||||||
|
/// apply the new options. The old invoker (a captured reference — exactly what the old child holds)
|
||||||
|
/// re-caches its stale pipeline AFTER the new <see cref="DriverCapabilityInvokerFactory.Create"/>'s
|
||||||
|
/// invalidate; the new invoker must nonetheless behave per the new config. Fails without the per-Create
|
||||||
|
/// generation stamp (both invokers share the (instance, host, capability) key and the stale entry wins).
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Respawn_interleaved_with_old_invoker_call_still_applies_new_options()
|
||||||
|
{
|
||||||
|
var builder = new DriverResiliencePipelineBuilder();
|
||||||
|
var factory = MakeFactory(builder);
|
||||||
|
|
||||||
|
// Old invoker: Read retries (tier-A default 3). Warm its cache.
|
||||||
|
var oldInvoker = factory.Create("d1", "Modbus", "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":3}}}");
|
||||||
|
await FailingRead(oldInvoker); // caches the retrying pipeline under the old generation
|
||||||
|
|
||||||
|
// Respawn with new options: Read no longer retries.
|
||||||
|
var newInvoker = factory.Create("d1", "Modbus", "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":0}}}");
|
||||||
|
|
||||||
|
// The lingering old child's late call lands AFTER the new Create's invalidate — re-caching the stale
|
||||||
|
// retrying pipeline under the OLD generation.
|
||||||
|
await FailingRead(oldInvoker);
|
||||||
|
|
||||||
|
// The new invoker must honor the new options (no retry) — a single attempt.
|
||||||
|
var attempts = 0;
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await newInvoker.ExecuteAsync(
|
||||||
|
DriverCapability.Read, "host-1",
|
||||||
|
async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); },
|
||||||
|
CancellationToken.None));
|
||||||
|
|
||||||
|
attempts.ShouldBe(1, "the new invoker must apply the new options (retryCount 0), not the stale re-cache");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task FailingRead(IDriverCapabilityInvoker invoker) =>
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await invoker.ExecuteAsync(
|
||||||
|
DriverCapability.Read, "host-1",
|
||||||
|
async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); },
|
||||||
|
CancellationToken.None));
|
||||||
|
|
||||||
private sealed class CapturingLogger : ILogger
|
private sealed class CapturingLogger : ILogger
|
||||||
{
|
{
|
||||||
public List<(LogLevel Level, string Message)> Entries { get; } = new();
|
public List<(LogLevel Level, string Message)> Entries { get; } = new();
|
||||||
|
|||||||
+164
-9
@@ -98,18 +98,23 @@ public sealed class DriverResilienceOptionsParserTests
|
|||||||
read.BreakerFailureThreshold.ShouldBe(tierDefault.BreakerFailureThreshold);
|
read.BreakerFailureThreshold.ShouldBe(tierDefault.BreakerFailureThreshold);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that bulkhead overrides are honored.</summary>
|
/// <summary>
|
||||||
|
/// 01/U-6: the deleted bulkhead knob's keys in stored JSON become ignored unknown keys — a
|
||||||
|
/// config carrying them parses clean with no diagnostic (migration-free contract).
|
||||||
|
/// </summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public void BulkheadOverrides_AreHonored()
|
public void BulkheadKeys_InStoredJson_AreIgnored()
|
||||||
{
|
{
|
||||||
var json = """
|
var json = """
|
||||||
{ "bulkheadMaxConcurrent": 100, "bulkheadMaxQueue": 500 }
|
{ "bulkheadMaxConcurrent": 100, "bulkheadMaxQueue": 500 }
|
||||||
""";
|
""";
|
||||||
|
|
||||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, json, out _);
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, json, out var diag);
|
||||||
|
|
||||||
options.BulkheadMaxConcurrent.ShouldBe(100);
|
diag.ShouldBeNull();
|
||||||
options.BulkheadMaxQueue.ShouldBe(500);
|
// Known capabilities untouched — the stale bulkhead keys are simply ignored.
|
||||||
|
options.Resolve(DriverCapability.Read).ShouldBe(
|
||||||
|
DriverResilienceOptions.GetTierDefaults(DriverTier.B)[DriverCapability.Read]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that unknown capability surfaces in diagnostic but does not fail.</summary>
|
/// <summary>Verifies that unknown capability surfaces in diagnostic but does not fail.</summary>
|
||||||
@@ -133,17 +138,18 @@ public sealed class DriverResilienceOptionsParserTests
|
|||||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that property names are case insensitive.</summary>
|
/// <summary>Verifies that top-level property names are case insensitive.</summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public void PropertyNames_AreCaseInsensitive()
|
public void PropertyNames_AreCaseInsensitive()
|
||||||
{
|
{
|
||||||
var json = """
|
var json = """
|
||||||
{ "BULKHEADMAXCONCURRENT": 42 }
|
{ "RECYCLEINTERVALSECONDS": 3600 }
|
||||||
""";
|
""";
|
||||||
|
|
||||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out _);
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.C, json, out var diag);
|
||||||
|
|
||||||
options.BulkheadMaxConcurrent.ShouldBe(42);
|
diag.ShouldBeNull();
|
||||||
|
options.RecycleIntervalSeconds.ShouldBe(3600);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that capability name is case insensitive.</summary>
|
/// <summary>Verifies that capability name is case insensitive.</summary>
|
||||||
@@ -222,4 +228,153 @@ public sealed class DriverResilienceOptionsParserTests
|
|||||||
options.RecycleIntervalSeconds.ShouldBeNull();
|
options.RecycleIntervalSeconds.ShouldBeNull();
|
||||||
diag.ShouldContain("must be positive");
|
diag.ShouldContain("must be positive");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- 01/S-6: range clamps (clamp + diagnostic, never brick) ----
|
||||||
|
|
||||||
|
/// <summary>A non-positive timeout is nonsense; it snaps to the capability's tier default with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void TimeoutSeconds_Zero_FallsBackToTierDefault_WithDiagnostic()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0}}}", out var diag);
|
||||||
|
|
||||||
|
var tierDefault = DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read];
|
||||||
|
options.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(tierDefault.TimeoutSeconds);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("timeoutSeconds");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A timeout above Polly's ceiling clamps to the max with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void TimeoutSeconds_AbovePollyMax_Clamped()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":999999}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(ResiliencePolicyRanges.MaxTimeoutSeconds);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("timeoutSeconds");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A negative retry count clamps to zero with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void RetryCount_Negative_ClampedToZero()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":-1}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(0);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("retryCount");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>An absurd retry count clamps to the operational cap with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void RetryCount_AboveCap_ClampedTo100()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":2000000000}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(ResiliencePolicyRanges.MaxRetryCount);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("retryCount");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A breaker threshold of 1 (below Polly's MinimumThroughput floor) raises to 2 with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void BreakerThreshold_One_ClampedToTwo()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":1}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).BreakerFailureThreshold.ShouldBe(
|
||||||
|
ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("breakerFailureThreshold");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A negative breaker threshold disables the breaker (0) with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void BreakerThreshold_Negative_DisablesBreaker()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":-5}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).BreakerFailureThreshold.ShouldBe(0);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("breakerFailureThreshold");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Multiple clamps in one policy all surface in the (appended) diagnostic string.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void MultipleClamps_AllSurfaceInDiagnostic()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A,
|
||||||
|
"{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0,\"retryCount\":-1,\"breakerFailureThreshold\":1}}}",
|
||||||
|
out var diag);
|
||||||
|
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("timeoutSeconds");
|
||||||
|
diag.ShouldContain("retryCount");
|
||||||
|
diag.ShouldContain("breakerFailureThreshold");
|
||||||
|
|
||||||
|
var read = options.Resolve(DriverCapability.Read);
|
||||||
|
read.TimeoutSeconds.ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read].TimeoutSeconds);
|
||||||
|
read.RetryCount.ShouldBe(0);
|
||||||
|
read.BreakerFailureThreshold.ShouldBe(ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 01/S-8 = 03/S12: write-shaped capabilities never retry ----
|
||||||
|
|
||||||
|
/// <summary>A Write retryCount override is forced to 0 with a diagnostic (writes are non-idempotent).</summary>
|
||||||
|
[Fact]
|
||||||
|
public void WriteRetryOverride_IsForcedToZero_WithDiagnostic()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Write\":{\"retryCount\":5}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Write).RetryCount.ShouldBe(0);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("Write never retries");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>An AlarmAcknowledge retryCount override is forced to 0 with a diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void AlarmAcknowledgeRetryOverride_IsForcedToZero_WithDiagnostic()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"AlarmAcknowledge\":{\"retryCount\":3}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.AlarmAcknowledge).RetryCount.ShouldBe(0);
|
||||||
|
diag.ShouldNotBeNull();
|
||||||
|
diag.ShouldContain("AlarmAcknowledge never retries");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A Read (idempotent) retryCount override is still honored — the invariant is write-shaped only.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void ReadRetryOverride_StillHonored()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":5}}}", out var diag);
|
||||||
|
|
||||||
|
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(5);
|
||||||
|
diag.ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>In-range overrides pass through untouched with no diagnostic.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void InRangeOverrides_PassThrough_NoDiagnostic()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A,
|
||||||
|
"{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":5,\"retryCount\":3,\"breakerFailureThreshold\":4}}}",
|
||||||
|
out var diag);
|
||||||
|
|
||||||
|
diag.ShouldBeNull();
|
||||||
|
var read = options.Resolve(DriverCapability.Read);
|
||||||
|
read.TimeoutSeconds.ShouldBe(5);
|
||||||
|
read.RetryCount.ShouldBe(3);
|
||||||
|
read.BreakerFailureThreshold.ShouldBe(4);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,34 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
|||||||
[Trait("Category", "Unit")]
|
[Trait("Category", "Unit")]
|
||||||
public sealed class DriverResilienceOptionsTests
|
public sealed class DriverResilienceOptionsTests
|
||||||
{
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// 01/U-6 knob-inertness guard (OVERALL theme #1): the public property set of
|
||||||
|
/// <see cref="DriverResilienceOptions"/> must be EXACTLY the pipeline-wired knobs. Every option a
|
||||||
|
/// parser can populate must map to a strategy the builder actually composes — a parsed-but-unapplied
|
||||||
|
/// knob (the bulkhead genre this pass deleted) is inertness the interface-forwarding and
|
||||||
|
/// unwrapped-dispatch guards can't catch. To add a knob: wire it in <c>DriverResiliencePipelineBuilder</c>,
|
||||||
|
/// add a behavior test that proves it engages, THEN add it to the expected set below.
|
||||||
|
/// (<c>RecycleIntervalSeconds</c> is itself Tier-C-dormant per 01/U-2 — a documented open question,
|
||||||
|
/// explicitly out of this pass's scope; it stays in the expected list as the recycle scheduler, not
|
||||||
|
/// the Polly pipeline, is its consumer.)
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Options_properties_are_exactly_the_pipeline_wired_set()
|
||||||
|
{
|
||||||
|
var expected = new[] { "Tier", "CapabilityPolicies", "RecycleIntervalSeconds" };
|
||||||
|
|
||||||
|
var actual = typeof(DriverResilienceOptions)
|
||||||
|
.GetProperties()
|
||||||
|
.Select(p => p.Name)
|
||||||
|
.OrderBy(n => n)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
actual.ShouldBe(expected.OrderBy(n => n).ToArray(),
|
||||||
|
"DriverResilienceOptions must expose only pipeline-wired knobs — a new option needs a builder " +
|
||||||
|
"strategy + a behavior test before it is added to the expected set (guards against the deleted " +
|
||||||
|
"bulkhead 'parsed-but-unapplied' genre; see 01/U-6).");
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that tier defaults cover every capability.</summary>
|
/// <summary>Verifies that tier defaults cover every capability.</summary>
|
||||||
/// <param name="tier">The driver tier to test.</param>
|
/// <param name="tier">The driver tier to test.</param>
|
||||||
[Theory]
|
[Theory]
|
||||||
|
|||||||
+79
@@ -325,6 +325,85 @@ public sealed class DriverResiliencePipelineBuilderTests
|
|||||||
e.Message.Contains("host-9"));
|
e.Message.Contains("host-9"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 01/S-6 builder invariant: building a pipeline from a hostile, directly-constructed
|
||||||
|
/// <see cref="CapabilityPolicy"/> (bypassing the parser) must NEVER throw Polly's
|
||||||
|
/// <c>ValidationException</c>. Sweeps the full matrix of {int.MinValue, -1, 0, 1, int.MaxValue}
|
||||||
|
/// across timeout/retry/breaker and asserts every built pipeline executes a trivial call.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Build_NeverThrows_ForHostileDirectOptions()
|
||||||
|
{
|
||||||
|
int[] hostile = { int.MinValue, -1, 0, 1, int.MaxValue };
|
||||||
|
var host = 0;
|
||||||
|
foreach (var t in hostile)
|
||||||
|
foreach (var r in hostile)
|
||||||
|
foreach (var b in hostile)
|
||||||
|
{
|
||||||
|
var options = new DriverResilienceOptions
|
||||||
|
{
|
||||||
|
Tier = DriverTier.A,
|
||||||
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||||
|
{
|
||||||
|
[DriverCapability.Read] = new(TimeoutSeconds: t, RetryCount: r, BreakerFailureThreshold: b),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
var builder = new DriverResiliencePipelineBuilder();
|
||||||
|
var pipeline = builder.GetOrCreate("hostile", $"h{host++}", DriverCapability.Read, options);
|
||||||
|
|
||||||
|
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(1), CancellationToken.None);
|
||||||
|
result.ShouldBe(1, $"pipeline built from hostile (t={t}, r={r}, b={b}) must execute, not throw");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 01/S-7 = 03/S13: a stale-epoch re-cache must not be served to a new generation. Deterministic
|
||||||
|
/// replay of the respawn race — the old child's late <c>GetOrCreate</c> re-caches under generation 1
|
||||||
|
/// AFTER the invalidate; the new invoker (generation 2) with new options must build + serve a pipeline
|
||||||
|
/// honoring the NEW options, not the poisoned old-generation entry. Generations make the interleaving
|
||||||
|
/// expressible sequentially — no timing dependence.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task StaleGeneration_ReCache_IsNotServed_ToNewGeneration()
|
||||||
|
{
|
||||||
|
var builder = new DriverResiliencePipelineBuilder();
|
||||||
|
var optsA = new DriverResilienceOptions
|
||||||
|
{
|
||||||
|
Tier = DriverTier.A,
|
||||||
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||||
|
{
|
||||||
|
[DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 3, BreakerFailureThreshold: 0),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
var optsB = new DriverResilienceOptions
|
||||||
|
{
|
||||||
|
Tier = DriverTier.A,
|
||||||
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||||
|
{
|
||||||
|
[DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 0, BreakerFailureThreshold: 0),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// gen 1 warms the cache with the retrying optsA.
|
||||||
|
builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1);
|
||||||
|
builder.Invalidate("i");
|
||||||
|
// The lingering old child's late call re-caches optsA under its OWN generation 1.
|
||||||
|
builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1);
|
||||||
|
|
||||||
|
// The new invoker (gen 2, optsB) must get a pipeline honoring optsB (no retry), not the poisoned one.
|
||||||
|
var newPipeline = builder.GetOrCreate("i", "h", DriverCapability.Read, optsB, optionsGeneration: 2);
|
||||||
|
var attempts = 0;
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await newPipeline.ExecuteAsync(async _ =>
|
||||||
|
{
|
||||||
|
attempts++;
|
||||||
|
await Task.Yield();
|
||||||
|
throw new InvalidOperationException("boom");
|
||||||
|
}));
|
||||||
|
|
||||||
|
attempts.ShouldBe(1, "the new generation must honor optsB (retryCount 0), not the stale optsA re-cache");
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Minimal in-memory <see cref="ILogger"/> capturing (level, formatted-message) pairs so the
|
/// <summary>Minimal in-memory <see cref="ILogger"/> capturing (level, formatted-message) pairs so the
|
||||||
/// resilience logging contract can be asserted without a real logging provider.</summary>
|
/// resilience logging contract can be asserted without a real logging provider.</summary>
|
||||||
private sealed class CapturingLogger : ILogger
|
private sealed class CapturingLogger : ILogger
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
using Shouldly;
|
||||||
|
using Xunit;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Load-bearing regression guard for 01/S-6 — an operator-authorable out-of-range
|
||||||
|
/// ResilienceConfig value must NEVER brick a healthy driver. Before the R2-02 hardening pass,
|
||||||
|
/// <c>"timeoutSeconds": 0</c> (→ <c>TimeSpan.Zero</c>, outside Polly's [10 ms, 24 h] window) and
|
||||||
|
/// <c>"breakerFailureThreshold": 1</c> (below Polly's MinimumThroughput floor of 2) each made
|
||||||
|
/// <see cref="DriverResiliencePipelineBuilder"/> throw <c>ValidationException</c> inside the
|
||||||
|
/// memoizing <c>GetOrAdd</c> factory on EVERY capability call for that (instance, host,
|
||||||
|
/// capability) — a permanent brick while the driver is healthy. The parser now clamps and the
|
||||||
|
/// builder belt-and-braces clamps, so a pipeline built from a parsed hostile config executes.
|
||||||
|
/// </summary>
|
||||||
|
[Trait("Category", "Unit")]
|
||||||
|
public sealed class ResilienceConfigBrickTests
|
||||||
|
{
|
||||||
|
/// <summary>A parsed <c>timeoutSeconds:0</c> override must not brick a capability call.</summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task ParsedConfig_WithZeroTimeout_DoesNotBrickCapabilityCalls()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0}}}", out _);
|
||||||
|
|
||||||
|
var pipeline = new DriverResiliencePipelineBuilder()
|
||||||
|
.GetOrCreate("i1", "h1", DriverCapability.Read, options);
|
||||||
|
|
||||||
|
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(42), CancellationToken.None);
|
||||||
|
result.ShouldBe(42);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A parsed <c>breakerFailureThreshold:1</c> override must not brick a capability call.</summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task ParsedConfig_WithBreakerThresholdOne_DoesNotBrickCapabilityCalls()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":1}}}", out _);
|
||||||
|
|
||||||
|
var pipeline = new DriverResiliencePipelineBuilder()
|
||||||
|
.GetOrCreate("i1", "h1", DriverCapability.Read, options);
|
||||||
|
|
||||||
|
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(42), CancellationToken.None);
|
||||||
|
result.ShouldBe(42);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Text.Json.Nodes;
|
||||||
using Shouldly;
|
using Shouldly;
|
||||||
using Xunit;
|
using Xunit;
|
||||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
using ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
||||||
@@ -6,6 +7,59 @@ using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|||||||
|
|
||||||
public class ResilienceFormModelTests
|
public class ResilienceFormModelTests
|
||||||
{
|
{
|
||||||
|
// ---- 04/C-7: non-lossy round-trip ----
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Unknown_top_level_key_survives_round_trip()
|
||||||
|
{
|
||||||
|
// bulkheadMaxConcurrent doubles as the 01/U-6 continuity proof: the deleted knob's key survives.
|
||||||
|
var json = """{"bulkheadMaxConcurrent":16,"capabilityPolicies":{"Read":{"timeoutSeconds":5}}}""";
|
||||||
|
|
||||||
|
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||||
|
|
||||||
|
back.ShouldNotBeNull();
|
||||||
|
var obj = JsonNode.Parse(back)!.AsObject();
|
||||||
|
obj["bulkheadMaxConcurrent"]!.GetValue<int>().ShouldBe(16);
|
||||||
|
obj["capabilityPolicies"]!["Read"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Unknown_capability_entry_survives_round_trip()
|
||||||
|
{
|
||||||
|
var json = """{"capabilityPolicies":{"Read":{"timeoutSeconds":5},"FutureCap":{"timeoutSeconds":9}}}""";
|
||||||
|
|
||||||
|
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||||
|
|
||||||
|
back.ShouldNotBeNull();
|
||||||
|
var obj = JsonNode.Parse(back)!.AsObject();
|
||||||
|
obj["capabilityPolicies"]!["FutureCap"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(9);
|
||||||
|
obj["capabilityPolicies"]!["Read"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Unknown_per_policy_field_survives_round_trip()
|
||||||
|
{
|
||||||
|
var json = """{"capabilityPolicies":{"Read":{"timeoutSeconds":5,"futureField":7}}}""";
|
||||||
|
|
||||||
|
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||||
|
|
||||||
|
back.ShouldNotBeNull();
|
||||||
|
var read = JsonNode.Parse(back)!.AsObject()["capabilityPolicies"]!["Read"]!.AsObject();
|
||||||
|
read["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||||
|
read["futureField"]!.GetValue<int>().ShouldBe(7);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Malformed_json_sets_ParseFailed_and_ToJson_returns_original_text()
|
||||||
|
{
|
||||||
|
const string json = "{ not json";
|
||||||
|
|
||||||
|
var m = ResilienceFormModel.FromJson(json);
|
||||||
|
|
||||||
|
m.ParseFailed.ShouldBeTrue();
|
||||||
|
m.ToJson().ShouldBe(json); // never overwrite what it couldn't read
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Blank_form_serializes_to_null()
|
public void Blank_form_serializes_to_null()
|
||||||
=> new ResilienceFormModel().ToJson().ShouldBeNull();
|
=> new ResilienceFormModel().ToJson().ShouldBeNull();
|
||||||
@@ -13,7 +67,7 @@ public class ResilienceFormModelTests
|
|||||||
[Fact]
|
[Fact]
|
||||||
public void Partial_override_round_trips()
|
public void Partial_override_round_trips()
|
||||||
{
|
{
|
||||||
var m = new ResilienceFormModel { BulkheadMaxConcurrent = 16 };
|
var m = new ResilienceFormModel { RecycleIntervalSeconds = 3600 };
|
||||||
m.Policies["Read"].TimeoutSeconds = 5;
|
m.Policies["Read"].TimeoutSeconds = 5;
|
||||||
m.Policies["Read"].RetryCount = 5;
|
m.Policies["Read"].RetryCount = 5;
|
||||||
|
|
||||||
@@ -21,7 +75,7 @@ public class ResilienceFormModelTests
|
|||||||
json.ShouldNotBeNull();
|
json.ShouldNotBeNull();
|
||||||
|
|
||||||
var back = ResilienceFormModel.FromJson(json);
|
var back = ResilienceFormModel.FromJson(json);
|
||||||
back.BulkheadMaxConcurrent.ShouldBe(16);
|
back.RecycleIntervalSeconds.ShouldBe(3600);
|
||||||
back.Policies["Read"].TimeoutSeconds.ShouldBe(5);
|
back.Policies["Read"].TimeoutSeconds.ShouldBe(5);
|
||||||
back.Policies["Write"].IsEmpty.ShouldBeTrue();
|
back.Policies["Write"].IsEmpty.ShouldBeTrue();
|
||||||
}
|
}
|
||||||
@@ -30,19 +84,48 @@ public class ResilienceFormModelTests
|
|||||||
public void Malformed_json_yields_empty_model()
|
public void Malformed_json_yields_empty_model()
|
||||||
{
|
{
|
||||||
var m = ResilienceFormModel.FromJson("{ not valid json");
|
var m = ResilienceFormModel.FromJson("{ not valid json");
|
||||||
m.BulkheadMaxConcurrent.ShouldBeNull();
|
m.RecycleIntervalSeconds.ShouldBeNull();
|
||||||
m.Policies["Read"].IsEmpty.ShouldBeTrue();
|
m.Policies["Read"].IsEmpty.ShouldBeTrue();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Validate_flags_out_of_range_values_with_capability_and_range()
|
||||||
|
{
|
||||||
|
var m = new ResilienceFormModel();
|
||||||
|
m.Policies["Read"].TimeoutSeconds = 0; // < MinTimeoutSeconds
|
||||||
|
m.Policies["Read"].RetryCount = -1; // < MinRetryCount
|
||||||
|
m.Policies["Read"].BreakerFailureThreshold = 1; // below Polly MinimumThroughput floor
|
||||||
|
|
||||||
|
var msgs = m.Validate();
|
||||||
|
|
||||||
|
msgs.Count.ShouldBe(3);
|
||||||
|
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("timeout"));
|
||||||
|
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("retries"));
|
||||||
|
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("breaker"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Validate_passes_in_range_and_blank()
|
||||||
|
{
|
||||||
|
var blank = new ResilienceFormModel();
|
||||||
|
blank.Validate().ShouldBeEmpty();
|
||||||
|
|
||||||
|
var inRange = new ResilienceFormModel();
|
||||||
|
inRange.Policies["Read"].TimeoutSeconds = 5;
|
||||||
|
inRange.Policies["Read"].RetryCount = 3;
|
||||||
|
inRange.Policies["Read"].BreakerFailureThreshold = 4;
|
||||||
|
inRange.Policies["Write"].BreakerFailureThreshold = 0; // 0 = breaker off, legal
|
||||||
|
inRange.Validate().ShouldBeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Emitted_json_is_consumable_by_the_runtime_parser()
|
public void Emitted_json_is_consumable_by_the_runtime_parser()
|
||||||
{
|
{
|
||||||
var m = new ResilienceFormModel { BulkheadMaxConcurrent = 16 };
|
var m = new ResilienceFormModel();
|
||||||
m.Policies["Read"].TimeoutSeconds = 7;
|
m.Policies["Read"].TimeoutSeconds = 7;
|
||||||
|
|
||||||
var opts = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, m.ToJson(), out var diag);
|
var opts = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, m.ToJson(), out var diag);
|
||||||
diag.ShouldBeNull();
|
diag.ShouldBeNull();
|
||||||
opts.BulkheadMaxConcurrent.ShouldBe(16);
|
|
||||||
opts.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(7);
|
opts.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(7);
|
||||||
opts.Resolve(DriverCapability.Write).RetryCount.ShouldBe(0);
|
opts.Resolve(DriverCapability.Write).RetryCount.ShouldBe(0);
|
||||||
}
|
}
|
||||||
|
|||||||
+5
@@ -75,6 +75,9 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
|||||||
reply!.Success.ShouldBeTrue();
|
reply!.Success.ShouldBeTrue();
|
||||||
driver.Writes.Count.ShouldBeGreaterThanOrEqualTo(1); // the invoker's call-site actually ran
|
driver.Writes.Count.ShouldBeGreaterThanOrEqualTo(1); // the invoker's call-site actually ran
|
||||||
invoker.Calls.ShouldContain(c => c.Capability == DriverCapability.Write && c.Host == "plc-7");
|
invoker.Calls.ShouldContain(c => c.Capability == DriverCapability.Write && c.Host == "plc-7");
|
||||||
|
// 03/S12: the dispatch site must default writes to NON-idempotent so the invoker's no-retry arm is
|
||||||
|
// authoritative — a command-shaped write (pulse, counter, supervisory) must never replay.
|
||||||
|
invoker.WriteCalls.ShouldContain(c => c.Host == "plc-7" && c.IsIdempotent == false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Records every capability + resolved-host key routed through it, then delegates to the
|
/// <summary>Records every capability + resolved-host key routed through it, then delegates to the
|
||||||
@@ -83,6 +86,7 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
|||||||
private sealed class RecordingInvoker : IDriverCapabilityInvoker
|
private sealed class RecordingInvoker : IDriverCapabilityInvoker
|
||||||
{
|
{
|
||||||
public ConcurrentQueue<(DriverCapability Capability, string Host)> Calls { get; } = new();
|
public ConcurrentQueue<(DriverCapability Capability, string Host)> Calls { get; } = new();
|
||||||
|
public ConcurrentQueue<(string Host, bool IsIdempotent)> WriteCalls { get; } = new();
|
||||||
|
|
||||||
public ValueTask<TResult> ExecuteAsync<TResult>(
|
public ValueTask<TResult> ExecuteAsync<TResult>(
|
||||||
DriverCapability capability, string hostName,
|
DriverCapability capability, string hostName,
|
||||||
@@ -105,6 +109,7 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
|||||||
Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
|
Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
Calls.Enqueue((DriverCapability.Write, hostName));
|
Calls.Enqueue((DriverCapability.Write, hostName));
|
||||||
|
WriteCalls.Enqueue((hostName, isIdempotent));
|
||||||
return callSite(cancellationToken);
|
return callSite(cancellationToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user