From 079692850483ead0aa46755b00e0bd0f7795b0e9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 10 Nov 2026 06:37:36 -0500 Subject: [PATCH] feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results. The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged. - Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill. - Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider carries the runner to DriverHostActor. - Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial); design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms; batched p99 <= 53 ms at 1,000. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- CLAUDE.md | 6 +- docker-dev/docker-compose.script-runner.yml | 3 +- docs/ScriptEditor.md | 6 +- docs/ScriptedAlarms.md | 2 + docs/VirtualTags.md | 33 ++- ...-09-out-of-process-script-runner-design.md | 137 ++++++++- .../ScriptedAlarmEngine.cs | 275 ++++++++++++++++-- ....MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj | 1 + .../OutOfProcessScriptRunner.cs | 185 ++++++++++-- .../WorkerConnection.cs | 174 ++++++++++- .../Protos/script_worker.proto | 9 + .../ScriptWorkerFramePipe.cs | 5 +- .../IScriptRunner.cs | 40 ++- .../ScriptBatchCoalescer.cs | 106 +++++++ .../ScriptRunnerOptions.cs | 5 +- .../ScriptingOptions.cs | 7 +- .../ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj | 2 + .../CalculationDriver.cs | 111 ++++++- .../CalculationEvaluator.cs | 56 ++++ .../Configuration/ScriptRunnerSelection.cs | 16 +- .../Engines/RoslynVirtualTagEvaluator.cs | 11 +- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 3 + .../Drivers/DriverHostActor.cs | 21 +- .../ServiceCollectionExtensions.cs | 6 +- .../WorkerHost.cs | 95 +++++- .../ScriptedAlarmEngineScriptRunnerTests.cs | 192 ++++++++++++ .../ScriptBatchCoalescerTests.cs | 103 +++++++ .../CalculationDriverBatchPumpTests.cs | 110 +++++++ .../ScriptRunnerSelectionTests.cs | 27 +- .../AlarmSeamTests.cs | 152 ++++++++++ .../BatchSubmitTests.cs | 172 +++++++++++ .../ScriptRunnerPerfTests.cs | 212 ++++++++++++++ ...B.MOM.WW.OtOpcUa.ScriptWorker.Tests.csproj | 2 + 33 files changed, 2168 insertions(+), 117 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptBatchCoalescer.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineScriptRunnerTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptBatchCoalescerTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverBatchPumpTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/AlarmSeamTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BatchSubmitTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 90a8fe1a..a9b9c1ee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -390,10 +390,12 @@ endpoints under `/api/script-analysis/*`, gated by the `ConfigEditor` policy See `docs/ScriptEditor.md` for the full guide. Scripts may use the `{{equip}}` token for equipment-relative tag paths that resolve per-equipment at deploy time — see the "Equipment-relative tag paths" section in `docs/ScriptEditor.md`. **Out-of-process script runner (lmxopcua#780, opt-in).** `Scripting:Runner` = `InProcess` (default, unchanged) | `OutOfProcess` | `Shadow` (not yet). -`OutOfProcess` runs the VirtualTag + Calculation seams' compiled scripts in `ZB.MOM.WW.OtOpcUa.ScriptWorker` processes +`OutOfProcess` runs the VirtualTag + Calculation seams' compiled scripts (and the scripted-alarm predicates when `Scripting:Runner:Seams` +lists `ScriptedAlarm` — never by default) in `ZB.MOM.WW.OtOpcUa.ScriptWorker` processes (`src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess`, protocol in `Core.Scripting.Protocol`), shipped in `script-worker/` beside the host (self-contained + started via its apphost on a `-r win-x64 --self-contained` publish, through the muxer otherwise); a runner-side failure publishes `BadResourceUnavailable` (ScadaBridge-visible, ScadaBridge#112) and the seam re-evaluates -when the worker is back. Optional `Scripting:Worker:PeCacheDir` = HMAC'd compiled-PE cache (restart without Roslyn); the `script-runner` +when the worker is back. `IScriptRunner.RunBatch` sends ready evaluations as one frame per worker (Calc pump, VT coalescer, +alarm dirty set); a run queued behind busy worker threads has its deadlines restarted when it starts, so it is never blamed for a sibling. Optional `Scripting:Worker:PeCacheDir` = HMAC'd compiled-PE cache (restart without Roslyn); the `script-runner` readiness check is at worst Degraded (never fails ready). The worker tests are `tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests` (`Category=ScriptWorker`, real processes; `Category=Perf` harness opt-in via `OTOPCUA_SCRIPT_PERF=1`). docker-dev overlay: `docker-dev/docker-compose.script-runner.yml`. See `docs/VirtualTags.md` diff --git a/docker-dev/docker-compose.script-runner.yml b/docker-dev/docker-compose.script-runner.yml index 457c9cfa..70a3ea99 100644 --- a/docker-dev/docker-compose.script-runner.yml +++ b/docker-dev/docker-compose.script-runner.yml @@ -1,5 +1,6 @@ # docker-dev/docker-compose.script-runner.yml — optional overlay: run the central pair's VirtualTag + Calculation -# scripts OUT OF PROCESS (lmxopcua#780, Scripting:Runner=OutOfProcess). The scripted-alarm seam stays in process. +# scripts OUT OF PROCESS (lmxopcua#780, Scripting:Runner=OutOfProcess). The scripted-alarm seam stays in process; to +# move it too (last in the rollout, design §14 #10) add `Scripting__Runner__Seams__2: "ScriptedAlarm"` below. # # Up (recreates only the two central nodes with the extra env; the image must be built from a branch that has the # runner, i.e. `docker compose -f docker-dev/docker-compose.yml build central-1` first): diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 5c46e437..5735ce1a 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -212,10 +212,12 @@ scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: run returns, so a run that OOMs the process first is not stopped — that needs an out-of-process runner (→ #750). - **Out of process (opt-in, lmxopcua#780).** `Scripting:Runner=OutOfProcess` moves the - virtual-tag and Calculation seams' compiled runs into `ScriptWorker` processes, where a + virtual-tag and Calculation seams' compiled runs — and the scripted-alarm predicates, + when `Scripting:Runner:Seams` lists `ScriptedAlarm` — into `ScriptWorker` processes, where a run past `T + Scripting:Worker:HardGrace` is stopped by killing its worker and an OOM kills a worker instead of the server. A run whose worker died under it reads - `BadResourceUnavailable`. See `docs/VirtualTags.md` §"Out-of-process runner". + `BadResourceUnavailable` (an alarm is frozen with that annotation, never cleared). See + `docs/VirtualTags.md` §"Out-of-process runner". - **Recursion.** Unbounded recursion fails the script with `ScriptStackExhaustedException` (Bad, script log) instead of overflowing the stack, which would terminate the whole server. The runtime compile inserts a diff --git a/docs/ScriptedAlarms.md b/docs/ScriptedAlarms.md index f538d72f..227c7a59 100644 --- a/docs/ScriptedAlarms.md +++ b/docs/ScriptedAlarms.md @@ -49,6 +49,8 @@ new ScriptedAlarmDefinition( Alarm predicates reuse the same Roslyn sandbox as virtual tags — `ScriptEvaluator` compiles the source, `ScriptRunGuard` runs it with the node's `Scripting:RunTimeout` budget (default 2 s — the same budget virtual tags get; before lmxopcua#779 the host built the engine without a budget and every predicate silently ran on `TimedScriptEvaluator.DefaultTimeout`, 250 ms), and `DependencyExtractor` statically harvests the tag paths the script reads. The sandbox rules (forbidden types, cancellation, logging sinks) are documented in [VirtualTags.md](VirtualTags.md); ScriptedAlarms does not redefine them. The known resource limits — unbounded script-side memory and the orphan-thread CPU-budget caveat — are documented in that file as well; per-publish assembly accretion was resolved by the Core.Scripting-008 collectible-`AssemblyLoadContext` rewrite and no longer requires periodic server restarts. +**Out of process (opt-in, lmxopcua#780).** With `Scripting:Runner=OutOfProcess` **and** `ScriptedAlarm` listed in `Scripting:Runner:Seams` (an unset list leaves alarms in process — they go last in the rollout), the predicates run in the pre-emptible `ScriptWorker` pool. `LoadAsync` still compiles every predicate, to PE bytes only, so a broken predicate fails the load exactly as before; the worker compiles and runs it. Each re-evaluation pass (and the startup-recovery pass) submits every dirty alarm's predicate in **one batch** and applies the results — Part 9 transition, persist-before-update, emissions — in the serial order under the engine's gate, so the transition sequence is the in-process one and a large engine pays one round trip per pass, not one per alarm. Failure semantics (design §9): a predicate that times out, is killed at its hard deadline or is quarantined freezes its condition with a `BadInternalError` quality annotation, as in process; one whose worker restarted under it, or that found no worker ready, freezes with **`BadResourceUnavailable`** (`0x80040000`) and is re-evaluated by itself when the worker is back. No outcome but a completed `bool` reaches the state machine, so a dead worker never clears an alarm. Quarantines (`ScriptedAlarmEngine.QuarantinedAlarmIds`) lift on a reload. See [VirtualTags.md](VirtualTags.md) §"Out-of-process runner". + `AlarmPredicateContext` (`AlarmPredicateContext.cs`) is the script's `ScriptContext` subclass: - `GetTag(path)` returns a `DataValueSnapshot` from the engine-maintained read cache. Missing path → `DataValueSnapshot(null, 0x80340000u, null, now)` (`BadNodeIdUnknown`). An empty path returns the same. diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index 76857ae9..c54f0a9b 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -46,10 +46,11 @@ The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocati `Scripting:Runner` selects where compiled scripts run: **`InProcess`** (the default and the rollback — everything above, unchanged), **`OutOfProcess`**, or `Shadow` (design P2; not implemented yet, so it runs `InProcess` with a -startup Warning). `Scripting:Runner:Seams` narrows `OutOfProcess` to a subset (`VirtualTag`, `Calculation`; the -`ScriptedAlarm` seam stays in process for now — alarms go last, design §14 #10). Under `OutOfProcess` the virtual-tag -and Calculation evaluators hand each compiled run to `OutOfProcessScriptRunner` -(`Core.Scripting.OutOfProcess`), which runs it in a pool of `ZB.MOM.WW.OtOpcUa.ScriptWorker` processes: +startup Warning). `Scripting:Runner:Seams` selects the seams (`VirtualTag`, `Calculation`, `ScriptedAlarm`). Unset or +empty means `VirtualTag` + `Calculation`: the **`ScriptedAlarm` seam runs out of process only when it is listed +explicitly** (alarms go last, design §14 #10). Under `OutOfProcess` the selected seams hand their compiled runs to +`OutOfProcessScriptRunner` (`Core.Scripting.OutOfProcess`), which runs them in a pool of `ZB.MOM.WW.OtOpcUa.ScriptWorker` +processes: - **K shards** (`Scripting:Worker:Shards`, default 2) placed by a stable FNV-1a hash of the key, plus an on-demand **isolation worker** that runs suspects one at a time. No worker starts until the first compiled evaluation, so a @@ -122,13 +123,33 @@ and Calculation evaluators hand each compiled run to `OutOfProcessScriptRunner` directly: `OutOfProcessScriptRunner.ResolveWorkerEntry` picks the apphost only when the worker's `runtimeconfig.json` lists `includedFrameworks`. Gated on windev against a real `-r win-x64` publish (`OTOPCUA_SCRIPTWORKER_TEST_PATH` points the worker suite at the published apphost). Not yet done (lmxopcua#780): the low-privilege worker account, pipe ACL, firewall - rule and PeCacheDir ACL (installer), the scripted-alarm seam and the per-seam batch submit (design §4.5). + rule and PeCacheDir ACL (installer). - **docker-dev.** `docker-dev/docker-compose.script-runner.yml` turns `OutOfProcess` on for the central pair's VirtualTag and Calculation seams. The 2026-11-10 live drill (runaway kill and quarantine, `kill -9` recovery, open circuit with readiness still 200) is recorded in the design doc's rollout section. - **Perf harness.** `ScriptRunnerPerfTests` (`Category=Perf`, skipped unless `OTOPCUA_SCRIPT_PERF=1`) measures InProcess against OutOfProcess at K = 1/2/4 against the accepted 5,000 evals/s/node gate and the design §8.2 budgets. The numbers - are in the design doc's rollout section (§12.1). The gate passes; the host-CPU and RSS budgets miss (lmxopcua#789). + are in the design doc's rollout section (§12.1, §12.2). The gate passes; the host-CPU and RSS budgets miss (lmxopcua#789). + It also measures batch submit (RunBatch against the same evaluations as concurrent RunAsync calls) and the + scripted-alarm pass latency (push → last transition, in process / out of process batched / out of process serial) + against the 250 ms alarm budget — design §12.3. +- **Batch submit (design §4.5).** `IScriptRunner.RunBatch` submits several evaluations at once: every item that is ready + to run (its key loaded in a live worker, not a suspect) goes out in ONE `EvalBatch` frame per worker, and results are + per item and streamed. The Calculation pump dequeues everything pending and submits it as one batch, publishing in + dequeue order (a tag queued twice ends the batch, so every queued evaluation still runs against the snapshot that + triggered it); the virtual-tag seam coalesces evaluations that become ready together (one actor per tag) into one + batch, with no timer; the scripted-alarm engine submits a pass's whole dirty set. In process the pump and the engine + stay one-at-a-time, unchanged. **Per-item deadlines and blame stay exact:** an evaluation that arrives while every + evaluation thread is busy is reported back as *deferred*, the host stops its clock, and restarts T and G when the worker + reports it *started* — so an item queued behind a runaway sibling is neither timed out nor struck, and at the kill it + is a victim (`script worker restarted`), not the culprit. +- **Scripted alarms** (only when `Scripting:Runner:Seams` lists `ScriptedAlarm`). The engine compiles each predicate at + load only to validate it (same Roslyn, sandbox and diagnostics, so a broken predicate still fails the load; no + assembly is loaded into the host), the worker compiles and runs it, and each re-evaluation pass submits its dirty set + as one batch and applies the results in the serial order under the engine's gate — the Part 9 sequence is the + in-process one. A predicate that did not complete freezes its condition (a dead worker never clears an alarm): timed + out, killed or quarantined → `BadInternalError` annotation, as in process; the runner's fault (worker restarted, none + ready) → `BadResourceUnavailable` annotation, and the alarm is re-evaluated by itself when the worker is back. - **Compile threads.** A worker compiles on `min(8, max(1, CPUs/2))` dedicated threads, never on the thread pool, so a burst of cold loads cannot starve its heartbeat or its pipe writer. The host load timeout still counts the time a load spends in that queue (lmxopcua#790). diff --git a/docs/plans/2026-11-09-out-of-process-script-runner-design.md b/docs/plans/2026-11-09-out-of-process-script-runner-design.md index e2b6e0ba..595df57b 100644 --- a/docs/plans/2026-11-09-out-of-process-script-runner-design.md +++ b/docs/plans/2026-11-09-out-of-process-script-runner-design.md @@ -640,7 +640,7 @@ per-shard state, restarts in the last hour and the quarantine list. Quarantines | Key | Default | Meaning | |---|---|---| | `Scripting:Runner` | `InProcess` | `InProcess` \| `OutOfProcess` \| `Shadow` | -| `Scripting:Runner:Seams` | all three | Optional per-seam subset for `OutOfProcess` / `Shadow` (e.g. `["Calculation"]`) | +| `Scripting:Runner:Seams` | VirtualTag + Calculation | Optional per-seam subset for `OutOfProcess` / `Shadow` (e.g. `["Calculation"]`). Unset selects VirtualTag and Calculation; `ScriptedAlarm` only when listed (§14 #10, as built in §12.3) | | `Scripting:Worker:Shards` | `2` | 1–8 | | `Scripting:Worker:HardGrace` | `max(1s, T)` | §5.1 | | `Scripting:Worker:HeapLimitBytes` | `536870912` | §5.2 | @@ -660,6 +660,9 @@ per-shard state, restarts in the last hour and the quarantine list. Quarantines `VirtualTagActor`), so the dispatcher-thread blocking is gone. The Calc pump and the alarm engine still run one script at a time; why that was not relaxed in-process is recorded in `docs/VirtualTags.md` §"Upstream-tag feed". `IScriptRunner` and batch-submit remain to do. + - **Batch submit landed in lmxopcua#780 (fourth P1 slice, §12.3):** `IScriptRunner.RunBatch`, used by the Calc pump, the + VirtualTag seam (a coalescer) and the scripted-alarm engine on the out-of-process path. In process the pump and the + engine stay one at a time, unchanged. 2. **P1, worker + protocol behind the flag.** - Ship `ScriptWorker`, `OutOfProcessScriptRunner` and the PE cache. - Default stays `InProcess`. Enable on the docker-dev rig only. @@ -686,6 +689,11 @@ per-shard state, restarts in the last hour and the quarantine list. Quarantines §10 gauges and histograms, the degraded-but-ready `script-runner` health check (§14 #12), and the orphan sweep by launch marker (pid + start time + image, host gone). Still open: self-contained worker publish, installer (account, pipe ACL, firewall), docker-dev live drill, scripted-alarm seam, the §4.5 batch submit, the §8 perf harness. + - **Fourth P1 slice (lmxopcua#780, record in §12.3):** the §4.5 batch submit (`IScriptRunner.RunBatch`, one `EvalBatch` + frame per worker; Calc pump, VT coalescer, alarm dirty set), exact per-item deadlines for work queued in the worker + (deferral / start notices), and the scripted-alarm seam behind `Scripting:Runner:Seams` — alarms run out of process + only when listed. Still open: installer (account, pipe ACL, firewall, PeCacheDir ACL), #789's remaining misses, then + P2 Shadow. - **Third P1 slice (lmxopcua#780, record in §12.1):** a self-contained worker publish for self-contained (Windows) Host publishes, started through its apphost; the docker-dev live drill (`docker-dev/docker-compose.script-runner.yml`); the §8 perf harness (`Category=Perf`, opt-in), which passes the accepted 5k evals/s/node gate; and two runner fixes @@ -963,6 +971,133 @@ Additional measurements: - Tests: `A_shard_that_cannot_start_logs_one_error_then_concise_rate_limited_retries_then_its_recovery`, with two intervals. Red when every failure is logged at Error with the exception. +### 12.3 Fourth P1 slice: batch submit, exact per-item deadlines, the scripted-alarm seam (2026-11-10) + +**Batch submit (§4.5, P0).** +- `IScriptRunner.RunBatch(invocations)` returns one task per item. Results are per item and streamed. + - The default implementation is `RunAsync` per item, started together. `InProcessScriptRunner` uses it. + - `OutOfProcessScriptRunner` sends every item that is ready to run as ONE `EvalBatch` frame per worker. Ready means + admitted, not quarantined, not a suspect, and already loaded in a live worker at this source and slot order. + - Anything else goes through the full `RunAsync` path on its own: a first evaluation that needs a compile, a worker + that is down, or a suspect bound for the one-at-a-time isolation worker. A batch therefore never waits for a load + or a restart as a whole. +- Seams that use it, on the out-of-process path only: + - **Calculation pump.** It dequeues everything pending (each entry's dependency snapshot was taken at enqueue), + submits one batch, and publishes in dequeue order. A tag queued twice ends the batch, so every queued evaluation + still runs, in order, against the snapshot that triggered it. The published sequence is the one-at-a-time pump's. + - **VirtualTag seam.** There is one actor per tag, so an upstream burst makes many tags ready at once from different + threads. `ScriptBatchCoalescer` hands those to one `RunBatch`. It has no timer: an evaluation on an idle seam is + submitted at once by its own caller. + - **Scripted alarms.** See below. +- In process, the Calc pump and the alarm engine stay one at a time, unchanged. The VirtualTag in-process path does + not touch the coalescer. + +**Per-item deadlines and blame stay exact under batching.** +- The host arms T and T + G at submit. That is when the run starts, provided the worker has an idle evaluation thread. +- A batch larger than the worker's scheduler, or work queued behind a runaway, used to wait in the worker with its + clock already running. It could then be timed out, take timeout strikes, and even be the key blamed at T + G. +- The worker's scheduler now hands an evaluation straight to an idle thread, or else puts it in a backlog and reports + it in `EvalResults.deferred_eval_ids`. + - The host stops that evaluation's clock. + - When a thread picks the evaluation up, its id rides back in `started_eval_ids` on the result frame of the run that + thread just finished. The host then restarts the full T, then G, from the start. + - The two notices are unordered on the wire, and the host handles either order. + - The deadlines are absolute timestamps that the timer re-checks, so a stale tick never fires early. +- A run queued behind a runaway is therefore neither timed out nor struck. At the kill it is a victim + (`script worker restarted`, `BadResourceUnavailable`, re-evaluated on return), not the culprit. +- Tests: + - `A_runaway_in_a_batch_is_blamed_alone_and_siblings_queued_behind_it_are_not_timed_out_or_struck` runs one evaluation + thread and a batch of a runaway plus three siblings. It is red with the host ignoring the deferral, and red with the + worker never deferring. + - `A_deferred_run_is_timed_and_killed_from_when_it_starts` is the positive control. It is red with a start notice that + does not re-arm the clock. +- This also changes plain `RunAsync` under saturation: a run that waits for a thread no longer times out on its wait. + That matches the in-process guard, whose budget clock starts when the pool picks the run up (§1.2). + +**Scripted-alarm seam (★10: alarms last).** +- `Scripting:Runner:Seams` unset now means VirtualTag + Calculation. **`ScriptedAlarm` runs out of process only when it + is listed.** +- `ScriptRunnerSelection` serves it. The Runtime's `DriverHostActor` gets it through the new `IScriptRunnerProvider` + and passes it to the engine (a new last parameter, `scriptedAlarmRunner`). +- Engine changes, all on the runner path only: + - `LoadAsync` compiles each predicate to PE bytes only (`CompileToImage`). A broken predicate still fails the load + exactly as in process, and no assembly is loaded into the host. Deviation: Roslyn still runs in the host at load + for alarms; the worker compiles again for execution. + - Startup recovery and every re-evaluation pass snapshot each dirty alarm's inputs under `_evalGate`, submit all + their predicates as one batch, and then apply the results in the serial order under the gate: Part 9 transition, + persist before update, emissions. A test shows the transition sequence is identical to an in-process engine's + for the same pushes. +- Failure semantics (§9, alarm column): + - Timed out, killed, refused or quarantined: the condition is frozen with a `BadInternalError` annotation, as today. + - The runner's fault (worker restarted, none ready): frozen with **`BadResourceUnavailable`**, consistent with + VT/Calc. The alarm is re-evaluated by itself when the runner's `Recovery` completes. + - A predicate that threw keeps its state with no annotation, as in process. + - No outcome but a completed `bool` reaches the state machine, so a dead worker never clears an alarm. +- Real-worker tests: + - `A_runaway_predicate_is_killed_and_quarantined_its_alarm_frozen_Bad_and_a_sibling_keeps_evaluating`; + - `A_worker_that_dies_under_a_predicate_freezes_it_BadResourceUnavailable_never_clears_it_and_it_is_re_evaluated_on_return`. + It is red when the runner's failure is read as `false`. +- Budget note: since lmxopcua#779 the host passes `Scripting:RunTimeout` (2 s) to the engine. The passes below are + still judged against the stricter 250 ms the design was written for. + +**Measured (§8 harness, `Batch_throughput_and_alarm_pass_latency`; macOS Arm64, 14 CPUs, load average about 14 from +other work on the box, 10 s phases):** + +| Batch size | RunBatch evals/s | RunAsync ×n evals/s | RunBatch p50 / p99 µs | RunAsync ×n p50 / p99 µs | +|---:|---:|---:|---:|---:| +| 1 | 38,296 | 40,980 | 24 / 41 | 23 / 35 | +| 10 | 147,035 | 118,516 | 57 / 121 | 70 / 142 | +| 100 | 369,304 | 232,967 | 215 / 1,095 | 363 / 1,341 | +| 500 | 463,033 | 256,232 | 818 / 2,541 | 1,701 / 3,178 | + +(OutOfProcess K = 2, one batch at a time. The in-process rows show no difference between the two, as expected.) + +| Alarms per pass | InProcess p50 / p99 ms | OutOfProcess batched p50 / p99 ms | OutOfProcess serial p50 / p99 ms | +|---:|---:|---:|---:| +| 100 | 0.4 / 1.5 | 0.4 / 2.1 | 3.6 / 4.9 | +| 500 | 2.2 / 4.7 | 1.7 / 4.8 | 16.1 / 18.0 | +| 1,000 | 4.4 / 7.7 | 2.5 / 41.0 | 28.1 / 33.4 | + +- A pass is push → the last of the N transitions it emits. "Serial" replays the pre-batch one-predicate-at-a-time pass + over the same runner. That is the N × round-trip cost §8.2 warned about: about 30 µs per alarm, against 2–4 µs + batched. +- Every configuration is far inside 250 ms. The one 41 ms p99 is a single loaded-box outlier. +- The §8.2 harness rerun on the same box after the scheduler change: + - K = 2 closed loop ×64: 366k evals/s; + - paced 10k/s: +5 % host CPU vs InProcess (budget ≤ 25 %); + - idle p50: 25 µs; + - kill latency: 14–19 ms; + - warm-cache restart of 1,000 scripts: 220 ms; + - every result correct. + +**windev** (Windows VM, 36 CPUs, .NET 10.0.8; the full `ScriptWorker.Tests` suite also passes there, 157/157): + +| Batch size | RunBatch evals/s | RunAsync ×n evals/s | RunBatch host CPU | RunAsync ×n host CPU | +|---:|---:|---:|---:|---:| +| 1 | 1,810 | 1,921 | 63 % | 65 % | +| 10 | 13,013 | 11,220 | 109 % | 157 % | +| 100 | 48,685 | 40,699 | 129 % | 224 % | +| 500 | 86,322 | 83,116 | 143 % | 289 % | + +| Alarms per pass | InProcess p50 / p99 ms | OutOfProcess batched p50 / p99 ms | OutOfProcess serial p50 / p99 ms | +|---:|---:|---:|---:| +| 100 | 8.2 / 39.0 | 2.9 / 5.1 | 51.7 / 64.8 | +| 500 | 42.3 / 53.3 | 7.4 / 35.7 | **236.5 / 273.3 (MISS)** | +| 1,000 | 36.6 / 84.2 | 11.7 / 52.7 | **466.8 / 519.4 (MISS)** | + +- On the VM, where one round trip costs about 0.5 ms, the pre-batch serial pass **misses the 250 ms budget from about + 500 alarms**. The batched pass stays at p99 ≤ 53 ms for 1,000 alarms, below the in-process engine's own figures. + That is the measurement behind §8's "batch submit is a prerequisite, not an optimisation", and the reason ★10 put + alarms last. +- On Windows, batching halves the host CPU of a 100–500 evaluation burst. That is the overlapped-I/O path that is still + #789's main miss. +- No predicate failed to complete in any pass on either host. +- Rerun of the §8.2 harness on windev: + - K = 2 closed loop ×64: 50k evals/s, gate PASS; + - paced 10k/s: +99 % host CPU vs InProcess (#789, unchanged in kind); + - kill latency: 9–19 ms; + - warm-cache restart of 1,000 scripts: 1.8 s. + --- ## 13. Test plan diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index d372c283..5204b00f 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -81,6 +81,17 @@ public sealed class ScriptedAlarmEngine : IDisposable /// private readonly ScriptRunGuard _guard; + /// + /// lmxopcua#780 — the out-of-process runner when Scripting:Runner=OutOfProcess lists the ScriptedAlarm seam; + /// null (the default) keeps the in-process guard path, unchanged. With a runner the predicates compile and run in + /// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch. + /// + private readonly IScriptRunner? _runner; + + // Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the + // runner's Recovery to re-evaluate them once. Guarded by _dirtyLock. + private readonly HashSet _recoveryArmed = new(StringComparer.Ordinal); + // ConcurrentDictionary, not a plain Dictionary: every mutation happens under // _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds, // RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request @@ -243,6 +254,8 @@ public sealed class ScriptedAlarmEngine : IDisposable /// default used to be TimedScriptEvaluator.DefaultTimeout (250 ms), which is what every production engine /// silently ran with because the host never passed a value. /// Run-guard budget + quarantine threshold; null ⇒ defaults. + /// lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of + /// process; null (the default) keeps the in-process guard path, unchanged. public ScriptedAlarmEngine( ITagUpstreamSource upstream, IAlarmStateStore store, @@ -250,8 +263,10 @@ public sealed class ScriptedAlarmEngine : IDisposable ILogger engineLogger, Func? clock = null, TimeSpan? scriptTimeout = null, - ScriptingOptions? scripting = null) + ScriptingOptions? scripting = null, + IScriptRunner? scriptRunner = null) { + _runner = scriptRunner; _upstream = upstream ?? throw new ArgumentNullException(nameof(upstream)); _store = store ?? throw new ArgumentNullException(nameof(store)); _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); @@ -268,6 +283,25 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The wall-clock budget each predicate run is given (diagnostics / tests). public TimeSpan ScriptTimeout => _scriptTimeout; + /// The out-of-process runner predicates run through, or null in process (diagnostics / tests). + public IScriptRunner? ScriptRunner => _runner; + + /// + /// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when + /// one is in use (diagnostics / tests). + /// + public IReadOnlyCollection QuarantinedAlarmIds => _runner is null + ? _guard.QuarantinedKeys + : [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.AlarmPredicate)]; + + /// + /// OPC UA BadResourceUnavailable — the annotation for a predicate the out-of-process RUNNER could not run + /// (its worker restarted under it, or no worker was ready): the condition is frozen exactly as for a script + /// fault, but the quality tells the operator the node is sick, not the script (design §9, §14 #3; the same split + /// the virtual-tag and Calculation seams make). + /// + private const uint StatusBadResourceUnavailable = 0x80040000u; + /// /// OPC UA BadInternalError — the quality a condition is annotated with while its predicate /// cannot be evaluated (timed out, refused because the previous run is still executing, or @@ -331,6 +365,8 @@ public sealed class ScriptedAlarmEngine : IDisposable // A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts // its own through the guard's source identity; this covers "redeployed to retry"). _guard.ResetAll(); + _runner?.Reset(ScriptContextKind.AlarmPredicate); + lock (_dirtyLock) _recoveryArmed.Clear(); var compileFailures = new List(); foreach (var def in definitions) @@ -350,7 +386,13 @@ public sealed class ScriptedAlarmEngine : IDisposable // The compiled script is run through _guard (timeout + quarantine) at evaluation time — // the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not // see that the run it abandoned was still executing. - var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource); + // + // Out of process (lmxopcua#780) the worker compiles and runs the predicate; here the source is only + // compiled to PE bytes and dropped — same Roslyn, sandbox and diagnostics, so a broken predicate still + // fails the load exactly as in process — and no assembly is loaded into the host. + ScriptEvaluator? evaluator = null; + if (_runner is null) evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource); + else _ = ScriptEvaluator.CompileToImage(def.PredicateScriptSource); var logger = _loggerFactory.Create(def.AlarmId); var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate); @@ -396,14 +438,32 @@ public sealed class ScriptedAlarmEngine : IDisposable // Any predicate emissions queue into `pending` and fire after the gate // is released — so a startup-recovery activation event can call back into // the engine without deadlocking. - foreach (var (alarmId, state) in _alarms) + if (_runner is null) { - var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false); - var seed = persisted ?? state.Condition; - var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending) - .ConfigureAwait(false); - _alarms[alarmId] = state with { Condition = afterPredicate }; - await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false); + foreach (var (alarmId, state) in _alarms) + { + var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false); + var seed = persisted ?? state.Condition; + var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending) + .ConfigureAwait(false); + _alarms[alarmId] = state with { Condition = afterPredicate }; + await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false); + } + } + else + { + // Out of process: every predicate goes in one batch, then each alarm is applied and saved in the + // same order the serial loop above uses. + var seeds = new List<(AlarmState State, AlarmConditionState Seed)>(_alarms.Count); + foreach (var (alarmId, state) in _alarms) + seeds.Add((state, await _store.LoadAsync(alarmId, ct).ConfigureAwait(false) ?? state.Condition)); + var after = await EvaluatePredicateBatchToStatesAsync(seeds, _clock(), ct, pending).ConfigureAwait(false); + for (var i = 0; i < seeds.Count; i++) + { + var state = seeds[i].State; + _alarms[state.Definition.AlarmId] = state with { Condition = after[i] }; + await _store.SaveAsync(after[i], ct).ConfigureAwait(false); + } } _loaded = true; @@ -614,7 +674,12 @@ public sealed class ScriptedAlarmEngine : IDisposable if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again _valueCache[path] = value; if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return; + MarkDirty(alarmIds); + } + /// Marks dirty and starts the drainer if none is running. + private void MarkDirty(IEnumerable alarmIds) + { bool startDrain; lock (_dirtyLock) { @@ -754,17 +819,25 @@ public sealed class ScriptedAlarmEngine : IDisposable // disposing store or mutating _alarms after clear is unsafe. if (_disposed) return; - foreach (var id in alarmIds) + if (_runner is not null) { - if (!_alarms.TryGetValue(id, out var state)) continue; - var newState = await EvaluatePredicateToStateAsync( - state, state.Condition, _clock(), ct, pending).ConfigureAwait(false); - if (!ReferenceEquals(newState, state.Condition)) + // Out of process (lmxopcua#780, design §4.5): the whole dirty set in one batch, applied in order. + await ReevaluateBatchAsync(alarmIds, ct, pending).ConfigureAwait(false); + } + else + { + foreach (var id in alarmIds) { - // Persist before updating in-memory so a store failure leaves - // both sides at the prior state. - await _store.SaveAsync(newState, ct).ConfigureAwait(false); - _alarms[id] = state with { Condition = newState }; + if (!_alarms.TryGetValue(id, out var state)) continue; + var newState = await EvaluatePredicateToStateAsync( + state, state.Condition, _clock(), ct, pending).ConfigureAwait(false); + if (!ReferenceEquals(newState, state.Condition)) + { + // Persist before updating in-memory so a store failure leaves + // both sides at the prior state. + await _store.SaveAsync(newState, ct).ConfigureAwait(false); + _alarms[id] = state with { Condition = newState }; + } } } } @@ -837,7 +910,7 @@ public sealed class ScriptedAlarmEngine : IDisposable try { predicateTrue = await _guard.RunAsync( - state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context, + state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator!, context, _scriptTimeout, ct) .ConfigureAwait(false); } @@ -890,6 +963,162 @@ public sealed class ScriptedAlarmEngine : IDisposable return result.State; } + /// + /// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the + /// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in + /// the original order, under _evalGate. Predicates are pure functions of the input snapshot taken here, + /// under the gate, so the state machine sees exactly the sequence the serial loop produces. The batch costs one + /// round trip for the whole set instead of one per alarm, which is what keeps a large engine inside its budget. + /// + private async Task ReevaluateBatchAsync(IReadOnlyList alarmIds, CancellationToken ct, List pending) + { + var items = new List<(AlarmState State, AlarmConditionState Seed)>(alarmIds.Count); + foreach (var id in alarmIds) + if (_alarms.TryGetValue(id, out var state)) items.Add((state, state.Condition)); + if (items.Count == 0) return; + + var after = await EvaluatePredicateBatchToStatesAsync(items, _clock(), ct, pending).ConfigureAwait(false); + for (var i = 0; i < items.Count; i++) + { + var (state, seed) = items[i]; + if (ReferenceEquals(after[i], seed)) continue; + // Persist before updating in-memory so a store failure leaves both sides at the prior state. + await _store.SaveAsync(after[i], ct).ConfigureAwait(false); + _alarms[state.Definition.AlarmId] = state with { Condition = after[i] }; + } + } + + /// + /// for a batch, through : the same readiness gate, + /// quality tracking and Part 9 application per alarm, in order, with every predicate that has to run submitted + /// together. Failure semantics (design §9): a script fault, timeout, kill or quarantine freezes the condition with a + /// BadInternalError annotation, exactly as in process; a runner failure (worker restarted, none ready) freezes + /// it with BadResourceUnavailable and re-evaluates it once the runner can run it again. A dead worker never + /// clears an alarm: no outcome but a completed bool reaches the state machine. + /// + private async Task EvaluatePredicateBatchToStatesAsync( + IReadOnlyList<(AlarmState State, AlarmConditionState Seed)> items, DateTime nowUtc, CancellationToken ct, + List pendingEmissions) + { + var runner = _runner!; + var after = new AlarmConditionState[items.Count]; + var scratches = new AlarmScratch[items.Count]; + var worst = new uint[items.Count]; + var slot = new int[items.Count]; + var invocations = new List(items.Count); + + // Pass 1, no side effects: snapshot each alarm's inputs and decide whether its predicate runs. + for (var i = 0; i < items.Count; i++) + { + var state = items[i].State; + var scratch = scratches[i] = _scratchByAlarmId.GetOrAdd( + state.Definition.AlarmId, _ => new AlarmScratch(state.Inputs, state.Logger, _clock)); + RefillReadCache(scratch.ReadCache, state.Inputs); + worst[i] = WorstInputStatus(scratch.ReadCache); + if (!AreInputsReady(scratch.ReadCache)) + { + slot[i] = -1; + continue; + } + slot[i] = invocations.Count; + // A copy: the runner may still read the inputs after the caller has its answer (a soft timeout), and the + // scratch is refilled by the next pass. + invocations.Add(new ScriptInvocation(state.Definition.AlarmId, ScriptContextKind.AlarmPredicate, + state.Definition.PredicateScriptSource, + new Dictionary(scratch.ReadCache, StringComparer.Ordinal), _scriptTimeout, state.Logger)); + } + + var outcomes = invocations.Count == 0 ? [] : runner.RunBatch(invocations, ct); + PredicateBatchSubmittedForTest?.Invoke(invocations.Count); + + // Pass 2, in order: exactly what the serial path does with each alarm. + for (var i = 0; i < items.Count; i++) + { + var (state, seed) = items[i]; + var scratch = scratches[i]; + if (slot[i] < 0) + { + if (TrackQualityBucket(state.Definition.AlarmId, worst[i])) + pendingEmissions.Add(BuildQualityEmission(state, seed, worst[i])); + after[i] = seed; + continue; + } + + var outcome = await outcomes[slot[i]].ConfigureAwait(false); + after[i] = ApplyPredicateOutcome(state, seed, scratch, worst[i], outcome, nowUtc, ct, pendingEmissions); + } + return after; + } + + /// One runner outcome onto one alarm (the second half of ). + private AlarmConditionState ApplyPredicateOutcome(AlarmState state, AlarmConditionState seed, AlarmScratch scratch, + uint worstStatus, ScriptRunOutcome outcome, DateTime nowUtc, CancellationToken ct, List pendingEmissions) + { + switch (outcome.Status) + { + case ScriptRunStatus.Unavailable: + // Not the script's fault: frozen, annotated BadResourceUnavailable, and evaluated again (against the + // inputs current then) when its worker is back. + ArmRecovery(state.Definition.AlarmId, outcome.Recovery); + return PredicateDidNotComplete(state, seed, scratch, outcome.Reason ?? ScriptRunOutcome.RunnerUnavailableReason, + pendingEmissions, StatusBadResourceUnavailable); + case ScriptRunStatus.Failed: + var reason = outcome.Reason ?? "evaluator failure"; + if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); + if (reason.StartsWith(ScriptThrewPrefix, StringComparison.Ordinal)) + { + // As in process: a predicate that threw logs an Error and keeps its state, with no quality annotation. + scratch.LastNonCompletion = null; + state.Logger.Error("Alarm predicate threw — state unchanged: {Reason}", reason[ScriptThrewPrefix.Length..]); + return seed; + } + // Timed out, killed, refused, quarantined, unsupported type, did not compile: frozen + BadInternalError. + return PredicateDidNotComplete(state, seed, scratch, reason, pendingEmissions); + } + + if (outcome.Value is not bool predicateTrue) + { + scratch.LastNonCompletion = null; + state.Logger.Error("Alarm predicate returned {Type} instead of a bool — state unchanged", outcome.Value?.GetType().Name ?? "null"); + return seed; + } + scratch.LastNonCompletion = null; + + // From here on, identical to the serial path: the bucket is tracked only once the predicate completed (I3). + var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); + var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); + var transition = result.Emission != EmissionKind.None + ? BuildEmission(state, result.State, result.Emission, worstStatus) + : null; + if (transition is not null) pendingEmissions.Add(transition); + else if (qualityBucketChanged) pendingEmissions.Add(BuildQualityEmission(state, result.State, worstStatus)); + return result.State; + } + + /// The prefix the runners put on a predicate that threw (the in-process path catches the exception instead). + private const string ScriptThrewPrefix = "script threw: "; + + /// + /// Re-evaluate once when completes — the runner's worker is + /// back (design §9 "re-evaluated on return"). At most one wait per alarm. + /// + private void ArmRecovery(string alarmId, Task? recovery) + { + if (recovery is null) return; + lock (_dirtyLock) + { + if (_disposed || !_recoveryArmed.Add(alarmId)) return; + } + _ = recovery.ContinueWith(_ => + { + lock (_dirtyLock) _recoveryArmed.Remove(alarmId); + if (!_disposed) MarkDirty([alarmId]); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + /// Test hook: the number of predicates each out-of-process batch pass submitted. + internal Action? PredicateBatchSubmittedForTest { get; set; } + /// /// The predicate did not return (timed out, refused because the previous run is still executing, or /// quarantined): hold the condition at , log the reason once per change of @@ -900,15 +1129,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// private AlarmConditionState PredicateDidNotComplete( AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason, - List pendingEmissions) + List pendingEmissions, uint status = StatusBadInternalError) { if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal)) { scratch.LastNonCompletion = reason; state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason); } - if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError)) - pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError)); + if (TrackQualityBucket(state.Definition.AlarmId, status)) + pendingEmissions.Add(BuildQualityEmission(state, seed, status)); return seed; } @@ -1339,7 +1568,7 @@ public sealed class ScriptedAlarmEngine : IDisposable private sealed record AlarmState( ScriptedAlarmDefinition Definition, - ScriptEvaluator Evaluator, + ScriptEvaluator? Evaluator, // null out of process: the worker holds the compiled form IReadOnlySet Inputs, IReadOnlyList TemplateTokens, ILogger Logger, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj index 5d1f6cd3..b784a0a9 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs index e4c1f64b..8cf22db5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs @@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot, TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null, - int compileThreads = 0) + int compileThreads = 0, int evalThreads = 0) { _compileThreads = compileThreads; ArgumentNullException.ThrowIfNull(options); @@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log); if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept); - var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards); + // Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the + // queue-behind-a-busy-thread case deterministic). + if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards); _shards = new WorkerShard[_worker.Shards]; for (var i = 0; i < _shards.Length; i++) { @@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, /// Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does. internal Action? SlotReleasingForTest { get; set; } + /// Test hook: runs once per batch frame submits, with the number of evaluations in it. + internal Action? BatchSubmittedForTest { get; set; } + /// Scoped keys currently treated as suspects (tests / diagnostics). internal IReadOnlyCollection SuspectKeys => _keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray(); @@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, // next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release // was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already // run by then. - var heldGate = gate; - var released = 0; - void ReleaseSlot() - { - if (Interlocked.Exchange(ref released, 1) != 0) return; - SlotReleasingForTest?.Invoke(key); - Interlocked.Exchange(ref state.InFlight, 0); - heldGate?.Release(); - } - var submitted = Stopwatch.GetTimestamp(); var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout, - _worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot); + _worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate)); handedOff = true; + return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return ScriptRunOutcome.Failed("evaluation cancelled"); + } + finally + { + if (!handedOff) + { + Interlocked.Exchange(ref state.InFlight, 0); + gate?.Release(); + } + } + // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace + // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the + // shard has no live worker (or for the isolation gate). + TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace; + } + + /// + /// + /// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script + /// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE + /// EvalBatch frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect + /// bound for the one-at-a-time isolation worker) takes the full path on its own, so a batch + /// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker + /// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway + /// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported + /// , design §9). + /// + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(invocations); + var outcomes = new Task[invocations.Count]; + Dictionary>? groups = null; + for (var i = 0; i < invocations.Count; i++) + { + var invocation = invocations[i]; + ArgumentNullException.ThrowIfNull(invocation); + switch (TryPrepareBatchItem(invocation, out var item, out var answered)) + { + case BatchPreparation.Submit: + groups ??= []; + if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = []; + group.Add((i, item)); + break; + case BatchPreparation.Answered: + outcomes[i] = Task.FromResult(answered!); + break; + default: + outcomes[i] = RunAsync(invocation, ct); + break; + } + } + if (groups is null) return outcomes; + + foreach (var (connection, group) in groups) + { + var submissions = new BatchSubmission[group.Count]; + for (var j = 0; j < group.Count; j++) + { + var item = group[j].Item; + submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation), + item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null)); + } + var submitted = Stopwatch.GetTimestamp(); + BatchSubmittedForTest?.Invoke(submissions.Length); + var pendings = connection.SubmitBatch(submissions); + for (var j = 0; j < group.Count; j++) + { + var (index, item) = group[j]; + outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct); + } + } + return outcomes; + } + + /// + /// The synchronous half of a batch item: everything decides before it submits, for the case + /// where nothing has to be waited for. Takes the key's single-flight slot when it returns + /// ; never when it returns anything else. + /// + private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered) + { + item = null; + answered = null; + if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full; + + var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key); + var state = Admit(key, invocation.Kind, invocation.Source); + if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full; + + var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)]; + if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full; + if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full; + + if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) + { + ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair("script", key)); + answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason); + return BatchPreparation.Answered; + } + state.ScriptLogger = invocation.ScriptLogger; + NoteLoad(entry); + item = new BatchItem(key, state, invocation, shard, connection, entry); + return BatchPreparation.Submit; + } + + /// + /// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation + /// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller + /// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation + /// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a + /// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found + /// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike, + /// quarantine, suspect) has already run by then. + /// + private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate) + { + var released = 0; + return () => + { + if (Interlocked.Exchange(ref released, 1) != 0) return; + SlotReleasingForTest?.Invoke(key); + Interlocked.Exchange(ref state.InFlight, 0); + heldGate?.Release(); + }; + } + + /// Waits for one submitted evaluation and turns its result into the caller's outcome. + private async Task AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation, + WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct) + { + try + { var result = ct.CanBeCanceled ? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false) : await pending.Outcome.ConfigureAwait(false); @@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, { return ScriptRunOutcome.Failed("evaluation cancelled"); } - finally - { - if (!handedOff) - { - Interlocked.Exchange(ref state.InFlight, 0); - gate?.Release(); - } - } - - // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace - // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the - // shard has no live worker (or for the isolation gate). - TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace; } /// @@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, => LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status())) .Select(s => new Measurement(value(s), new KeyValuePair("shard", s.Label))); + private enum BatchPreparation + { + /// Run it through the full path on its own. + Full, + + /// Answered without running (refused). + Answered, + + /// Submit it with the batch; its slot is taken. + Submit, + } + + private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard, + WorkerConnection Connection, WorkerConnection.LoadEntry Entry); + private sealed class KeyState(string source, ScriptContextKind kind) { private byte[]? _sha; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs index 009421e4..9b0d91d0 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs @@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i } } +/// One evaluation of a . +/// The scoped key. +/// The request (its id is assigned on submit). +/// T. +/// G. +/// Runs once when the worker has answered or ended, before the caller resumes (the slot release). +internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished); + /// What the host needs to launch one worker. /// The shard label (0, 1, … or iso). /// The worker entry assembly (.dll, run through the dotnet muxer) or apphost. @@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable return pending; } + /// + /// Submits several evaluations as ONE frame (lmxopcua#780 batch submit, design §4.5). Each + /// item gets its own deadlines and slot release exactly as through ; + /// an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts. + /// + /// The evaluations (their ids are assigned here). + /// One pending evaluation per item, in order. + public PendingEval[] SubmitBatch(IReadOnlyList items) + { + ArgumentNullException.ThrowIfNull(items); + var pendings = new PendingEval[items.Count]; + var batch = new EvalBatch(); + Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp()); + for (var i = 0; i < items.Count; i++) + { + var item = items[i]; + item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId); + var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher); + pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline); + _pending[item.Request.EvalId] = pending; + batch.Items.Add(item.Request); + pendings[i] = pending; + } + if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted)) + { + foreach (var pending in pendings) + if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null); + } + return pendings; + } + + /// + /// The compiled form of in this worker when it is loaded at + /// with and compiled cleanly — the batch fast path, which never waits for a load. + /// + /// The scoped key. + /// The source hash (hex). + /// The slot order. + /// The entry, or null when the key needs a (re)load or its load failed. + public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList inputPaths) + => _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths) + && existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut + ? existing + : null; + private void OnHardDeadline(PendingEval pending) { _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}", @@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable // An eval id with no outstanding request is a protocol violation (design §6 "what the host // trusts"), except for one that was already completed by a kill race — ignored then. if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result); - else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId)) - throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}"); + else CheckIssued(result.EvalId, "result"); + } + // Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines + // restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time. + // The two notices are unordered with respect to each other (see PendingEval.Start). + foreach (var id in frame.Results.StartedEvalIds) + { + if (_pending.TryGetValue(id, out var started)) started.Start(); + else CheckIssued(id, "start notice"); + } + foreach (var id in frame.Results.DeferredEvalIds) + { + if (_pending.TryGetValue(id, out var deferred)) deferred.Defer(); + else CheckIssued(id, "deferral notice"); } break; case Envelope.BodyOneofCase.LoadStarted: @@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable } } + /// An id the host never issued is a protocol violation; one it issued and already completed is a benign race. + private void CheckIssued(ulong evalId, string what) + { + if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId)) + throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}"); + } + private async Task PipeEndedAsync(WorkerExitReason? fault) { // The pipe is gone: the worker is dead or dying. Make sure, then report. @@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable /// WaitAsync(timeout) timer, the hard-deadline CancellationTokenSource and their continuations were a /// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G. /// +/// +/// The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind +/// busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full +/// T, then G, from the start — without a stale tick firing early. +/// internal sealed class PendingEval { /// What yields when the soft deadline passed first. Never on the wire. @@ -926,9 +1003,15 @@ internal sealed class PendingEval private readonly TaskCompletionSource _outcome; private readonly ResultDispatcher? _dispatcher; private readonly Action? _onFinished; + private readonly object _clock = new(); private Timer? _timer; private Action? _onHardDeadline; + private TimeSpan _soft; private TimeSpan _grace; + private long _softDue; + private long _hardDue; + private bool _deferred; + private bool _started; private int _phase; private int _done; @@ -954,6 +1037,12 @@ internal sealed class PendingEval /// T + G, for the kill log line. public TimeSpan HardDeadline { get; private set; } + /// Whether the worker reported this run as queued and has not reported it started (tests / diagnostics). + internal bool IsDeferred + { + get { lock (_clock) return _deferred; } + } + /// The worker's result, or null when the worker ended first. public Task Completion => _completion.Task; @@ -966,27 +1055,96 @@ internal sealed class PendingEval /// Starts the deadline timer. Called once, before the request is sent. internal void Arm(TimeSpan soft, TimeSpan grace, Action onHardDeadline) { + _soft = soft; _grace = grace; _onHardDeadline = onHardDeadline; HardDeadline = soft + grace; + var now = Stopwatch.GetTimestamp(); + _softDue = now + Ticks(soft); + _hardDue = _softDue + Ticks(grace); // Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace. _timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite); _timer.Change(soft, Timeout.InfiniteTimeSpan); } + /// + /// The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running, + /// so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire. + /// + internal void Defer() + { + lock (_clock) + { + if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return; + _deferred = true; + TryChange(Timeout.InfiniteTimeSpan); + } + } + + /// + /// The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline + /// had already been reported). A run that was never deferred keeps the clock it has had since submit. + /// + internal void Start() + { + lock (_clock) + { + _started = true; + if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return; + _deferred = false; + var now = Stopwatch.GetTimestamp(); + if (Volatile.Read(ref _phase) == 0) + { + _softDue = now + Ticks(_soft); + _hardDue = _softDue + Ticks(_grace); + TryChange(_soft); + } + else + { + // Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets + // the whole T + G from its start before it can be blamed for a kill. + _hardDue = now + Ticks(_soft + _grace); + TryChange(_soft + _grace); + } + } + } + private void OnTimer() { if (Volatile.Read(ref _done) != 0) return; - if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0) + bool soft = false, hard = false; + lock (_clock) { - _outcome.TrySetResult(SoftTimedOut); - try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); } - catch (ObjectDisposedException) { /* answered meanwhile */ } - return; + if (_deferred || Volatile.Read(ref _done) != 0) return; + var now = Stopwatch.GetTimestamp(); + if (Volatile.Read(ref _phase) == 0) + { + // A tick armed before a deferral/start can land early: re-arm for what is left. + if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; } + Interlocked.Exchange(ref _phase, 1); + soft = true; + TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero); + } + else if (Volatile.Read(ref _phase) == 1) + { + if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; } + Interlocked.Exchange(ref _phase, 2); + hard = true; + } } - if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this); + // Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult. + if (soft) _outcome.TrySetResult(SoftTimedOut); + if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this); } + private void TryChange(TimeSpan due) + { + try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); } + catch (ObjectDisposedException) { /* answered meanwhile */ } + } + + private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency); + /// /// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the /// reader, then hand the caller's resumption to the dispatcher (or the pool). diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto index 2afa1d78..7e918208 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto @@ -134,8 +134,17 @@ message EvalResult { uint32 run_micros = 6; } +// Per-item deadlines are exact for queued work too (lmxopcua#780 batch submit): the host arms an evaluation's soft and +// hard deadlines when it submits, which is when the run starts as long as the worker has an idle evaluation thread. An +// evaluation that arrives while every thread is busy (a batch larger than the scheduler, or work queued behind a +// runaway) is reported in `deferred_eval_ids`; the host then stops its clock, and restarts it — the full T, then G — +// when the evaluation appears in `started_eval_ids`, which an evaluation thread sends alongside the result of the run it +// just finished. So a run waiting behind a sibling is never timed out, struck or killed for the sibling's time. +// Either list may be the only content of a frame. A host that predates these fields ignores them. message EvalResults { repeated EvalResult items = 1; + repeated uint64 started_eval_ids = 2; + repeated uint64 deferred_eval_ids = 3; } message ScriptError { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs index 09e6fa16..98a90ba7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs @@ -233,7 +233,10 @@ public sealed class ScriptWorkerFrameWriter break; case Envelope.BodyOneofCase.Results: if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; } - (results ??= new EvalResults()).Items.AddRange(envelope.Results.Items); + results ??= new EvalResults(); + results.Items.AddRange(envelope.Results.Items); + results.StartedEvalIds.AddRange(envelope.Results.StartedEvalIds); + results.DeferredEvalIds.AddRange(envelope.Results.DeferredEvalIds); break; default: if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptRunner.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptRunner.cs index 048a0229..118eaba7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptRunner.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptRunner.cs @@ -28,6 +28,31 @@ public interface IScriptRunner /// The outcome; never faults for a script failure. Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default); + /// + /// Submits several evaluations at once (design §4.5): the out-of-process runner sends every one that is ready to + /// run as ONE request frame per worker, instead of one frame and one submission per evaluation. Results are + /// per item and streamed — each task completes when its own evaluation does, so a slow item never holds a fast + /// sibling's result — and every item keeps exactly the semantics of : its own budget, its + /// own single-flight slot, and its own blame (a runaway item is killed and struck alone; an item queued behind + /// it is neither timed out nor struck for its time). + /// + /// + /// The default implementation — what uses — is per + /// item, started together. A key that appears twice in one batch is refused for its second item exactly as two + /// concurrent calls would be; callers that may repeat a key split the batch there. + /// Never throws for script faults; the returned tasks never fault. + /// + /// What to run, in order. + /// Caller cancellation (shutdown); wins over every item's budget. + /// One task per invocation, in the same order. + IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(invocations); + var outcomes = new Task[invocations.Count]; + for (var i = 0; i < outcomes.Length; i++) outcomes[i] = RunAsync(invocations[i], ct); + return outcomes; + } + /// /// The deploy boundary for one seam: lifts every quarantine and forgets every remembered compile failure of /// , and drops its compiled scripts. @@ -50,7 +75,7 @@ public enum ScriptContextKind /// A Calculation tag: VirtualTagContext → object?. Calculation = 2, - /// A scripted-alarm predicate: AlarmPredicateContext → bool (not run out of process yet). + /// A scripted-alarm predicate: AlarmPredicateContext → bool. AlarmPredicate = 3, } @@ -142,3 +167,16 @@ public static class ScriptRunOutcomeExtensions }; } } + +/// +/// Which each script seam uses on this node (lmxopcua#780), for components that cannot +/// reference the Host's selection directly (the Runtime's scripted-alarm host). Null for a seam means its in-process +/// guard path. +/// +public interface IScriptRunnerProvider +{ + /// The runner must use, or null for its in-process path. + /// The seam. + /// The runner, or null. + IScriptRunner? ForSeam(ScriptSeam seam); +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptBatchCoalescer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptBatchCoalescer.cs new file mode 100644 index 00000000..f2a0359c --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptBatchCoalescer.cs @@ -0,0 +1,106 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Opportunistic batching for a seam whose evaluations are submitted independently — one virtual-tag actor each — +/// but often become ready together (one upstream burst fans out to many tags): evaluations submitted while another +/// submission is in progress are handed to together, in arrival order, by the +/// submitter already holding the flush (lmxopcua#780, design §4.5). +/// +/// +/// There is no timer: an evaluation on an idle seam is submitted at once by its own caller, so batching never +/// adds latency. Per-item semantics are 's — each item keeps its own budget, slot +/// and blame. +/// Each caller's task completes synchronously on the thread that completes its run (the runner's result +/// dispatcher), exactly as awaiting directly does, so there is no extra thread +/// hop. +/// A key submitted twice concurrently is refused for the second exactly as two concurrent +/// calls would be; the virtual-tag actor never has two evaluations in flight. +/// +public sealed class ScriptBatchCoalescer +{ + private readonly IScriptRunner _runner; + private readonly object _lock = new(); + private List<(ScriptInvocation Invocation, TaskCompletionSource Done)> _queue = []; + private bool _flushing; + + /// Initializes the coalescer. + /// The runner every batch goes to. + public ScriptBatchCoalescer(IScriptRunner runner) + { + _runner = runner ?? throw new ArgumentNullException(nameof(runner)); + } + + /// Test hook: runs once per call, with the batch size. + internal Action? BatchSubmittedForTest { get; set; } + + /// Submits one evaluation, together with any submitted concurrently. + /// What to run. + /// The outcome; never faults. + public Task RunAsync(ScriptInvocation invocation) + { + ArgumentNullException.ThrowIfNull(invocation); + // Synchronous continuations: the caller resumes where its run completed, as with a direct await of RunAsync. + var done = new TaskCompletionSource(TaskCreationOptions.None); + lock (_lock) + { + _queue.Add((invocation, done)); + if (_flushing) return done.Task; + _flushing = true; + } + Flush(); + return done.Task; + } + + private void Flush() + { + while (true) + { + List<(ScriptInvocation Invocation, TaskCompletionSource Done)> batch; + lock (_lock) + { + if (_queue.Count == 0) + { + _flushing = false; + return; + } + batch = _queue; + _queue = []; + } + + IReadOnlyList> outcomes; + try + { + var invocations = new ScriptInvocation[batch.Count]; + for (var i = 0; i < invocations.Length; i++) invocations[i] = batch[i].Invocation; + BatchSubmittedForTest?.Invoke(invocations.Length); + outcomes = _runner.RunBatch(invocations); + } + catch (Exception ex) + { + // A defect in the runner must not strand the callers (nor the flush flag). + foreach (var (_, done) in batch) done.TrySetResult(ScriptRunOutcome.Failed($"script runner threw: {ex.Message}")); + continue; + } + + for (var i = 0; i < batch.Count; i++) Link(outcomes[i], batch[i].Done); + } + } + + private static void Link(Task outcome, TaskCompletionSource done) + { + if (outcome.IsCompleted) + { + Settle(outcome, done); + return; + } + outcome.ContinueWith(static (t, s) => Settle(t, (TaskCompletionSource)s!), done, + CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + private static void Settle(Task outcome, TaskCompletionSource done) + { + if (outcome.IsCompletedSuccessfully) done.TrySetResult(outcome.Result); + else done.TrySetResult(ScriptRunOutcome.Failed( + $"script runner threw: {outcome.Exception?.GetBaseException().Message ?? "cancelled"}")); + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunnerOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunnerOptions.cs index c8e7e431..98a33c4a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunnerOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunnerOptions.cs @@ -32,7 +32,10 @@ public enum ScriptSeam /// Calculation tags (CalculationEvaluator). Calculation = 1, - /// Scripted-alarm predicates (ScriptedAlarmEngine). Last in the rollout (design §14 #10). + /// + /// Scripted-alarm predicates (ScriptedAlarmEngine). Last in the rollout (design §14 #10): out of process only + /// when Scripting:Runner:Seams lists it — an unset list does not select it. + /// ScriptedAlarm = 2, } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs index bae47f1f..cb657b0b 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -80,7 +80,8 @@ public sealed class ScriptingOptions /// /// The seams applies to when it is not - /// (Scripting:Runner:Seams, e.g. ["Calculation"]). Null or empty means all three. Bound by + /// (Scripting:Runner:Seams, e.g. ["Calculation"]). Null or empty means VirtualTag and Calculation: + /// scripted alarms run out of process only when listed explicitly (design §14 #10, alarms last). Bound by /// ScriptingOptionsBinding from the documented key, because Scripting:Runner is itself a value. /// public IReadOnlyList? RunnerSeams { get; set; } @@ -91,10 +92,10 @@ public sealed class ScriptingOptions /// Whether is selected for out-of-process execution by these options. /// The seam. /// true when is and the seam is in - /// (or that list is empty). + /// (or that list is empty and the seam is not ). public bool IsOutOfProcess(ScriptSeam seam) => Runner == ScriptRunnerMode.OutOfProcess - && (RunnerSeams is not { Count: > 0 } seams || seams.Contains(seam)); + && (RunnerSeams is { Count: > 0 } seams ? seams.Contains(seam) : seam != ScriptSeam.ScriptedAlarm); /// Upper bound applies to . public const double MaxStuckOrphanQuarantineFactor = 1000; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj index 05efb3c4..012857c6 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj @@ -26,6 +26,8 @@ + + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index 76156fc8..dbb234d7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -522,6 +522,14 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// false timeouts under pool starvation). private async Task RunEvaluationPumpAsync() { + // lmxopcua#780 (design §4.5): with an out-of-process runner a run costs a pipe round trip, so the pump submits + // everything queued as one batch and publishes in dequeue order. In process it stays one at a time, unchanged. + if (_evaluator.SubmitsBatches) + { + await RunEvaluationBatchPumpAsync().ConfigureAwait(false); + return; + } + while (true) { PendingEvaluation next; @@ -557,6 +565,73 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba } } + /// + /// The pump on the out-of-process path (lmxopcua#780, design §4.5): dequeue everything pending — each entry's + /// dependency snapshot was captured at enqueue — submit it as ONE batch, and publish in dequeue order. A calc tag + /// queued more than once (a burst of changes) ends the batch at its second entry, so every queued evaluation still + /// runs, in order, against the snapshot that triggered it, exactly as the one-at-a-time pump would run them. + /// + private async Task RunEvaluationBatchPumpAsync() + { + while (true) + { + List batch; + lock (_lock) + { + if (!_running) _pending.Clear(); + if (_pending.Count == 0) + { + _pumpScheduled = false; + return; + } + batch = []; + var inBatch = new HashSet(StringComparer.Ordinal); + while (_pending.Count > 0 && inBatch.Add(_pending.Peek().Definition.RawPath)) batch.Add(_pending.Dequeue()); + } + + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed + try + { + await RunEvaluationBatchAsync(batch).ConfigureAwait(false); + } + catch (Exception ex) + { + // Defect path only (the evaluator never throws for script faults); must not strand the queue. + _logger.LogError(ex, "Calculation {Driver}: batch evaluation of {Count} tag(s) failed", _driverInstanceId, batch.Count); + } + finally + { + _evalGate.Release(); + } + } + } + + /// Runs one batch and publishes every outcome in dequeue order. Under , not . + /// The prepared evaluations; no calc tag twice. + private async Task RunEvaluationBatchAsync(IReadOnlyList batch) + { + // Hold-on-Bad entries never reach the runner; the rest are submitted together. + var requests = new List<(string, string, IReadOnlyDictionary)>(batch.Count); + var slot = new int[batch.Count]; + for (var i = 0; i < batch.Count; i++) + { + if (!InputQuality.IsUsable(batch[i].WorstInputStatus)) { slot[i] = -1; continue; } + slot[i] = requests.Count; + requests.Add((batch[i].Definition.RawPath, batch[i].Definition.ScriptSource, batch[i].Dependencies)); + } + var results = _evaluator.EvaluateBatch(requests, _evalCts.Token); + BatchSubmittedForTest?.Invoke(requests.Count); + + for (var i = 0; i < batch.Count; i++) + { + if (slot[i] < 0) HoldOnBadInput(batch[i]); + else ApplyEvaluationResult(batch[i], await results[slot[i]].ConfigureAwait(false)); + } + } + + /// Test hook: the number of compiled evaluations each batch pass submitted. + internal Action? BatchSubmittedForTest { get; set; } + /// Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task /// completes). @@ -610,16 +685,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // sub-code. Uncertain IS usable: the script runs and the output inherits the Uncertain code. if (!InputQuality.IsUsable(pending.WorstInputStatus)) { - (object? Value, uint StatusCode)? badPublish; - lock (_lock) - { - if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return; - badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc); - } - - // No script log: an input dying is not a script fault, and the script log is the per-failure - // SCRIPT diagnostic. Here the status code is the diagnostic. - if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc); + HoldOnBadInput(pending); return; } @@ -627,7 +693,32 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // then the input's real ones rather than a fabricated Good stamped at evaluation time. var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) .ConfigureAwait(false); + ApplyEvaluationResult(pending, result); + } + /// A Bad input holds the script: publish the input's own status (once per transition), no script log. + /// The prepared evaluation whose worst input is Bad. + private void HoldOnBadInput(PendingEvaluation pending) + { + var def = pending.Definition; + (object? Value, uint StatusCode)? badPublish; + lock (_lock) + { + if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return; + badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc); + } + + // No script log: an input dying is not a script fault, and the script log is the per-failure + // SCRIPT diagnostic. Here the status code is the diagnostic. + if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc); + } + + /// Record one evaluation's outcome and publish it. Under , not . + /// The prepared evaluation. + /// Its outcome. + private void ApplyEvaluationResult(PendingEvaluation pending, VirtualTagEvalResult result) + { + var def = pending.Definition; string? failureToLog = null; (object? Value, uint StatusCode)? toPublish; lock (_lock) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 6af66db6..01a1fd05 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -283,6 +283,62 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO } } + /// + /// Whether compiled runs go to an (the out-of-process seam), so the driver's pump can + /// submit everything it has dequeued as one batch (lmxopcua#780, design §4.5). False on the in-process path, whose + /// pump stays one evaluation at a time, unchanged. + /// + public bool SubmitsBatches => _runner is not null && !_disposed; + + /// + /// Evaluates several calc tags at once (design §4.5): the passthrough fast path and the trivial failures are answered + /// inline exactly as answers them, and every compiled run is handed to the runner in ONE + /// . Results are per item; each item keeps 's + /// outcome, budget and blame. Without a runner it is per item, in order. + /// + /// The evaluations: calc-tag id, script source, dependency snapshots. A calc tag must not appear twice. + /// Caller cancellation (driver shutdown). + /// One task per request, in order; none faults. + public IReadOnlyList> EvaluateBatch( + IReadOnlyList<(string CalcTagId, string ScriptSource, IReadOnlyDictionary Dependencies)> requests, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(requests); + var results = new Task[requests.Count]; + if (_runner is null || _disposed) + { + for (var i = 0; i < requests.Count; i++) + results[i] = EvaluateAsync(requests[i].CalcTagId, requests[i].ScriptSource, requests[i].Dependencies, ct); + return results; + } + + List? invocations = null; + List? indexes = null; + for (var i = 0; i < requests.Count; i++) + { + var (id, source, deps) = requests[i]; + // Exactly the fast answers EvaluateAsync gives before it reaches the runner. + if (string.IsNullOrWhiteSpace(source) || PassthroughScript.TryMatch(source, out _)) + { + results[i] = EvaluateAsync(id, source, deps, ct); + continue; + } + var scriptLog = _scriptRoot + .ForContext(ScriptLoggerFactory.ScriptIdProperty, id) + .ForContext(ScriptLoggerFactory.VirtualTagIdProperty, id); + (invocations ??= []).Add(new ScriptInvocation(id, ScriptContextKind.Calculation, source, deps, _runTimeout, scriptLog)); + (indexes ??= []).Add(i); + } + if (invocations is null) return results; + + var outcomes = _runner.RunBatch(invocations, ct); + for (var j = 0; j < outcomes.Count; j++) results[indexes![j]] = ToResultAsync(outcomes[j]); + return results; + + static async Task ToResultAsync(Task outcome) + => (await outcome.ConfigureAwait(false)).ToVirtualTagEvalResult(); + } + /// Record a compile failure for and log it ONCE — the next trigger /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning. private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs index 2039c255..ebac1ffd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs @@ -12,10 +12,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the /// workers. /// -public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable +public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable { - /// Seams the out-of-process runner can serve in this build. Scripted alarms go last (design §14 #10). - public static readonly IReadOnlyList SupportedOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation]; + /// Seams the out-of-process runner can serve in this build. + public static readonly IReadOnlyList SupportedOutOfProcessSeams = + [ScriptSeam.VirtualTag, ScriptSeam.Calculation, ScriptSeam.ScriptedAlarm]; + + /// + /// The seams an unset or empty Scripting:Runner:Seams selects. Scripted alarms are not among them: they go + /// out of process only when listed explicitly, last in the rollout (design §14 #10 — the alarm budget and the + /// engine refactor). + /// + public static readonly IReadOnlyList DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation]; private readonly OutOfProcessScriptRunner? _runner; private readonly HashSet _seams; @@ -94,5 +102,5 @@ public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable public void Dispose() => _runner?.Dispose(); private static IEnumerable RequestedSeams(ScriptingOptions options) - => options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : Enum.GetValues(); + => options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index cc2c5095..0c130b2b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -54,6 +54,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac private readonly TimeSpan _runTimeout; private readonly ScriptRunGuard _guard; private readonly IScriptRunner? _runner; + private readonly ScriptBatchCoalescer? _batches; private bool _disposed; /// Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout. @@ -79,6 +80,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac _runTimeout = runTimeout ?? (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout; _guard = new ScriptRunGuard(scripting, _scriptRoot); _runner = scriptRunner; + _batches = scriptRunner is null ? null : new ScriptBatchCoalescer(scriptRunner); } /// The out-of-process runner this evaluator delegates compiled runs to, or null (in process). @@ -87,6 +89,9 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac /// The run guard every compiled evaluation goes through (diagnostics / tests). public ScriptRunGuard RunGuard => _guard; + /// The coalescer the out-of-process path submits through, or null in process (tests). + internal ScriptBatchCoalescer? Batches => _batches; + // Value-only overload. Every dependency reaches the script stamped Good with an // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot // overload instead; this one remains for callers that genuinely hold only @@ -161,9 +166,11 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac // lmxopcua#780 — OutOfProcess: the compiled run goes to the pre-emptible worker pool, which compiles in the // worker and owns timeouts, kills and quarantine for this seam. An Unavailable outcome (worker restarted, no // worker ready) publishes BadResourceUnavailable; everything else keeps today's Bad contract. - if (_runner is not null) + // Every tag has its own actor, so tags an upstream burst made ready together arrive here concurrently; the + // coalescer submits those as one batch (design §4.5) and an evaluation on an idle seam at once. + if (_batches is not null) { - var outcome = await _runner.RunAsync(new ScriptInvocation( + var outcome = await _batches.RunAsync(new ScriptInvocation( virtualTagId, ScriptContextKind.VirtualTag, expression, readCache, _runTimeout, scriptLog)).ConfigureAwait(false); return outcome.ToVirtualTagEvalResult(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 4277552e..ae81785d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -385,6 +385,9 @@ if (hasDriver) Log.Information("Scripting runner: {RunnerMode}", scriptingOptions.Runner); builder.Services.AddSingleton(sp => ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection.Create( sp.GetRequiredService(), Log.Logger, sp.GetService()?.Logger)); + // The Runtime's scripted-alarm host resolves its runner through this (it cannot reference the Host). + builder.Services.AddSingleton(sp => + sp.GetRequiredService()); // lmxopcua#777: the ONE operator-write timeout (DriverWrites:Timeout) every write bound is derived from — the // driver child's cancel + abandon, its queue budget, DriverHostActor's Ask and the inbound-write gateway's Ask. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 781b0348..6879a972 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -206,6 +206,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// Node-wide script run-guard options for the scripted-alarm engine; null ⇒ compiled defaults. private readonly ScriptingOptions? _scripting; + // lmxopcua#780 — the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm; null keeps the engine's + // in-process guard path. + private readonly IScriptRunner? _scriptedAlarmRunner; + /// /// Driver children this host has stopped (, an operator Restart) and is watching /// until they terminate. A value burst from one of them is DROPPED in : @@ -771,7 +775,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS ScriptingOptions? scripting = null, NativeAlarmAckPreflightView? ackPreflightView = null, DriverWriteBounds? writeBounds = null, - TimeProvider? writeClock = null) => + TimeProvider? writeClock = null, + IScriptRunner? scriptedAlarmRunner = null) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -785,7 +790,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView, sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay, sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView, - scripting, ackPreflightView, writeBounds, writeClock)); + scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -916,8 +921,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS ScriptingOptions? scripting = null, NativeAlarmAckPreflightView? ackPreflightView = null, DriverWriteBounds? writeBounds = null, - TimeProvider? writeClock = null) + TimeProvider? writeClock = null, + IScriptRunner? scriptedAlarmRunner = null) { + _scriptedAlarmRunner = scriptedAlarmRunner; _writeBounds = writeBounds ?? DriverWriteBounds.Default; _writeClock = writeClock ?? TimeProvider.System; _ackPreflightView = ackPreflightView; @@ -1242,13 +1249,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS var store = _alarmStateStore; var scriptLogger = _scriptRootLogger.Logger; var scripting = _scripting; + var alarmRunner = _scriptedAlarmRunner; _scriptedAlarmHost = Context.ActorOf( ScriptedAlarmHostActor.Props( _opcUaPublishActor, _dependencyMux, () => { var upstream = new DependencyMuxTagUpstreamSource(); - return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting)); + return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting, alarmRunner)); }, _localNode, telemetryHub: _telemetryHub), "scripted-alarm-host"); @@ -1265,15 +1273,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// The condition-state store. /// The root script logger. /// The node-wide Scripting options; null ⇒ defaults. + /// The out-of-process runner for the ScriptedAlarm seam; null ⇒ in process. /// A new, unloaded engine. internal static ScriptedAlarmEngine CreateScriptedAlarmEngine( ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.ITagUpstreamSource upstream, IAlarmStateStore store, - Serilog.ILogger scriptLogger, ScriptingOptions? scripting) + Serilog.ILogger scriptLogger, ScriptingOptions? scripting, IScriptRunner? scriptRunner = null) { var budget = (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout; return new ScriptedAlarmEngine( upstream, store, new ScriptLoggerFactory(scriptLogger), scriptLogger, - scriptTimeout: budget, scripting: scripting); + scriptTimeout: budget, scripting: scripting, scriptRunner: scriptRunner); } private void Bootstrap() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 2f3239b8..5d287f2d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -664,7 +664,11 @@ public static class ServiceCollectionExtensions // The one configured write timeout (DriverWrites:Timeout, bound + validated in Program.cs) the // children's write bounds AND this host's write Ask are derived from (lmxopcua#777). Unbound in // test graphs ⇒ null ⇒ DriverWriteBounds.Default. - writeBounds: resolver.GetService()), + writeBounds: resolver.GetService(), + // lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under + // the InProcess default, when the seam is not listed, and in test graphs that register no provider. + scriptedAlarmRunner: resolver.GetService() + ?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)), DriverHostActorName); registry.Register(driverHost); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs index fe239452..b7e0751d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs @@ -34,8 +34,22 @@ internal sealed class WorkerHost private readonly int _parentPid; private readonly ConcurrentDictionary _scripts = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _running = new(); + /// + /// Hand-off to an IDLE evaluation thread: an evaluation is put here only when a thread is known to be waiting + /// for it (), so whatever is put here starts at once. Everything else waits in + /// and is reported to the host as deferred (lmxopcua#780 batch submit: per-item deadlines + /// stay exact, see the proto's EvalResults). + /// private readonly BlockingCollection _queue = new(new ConcurrentQueue()); + /// Evaluations that arrived while every evaluation thread was busy, in arrival order. Under . + private readonly Queue _backlog = new(); + + private readonly object _schedLock = new(); + + /// Evaluation threads committed to taking from , minus hand-offs not yet taken. Under . + private int _idle; + /// /// Loads waiting for a compile thread. Compiles are long, synchronous and CPU-bound, so they run on /// dedicated threads, never on the thread pool: a burst of cold loads (a node start @@ -225,8 +239,20 @@ internal sealed class WorkerHost if (_scripts.TryRemove(key, out var gone)) gone.Dispose(); return null; case Envelope.BodyOneofCase.Eval: - foreach (var item in frame.Eval.Items) Admit(item); + { + List? deferred = null; + foreach (var item in frame.Eval.Items) + if (!Admit(item)) (deferred ??= []).Add(item.EvalId); + // One notice per incoming frame. It can reach the host after the deferred run's own start notice (an + // evaluation thread may pick the run up before this is queued); the host treats the two as unordered. + if (deferred is not null) + { + var notice = new EvalResults(); + notice.DeferredEvalIds.AddRange(deferred); + Send(new Envelope { Results = notice }); + } return null; + } case Envelope.BodyOneofCase.Cancel: if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel)) { @@ -288,38 +314,72 @@ internal sealed class WorkerHost Send(new Envelope { LoadResult = result }); } - private void Admit(EvalRequest request) + /// + /// Schedules one evaluation: straight to an idle evaluation thread when there is one (it starts now, so the + /// host's clock, started at submit, is right), otherwise into the backlog. + /// + /// The evaluation. + /// False when it was deferred to the backlog (the host must stop its clock until it starts). + private bool Admit(EvalRequest request) { - if (_queue.Count >= _queueCapacity) + lock (_schedLock) { - SendResult(new EvalResult + if (_backlog.Count + _queue.Count < _queueCapacity) { - EvalId = request.EvalId, - Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" }, - }); - return; + if (_idle > 0 && _backlog.Count == 0) + { + _idle--; + _queue.Add(request); + return true; + } + _backlog.Enqueue(request); + return false; + } } - _queue.Add(request); + // Full: answered at once, outside the lock (a send may write to the pipe on this thread). + SendResult(new EvalResult + { + EvalId = request.EvalId, + Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" }, + }); + return true; } private void EvalThreadLoop() { - foreach (var request in _queue.GetConsumingEnumerable()) + EvalResult? finished = null; + while (true) { - EvalResult result; + // A thread that just finished a run takes the oldest deferred evaluation, and its start notice rides on + // the finished run's result frame. With nothing deferred it goes idle and waits for a hand-off. + EvalRequest? request = null; + lock (_schedLock) + { + if (finished is not null && _backlog.TryDequeue(out var next)) request = next; + else _idle++; + } + if (finished is not null) + { + var frame = new EvalResults(); + frame.Items.Add(finished); + if (request is not null) frame.StartedEvalIds.Add(request.EvalId); + Send(new Envelope { Results = frame }); + finished = null; + } + request ??= _queue.Take(); + try { - result = Evaluate(request); + finished = Evaluate(request); } catch (Exception ex) { - result = new EvalResult + finished = new EvalResult { EvalId = request.EvalId, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = ex.Message }, }; } - SendResult(result); } } @@ -431,7 +491,7 @@ internal sealed class WorkerHost Heartbeat = new Heartbeat { Inflight = (uint)_running.Count, - Queued = (uint)_queue.Count, + Queued = (uint)QueuedCount(), HeapBytes = GC.GetTotalMemory(false), Gen2Count = (uint)GC.CollectionCount(2), }, @@ -489,6 +549,11 @@ internal sealed class WorkerHost Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed); } + private int QueuedCount() + { + lock (_schedLock) return _backlog.Count + _queue.Count; + } + private void SendResult(EvalResult result) { var results = new EvalResults(); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineScriptRunnerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineScriptRunnerTests.cs new file mode 100644 index 00000000..fe216d29 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineScriptRunnerTests.cs @@ -0,0 +1,192 @@ +using System.Collections.Concurrent; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// lmxopcua#780 — the scripted-alarm seam on an (design §4.5, §9). With a runner the engine +/// submits a pass's whole dirty set as ONE batch and applies the results in the serial order, so the Part 9 sequence +/// is the in-process one; and a predicate that did not complete freezes its condition with an annotation that says +/// whose fault it was: BadInternalError for the script (timeout, kill, quarantine), BadResourceUnavailable +/// for the runner (worker restarted, none ready) — re-evaluated by itself when the runner is back. A dead worker never +/// clears an alarm. +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineScriptRunnerTests +{ + private const uint StatusBadInternalError = 0x80020000u; + private const uint StatusBadResourceUnavailable = 0x80040000u; + + /// Answers each predicate through and records every batch. + private sealed class FakeRunner : IScriptRunner + { + public Func Answer { get; set; } = _ => ScriptRunOutcome.Ok(false); + + public ConcurrentQueue> Batches { get; } = new(); + + public List Resets { get; } = []; + + public int SingleRuns; + + public Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default) + { + Interlocked.Increment(ref SingleRuns); + return Task.FromResult(Answer(invocation)); + } + + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + invocations.ShouldAllBe(i => i.Kind == ScriptContextKind.AlarmPredicate); + Batches.Enqueue(invocations.Select(i => i.Key).ToArray()); + return invocations.Select(i => Task.FromResult(Answer(i))).ToArray(); + } + + public void Reset(ScriptContextKind kind) => Resets.Add(kind); + + public IReadOnlyCollection QuarantinedKeys(ScriptContextKind kind) => []; + } + + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, IScriptRunner? runner) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: TimeSpan.FromMilliseconds(250), scriptRunner: runner); + } + + /// The fake's stand-in for the five predicates: Temp > threshold, threshold from the alarm id. + private static ScriptRunOutcome Threshold(ScriptInvocation invocation) + => ScriptRunOutcome.Ok((int)invocation.Inputs["Temp"].Value! > int.Parse(invocation.Key[1..])); + + private static readonly ScriptedAlarmDefinition[] Five = + [.. new[] { 10, 20, 30, 40, 50 }.Select(t => Alarm($"a{t}", $"return (int)ctx.GetTag(\"Temp\").Value > {t};"))]; + + private static async Task WaitFor(Func probe, string what) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + if (probe()) return; + await Task.Delay(10); + } + throw new TimeoutException(what); + } + + [Fact] + public async Task A_dirty_set_is_one_batch_and_the_transitions_come_out_exactly_as_in_process() + { + var ct = TestContext.Current.CancellationToken; + var runner = new FakeRunner { Answer = Threshold }; + var outOfProcess = new FakeUpstream(); + var inProcess = new FakeUpstream(); + outOfProcess.Set("Temp", 0); + inProcess.Set("Temp", 0); + using var oop = Build(outOfProcess, runner); + using var ip = Build(inProcess, null); + var oopEvents = new ConcurrentQueue<(string, EmissionKind)>(); + var ipEvents = new ConcurrentQueue<(string, EmissionKind)>(); + oop.OnEvent += (_, e) => oopEvents.Enqueue((e.AlarmId, e.Emission)); + ip.OnEvent += (_, e) => ipEvents.Enqueue((e.AlarmId, e.Emission)); + + await oop.LoadAsync(Five, ct); + await ip.LoadAsync(Five, ct); + runner.Batches.Count.ShouldBe(1, "startup recovery evaluates every alarm in one batch"); + runner.Resets.ShouldBe([ScriptContextKind.AlarmPredicate], "a load is the seam's deploy boundary"); + + foreach (var temp in new[] { 35, 60, 15, 0 }) + { + var before = runner.Batches.Count; + outOfProcess.Push("Temp", temp); + inProcess.Push("Temp", temp); + await WaitFor(() => runner.Batches.Count > before && oop.GetAllStates().Count == 5, "the out-of-process pass"); + await WaitFor(() => ipEvents.Count == oopEvents.Count, "the in-process pass to catch up"); + } + + runner.Batches.Skip(1).ShouldAllBe(b => b.Count == 5, "each push re-evaluates the whole dirty set in one batch"); + runner.SingleRuns.ShouldBe(0); + oopEvents.ToArray().ShouldBe(ipEvents.ToArray(), "the same Part 9 sequence, in the same order"); + oopEvents.Count(e => e.Item2 == EmissionKind.Activated).ShouldBe(5); + oop.GetAllStates().Select(s => (s.AlarmId, s.Active)).OrderBy(x => x.AlarmId) + .ShouldBe(ip.GetAllStates().Select(s => (s.AlarmId, s.Active)).OrderBy(x => x.AlarmId)); + } + + [Fact] + public async Task A_predicate_that_did_not_complete_is_frozen_and_annotated_by_whose_fault_it_was() + { + var ct = TestContext.Current.CancellationToken; + var runner = new FakeRunner { Answer = _ => ScriptRunOutcome.Ok(true) }; + var up = new FakeUpstream(); + up.Set("Temp", 1); + using var eng = Build(up, runner); + var events = new ConcurrentQueue(); + eng.OnEvent += (_, e) => events.Enqueue(e); + await eng.LoadAsync([Alarm("a", "return (int)ctx.GetTag(\"Temp\").Value > 0;")], ct); + eng.GetState("a")!.Active.ShouldBe(AlarmActiveState.Active); + events.Clear(); + + // The runner, not the script: frozen ACTIVE (a dead worker never clears an alarm), BadResourceUnavailable. + var recovered = new TaskCompletionSource(); + runner.Answer = _ => ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerRestartedReason) with { Recovery = recovered.Task }; + up.Push("Temp", 2); + await WaitFor(() => !events.IsEmpty, "the runner-failure annotation"); + var annotation = events.Single(); + annotation.Emission.ShouldBe(EmissionKind.QualityChanged); + annotation.WorstInputStatusCode.ShouldBe(StatusBadResourceUnavailable); + eng.GetState("a")!.Active.ShouldBe(AlarmActiveState.Active, "not cleared because the worker died"); + + // The worker is back: re-evaluated by itself, against the current inputs, with no new push. + events.Clear(); + runner.Answer = _ => ScriptRunOutcome.Ok(false); + var batches = runner.Batches.Count; + recovered.SetResult(); + await WaitFor(() => runner.Batches.Count > batches && eng.GetState("a")!.Active == AlarmActiveState.Inactive, "the recovery re-evaluation"); + await WaitFor(() => events.Any(e => e.Emission == EmissionKind.Cleared), "the clear"); + + // The completed run restored the quality on the transition itself (no separate annotation). + events.Single().WorstInputStatusCode.ShouldBe(0u); + + // The script: a timeout (or kill, or quarantine) freezes it with BadInternalError, exactly as in process. + events.Clear(); + runner.Answer = _ => ScriptRunOutcome.Failed("script timed out after 0.3s"); + up.Push("Temp", 3); + await WaitFor(() => !events.IsEmpty, "the script-failure annotation"); + events.Single().WorstInputStatusCode.ShouldBe(StatusBadInternalError); + eng.GetState("a")!.Active.ShouldBe(AlarmActiveState.Inactive); + + // A predicate that threw keeps its state with no annotation (as in process: its run completed). + events.Clear(); + runner.Answer = _ => ScriptRunOutcome.Ok(true); + up.Push("Temp", 4); + await WaitFor(() => eng.GetState("a")!.Active == AlarmActiveState.Active, "a completed run"); + events.Clear(); + runner.Answer = _ => ScriptRunOutcome.Failed("script threw: boom"); + var before = runner.Batches.Count; + up.Push("Temp", 5); + await WaitFor(() => runner.Batches.Count > before, "the throwing pass"); + await Task.Delay(100, ct); + events.ShouldBeEmpty(); + eng.GetState("a")!.Active.ShouldBe(AlarmActiveState.Active); + } + + [Fact] + public async Task Alarms_whose_inputs_are_not_ready_never_reach_the_runner() + { + var ct = TestContext.Current.CancellationToken; + var runner = new FakeRunner { Answer = _ => ScriptRunOutcome.Ok(true) }; + var up = new FakeUpstream(); + up.Set("A", 1); + up.Set("B", 0, 0x80000000u); // Bad + using var eng = Build(up, runner); + await eng.LoadAsync([Alarm("good", "return (int)ctx.GetTag(\"A\").Value > 0;"), Alarm("bad", "return (int)ctx.GetTag(\"B\").Value > 0;")], ct); + runner.Batches.Single().ShouldBe(["good"], "a Bad input holds the predicate, as in process"); + eng.GetState("bad")!.Active.ShouldBe(AlarmActiveState.Inactive); + eng.GetState("good")!.Active.ShouldBe(AlarmActiveState.Active); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptBatchCoalescerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptBatchCoalescerTests.cs new file mode 100644 index 00000000..27490a9d --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptBatchCoalescerTests.cs @@ -0,0 +1,103 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// lmxopcua#780 (design §4.5) — the virtual-tag seam's opportunistic batching: an evaluation on an idle seam is +/// submitted at once by its own caller (no timer, no added latency), and evaluations submitted while a submission +/// is in progress go to the runner together, in arrival order, each caller getting its own outcome. +/// +[Trait("Category", "Unit")] +public sealed class ScriptBatchCoalescerTests +{ + private static readonly IReadOnlyDictionary NoInputs = new Dictionary(); + + private static ScriptInvocation Vt(string key) => new(key, ScriptContextKind.VirtualTag, "return 1;", NoInputs, TimeSpan.FromSeconds(1)); + + /// Answers each key with itself; the first blocks until released. + private sealed class BlockingRunner : IScriptRunner + { + private int _calls; + + public ManualResetEventSlim Entered { get; } = new(); + + public ManualResetEventSlim Release { get; } = new(); + + public ConcurrentQueue Batches { get; } = new(); + + public bool BlockFirst { get; init; } = true; + + public Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default) + => throw new InvalidOperationException("the coalescer must submit through RunBatch"); + + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + Batches.Enqueue(invocations.Select(i => i.Key).ToArray()); + if (BlockFirst && Interlocked.Increment(ref _calls) == 1) + { + Entered.Set(); + Release.Wait(TimeSpan.FromSeconds(10)).ShouldBeTrue(); + } + return invocations.Select(i => Task.FromResult(ScriptRunOutcome.Ok(i.Key))).ToArray(); + } + + public void Reset(ScriptContextKind kind) { } + + public IReadOnlyCollection QuarantinedKeys(ScriptContextKind kind) => []; + } + + [Fact] + public async Task An_evaluation_on_an_idle_seam_is_submitted_at_once_by_its_own_caller() + { + var runner = new BlockingRunner { BlockFirst = false }; + var coalescer = new ScriptBatchCoalescer(runner); + var outcome = coalescer.RunAsync(Vt("only")); + outcome.IsCompletedSuccessfully.ShouldBeTrue("submitted synchronously, no timer"); + (await outcome).Value.ShouldBe("only"); + runner.Batches.ToArray().ShouldBe([["only"]]); + } + + [Fact] + public async Task Evaluations_submitted_while_a_submission_is_in_progress_go_together_in_arrival_order() + { + var runner = new BlockingRunner(); + var coalescer = new ScriptBatchCoalescer(runner); + var first = Task.Run(() => coalescer.RunAsync(Vt("first")), TestContext.Current.CancellationToken); + runner.Entered.Wait(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken).ShouldBeTrue(); + + var queued = Enumerable.Range(0, 10).Select(i => coalescer.RunAsync(Vt($"k{i}"))).ToArray(); + queued.ShouldAllBe(t => !t.IsCompleted, "queued behind the submission in progress"); + runner.Release.Set(); + + (await first).Value.ShouldBe("first"); + var outcomes = await Task.WhenAll(queued); + outcomes.Select(o => o.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)$"k{i}")); + runner.Batches.ToArray().ShouldBe([["first"], [.. Enumerable.Range(0, 10).Select(i => $"k{i}")]]); + } + + [Fact] + public async Task A_runner_that_throws_does_not_strand_its_callers() + { + var coalescer = new ScriptBatchCoalescer(new ThrowingRunner()); + var outcome = await coalescer.RunAsync(Vt("x")); + outcome.Status.ShouldBe(ScriptRunStatus.Failed); + outcome.Reason!.ShouldContain("boom"); + (await coalescer.RunAsync(Vt("y"))).Status.ShouldBe(ScriptRunStatus.Failed, "the flush flag was released"); + } + + private sealed class ThrowingRunner : IScriptRunner + { + public Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default) => throw new NotSupportedException(); + + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + => throw new InvalidOperationException("boom"); + + public void Reset(ScriptContextKind kind) { } + + public IReadOnlyCollection QuarantinedKeys(ScriptContextKind kind) => []; + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverBatchPumpTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverBatchPumpTests.cs new file mode 100644 index 00000000..2cc5578f --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverBatchPumpTests.cs @@ -0,0 +1,110 @@ +using System.Collections.Concurrent; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// lmxopcua#780 batch submit (design §4.5) in the Calculation pump: with an out-of-process runner the pump hands +/// everything it dequeued to ONE and publishes in dequeue order. A tag queued +/// twice ends the batch, so every queued evaluation still runs, in order, against the snapshot that triggered it — +/// the published sequence is exactly the one-at-a-time pump's. +/// +public sealed class CalculationDriverBatchPumpTests +{ + private const uint StatusGood = 0u; + + /// + /// Answers src/a × n for tag calc/tn. The first batch is held until , so the + /// test can queue more triggers behind it. + /// + private sealed class GatedRunner : IScriptRunner + { + private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _batches; + + public ConcurrentQueue Batches { get; } = new(); + + public int SingleRuns; + + public void Release() => _gate.SetResult(); + + public Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default) + { + Interlocked.Increment(ref SingleRuns); + return Task.FromResult(Answer(invocation)); + } + + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + Batches.Enqueue(invocations.Select(i => i.Key).ToArray()); + var held = Interlocked.Increment(ref _batches) == 1; + return invocations.Select(i => held ? Later(i) : Task.FromResult(Answer(i))).ToArray(); + } + + private async Task Later(ScriptInvocation invocation) + { + await _gate.Task.ConfigureAwait(false); + return Answer(invocation); + } + + private static ScriptRunOutcome Answer(ScriptInvocation invocation) + => ScriptRunOutcome.Ok((double)invocation.Inputs["src/a"].Value! * int.Parse(invocation.Key["calc/t".Length..])); + + public void Reset(ScriptContextKind kind) { } + + public IReadOnlyCollection QuarantinedKeys(ScriptContextKind kind) => []; + } + + private static RawTagEntry CalcTag(string rawPath, int factor) + { + var source = $"return (double)ctx.GetTag(\"src/a\").Value * {factor};"; + var json = $"{{\"scriptId\":\"s{factor}\",\"changeTriggered\":true,\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}"; + return new RawTagEntry(rawPath, json, WriteIdempotent: false); + } + + private static async Task Settle(CalculationDriver driver) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline && !driver.IsEvaluationIdle) await Task.Delay(10); + driver.IsEvaluationIdle.ShouldBeTrue(); + } + + [Fact] + public async Task The_pump_submits_what_it_dequeued_as_one_batch_and_publishes_in_dequeue_order() + { + var runner = new GatedRunner(); + var driver = new CalculationDriver( + new CalculationDriverOptions { RawTags = [CalcTag("calc/t1", 1), CalcTag("calc/t2", 2), CalcTag("calc/t3", 3)], ScriptRunner = runner }, + "calc-drv", new ScriptRootLogger(new LoggerConfiguration().CreateLogger())); + var published = new ConcurrentQueue<(string Tag, object? Value, uint Status)>(); + driver.OnDataChange += (_, e) => published.Enqueue((e.FullReference, e.Snapshot.Value, e.Snapshot.StatusCode)); + var sizes = new ConcurrentQueue(); + driver.BatchSubmittedForTest = sizes.Enqueue; + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && runner.Batches.IsEmpty) await Task.Delay(5, TestContext.Current.CancellationToken); + // Two more triggers queue behind the held batch: six evaluations, each tag twice. + driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow); + driver.OnDependencyValue("src/a", 3.0, StatusGood, DateTime.UtcNow); + runner.Release(); + await Settle(driver); + + runner.SingleRuns.ShouldBe(0); + runner.Batches.ToArray().ShouldBe( + [["calc/t1", "calc/t2", "calc/t3"], ["calc/t1", "calc/t2", "calc/t3"], ["calc/t1", "calc/t2", "calc/t3"]], + "a tag queued twice ends the batch; every queued evaluation still runs"); + sizes.ToArray().ShouldBe([3, 3, 3]); + published.ToArray().ShouldBe( + [ + ("calc/t1", 1.0, StatusGood), ("calc/t2", 2.0, StatusGood), ("calc/t3", 3.0, StatusGood), + ("calc/t1", 2.0, StatusGood), ("calc/t2", 4.0, StatusGood), ("calc/t3", 6.0, StatusGood), + ("calc/t1", 3.0, StatusGood), ("calc/t2", 6.0, StatusGood), ("calc/t3", 9.0, StatusGood), + ], "dequeue order, each against the snapshot that triggered it"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ScriptRunnerSelectionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ScriptRunnerSelectionTests.cs index 0cf7688c..1f458b1c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ScriptRunnerSelectionTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ScriptRunnerSelectionTests.cs @@ -139,13 +139,32 @@ public sealed class ScriptRunnerSelectionTests [Fact] public void OutOfProcess_selects_only_the_supported_requested_seams() { - var all = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess }; - ScriptRunnerSelection.Validate(all, "/srv").ShouldContain(w => w.Contains("ScriptedAlarm seam cannot run out of process yet")); - using (var selection = ScriptRunnerSelection.Create(all, Silent, null)) + // Unset Seams: VirtualTag + Calculation. Scripted alarms stay in process unless listed (design §14 #10). + var unset = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess }; + ScriptRunnerSelection.Validate(unset, "/srv").ShouldNotContain(w => w.Contains("cannot run out of process")); + unset.IsOutOfProcess(ScriptSeam.ScriptedAlarm).ShouldBeFalse(); + unset.IsOutOfProcess(ScriptSeam.VirtualTag).ShouldBeTrue(); + using (var selection = ScriptRunnerSelection.Create(unset, Silent, null)) { selection.ForSeam(ScriptSeam.VirtualTag).ShouldNotBeNull(); selection.ForSeam(ScriptSeam.Calculation).ShouldBeSameAs(selection.ForSeam(ScriptSeam.VirtualTag), "one node-wide runner"); - selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull(); + selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull("alarms go out of process only when listed explicitly"); + } + + // Listed explicitly: the alarm seam shares the node's one runner. + var withAlarms = new ScriptingOptions + { + Runner = ScriptRunnerMode.OutOfProcess, + RunnerSeams = [ScriptSeam.VirtualTag, ScriptSeam.ScriptedAlarm], + }; + withAlarms.IsOutOfProcess(ScriptSeam.ScriptedAlarm).ShouldBeTrue(); + ScriptRunnerSelection.Validate(withAlarms, "/srv").ShouldNotContain(w => w.Contains("cannot run out of process")); + using (var selection = ScriptRunnerSelection.Create(withAlarms, Silent, null)) + { + selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldNotBeNull(); + selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeSameAs(selection.ForSeam(ScriptSeam.VirtualTag), "one node-wide runner"); + selection.ForSeam(ScriptSeam.Calculation).ShouldBeNull(); + ((IScriptRunnerProvider)selection).ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeSameAs(selection.ForSeam(ScriptSeam.ScriptedAlarm)); } var calcOnly = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess, RunnerSeams = [ScriptSeam.Calculation] }; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/AlarmSeamTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/AlarmSeamTests.cs new file mode 100644 index 00000000..26e3d9b0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/AlarmSeamTests.cs @@ -0,0 +1,152 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; + +namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; + +/// A minimal : set or push a value; push notifies subscribers synchronously. +internal sealed class TestUpstream : ITagUpstreamSource +{ + private readonly ConcurrentDictionary _values = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary>> _subs = new(StringComparer.Ordinal); + + public void Set(string path, object? value, uint status = 0u) + => _values[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow); + + public void Push(string path, object? value, uint status = 0u) + { + Set(path, value, status); + if (!_subs.TryGetValue(path, out var list)) return; + Action[] observers; + lock (list) observers = [.. list]; + foreach (var observer in observers) observer(path, _values[path]); + } + + public DataValueSnapshot ReadTag(string path) + => _values.TryGetValue(path, out var v) ? v : new DataValueSnapshot(null, 0x80320000u, null, DateTime.UtcNow); + + public IDisposable SubscribeTag(string path, Action observer) + { + var list = _subs.GetOrAdd(path, _ => []); + lock (list) list.Add(observer); + return new Unsubscribe(() => { lock (list) list.Remove(observer); }); + } + + private sealed class Unsubscribe(Action dispose) : IDisposable + { + public void Dispose() => dispose(); + } +} + +/// +/// lmxopcua#780 — the scripted-alarm seam over the REAL worker (design §9, alarm column): a runaway predicate is killed +/// at T + G and quarantined, its condition frozen with a BadInternalError annotation and never transitioned; a +/// sibling alarm on the same shard keeps evaluating; and a worker that dies under an in-flight predicate freezes its +/// condition BadResourceUnavailable — it never clears it — and the alarm is re-evaluated by itself on return. +/// +[Collection(ScriptWorkerCollection.Name)] +[Trait("Category", "ScriptWorker")] +public sealed class AlarmSeamTests +{ + private const uint StatusBadInternalError = 0x80020000u; + private const uint StatusBadResourceUnavailable = 0x80040000u; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner runner, TimeSpan budget) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: budget, scriptRunner: runner); + } + + [Fact] + public async Task A_runaway_predicate_is_killed_and_quarantined_its_alarm_frozen_Bad_and_a_sibling_keeps_evaluating() + { + var budget = TimeSpan.FromMilliseconds(300); + var log = new CapturingSink(); + await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2), log.CreateLogger()); + var up = new TestUpstream(); + up.Set("x", 0); + using var engine = Engine(up, runner, budget); + var events = new ConcurrentQueue(); + engine.OnEvent += (_, e) => events.Enqueue(e); + await engine.LoadAsync( + [ + Alarm("runaway", "if ((int)ctx.GetTag(\"x\").Value > 5) { while (true) { } } return false;"), + Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;"), + ], Ct); + var first = runner.Shards[0].Current.ShouldNotBeNull(); + + up.Push("x", 10); + await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Activated }), TimeSpan.FromSeconds(10), "the sibling's activation"); + await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "runaway", Emission: EmissionKind.QualityChanged }), TimeSpan.FromSeconds(10), "the runaway's annotation"); + events.Single(e => e.AlarmId == "runaway").WorstInputStatusCode.ShouldBe(StatusBadInternalError); + (await first.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).CulpritKey + .ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.AlarmPredicate, "runaway")); + + // The second kill quarantines it; the sibling (same shard, worker restarted twice) keeps evaluating. Re-pushed: + // Exited completes before the dead worker's evaluations are released, so the first push after it can find the + // runaway's slot still held and be refused (not a strike, exactly as in process) — the next push runs it. + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60); + for (var x = 11; !engine.QuarantinedAlarmIds.Contains("runaway"); x++) + { + if (DateTime.UtcNow > deadline) + throw new TimeoutException("no kill quarantine; events: " + + string.Join(", ", events.Select(e => $"{e.AlarmId}:{e.Emission}:{e.WorstInputStatusCode:X8}")) + + Environment.NewLine + string.Join(Environment.NewLine, log.Rendered)); + up.Push("x", x); + await Task.Delay(TimeSpan.FromSeconds(2), Ct); + } + engine.QuarantinedAlarmIds.ShouldBe(["runaway"]); + up.Push("x", 0); + await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Cleared }), TimeSpan.FromSeconds(60), "the sibling's clear"); + + engine.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "a predicate that never returned never transitions"); + events.Where(e => e.AlarmId == "runaway").ShouldAllBe(e => e.Emission == EmissionKind.QualityChanged); + + // A redeploy (reload) lifts the quarantine, as in process. + await engine.LoadAsync([Alarm("runaway", "return (int)ctx.GetTag(\"x\").Value > 5;"), Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;")], Ct); + engine.QuarantinedAlarmIds.ShouldBeEmpty(); + } + + [Fact] + public async Task A_worker_that_dies_under_a_predicate_freezes_it_BadResourceUnavailable_never_clears_it_and_it_is_re_evaluated_on_return() + { + await using var runner = Scripts.Runner(Scripts.Options(probation: 1)); + var up = new TestUpstream(); + up.Set("ms", 1); + using var engine = Engine(up, runner, TimeSpan.FromSeconds(5)); + var events = new ConcurrentQueue(); + engine.OnEvent += (_, e) => events.Enqueue(e); + await engine.LoadAsync([Alarm("slow", + "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms > 0;")], Ct); + engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active); + var worker = runner.Shards[0].Current.ShouldNotBeNull(); + events.Clear(); + + up.Push("ms", 1500); + await Task.Delay(300, Ct); // the predicate is spinning in the worker + Process.GetProcessById(worker.Pid).Kill(); + + await Scripts.Eventually(() => events.Any(e => e.Emission == EmissionKind.QualityChanged), TimeSpan.FromSeconds(10), "the runner-failure annotation"); + events.First().WorstInputStatusCode.ShouldBe(StatusBadResourceUnavailable); + engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active, "a dead worker never clears an alarm"); + + // No further input change: the runner's recovery re-evaluates it (in the isolation worker) and the quality returns. + await Scripts.Eventually(() => events.Any(e => e is { Emission: EmissionKind.QualityChanged, WorstInputStatusCode: 0u }), + TimeSpan.FromSeconds(60), "the re-evaluation on return"); + engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active); + events.ShouldNotContain(e => e.Emission == EmissionKind.Cleared); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BatchSubmitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BatchSubmitTests.cs new file mode 100644 index 00000000..e3d2bc87 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BatchSubmitTests.cs @@ -0,0 +1,172 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; + +namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; + +/// +/// lmxopcua#780 batch submit (design §4.5) against the real worker: sends one frame +/// per worker and answers per item, and every item keeps its own deadlines and blame — in particular an item that has to +/// wait in the worker behind a runaway sibling is neither timed out, nor struck, nor blamed for the kill. +/// +[Collection(ScriptWorkerCollection.Name)] +[Trait("Category", "ScriptWorker")] +public sealed class BatchSubmitTests +{ + /// Runs away when input spin is true, otherwise returns n + 1. + private const string MaybeRunaway = "if ((bool)ctx.GetTag(\"spin\").Value) { while (true) { } } return (int)ctx.GetTag(\"n\").Value + 1;"; + + /// Spins for input ms milliseconds, then returns it. + private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static IReadOnlyDictionary Inputs(params (string Path, object Value)[] values) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var (path, value) in values) map[path] = new DataValueSnapshot(value, 0, DateTime.UtcNow, DateTime.UtcNow); + return map; + } + + private static ScriptInvocation Item(string key, bool spin, int n, TimeSpan? timeout = null) + => Scripts.Vt(key, MaybeRunaway, timeout, Inputs(("spin", spin), ("n", n))); + + private static OutOfProcessScriptRunner Runner(ScriptingOptions options, int evalThreads) + => new(options, new LoggerConfiguration().CreateLogger(), null, + heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60), evalThreads: evalThreads); + + [Fact] + public async Task A_batch_of_loaded_scripts_goes_out_as_one_frame_and_every_item_gets_its_own_result() + { + await using var runner = Runner(Scripts.Options(), evalThreads: 4); + // First evaluations load (compile) each key; those take the full path one by one. + var warm = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i)).ToArray(), Ct)); + warm.ShouldAllBe(o => o.Status == ScriptRunStatus.Ok); + + var frames = new ConcurrentQueue(); + runner.BatchSubmittedForTest = frames.Enqueue; + var outcomes = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i * 10)).ToArray(), Ct)); + + frames.ToArray().ShouldBe([40], "every loaded item rides one EvalBatch frame"); + for (var i = 0; i < 40; i++) + { + outcomes[i].Status.ShouldBe(ScriptRunStatus.Ok, outcomes[i].Reason); + outcomes[i].Value.ShouldBe(i * 10 + 1, $"item {i} got its own result"); + } + } + + [Fact] + public async Task A_batch_answers_exactly_what_the_same_evaluations_answer_one_by_one() + { + // Parity: Ok, a throw, a compile error, a duplicate key (refused) and a fast item, batched vs RunAsync. + ScriptInvocation[] Items() => + [ + Scripts.Vt("ok", "return (int)ctx.GetTag(\"n\").Value * 2;", inputs: Inputs(("n", 21))), + Scripts.Vt("throws", "throw new System.InvalidOperationException(\"boom\");"), + Scripts.Vt("broken", "return undefinedThing;"), + Scripts.Vt("fast", "return 5;"), + ]; + + await using var runner = Runner(Scripts.Options(), evalThreads: 2); + var single = new List(); + foreach (var item in Items()) single.Add(await runner.RunAsync(item, Ct)); + var batched = await Task.WhenAll(runner.RunBatch(Items(), Ct)); + + batched.Select(o => (o.Status, o.Value, o.Reason)).ShouldBe(single.Select(o => (o.Status, o.Value, o.Reason))); + single[0].Value.ShouldBe(42); + single[1].Reason!.ShouldStartWith("script threw: "); + single[2].Reason!.ShouldStartWith("compile error: "); + + // In process, the default RunBatch is RunAsync per item: same answers. + using var inProcess = new InProcessScriptRunner(); + var inProcessBatched = await Task.WhenAll(((IScriptRunner)inProcess).RunBatch(Items(), Ct)); + inProcessBatched.Select(o => (o.Status, o.Value)).ShouldBe(single.Select(o => (o.Status, o.Value))); + + // A key twice in one batch: the second is refused, exactly like two concurrent RunAsync calls. + var twice = await Task.WhenAll(runner.RunBatch([Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 200))), Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 1)))], Ct)); + twice.Count(o => o.Reason == ScriptRunGuard.PreviousRunStillRunningReason).ShouldBe(1); + } + + /// + /// The blame rule (design §5.3.1) under batching: with ONE evaluation thread, a runaway at the head of a batch holds + /// the thread and its siblings wait in the worker. The siblings are the kill's victims — Unavailable "worker + /// restarted", no timeout, no strike — and only the runaway is killed and quarantined. Mutation check: with the + /// host ignoring the worker's deferral notice, the siblings' clocks run from submit, they time out at T with the + /// runaway, and at T + G any of the four can be the one blamed. + /// + [Fact] + public async Task A_runaway_in_a_batch_is_blamed_alone_and_siblings_queued_behind_it_are_not_timed_out_or_struck() + { + var softT = TimeSpan.FromMilliseconds(400); + var grace = TimeSpan.FromMilliseconds(500); + await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 1), evalThreads: 1); + var keys = new[] { "runaway", "sib1", "sib2", "sib3" }; + foreach (var key in keys) (await runner.RunAsync(Item(key, false, 1, softT), Ct)).Value.ShouldBe(2); + var worker = runner.Shards[0].Current.ShouldNotBeNull(); + + var frames = new ConcurrentQueue(); + runner.BatchSubmittedForTest = frames.Enqueue; + var sw = Stopwatch.StartNew(); + var tasks = runner.RunBatch([Item("runaway", true, 0, softT), Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct); + frames.ToArray().ShouldBe([4]); + + var runaway = await tasks[0]; + runaway.Status.ShouldBe(ScriptRunStatus.Failed); + runaway.Reason!.ShouldStartWith("script timed out after "); + // Nothing has answered the queued siblings at the runaway's soft deadline. + tasks.Skip(1).ShouldAllBe(t => !t.IsCompleted, "a run waiting behind a busy thread is not timed out at its submit-relative T"); + + var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct); + exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline); + exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway")); + sw.Elapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50)); + + var siblings = await Task.WhenAll(tasks.Skip(1)); + foreach (var sibling in siblings) + { + sibling.Status.ShouldBe(ScriptRunStatus.Unavailable, "a victim of another key's kill, not a script failure"); + sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); + } + await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count > 0, TimeSpan.FromSeconds(5), "the kill quarantine"); + runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"], "the siblings took no strike"); + + // The siblings run again on the restarted worker, with their own inputs. + var again = await Task.WhenAll(runner.RunBatch([Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct)); + again.Select(o => o.Value).ShouldBe([11, 21, 31]); + } + + /// + /// The positive control for the deferral: a run that waited behind a sibling still gets its deadlines — the full T, + /// then G — counted from when it STARTED. Mutation check: a start notice that does not re-arm the clock leaves the + /// deferred runaway with no deadline at all; it is never answered or killed. + /// + [Fact] + public async Task A_deferred_run_is_timed_and_killed_from_when_it_starts() + { + var softT = TimeSpan.FromMilliseconds(400); + var grace = TimeSpan.FromMilliseconds(500); + var head = TimeSpan.FromMilliseconds(300); + await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 5), evalThreads: 1); + (await runner.RunAsync(Scripts.Vt("head", Spin, softT, Inputs(("ms", 1))), Ct)).Value.ShouldBe(1); + (await runner.RunAsync(Item("tail", false, 1, softT), Ct)).Value.ShouldBe(2); + var worker = runner.Shards[0].Current.ShouldNotBeNull(); + + var sw = Stopwatch.StartNew(); + var tasks = runner.RunBatch([Scripts.Vt("head", Spin, softT, Inputs(("ms", (int)head.TotalMilliseconds))), Item("tail", true, 0, softT)], Ct); + + (await tasks[0]).Value.ShouldBe((int)head.TotalMilliseconds, "the head ran inside its budget"); + var tail = await tasks[1]; + var answered = sw.Elapsed; + tail.Reason!.ShouldStartWith("script timed out after "); + answered.ShouldBeGreaterThanOrEqualTo(head + softT - TimeSpan.FromMilliseconds(50), "T counts from the run's start, after the head"); + + var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct); + exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "tail")); + sw.Elapsed.ShouldBeGreaterThanOrEqualTo(head + softT + grace - TimeSpan.FromMilliseconds(50), "G counts from the run's start too"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs index 3f82550a..69810ac5 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs @@ -3,6 +3,7 @@ using System.Text; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; @@ -261,6 +262,217 @@ public sealed class ScriptRunnerPerfTests k2Closed.EvalsPerSecond.ShouldBeGreaterThanOrEqualTo(AcceptedGateEvalsPerSecond, "the accepted 5k evals/s/node gate at K=2"); } + /// + /// lmxopcua#780 batch submit (design §4.5): a closed loop of one batch at a time, + /// distinct loaded keys per batch, either through or as the same evaluations + /// submitted as concurrent calls. Latency is per batch (submit → last result). + /// + private static async Task BatchLoopAsync(IScriptRunner runner, int batchSize, bool viaBatch) + { + var deadline = Stopwatch.GetTimestamp() + (long)(Phase.TotalSeconds * Stopwatch.Frequency); + var samples = new List(100_000); + long ok = 0, notOk = 0; + string? firstFailure = null; + var offset = 0; + var x = 0; + var cpu0 = Process.GetCurrentProcess().TotalProcessorTime; + var wall = Stopwatch.StartNew(); + while (Stopwatch.GetTimestamp() < deadline) + { + var invocations = new ScriptInvocation[batchSize]; + var keys = new int[batchSize]; + for (var i = 0; i < batchSize; i++) + { + keys[i] = (offset + i) % Keys; + invocations[i] = Invocation(keys[i], x); + } + offset = (offset + batchSize) % Keys; + var t0 = Stopwatch.GetTimestamp(); + var outcomes = await Task.WhenAll(viaBatch + ? runner.RunBatch(invocations, Ct) + : invocations.Select(inv => runner.RunAsync(inv, Ct)).ToArray()); + samples.Add(Stopwatch.GetElapsedTime(t0).TotalMicroseconds); + for (var i = 0; i < batchSize; i++) + { + var o = outcomes[i]; + if (o.Status == ScriptRunStatus.Ok && Equals(o.Value, x + keys[i] % DistinctSources)) ok++; + else + { + notOk++; + firstFailure ??= $"{o.Status}: {o.Reason} (value {o.Value})"; + } + } + x++; + } + var elapsed = wall.Elapsed; + var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / elapsed.TotalMilliseconds * 100; + samples.Sort(); + return new PhaseResult((ok + notOk) / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure); + } + + /// + /// A runner that turns a batch back into one evaluation after another — the serial pass the alarm engine and the + /// Calc pump made before batch submit, which design §8.2 warns costs N round trips (the comparison row). + /// + private sealed class SerialRunner(IScriptRunner inner) : IScriptRunner + { + public Task RunAsync(ScriptInvocation invocation, CancellationToken ct = default) => inner.RunAsync(invocation, ct); + + public IReadOnlyList> RunBatch(IReadOnlyList invocations, CancellationToken ct = default) + { + var results = new Task[invocations.Count]; + Task previous = Task.CompletedTask; + for (var i = 0; i < results.Length; i++) previous = results[i] = After(previous, invocations[i], ct); + return results; + } + + private async Task After(Task previous, ScriptInvocation invocation, CancellationToken ct) + { + await previous.ConfigureAwait(false); + return await inner.RunAsync(invocation, ct).ConfigureAwait(false); + } + + public void Reset(ScriptContextKind kind) => inner.Reset(kind); + + public IReadOnlyCollection QuarantinedKeys(ScriptContextKind kind) => inner.QuarantinedKeys(kind); + } + + /// + /// Scripted-alarm pass latency: alarms on one input, every push flips all of them, and a + /// sample is the time from the push to the last of the N transitions the pass emits (the whole dirty set through + /// the engine's gate). The design's alarm budget is 250 ms (§1.1; the host now passes Scripting:RunTimeout, + /// 2 s by default, lmxopcua#779), so the pass is judged against the stricter 250 ms. + /// + private static async Task<(double P50Ms, double P99Ms, double MaxMs, double CpuPercent, int NotCompleted)> AlarmPassLatencyAsync( + IScriptRunner? runner, int alarms, int pushes) + { + var up = new TestUpstream(); + up.Set("x", 0); + var logger = new Serilog.LoggerConfiguration().CreateLogger(); + using var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: TimeSpan.FromMilliseconds(250), scriptRunner: runner); + var transitions = 0; + var notCompleted = 0; + var target = int.MaxValue; + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + engine.OnEvent += (_, e) => + { + // A Bad annotation means a predicate did not complete in that pass (timed out, refused): counted, reported. + if (e.Emission == EmissionKind.QualityChanged && e.WorstInputStatusCode != 0) Interlocked.Increment(ref notCompleted); + if (e.Emission is not (EmissionKind.Activated or EmissionKind.Cleared)) return; + if (Interlocked.Increment(ref transitions) == Volatile.Read(ref target)) done.TrySetResult(); + }; + await engine.LoadAsync(Enumerable.Range(0, alarms).Select(i => new ScriptedAlarmDefinition( + $"alarm-{i}", "Plant/Perf", $"alarm-{i}", AlarmKind.AlarmCondition, AlarmSeverity.High, "perf", + $"return (int)ctx.GetTag(\"x\").Value > {-(i % DistinctSources)};")).ToArray(), Ct); + var samples = new List(pushes); + var cpu0 = Process.GetCurrentProcess().TotalProcessorTime; + var wall = Stopwatch.StartNew(); + for (var p = 0; p < pushes + 2; p++) + { + // Alternate x = -1000 (every predicate false) and x = 1000 (every predicate true). The pass is expected to + // transition exactly the alarms not already in the target state — counted from the engine, so an alarm + // that did not complete an earlier pass cannot leave the count short. The first two passes are warm-up. + var activate = p % 2 == 1; + var expected = engine.GetAllStates().Count(c => (c.Active == AlarmActiveState.Active) != activate); + Interlocked.Exchange(ref transitions, 0); + done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Volatile.Write(ref target, expected); + var t0 = Stopwatch.GetTimestamp(); + up.Push("x", activate ? 1000 : -1000); + if (expected == 0) continue; + try + { + await done.Task.WaitAsync(TimeSpan.FromSeconds(60), Ct); + } + catch (TimeoutException) + { + throw new TimeoutException($"alarm pass {p} ({(runner is null ? "in process" : runner.GetType().Name)}, {alarms} alarms): " + + $"{Volatile.Read(ref transitions)} of {expected} transitions in 60 s, " + + $"{Volatile.Read(ref notCompleted)} predicate(s) did not complete so far"); + } + if (p >= 2) samples.Add(Stopwatch.GetElapsedTime(t0).TotalMilliseconds); + } + var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / wall.Elapsed.TotalMilliseconds * 100; + samples.Sort(); + return (Percentile(samples, 0.50), Percentile(samples, 0.99), samples[^1], cpu, notCompleted); + } + + /// + /// Batch throughput (RunBatch vs the same evaluations as concurrent RunAsync calls, K = 2) and scripted-alarm pass + /// latency (in process, out of process batched, and out of process serial). Report-only, except that every + /// result must be correct. + /// + [Fact] + public async Task Batch_throughput_and_alarm_pass_latency() + { + RequireOptIn(); + var report = new StringBuilder() + .AppendLine($"### Batch submit throughput ({Env()})") + .AppendLine() + .AppendLine("| Runner | batch | submitted as | evals/s | batch p50 µs | batch p99 µs | host CPU | ok / not ok |") + .AppendLine("|---|---:|---|---:|---:|---:|---:|---|"); + + foreach (var (name, create) in new (string, Func)[] + { + ("InProcess", () => new InProcessScriptRunner(new ScriptingOptions())), + ("OutOfProcess K=2", () => Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger())), + }) + { + var runner = create(); + try + { + await WarmAsync(runner); + foreach (var size in new[] { 1, 10, 100, 500 }) + { + foreach (var viaBatch in new[] { true, false }) + { + var r = await BatchLoopAsync(runner, size, viaBatch); + r.NotOk.ShouldBe(0, $"{name} batch {size}: {r.FirstFailure}{Environment.NewLine}{RunnerLog()}"); + report.AppendLine($"| {name} | {size} | {(viaBatch ? "RunBatch" : "RunAsync ×n")} | {r.EvalsPerSecond,9:N0} | " + + $"{r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |"); + } + } + } + finally + { + switch (runner) + { + case IAsyncDisposable a: await a.DisposeAsync(); break; + case IDisposable d: d.Dispose(); break; + } + } + } + + report.AppendLine() + .AppendLine($"### Scripted-alarm pass latency, push → last transition ({Env()})") + .AppendLine() + .AppendLine("| Engine | alarms | pass p50 ms | pass p99 ms | max ms | per alarm (p50) µs | host CPU | not completed | 250 ms |") + .AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---|"); + const int pushes = 40; + await using (var shared = Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger())) + { + foreach (var alarms in new[] { 100, 500, 1000 }) + { + foreach (var (name, runner) in new (string, IScriptRunner?)[] + { + ("InProcess (serial)", null), + ("OutOfProcess K=2, batched", shared), + ("OutOfProcess K=2, serial", new SerialRunner(shared)), + }) + { + var (p50, p99, max, cpu, notCompleted) = await AlarmPassLatencyAsync(runner, alarms, pushes); + report.AppendLine($"| {name} | {alarms:N0} | {p50,7:N1} | {p99,7:N1} | {max,7:N1} | {p50 * 1000 / alarms,6:N0} | " + + $"{cpu,5:N0} % | {notCompleted:N0} | {(p99 <= 250 ? "PASS" : "MISS")} |"); + } + } + } + report.AppendLine() + .AppendLine("Each push flips every alarm; a pass is the engine's whole dirty set under its gate. \"serial\" replays the " + + "pre-batch one-predicate-at-a-time pass over the same runner."); + Report(report); + } + private static ScriptingOptions PerfOptions(int shards, string? peCacheDir = null) { var options = Scripts.Options(shards: shards, hardGrace: TimeSpan.FromSeconds(2), peCacheDir: peCacheDir, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests.csproj b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests.csproj index eaa5e773..7c142d26 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests.csproj +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests.csproj @@ -22,6 +22,8 @@ + +