dohertj2/lmxopcua
1
0
Fork 0
Code Issues 26 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run
Details

Browse Source 
- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 06:37:36 -05:00
parent 11a5c1d4e2
commit 0796928504
33 changed files with 2168 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs
+252 -23
View File
@@ -81,6 +81,17 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// </summary>
private readonly ScriptRunGuard _guard;
/// <summary>
/// lmxopcua#780 — the out-of-process runner when <c>Scripting:Runner=OutOfProcess</c> lists the ScriptedAlarm seam;
/// null (the default) keeps the in-process guard path, unchanged. With a runner the predicates compile and run in
/// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch.
/// </summary>
private readonly IScriptRunner? _runner;
// Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the
// runner's Recovery to re-evaluate them once. Guarded by _dirtyLock.
private readonly HashSet<string> _recoveryArmed = new(StringComparer.Ordinal);
// ConcurrentDictionary, not a plain Dictionary: every mutation happens under
// _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds,
// RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request
@@ -243,6 +254,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// default used to be <c>TimedScriptEvaluator.DefaultTimeout</c> (250 ms), which is what every production engine
/// silently ran with because the host never passed a value.</param>
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
public ScriptedAlarmEngine(
ITagUpstreamSource upstream,
IAlarmStateStore store,
@@ -250,8 +263,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
ILogger engineLogger,
Func<DateTime>? clock = null,
TimeSpan? scriptTimeout = null,
ScriptingOptions? scripting = null)
ScriptingOptions? scripting = null,
IScriptRunner? scriptRunner = null)
{
_runner = scriptRunner;
_upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
_store = store ?? throw new ArgumentNullException(nameof(store));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
@@ -268,6 +283,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// <summary>The wall-clock budget each predicate run is given (diagnostics / tests).</summary>
public TimeSpan ScriptTimeout => _scriptTimeout;
/// <summary>The out-of-process runner predicates run through, or null in process (diagnostics / tests).</summary>
public IScriptRunner? ScriptRunner => _runner;
/// <summary>
/// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when
/// one is in use (diagnostics / tests).
/// </summary>
public IReadOnlyCollection<string> QuarantinedAlarmIds => _runner is null
? _guard.QuarantinedKeys
: [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.AlarmPredicate)];
/// <summary>
/// OPC UA <c>BadResourceUnavailable</c> — the annotation for a predicate the out-of-process RUNNER could not run
/// (its worker restarted under it, or no worker was ready): the condition is frozen exactly as for a script
/// fault, but the quality tells the operator the node is sick, not the script (design §9, §14 #3; the same split
/// the virtual-tag and Calculation seams make).
/// </summary>
private const uint StatusBadResourceUnavailable = 0x80040000u;
/// <summary>
/// OPC UA <c>BadInternalError</c> — the quality a condition is annotated with while its predicate
/// cannot be evaluated (timed out, refused because the previous run is still executing, or
@@ -331,6 +365,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
// A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts
// its own through the guard's source identity; this covers "redeployed to retry").
_guard.ResetAll();
_runner?.Reset(ScriptContextKind.AlarmPredicate);
lock (_dirtyLock) _recoveryArmed.Clear();
var compileFailures = new List<string>();
foreach (var def in definitions)
@@ -350,7 +386,13 @@ public sealed class ScriptedAlarmEngine : IDisposable
// The compiled script is run through _guard (timeout + quarantine) at evaluation time —
// the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not
// see that the run it abandoned was still executing.
var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
//
// Out of process (lmxopcua#780) the worker compiles and runs the predicate; here the source is only
// compiled to PE bytes and dropped — same Roslyn, sandbox and diagnostics, so a broken predicate still
// fails the load exactly as in process — and no assembly is loaded into the host.
ScriptEvaluator<AlarmPredicateContext, bool>? evaluator = null;
if (_runner is null) evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
else _ = ScriptEvaluator<AlarmPredicateContext, bool>.CompileToImage(def.PredicateScriptSource);
var logger = _loggerFactory.Create(def.AlarmId);
var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate);
@@ -396,14 +438,32 @@ public sealed class ScriptedAlarmEngine : IDisposable
// Any predicate emissions queue into `pending` and fire after the gate
// is released — so a startup-recovery activation event can call back into
// the engine without deadlocking.
foreach (var (alarmId, state) in _alarms)
if (_runner is null)
{
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
var seed = persisted ?? state.Condition;
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
.ConfigureAwait(false);
_alarms[alarmId] = state with { Condition = afterPredicate };
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
foreach (var (alarmId, state) in _alarms)
{
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
var seed = persisted ?? state.Condition;
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
.ConfigureAwait(false);
_alarms[alarmId] = state with { Condition = afterPredicate };
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
}
}
else
{
// Out of process: every predicate goes in one batch, then each alarm is applied and saved in the
// same order the serial loop above uses.
var seeds = new List<(AlarmState State, AlarmConditionState Seed)>(_alarms.Count);
foreach (var (alarmId, state) in _alarms)
seeds.Add((state, await _store.LoadAsync(alarmId, ct).ConfigureAwait(false) ?? state.Condition));
var after = await EvaluatePredicateBatchToStatesAsync(seeds, _clock(), ct, pending).ConfigureAwait(false);
for (var i = 0; i < seeds.Count; i++)
{
var state = seeds[i].State;
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
}
}
_loaded = true;
@@ -614,7 +674,12 @@ public sealed class ScriptedAlarmEngine : IDisposable
if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
_valueCache[path] = value;
if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
MarkDirty(alarmIds);
}
/// <summary>Marks <paramref name="alarmIds"/> dirty and starts the drainer if none is running.</summary>
private void MarkDirty(IEnumerable<string> alarmIds)
{
bool startDrain;
lock (_dirtyLock)
{
@@ -754,17 +819,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
// disposing store or mutating _alarms after clear is unsafe.
if (_disposed) return;
foreach (var id in alarmIds)
if (_runner is not null)
{
if (!_alarms.TryGetValue(id, out var state)) continue;
var newState = await EvaluatePredicateToStateAsync(
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
if (!ReferenceEquals(newState, state.Condition))
// Out of process (lmxopcua#780, design §4.5): the whole dirty set in one batch, applied in order.
await ReevaluateBatchAsync(alarmIds, ct, pending).ConfigureAwait(false);
}
else
{
foreach (var id in alarmIds)
{
// Persist before updating in-memory so a store failure leaves
// both sides at the prior state.
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
_alarms[id] = state with { Condition = newState };
if (!_alarms.TryGetValue(id, out var state)) continue;
var newState = await EvaluatePredicateToStateAsync(
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
if (!ReferenceEquals(newState, state.Condition))
{
// Persist before updating in-memory so a store failure leaves
// both sides at the prior state.
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
_alarms[id] = state with { Condition = newState };
}
}
}
}
@@ -837,7 +910,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
try
{
predicateTrue = await _guard.RunAsync(
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context,
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator!, context,
_scriptTimeout, ct)
.ConfigureAwait(false);
}
@@ -890,6 +963,162 @@ public sealed class ScriptedAlarmEngine : IDisposable
return result.State;
}
/// <summary>
/// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the
/// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in
/// the original order, under <c>_evalGate</c>. Predicates are pure functions of the input snapshot taken here,
/// under the gate, so the state machine sees exactly the sequence the serial loop produces. The batch costs one
/// round trip for the whole set instead of one per alarm, which is what keeps a large engine inside its budget.
/// </summary>
private async Task ReevaluateBatchAsync(IReadOnlyList<string> alarmIds, CancellationToken ct, List<ScriptedAlarmEvent> pending)
{
var items = new List<(AlarmState State, AlarmConditionState Seed)>(alarmIds.Count);
foreach (var id in alarmIds)
if (_alarms.TryGetValue(id, out var state)) items.Add((state, state.Condition));
if (items.Count == 0) return;
var after = await EvaluatePredicateBatchToStatesAsync(items, _clock(), ct, pending).ConfigureAwait(false);
for (var i = 0; i < items.Count; i++)
{
var (state, seed) = items[i];
if (ReferenceEquals(after[i], seed)) continue;
// Persist before updating in-memory so a store failure leaves both sides at the prior state.
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
}
}
/// <summary>
/// <see cref="EvaluatePredicateToStateAsync"/> for a batch, through <see cref="_runner"/>: the same readiness gate,
/// quality tracking and Part 9 application per alarm, in order, with every predicate that has to run submitted
/// together. Failure semantics (design §9): a script fault, timeout, kill or quarantine freezes the condition with a
/// <c>BadInternalError</c> annotation, exactly as in process; a runner failure (worker restarted, none ready) freezes
/// it with <c>BadResourceUnavailable</c> and re-evaluates it once the runner can run it again. A dead worker never
/// clears an alarm: no outcome but a completed <c>bool</c> reaches the state machine.
/// </summary>
private async Task<AlarmConditionState[]> EvaluatePredicateBatchToStatesAsync(
IReadOnlyList<(AlarmState State, AlarmConditionState Seed)> items, DateTime nowUtc, CancellationToken ct,
List<ScriptedAlarmEvent> pendingEmissions)
{
var runner = _runner!;
var after = new AlarmConditionState[items.Count];
var scratches = new AlarmScratch[items.Count];
var worst = new uint[items.Count];
var slot = new int[items.Count];
var invocations = new List<ScriptInvocation>(items.Count);
// Pass 1, no side effects: snapshot each alarm's inputs and decide whether its predicate runs.
for (var i = 0; i < items.Count; i++)
{
var state = items[i].State;
var scratch = scratches[i] = _scratchByAlarmId.GetOrAdd(
state.Definition.AlarmId, _ => new AlarmScratch(state.Inputs, state.Logger, _clock));
RefillReadCache(scratch.ReadCache, state.Inputs);
worst[i] = WorstInputStatus(scratch.ReadCache);
if (!AreInputsReady(scratch.ReadCache))
{
slot[i] = -1;
continue;
}
slot[i] = invocations.Count;
// A copy: the runner may still read the inputs after the caller has its answer (a soft timeout), and the
// scratch is refilled by the next pass.
invocations.Add(new ScriptInvocation(state.Definition.AlarmId, ScriptContextKind.AlarmPredicate,
state.Definition.PredicateScriptSource,
new Dictionary<string, DataValueSnapshot>(scratch.ReadCache, StringComparer.Ordinal), _scriptTimeout, state.Logger));
}
var outcomes = invocations.Count == 0 ? [] : runner.RunBatch(invocations, ct);
PredicateBatchSubmittedForTest?.Invoke(invocations.Count);
// Pass 2, in order: exactly what the serial path does with each alarm.
for (var i = 0; i < items.Count; i++)
{
var (state, seed) = items[i];
var scratch = scratches[i];
if (slot[i] < 0)
{
if (TrackQualityBucket(state.Definition.AlarmId, worst[i]))
pendingEmissions.Add(BuildQualityEmission(state, seed, worst[i]));
after[i] = seed;
continue;
}
var outcome = await outcomes[slot[i]].ConfigureAwait(false);
after[i] = ApplyPredicateOutcome(state, seed, scratch, worst[i], outcome, nowUtc, ct, pendingEmissions);
}
return after;
}
/// <summary>One runner outcome onto one alarm (the second half of <see cref="EvaluatePredicateToStateAsync"/>).</summary>
private AlarmConditionState ApplyPredicateOutcome(AlarmState state, AlarmConditionState seed, AlarmScratch scratch,
uint worstStatus, ScriptRunOutcome outcome, DateTime nowUtc, CancellationToken ct, List<ScriptedAlarmEvent> pendingEmissions)
{
switch (outcome.Status)
{
case ScriptRunStatus.Unavailable:
// Not the script's fault: frozen, annotated BadResourceUnavailable, and evaluated again (against the
// inputs current then) when its worker is back.
ArmRecovery(state.Definition.AlarmId, outcome.Recovery);
return PredicateDidNotComplete(state, seed, scratch, outcome.Reason ?? ScriptRunOutcome.RunnerUnavailableReason,
pendingEmissions, StatusBadResourceUnavailable);
case ScriptRunStatus.Failed:
var reason = outcome.Reason ?? "evaluator failure";
if (ct.IsCancellationRequested) throw new OperationCanceledException(ct);
if (reason.StartsWith(ScriptThrewPrefix, StringComparison.Ordinal))
{
// As in process: a predicate that threw logs an Error and keeps its state, with no quality annotation.
scratch.LastNonCompletion = null;
state.Logger.Error("Alarm predicate threw — state unchanged: {Reason}", reason[ScriptThrewPrefix.Length..]);
return seed;
}
// Timed out, killed, refused, quarantined, unsupported type, did not compile: frozen + BadInternalError.
return PredicateDidNotComplete(state, seed, scratch, reason, pendingEmissions);
}
if (outcome.Value is not bool predicateTrue)
{
scratch.LastNonCompletion = null;
state.Logger.Error("Alarm predicate returned {Type} instead of a bool — state unchanged", outcome.Value?.GetType().Name ?? "null");
return seed;
}
scratch.LastNonCompletion = null;
// From here on, identical to the serial path: the bucket is tracked only once the predicate completed (I3).
var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus);
var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc);
var transition = result.Emission != EmissionKind.None
? BuildEmission(state, result.State, result.Emission, worstStatus)
: null;
if (transition is not null) pendingEmissions.Add(transition);
else if (qualityBucketChanged) pendingEmissions.Add(BuildQualityEmission(state, result.State, worstStatus));
return result.State;
}
/// <summary>The prefix the runners put on a predicate that threw (the in-process path catches the exception instead).</summary>
private const string ScriptThrewPrefix = "script threw: ";
/// <summary>
/// Re-evaluate <paramref name="alarmId"/> once when <paramref name="recovery"/> completes — the runner's worker is
/// back (design §9 "re-evaluated on return"). At most one wait per alarm.
/// </summary>
private void ArmRecovery(string alarmId, Task? recovery)
{
if (recovery is null) return;
lock (_dirtyLock)
{
if (_disposed || !_recoveryArmed.Add(alarmId)) return;
}
_ = recovery.ContinueWith(_ =>
{
lock (_dirtyLock) _recoveryArmed.Remove(alarmId);
if (!_disposed) MarkDirty([alarmId]);
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
/// <summary>Test hook: the number of predicates each out-of-process batch pass submitted.</summary>
internal Action<int>? PredicateBatchSubmittedForTest { get; set; }
/// <summary>
/// The predicate did not return (timed out, refused because the previous run is still executing, or
/// quarantined): hold the condition at <paramref name="seed"/>, log the reason <b>once per change of
@@ -900,15 +1129,15 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// </summary>
private AlarmConditionState PredicateDidNotComplete(
AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason,
List<ScriptedAlarmEvent> pendingEmissions)
List<ScriptedAlarmEvent> pendingEmissions, uint status = StatusBadInternalError)
{
if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal))
{
scratch.LastNonCompletion = reason;
state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason);
}
if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError))
pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError));
if (TrackQualityBucket(state.Definition.AlarmId, status))
pendingEmissions.Add(BuildQualityEmission(state, seed, status));
return seed;
}
@@ -1339,7 +1568,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
private sealed record AlarmState(
ScriptedAlarmDefinition Definition,
ScriptEvaluator<AlarmPredicateContext, bool> Evaluator,
ScriptEvaluator<AlarmPredicateContext, bool>? Evaluator, // null out of process: the worker holds the compiled form
IReadOnlySet<string> Inputs,
IReadOnlyList<string> TemplateTokens,
ILogger Logger,