feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 06:37:36 -05:00
parent 11a5c1d4e2
commit 0796928504
33 changed files with 2168 additions and 117 deletions
@@ -81,6 +81,17 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// </summary>
private readonly ScriptRunGuard _guard;
/// <summary>
/// lmxopcua#780 — the out-of-process runner when <c>Scripting:Runner=OutOfProcess</c> lists the ScriptedAlarm seam;
/// null (the default) keeps the in-process guard path, unchanged. With a runner the predicates compile and run in
/// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch.
/// </summary>
private readonly IScriptRunner? _runner;
// Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the
// runner's Recovery to re-evaluate them once. Guarded by _dirtyLock.
private readonly HashSet<string> _recoveryArmed = new(StringComparer.Ordinal);
// ConcurrentDictionary, not a plain Dictionary: every mutation happens under
// _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds,
// RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request
@@ -243,6 +254,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// default used to be <c>TimedScriptEvaluator.DefaultTimeout</c> (250 ms), which is what every production engine
/// silently ran with because the host never passed a value.</param>
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
public ScriptedAlarmEngine(
ITagUpstreamSource upstream,
IAlarmStateStore store,
@@ -250,8 +263,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
ILogger engineLogger,
Func<DateTime>? clock = null,
TimeSpan? scriptTimeout = null,
ScriptingOptions? scripting = null)
ScriptingOptions? scripting = null,
IScriptRunner? scriptRunner = null)
{
_runner = scriptRunner;
_upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
_store = store ?? throw new ArgumentNullException(nameof(store));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
@@ -268,6 +283,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// <summary>The wall-clock budget each predicate run is given (diagnostics / tests).</summary>
public TimeSpan ScriptTimeout => _scriptTimeout;
/// <summary>The out-of-process runner predicates run through, or null in process (diagnostics / tests).</summary>
public IScriptRunner? ScriptRunner => _runner;
/// <summary>
/// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when
/// one is in use (diagnostics / tests).
/// </summary>
public IReadOnlyCollection<string> QuarantinedAlarmIds => _runner is null
? _guard.QuarantinedKeys
: [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.AlarmPredicate)];
/// <summary>
/// OPC UA <c>BadResourceUnavailable</c> — the annotation for a predicate the out-of-process RUNNER could not run
/// (its worker restarted under it, or no worker was ready): the condition is frozen exactly as for a script
/// fault, but the quality tells the operator the node is sick, not the script (design §9, §14 #3; the same split
/// the virtual-tag and Calculation seams make).
/// </summary>
private const uint StatusBadResourceUnavailable = 0x80040000u;
/// <summary>
/// OPC UA <c>BadInternalError</c> — the quality a condition is annotated with while its predicate
/// cannot be evaluated (timed out, refused because the previous run is still executing, or
@@ -331,6 +365,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
// A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts
// its own through the guard's source identity; this covers "redeployed to retry").
_guard.ResetAll();
_runner?.Reset(ScriptContextKind.AlarmPredicate);
lock (_dirtyLock) _recoveryArmed.Clear();
var compileFailures = new List<string>();
foreach (var def in definitions)
@@ -350,7 +386,13 @@ public sealed class ScriptedAlarmEngine : IDisposable
// The compiled script is run through _guard (timeout + quarantine) at evaluation time —
// the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not
// see that the run it abandoned was still executing.
var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
//
// Out of process (lmxopcua#780) the worker compiles and runs the predicate; here the source is only
// compiled to PE bytes and dropped — same Roslyn, sandbox and diagnostics, so a broken predicate still
// fails the load exactly as in process — and no assembly is loaded into the host.
ScriptEvaluator<AlarmPredicateContext, bool>? evaluator = null;
if (_runner is null) evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
else _ = ScriptEvaluator<AlarmPredicateContext, bool>.CompileToImage(def.PredicateScriptSource);
var logger = _loggerFactory.Create(def.AlarmId);
var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate);
@@ -396,14 +438,32 @@ public sealed class ScriptedAlarmEngine : IDisposable
// Any predicate emissions queue into `pending` and fire after the gate
// is released — so a startup-recovery activation event can call back into
// the engine without deadlocking.
foreach (var (alarmId, state) in _alarms)
if (_runner is null)
{
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
var seed = persisted ?? state.Condition;
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
.ConfigureAwait(false);
_alarms[alarmId] = state with { Condition = afterPredicate };
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
foreach (var (alarmId, state) in _alarms)
{
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
var seed = persisted ?? state.Condition;
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
.ConfigureAwait(false);
_alarms[alarmId] = state with { Condition = afterPredicate };
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
}
}
else
{
// Out of process: every predicate goes in one batch, then each alarm is applied and saved in the
// same order the serial loop above uses.
var seeds = new List<(AlarmState State, AlarmConditionState Seed)>(_alarms.Count);
foreach (var (alarmId, state) in _alarms)
seeds.Add((state, await _store.LoadAsync(alarmId, ct).ConfigureAwait(false) ?? state.Condition));
var after = await EvaluatePredicateBatchToStatesAsync(seeds, _clock(), ct, pending).ConfigureAwait(false);
for (var i = 0; i < seeds.Count; i++)
{
var state = seeds[i].State;
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
}
}
_loaded = true;
@@ -614,7 +674,12 @@ public sealed class ScriptedAlarmEngine : IDisposable
if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
_valueCache[path] = value;
if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
MarkDirty(alarmIds);
}
/// <summary>Marks <paramref name="alarmIds"/> dirty and starts the drainer if none is running.</summary>
private void MarkDirty(IEnumerable<string> alarmIds)
{
bool startDrain;
lock (_dirtyLock)
{
@@ -754,17 +819,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
// disposing store or mutating _alarms after clear is unsafe.
if (_disposed) return;
foreach (var id in alarmIds)
if (_runner is not null)
{
if (!_alarms.TryGetValue(id, out var state)) continue;
var newState = await EvaluatePredicateToStateAsync(
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
if (!ReferenceEquals(newState, state.Condition))
// Out of process (lmxopcua#780, design §4.5): the whole dirty set in one batch, applied in order.
await ReevaluateBatchAsync(alarmIds, ct, pending).ConfigureAwait(false);
}
else
{
foreach (var id in alarmIds)
{
// Persist before updating in-memory so a store failure leaves
// both sides at the prior state.
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
_alarms[id] = state with { Condition = newState };
if (!_alarms.TryGetValue(id, out var state)) continue;
var newState = await EvaluatePredicateToStateAsync(
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
if (!ReferenceEquals(newState, state.Condition))
{
// Persist before updating in-memory so a store failure leaves
// both sides at the prior state.
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
_alarms[id] = state with { Condition = newState };
}
}
}
}
@@ -837,7 +910,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
try
{
predicateTrue = await _guard.RunAsync(
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context,
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator!, context,
_scriptTimeout, ct)
.ConfigureAwait(false);
}
@@ -890,6 +963,162 @@ public sealed class ScriptedAlarmEngine : IDisposable
return result.State;
}
/// <summary>
/// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the
/// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in
/// the original order, under <c>_evalGate</c>. Predicates are pure functions of the input snapshot taken here,
/// under the gate, so the state machine sees exactly the sequence the serial loop produces. The batch costs one
/// round trip for the whole set instead of one per alarm, which is what keeps a large engine inside its budget.
/// </summary>
private async Task ReevaluateBatchAsync(IReadOnlyList<string> alarmIds, CancellationToken ct, List<ScriptedAlarmEvent> pending)
{
var items = new List<(AlarmState State, AlarmConditionState Seed)>(alarmIds.Count);
foreach (var id in alarmIds)
if (_alarms.TryGetValue(id, out var state)) items.Add((state, state.Condition));
if (items.Count == 0) return;
var after = await EvaluatePredicateBatchToStatesAsync(items, _clock(), ct, pending).ConfigureAwait(false);
for (var i = 0; i < items.Count; i++)
{
var (state, seed) = items[i];
if (ReferenceEquals(after[i], seed)) continue;
// Persist before updating in-memory so a store failure leaves both sides at the prior state.
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
}
}
/// <summary>
/// <see cref="EvaluatePredicateToStateAsync"/> for a batch, through <see cref="_runner"/>: the same readiness gate,
/// quality tracking and Part 9 application per alarm, in order, with every predicate that has to run submitted
/// together. Failure semantics (design §9): a script fault, timeout, kill or quarantine freezes the condition with a
/// <c>BadInternalError</c> annotation, exactly as in process; a runner failure (worker restarted, none ready) freezes
/// it with <c>BadResourceUnavailable</c> and re-evaluates it once the runner can run it again. A dead worker never
/// clears an alarm: no outcome but a completed <c>bool</c> reaches the state machine.
/// </summary>
private async Task<AlarmConditionState[]> EvaluatePredicateBatchToStatesAsync(
IReadOnlyList<(AlarmState State, AlarmConditionState Seed)> items, DateTime nowUtc, CancellationToken ct,
List<ScriptedAlarmEvent> pendingEmissions)
{
var runner = _runner!;
var after = new AlarmConditionState[items.Count];
var scratches = new AlarmScratch[items.Count];
var worst = new uint[items.Count];
var slot = new int[items.Count];
var invocations = new List<ScriptInvocation>(items.Count);
// Pass 1, no side effects: snapshot each alarm's inputs and decide whether its predicate runs.
for (var i = 0; i < items.Count; i++)
{
var state = items[i].State;
var scratch = scratches[i] = _scratchByAlarmId.GetOrAdd(
state.Definition.AlarmId, _ => new AlarmScratch(state.Inputs, state.Logger, _clock));
RefillReadCache(scratch.ReadCache, state.Inputs);
worst[i] = WorstInputStatus(scratch.ReadCache);
if (!AreInputsReady(scratch.ReadCache))
{
slot[i] = -1;
continue;
}
slot[i] = invocations.Count;
// A copy: the runner may still read the inputs after the caller has its answer (a soft timeout), and the
// scratch is refilled by the next pass.
invocations.Add(new ScriptInvocation(state.Definition.AlarmId, ScriptContextKind.AlarmPredicate,
state.Definition.PredicateScriptSource,
new Dictionary<string, DataValueSnapshot>(scratch.ReadCache, StringComparer.Ordinal), _scriptTimeout, state.Logger));
}
var outcomes = invocations.Count == 0 ? [] : runner.RunBatch(invocations, ct);
PredicateBatchSubmittedForTest?.Invoke(invocations.Count);
// Pass 2, in order: exactly what the serial path does with each alarm.
for (var i = 0; i < items.Count; i++)
{
var (state, seed) = items[i];
var scratch = scratches[i];
if (slot[i] < 0)
{
if (TrackQualityBucket(state.Definition.AlarmId, worst[i]))
pendingEmissions.Add(BuildQualityEmission(state, seed, worst[i]));
after[i] = seed;
continue;
}
var outcome = await outcomes[slot[i]].ConfigureAwait(false);
after[i] = ApplyPredicateOutcome(state, seed, scratch, worst[i], outcome, nowUtc, ct, pendingEmissions);
}
return after;
}
/// <summary>One runner outcome onto one alarm (the second half of <see cref="EvaluatePredicateToStateAsync"/>).</summary>
private AlarmConditionState ApplyPredicateOutcome(AlarmState state, AlarmConditionState seed, AlarmScratch scratch,
uint worstStatus, ScriptRunOutcome outcome, DateTime nowUtc, CancellationToken ct, List<ScriptedAlarmEvent> pendingEmissions)
{
switch (outcome.Status)
{
case ScriptRunStatus.Unavailable:
// Not the script's fault: frozen, annotated BadResourceUnavailable, and evaluated again (against the
// inputs current then) when its worker is back.
ArmRecovery(state.Definition.AlarmId, outcome.Recovery);
return PredicateDidNotComplete(state, seed, scratch, outcome.Reason ?? ScriptRunOutcome.RunnerUnavailableReason,
pendingEmissions, StatusBadResourceUnavailable);
case ScriptRunStatus.Failed:
var reason = outcome.Reason ?? "evaluator failure";
if (ct.IsCancellationRequested) throw new OperationCanceledException(ct);
if (reason.StartsWith(ScriptThrewPrefix, StringComparison.Ordinal))
{
// As in process: a predicate that threw logs an Error and keeps its state, with no quality annotation.
scratch.LastNonCompletion = null;
state.Logger.Error("Alarm predicate threw — state unchanged: {Reason}", reason[ScriptThrewPrefix.Length..]);
return seed;
}
// Timed out, killed, refused, quarantined, unsupported type, did not compile: frozen + BadInternalError.
return PredicateDidNotComplete(state, seed, scratch, reason, pendingEmissions);
}
if (outcome.Value is not bool predicateTrue)
{
scratch.LastNonCompletion = null;
state.Logger.Error("Alarm predicate returned {Type} instead of a bool — state unchanged", outcome.Value?.GetType().Name ?? "null");
return seed;
}
scratch.LastNonCompletion = null;
// From here on, identical to the serial path: the bucket is tracked only once the predicate completed (I3).
var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus);
var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc);
var transition = result.Emission != EmissionKind.None
? BuildEmission(state, result.State, result.Emission, worstStatus)
: null;
if (transition is not null) pendingEmissions.Add(transition);
else if (qualityBucketChanged) pendingEmissions.Add(BuildQualityEmission(state, result.State, worstStatus));
return result.State;
}
/// <summary>The prefix the runners put on a predicate that threw (the in-process path catches the exception instead).</summary>
private const string ScriptThrewPrefix = "script threw: ";
/// <summary>
/// Re-evaluate <paramref name="alarmId"/> once when <paramref name="recovery"/> completes — the runner's worker is
/// back (design §9 "re-evaluated on return"). At most one wait per alarm.
/// </summary>
private void ArmRecovery(string alarmId, Task? recovery)
{
if (recovery is null) return;
lock (_dirtyLock)
{
if (_disposed || !_recoveryArmed.Add(alarmId)) return;
}
_ = recovery.ContinueWith(_ =>
{
lock (_dirtyLock) _recoveryArmed.Remove(alarmId);
if (!_disposed) MarkDirty([alarmId]);
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
/// <summary>Test hook: the number of predicates each out-of-process batch pass submitted.</summary>
internal Action<int>? PredicateBatchSubmittedForTest { get; set; }
/// <summary>
/// The predicate did not return (timed out, refused because the previous run is still executing, or
/// quarantined): hold the condition at <paramref name="seed"/>, log the reason <b>once per change of
@@ -900,15 +1129,15 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// </summary>
private AlarmConditionState PredicateDidNotComplete(
AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason,
List<ScriptedAlarmEvent> pendingEmissions)
List<ScriptedAlarmEvent> pendingEmissions, uint status = StatusBadInternalError)
{
if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal))
{
scratch.LastNonCompletion = reason;
state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason);
}
if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError))
pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError));
if (TrackQualityBucket(state.Definition.AlarmId, status))
pendingEmissions.Add(BuildQualityEmission(state, seed, status));
return seed;
}
@@ -1339,7 +1568,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
private sealed record AlarmState(
ScriptedAlarmDefinition Definition,
ScriptEvaluator<AlarmPredicateContext, bool> Evaluator,
ScriptEvaluator<AlarmPredicateContext, bool>? Evaluator, // null out of process: the worker holds the compiled form
IReadOnlySet<string> Inputs,
IReadOnlyList<string> TemplateTokens,
ILogger Logger,
@@ -22,6 +22,7 @@
<ItemGroup>
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests"/>
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests"/>
</ItemGroup>
</Project>
@@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
int compileThreads = 0)
int compileThreads = 0, int evalThreads = 0)
{
_compileThreads = compileThreads;
ArgumentNullException.ThrowIfNull(options);
@@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
// Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
// queue-behind-a-busy-thread case deterministic).
if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
_shards = new WorkerShard[_worker.Shards];
for (var i = 0; i < _shards.Length; i++)
{
@@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
/// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
internal Action<string>? SlotReleasingForTest { get; set; }
/// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
internal Action<int>? BatchSubmittedForTest { get; set; }
/// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
internal IReadOnlyCollection<string> SuspectKeys =>
_keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
@@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
// run by then.
var heldGate = gate;
var released = 0;
void ReleaseSlot()
{
if (Interlocked.Exchange(ref released, 1) != 0) return;
SlotReleasingForTest?.Invoke(key);
Interlocked.Exchange(ref state.InFlight, 0);
heldGate?.Release();
}
var submitted = Stopwatch.GetTimestamp();
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
_worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
_worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
handedOff = true;
return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return ScriptRunOutcome.Failed("evaluation cancelled");
}
finally
{
if (!handedOff)
{
Interlocked.Exchange(ref state.InFlight, 0);
gate?.Release();
}
}
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
// shard has no live worker (or for the isolation gate).
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
}
/// <inheritdoc />
/// <remarks>
/// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
/// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
/// <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
/// bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
/// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
/// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
/// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
/// <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
/// </remarks>
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(invocations);
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
for (var i = 0; i < invocations.Count; i++)
{
var invocation = invocations[i];
ArgumentNullException.ThrowIfNull(invocation);
switch (TryPrepareBatchItem(invocation, out var item, out var answered))
{
case BatchPreparation.Submit:
groups ??= [];
if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
group.Add((i, item));
break;
case BatchPreparation.Answered:
outcomes[i] = Task.FromResult(answered!);
break;
default:
outcomes[i] = RunAsync(invocation, ct);
break;
}
}
if (groups is null) return outcomes;
foreach (var (connection, group) in groups)
{
var submissions = new BatchSubmission[group.Count];
for (var j = 0; j < group.Count; j++)
{
var item = group[j].Item;
submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
}
var submitted = Stopwatch.GetTimestamp();
BatchSubmittedForTest?.Invoke(submissions.Length);
var pendings = connection.SubmitBatch(submissions);
for (var j = 0; j < group.Count; j++)
{
var (index, item) = group[j];
outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
}
}
return outcomes;
}
/// <summary>
/// The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
/// where nothing has to be waited for. Takes the key's single-flight slot when it returns
/// <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
/// </summary>
private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
{
item = null;
answered = null;
if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
var state = Admit(key, invocation.Kind, invocation.Source);
if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
{
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
return BatchPreparation.Answered;
}
state.ScriptLogger = invocation.ScriptLogger;
NoteLoad(entry);
item = new BatchItem(key, state, invocation, shard, connection, entry);
return BatchPreparation.Submit;
}
/// <summary>
/// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
/// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
/// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
/// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
/// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
/// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
/// quarantine, suspect) has already run by then.
/// </summary>
private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
{
var released = 0;
return () =>
{
if (Interlocked.Exchange(ref released, 1) != 0) return;
SlotReleasingForTest?.Invoke(key);
Interlocked.Exchange(ref state.InFlight, 0);
heldGate?.Release();
};
}
/// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
{
try
{
var result = ct.CanBeCanceled
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
: await pending.Outcome.ConfigureAwait(false);
@@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
{
return ScriptRunOutcome.Failed("evaluation cancelled");
}
finally
{
if (!handedOff)
{
Interlocked.Exchange(ref state.InFlight, 0);
gate?.Release();
}
}
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
// shard has no live worker (or for the isolation gate).
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
}
/// <summary>
@@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
private enum BatchPreparation
{
/// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
Full,
/// <summary>Answered without running (refused).</summary>
Answered,
/// <summary>Submit it with the batch; its slot is taken.</summary>
Submit,
}
private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
private sealed class KeyState(string source, ScriptContextKind kind)
{
private byte[]? _sha;
@@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
}
}
/// <summary>One evaluation of a <see cref="WorkerConnection.SubmitBatch"/>.</summary>
/// <param name="Key">The scoped key.</param>
/// <param name="Request">The request (its id is assigned on submit).</param>
/// <param name="SoftTimeout">T.</param>
/// <param name="HardGrace">G.</param>
/// <param name="OnFinished">Runs once when the worker has answered or ended, before the caller resumes (the slot release).</param>
internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished);
/// <summary>What the host needs to launch one worker.</summary>
/// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
/// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
@@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable
return pending;
}
/// <summary>
/// Submits several evaluations as ONE <see cref="EvalBatch"/> frame (lmxopcua#780 batch submit, design §4.5). Each
/// item gets its own deadlines and slot release exactly as through <see cref="Submit(string, EvalRequest, TimeSpan?, TimeSpan, Action?)"/>;
/// an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts.
/// </summary>
/// <param name="items">The evaluations (their ids are assigned here).</param>
/// <returns>One pending evaluation per item, in order.</returns>
public PendingEval[] SubmitBatch(IReadOnlyList<BatchSubmission> items)
{
ArgumentNullException.ThrowIfNull(items);
var pendings = new PendingEval[items.Count];
var batch = new EvalBatch();
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
for (var i = 0; i < items.Count; i++)
{
var item = items[i];
item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher);
pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline);
_pending[item.Request.EvalId] = pending;
batch.Items.Add(item.Request);
pendings[i] = pending;
}
if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted))
{
foreach (var pending in pendings)
if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null);
}
return pendings;
}
/// <summary>
/// The compiled form of <paramref name="key"/> in this worker when it is loaded at <paramref name="sourceHashHex"/>
/// with <paramref name="inputPaths"/> and compiled cleanly — the batch fast path, which never waits for a load.
/// </summary>
/// <param name="key">The scoped key.</param>
/// <param name="sourceHashHex">The source hash (hex).</param>
/// <param name="inputPaths">The slot order.</param>
/// <returns>The entry, or null when the key needs a (re)load or its load failed.</returns>
public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList<string> inputPaths)
=> _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths)
&& existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut
? existing
: null;
private void OnHardDeadline(PendingEval pending)
{
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
@@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
// trusts"), except for one that was already completed by a kill race — ignored then.
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
else CheckIssued(result.EvalId, "result");
}
// Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines
// restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time.
// The two notices are unordered with respect to each other (see PendingEval.Start).
foreach (var id in frame.Results.StartedEvalIds)
{
if (_pending.TryGetValue(id, out var started)) started.Start();
else CheckIssued(id, "start notice");
}
foreach (var id in frame.Results.DeferredEvalIds)
{
if (_pending.TryGetValue(id, out var deferred)) deferred.Defer();
else CheckIssued(id, "deferral notice");
}
break;
case Envelope.BodyOneofCase.LoadStarted:
@@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
}
}
/// <summary>An id the host never issued is a protocol violation; one it issued and already completed is a benign race.</summary>
private void CheckIssued(ulong evalId, string what)
{
if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId))
throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}");
}
private async Task PipeEndedAsync(WorkerExitReason? fault)
{
// The pipe is gone: the worker is dead or dying. Make sure, then report.
@@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
/// <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
/// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
/// </summary>
/// <remarks>
/// The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind
/// busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full
/// T, then G, from the start — without a stale tick firing early.
/// </remarks>
internal sealed class PendingEval
{
/// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
@@ -926,9 +1003,15 @@ internal sealed class PendingEval
private readonly TaskCompletionSource<EvalResult?> _outcome;
private readonly ResultDispatcher? _dispatcher;
private readonly Action? _onFinished;
private readonly object _clock = new();
private Timer? _timer;
private Action<PendingEval>? _onHardDeadline;
private TimeSpan _soft;
private TimeSpan _grace;
private long _softDue;
private long _hardDue;
private bool _deferred;
private bool _started;
private int _phase;
private int _done;
@@ -954,6 +1037,12 @@ internal sealed class PendingEval
/// <summary>T + G, for the kill log line.</summary>
public TimeSpan HardDeadline { get; private set; }
/// <summary>Whether the worker reported this run as queued and has not reported it started (tests / diagnostics).</summary>
internal bool IsDeferred
{
get { lock (_clock) return _deferred; }
}
/// <summary>The worker's result, or null when the worker ended first.</summary>
public Task<EvalResult?> Completion => _completion.Task;
@@ -966,27 +1055,96 @@ internal sealed class PendingEval
/// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
{
_soft = soft;
_grace = grace;
_onHardDeadline = onHardDeadline;
HardDeadline = soft + grace;
var now = Stopwatch.GetTimestamp();
_softDue = now + Ticks(soft);
_hardDue = _softDue + Ticks(grace);
// Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
_timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
_timer.Change(soft, Timeout.InfiniteTimeSpan);
}
/// <summary>
/// The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running,
/// so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire.
/// </summary>
internal void Defer()
{
lock (_clock)
{
if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return;
_deferred = true;
TryChange(Timeout.InfiniteTimeSpan);
}
}
/// <summary>
/// The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline
/// had already been reported). A run that was never deferred keeps the clock it has had since submit.
/// </summary>
internal void Start()
{
lock (_clock)
{
_started = true;
if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return;
_deferred = false;
var now = Stopwatch.GetTimestamp();
if (Volatile.Read(ref _phase) == 0)
{
_softDue = now + Ticks(_soft);
_hardDue = _softDue + Ticks(_grace);
TryChange(_soft);
}
else
{
// Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets
// the whole T + G from its start before it can be blamed for a kill.
_hardDue = now + Ticks(_soft + _grace);
TryChange(_soft + _grace);
}
}
}
private void OnTimer()
{
if (Volatile.Read(ref _done) != 0) return;
if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
bool soft = false, hard = false;
lock (_clock)
{
_outcome.TrySetResult(SoftTimedOut);
try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
catch (ObjectDisposedException) { /* answered meanwhile */ }
return;
if (_deferred || Volatile.Read(ref _done) != 0) return;
var now = Stopwatch.GetTimestamp();
if (Volatile.Read(ref _phase) == 0)
{
// A tick armed before a deferral/start can land early: re-arm for what is left.
if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; }
Interlocked.Exchange(ref _phase, 1);
soft = true;
TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero);
}
else if (Volatile.Read(ref _phase) == 1)
{
if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; }
Interlocked.Exchange(ref _phase, 2);
hard = true;
}
}
if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
// Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
if (soft) _outcome.TrySetResult(SoftTimedOut);
if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
}
private void TryChange(TimeSpan due)
{
try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); }
catch (ObjectDisposedException) { /* answered meanwhile */ }
}
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
/// <summary>
/// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
/// reader, then hand the caller's resumption to the dispatcher (or the pool).
@@ -134,8 +134,17 @@ message EvalResult {
uint32 run_micros = 6;
}
// Per-item deadlines are exact for queued work too (lmxopcua#780 batch submit): the host arms an evaluation's soft and
// hard deadlines when it submits, which is when the run starts as long as the worker has an idle evaluation thread. An
// evaluation that arrives while every thread is busy (a batch larger than the scheduler, or work queued behind a
// runaway) is reported in `deferred_eval_ids`; the host then stops its clock, and restarts it — the full T, then G —
// when the evaluation appears in `started_eval_ids`, which an evaluation thread sends alongside the result of the run it
// just finished. So a run waiting behind a sibling is never timed out, struck or killed for the sibling's time.
// Either list may be the only content of a frame. A host that predates these fields ignores them.
message EvalResults {
repeated EvalResult items = 1;
repeated uint64 started_eval_ids = 2;
repeated uint64 deferred_eval_ids = 3;
}
message ScriptError {
@@ -233,7 +233,10 @@ public sealed class ScriptWorkerFrameWriter
break;
case Envelope.BodyOneofCase.Results:
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
(results ??= new EvalResults()).Items.AddRange(envelope.Results.Items);
results ??= new EvalResults();
results.Items.AddRange(envelope.Results.Items);
results.StartedEvalIds.AddRange(envelope.Results.StartedEvalIds);
results.DeferredEvalIds.AddRange(envelope.Results.DeferredEvalIds);
break;
default:
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
@@ -28,6 +28,31 @@ public interface IScriptRunner
/// <returns>The outcome; never faults for a script failure.</returns>
Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default);
/// <summary>
/// Submits several evaluations at once (design §4.5): the out-of-process runner sends every one that is ready to
/// run as ONE request frame per worker, instead of one frame and one submission per evaluation. Results are
/// per item and streamed — each task completes when its own evaluation does, so a slow item never holds a fast
/// sibling's result — and every item keeps exactly the semantics of <see cref="RunAsync"/>: its own budget, its
/// own single-flight slot, and its own blame (a runaway item is killed and struck alone; an item queued behind
/// it is neither timed out nor struck for its time).
/// </summary>
/// <remarks>
/// The default implementation — what <see cref="InProcessScriptRunner"/> uses — is <see cref="RunAsync"/> per
/// item, started together. A key that appears twice in one batch is refused for its second item exactly as two
/// concurrent <see cref="RunAsync"/> calls would be; callers that may repeat a key split the batch there.
/// Never throws for script faults; the returned tasks never fault.
/// </remarks>
/// <param name="invocations">What to run, in order.</param>
/// <param name="ct">Caller cancellation (shutdown); wins over every item's budget.</param>
/// <returns>One task per invocation, in the same order.</returns>
IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(invocations);
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
for (var i = 0; i < outcomes.Length; i++) outcomes[i] = RunAsync(invocations[i], ct);
return outcomes;
}
/// <summary>
/// The deploy boundary for one seam: lifts every quarantine and forgets every remembered compile failure of
/// <paramref name="kind"/>, and drops its compiled scripts.
@@ -50,7 +75,7 @@ public enum ScriptContextKind
/// <summary>A Calculation tag: <c>VirtualTagContext</c> → <c>object?</c>.</summary>
Calculation = 2,
/// <summary>A scripted-alarm predicate: <c>AlarmPredicateContext</c> → <c>bool</c> (not run out of process yet).</summary>
/// <summary>A scripted-alarm predicate: <c>AlarmPredicateContext</c> → <c>bool</c>.</summary>
AlarmPredicate = 3,
}
@@ -142,3 +167,16 @@ public static class ScriptRunOutcomeExtensions
};
}
}
/// <summary>
/// Which <see cref="IScriptRunner"/> each script seam uses on this node (lmxopcua#780), for components that cannot
/// reference the Host's selection directly (the Runtime's scripted-alarm host). Null for a seam means its in-process
/// guard path.
/// </summary>
public interface IScriptRunnerProvider
{
/// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
/// <param name="seam">The seam.</param>
/// <returns>The runner, or null.</returns>
IScriptRunner? ForSeam(ScriptSeam seam);
}
@@ -0,0 +1,106 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// <summary>
/// Opportunistic batching for a seam whose evaluations are submitted independently — one virtual-tag actor each —
/// but often become ready together (one upstream burst fans out to many tags): evaluations submitted while another
/// submission is in progress are handed to <see cref="IScriptRunner.RunBatch"/> together, in arrival order, by the
/// submitter already holding the flush (lmxopcua#780, design §4.5).
/// </summary>
/// <remarks>
/// <para>There is no timer: an evaluation on an idle seam is submitted at once by its own caller, so batching never
/// adds latency. Per-item semantics are <see cref="IScriptRunner.RunBatch"/>'s — each item keeps its own budget, slot
/// and blame.</para>
/// <para>Each caller's task completes synchronously on the thread that completes its run (the runner's result
/// dispatcher), exactly as awaiting <see cref="IScriptRunner.RunAsync"/> directly does, so there is no extra thread
/// hop.</para>
/// <para>A key submitted twice concurrently is refused for the second exactly as two concurrent
/// <see cref="IScriptRunner.RunAsync"/> calls would be; the virtual-tag actor never has two evaluations in flight.</para>
/// </remarks>
public sealed class ScriptBatchCoalescer
{
private readonly IScriptRunner _runner;
private readonly object _lock = new();
private List<(ScriptInvocation Invocation, TaskCompletionSource<ScriptRunOutcome> Done)> _queue = [];
private bool _flushing;
/// <summary>Initializes the coalescer.</summary>
/// <param name="runner">The runner every batch goes to.</param>
public ScriptBatchCoalescer(IScriptRunner runner)
{
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
}
/// <summary>Test hook: runs once per <see cref="IScriptRunner.RunBatch"/> call, with the batch size.</summary>
internal Action<int>? BatchSubmittedForTest { get; set; }
/// <summary>Submits one evaluation, together with any submitted concurrently.</summary>
/// <param name="invocation">What to run.</param>
/// <returns>The outcome; never faults.</returns>
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation)
{
ArgumentNullException.ThrowIfNull(invocation);
// Synchronous continuations: the caller resumes where its run completed, as with a direct await of RunAsync.
var done = new TaskCompletionSource<ScriptRunOutcome>(TaskCreationOptions.None);
lock (_lock)
{
_queue.Add((invocation, done));
if (_flushing) return done.Task;
_flushing = true;
}
Flush();
return done.Task;
}
private void Flush()
{
while (true)
{
List<(ScriptInvocation Invocation, TaskCompletionSource<ScriptRunOutcome> Done)> batch;
lock (_lock)
{
if (_queue.Count == 0)
{
_flushing = false;
return;
}
batch = _queue;
_queue = [];
}
IReadOnlyList<Task<ScriptRunOutcome>> outcomes;
try
{
var invocations = new ScriptInvocation[batch.Count];
for (var i = 0; i < invocations.Length; i++) invocations[i] = batch[i].Invocation;
BatchSubmittedForTest?.Invoke(invocations.Length);
outcomes = _runner.RunBatch(invocations);
}
catch (Exception ex)
{
// A defect in the runner must not strand the callers (nor the flush flag).
foreach (var (_, done) in batch) done.TrySetResult(ScriptRunOutcome.Failed($"script runner threw: {ex.Message}"));
continue;
}
for (var i = 0; i < batch.Count; i++) Link(outcomes[i], batch[i].Done);
}
}
private static void Link(Task<ScriptRunOutcome> outcome, TaskCompletionSource<ScriptRunOutcome> done)
{
if (outcome.IsCompleted)
{
Settle(outcome, done);
return;
}
outcome.ContinueWith(static (t, s) => Settle(t, (TaskCompletionSource<ScriptRunOutcome>)s!), done,
CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
private static void Settle(Task<ScriptRunOutcome> outcome, TaskCompletionSource<ScriptRunOutcome> done)
{
if (outcome.IsCompletedSuccessfully) done.TrySetResult(outcome.Result);
else done.TrySetResult(ScriptRunOutcome.Failed(
$"script runner threw: {outcome.Exception?.GetBaseException().Message ?? "cancelled"}"));
}
}
@@ -32,7 +32,10 @@ public enum ScriptSeam
/// <summary>Calculation tags (<c>CalculationEvaluator</c>).</summary>
Calculation = 1,
/// <summary>Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10).</summary>
/// <summary>
/// Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10): out of process only
/// when <c>Scripting:Runner:Seams</c> lists it — an unset list does not select it.
/// </summary>
ScriptedAlarm = 2,
}
@@ -80,7 +80,8 @@ public sealed class ScriptingOptions
/// <summary>
/// The seams <see cref="Runner"/> applies to when it is not <see cref="ScriptRunnerMode.InProcess"/>
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means all three. Bound by
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means VirtualTag and Calculation:
/// scripted alarms run out of process only when listed explicitly (design §14 #10, alarms last). Bound by
/// <c>ScriptingOptionsBinding</c> from the documented key, because <c>Scripting:Runner</c> is itself a value.
/// </summary>
public IReadOnlyList<ScriptSeam>? RunnerSeams { get; set; }
@@ -91,10 +92,10 @@ public sealed class ScriptingOptions
/// <summary>Whether <paramref name="seam"/> is selected for out-of-process execution by these options.</summary>
/// <param name="seam">The seam.</param>
/// <returns><c>true</c> when <see cref="Runner"/> is <see cref="ScriptRunnerMode.OutOfProcess"/> and the seam is in
/// <see cref="RunnerSeams"/> (or that list is empty).</returns>
/// <see cref="RunnerSeams"/> (or that list is empty and the seam is not <see cref="ScriptSeam.ScriptedAlarm"/>).</returns>
public bool IsOutOfProcess(ScriptSeam seam)
=> Runner == ScriptRunnerMode.OutOfProcess
&& (RunnerSeams is not { Count: > 0 } seams || seams.Contains(seam));
&& (RunnerSeams is { Count: > 0 } seams ? seams.Contains(seam) : seam != ScriptSeam.ScriptedAlarm);
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="StuckOrphanQuarantineFactor"/>.</summary>
public const double MaxStuckOrphanQuarantineFactor = 1000;
@@ -26,6 +26,8 @@
<ItemGroup>
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests"/>
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests"/>
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Host.Tests"/>
</ItemGroup>
</Project>
@@ -522,6 +522,14 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
/// false timeouts under pool starvation).</summary>
private async Task RunEvaluationPumpAsync()
{
// lmxopcua#780 (design §4.5): with an out-of-process runner a run costs a pipe round trip, so the pump submits
// everything queued as one batch and publishes in dequeue order. In process it stays one at a time, unchanged.
if (_evaluator.SubmitsBatches)
{
await RunEvaluationBatchPumpAsync().ConfigureAwait(false);
return;
}
while (true)
{
PendingEvaluation next;
@@ -557,6 +565,73 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
}
}
/// <summary>
/// The pump on the out-of-process path (lmxopcua#780, design §4.5): dequeue everything pending — each entry's
/// dependency snapshot was captured at enqueue — submit it as ONE batch, and publish in dequeue order. A calc tag
/// queued more than once (a burst of changes) ends the batch at its second entry, so every queued evaluation still
/// runs, in order, against the snapshot that triggered it, exactly as the one-at-a-time pump would run them.
/// </summary>
private async Task RunEvaluationBatchPumpAsync()
{
while (true)
{
List<PendingEvaluation> batch;
lock (_lock)
{
if (!_running) _pending.Clear();
if (_pending.Count == 0)
{
_pumpScheduled = false;
return;
}
batch = [];
var inBatch = new HashSet<string>(StringComparer.Ordinal);
while (_pending.Count > 0 && inBatch.Add(_pending.Peek().Definition.RawPath)) batch.Add(_pending.Dequeue());
}
if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
try
{
await RunEvaluationBatchAsync(batch).ConfigureAwait(false);
}
catch (Exception ex)
{
// Defect path only (the evaluator never throws for script faults); must not strand the queue.
_logger.LogError(ex, "Calculation {Driver}: batch evaluation of {Count} tag(s) failed", _driverInstanceId, batch.Count);
}
finally
{
_evalGate.Release();
}
}
}
/// <summary>Runs one batch and publishes every outcome in dequeue order. Under <see cref="_evalGate"/>, not <see cref="_lock"/>.</summary>
/// <param name="batch">The prepared evaluations; no calc tag twice.</param>
private async Task RunEvaluationBatchAsync(IReadOnlyList<PendingEvaluation> batch)
{
// Hold-on-Bad entries never reach the runner; the rest are submitted together.
var requests = new List<(string, string, IReadOnlyDictionary<string, DataValueSnapshot>)>(batch.Count);
var slot = new int[batch.Count];
for (var i = 0; i < batch.Count; i++)
{
if (!InputQuality.IsUsable(batch[i].WorstInputStatus)) { slot[i] = -1; continue; }
slot[i] = requests.Count;
requests.Add((batch[i].Definition.RawPath, batch[i].Definition.ScriptSource, batch[i].Dependencies));
}
var results = _evaluator.EvaluateBatch(requests, _evalCts.Token);
BatchSubmittedForTest?.Invoke(requests.Count);
for (var i = 0; i < batch.Count; i++)
{
if (slot[i] < 0) HoldOnBadInput(batch[i]);
else ApplyEvaluationResult(batch[i], await results[slot[i]].ConfigureAwait(false));
}
}
/// <summary>Test hook: the number of compiled evaluations each batch pass submitted.</summary>
internal Action<int>? BatchSubmittedForTest { get; set; }
/// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
/// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
/// completes).</summary>
@@ -610,16 +685,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
// sub-code. Uncertain IS usable: the script runs and the output inherits the Uncertain code.
if (!InputQuality.IsUsable(pending.WorstInputStatus))
{
(object? Value, uint StatusCode)? badPublish;
lock (_lock)
{
if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
}
// No script log: an input dying is not a script fault, and the script log is the per-failure
// SCRIPT diagnostic. Here the status code is the diagnostic.
if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
HoldOnBadInput(pending);
return;
}
@@ -627,7 +693,32 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
// then the input's real ones rather than a fabricated Good stamped at evaluation time.
var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
.ConfigureAwait(false);
ApplyEvaluationResult(pending, result);
}
/// <summary>A Bad input holds the script: publish the input's own status (once per transition), no script log.</summary>
/// <param name="pending">The prepared evaluation whose worst input is Bad.</param>
private void HoldOnBadInput(PendingEvaluation pending)
{
var def = pending.Definition;
(object? Value, uint StatusCode)? badPublish;
lock (_lock)
{
if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
}
// No script log: an input dying is not a script fault, and the script log is the per-failure
// SCRIPT diagnostic. Here the status code is the diagnostic.
if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
}
/// <summary>Record one evaluation's outcome and publish it. Under <see cref="_evalGate"/>, not <see cref="_lock"/>.</summary>
/// <param name="pending">The prepared evaluation.</param>
/// <param name="result">Its outcome.</param>
private void ApplyEvaluationResult(PendingEvaluation pending, VirtualTagEvalResult result)
{
var def = pending.Definition;
string? failureToLog = null;
(object? Value, uint StatusCode)? toPublish;
lock (_lock)
@@ -283,6 +283,62 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
}
}
/// <summary>
/// Whether compiled runs go to an <see cref="IScriptRunner"/> (the out-of-process seam), so the driver's pump can
/// submit everything it has dequeued as one batch (lmxopcua#780, design §4.5). False on the in-process path, whose
/// pump stays one evaluation at a time, unchanged.
/// </summary>
public bool SubmitsBatches => _runner is not null && !_disposed;
/// <summary>
/// Evaluates several calc tags at once (design §4.5): the passthrough fast path and the trivial failures are answered
/// inline exactly as <see cref="EvaluateAsync"/> answers them, and every compiled run is handed to the runner in ONE
/// <see cref="IScriptRunner.RunBatch"/>. Results are per item; each item keeps <see cref="EvaluateAsync"/>'s
/// outcome, budget and blame. Without a runner it is <see cref="EvaluateAsync"/> per item, in order.
/// </summary>
/// <param name="requests">The evaluations: calc-tag id, script source, dependency snapshots. A calc tag must not appear twice.</param>
/// <param name="ct">Caller cancellation (driver shutdown).</param>
/// <returns>One task per request, in order; none faults.</returns>
public IReadOnlyList<Task<VirtualTagEvalResult>> EvaluateBatch(
IReadOnlyList<(string CalcTagId, string ScriptSource, IReadOnlyDictionary<string, DataValueSnapshot> Dependencies)> requests,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(requests);
var results = new Task<VirtualTagEvalResult>[requests.Count];
if (_runner is null || _disposed)
{
for (var i = 0; i < requests.Count; i++)
results[i] = EvaluateAsync(requests[i].CalcTagId, requests[i].ScriptSource, requests[i].Dependencies, ct);
return results;
}
List<ScriptInvocation>? invocations = null;
List<int>? indexes = null;
for (var i = 0; i < requests.Count; i++)
{
var (id, source, deps) = requests[i];
// Exactly the fast answers EvaluateAsync gives before it reaches the runner.
if (string.IsNullOrWhiteSpace(source) || PassthroughScript.TryMatch(source, out _))
{
results[i] = EvaluateAsync(id, source, deps, ct);
continue;
}
var scriptLog = _scriptRoot
.ForContext(ScriptLoggerFactory.ScriptIdProperty, id)
.ForContext(ScriptLoggerFactory.VirtualTagIdProperty, id);
(invocations ??= []).Add(new ScriptInvocation(id, ScriptContextKind.Calculation, source, deps, _runTimeout, scriptLog));
(indexes ??= []).Add(i);
}
if (invocations is null) return results;
var outcomes = _runner.RunBatch(invocations, ct);
for (var j = 0; j < outcomes.Count; j++) results[indexes![j]] = ToResultAsync(outcomes[j]);
return results;
static async Task<VirtualTagEvalResult> ToResultAsync(Task<ScriptRunOutcome> outcome)
=> (await outcome.ConfigureAwait(false)).ToVirtualTagEvalResult();
}
/// <summary>Record a compile failure for <paramref name="sourceKey"/> and log it ONCE — the next trigger
/// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning.</summary>
private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what)
@@ -12,10 +12,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
/// workers.
/// </summary>
public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
{
/// <summary>Seams the out-of-process runner can serve in this build. Scripted alarms go last (design §14 #10).</summary>
public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
/// <summary>Seams the out-of-process runner can serve in this build.</summary>
public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams =
[ScriptSeam.VirtualTag, ScriptSeam.Calculation, ScriptSeam.ScriptedAlarm];
/// <summary>
/// The seams an unset or empty <c>Scripting:Runner:Seams</c> selects. Scripted alarms are not among them: they go
/// out of process only when listed explicitly, last in the rollout (design §14 #10 — the alarm budget and the
/// engine refactor).
/// </summary>
public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
private readonly OutOfProcessScriptRunner? _runner;
private readonly HashSet<ScriptSeam> _seams;
@@ -94,5 +102,5 @@ public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
public void Dispose() => _runner?.Dispose();
private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : Enum.GetValues<ScriptSeam>();
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;
}
@@ -54,6 +54,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
private readonly TimeSpan _runTimeout;
private readonly ScriptRunGuard _guard;
private readonly IScriptRunner? _runner;
private readonly ScriptBatchCoalescer? _batches;
private bool _disposed;
/// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
@@ -79,6 +80,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
_runTimeout = runTimeout ?? (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
_guard = new ScriptRunGuard(scripting, _scriptRoot);
_runner = scriptRunner;
_batches = scriptRunner is null ? null : new ScriptBatchCoalescer(scriptRunner);
}
/// <summary>The out-of-process runner this evaluator delegates compiled runs to, or null (in process).</summary>
@@ -87,6 +89,9 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
/// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
public ScriptRunGuard RunGuard => _guard;
/// <summary>The coalescer the out-of-process path submits through, or null in process (tests).</summary>
internal ScriptBatchCoalescer? Batches => _batches;
// Value-only overload. Every dependency reaches the script stamped Good with an
// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
// overload instead; this one remains for callers that genuinely hold only
@@ -161,9 +166,11 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
// lmxopcua#780 — OutOfProcess: the compiled run goes to the pre-emptible worker pool, which compiles in the
// worker and owns timeouts, kills and quarantine for this seam. An Unavailable outcome (worker restarted, no
// worker ready) publishes BadResourceUnavailable; everything else keeps today's Bad contract.
if (_runner is not null)
// Every tag has its own actor, so tags an upstream burst made ready together arrive here concurrently; the
// coalescer submits those as one batch (design §4.5) and an evaluation on an idle seam at once.
if (_batches is not null)
{
var outcome = await _runner.RunAsync(new ScriptInvocation(
var outcome = await _batches.RunAsync(new ScriptInvocation(
virtualTagId, ScriptContextKind.VirtualTag, expression, readCache, _runTimeout, scriptLog)).ConfigureAwait(false);
return outcome.ToVirtualTagEvalResult();
}
@@ -385,6 +385,9 @@ if (hasDriver)
Log.Information("Scripting runner: {RunnerMode}", scriptingOptions.Runner);
builder.Services.AddSingleton(sp => ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection.Create(
sp.GetRequiredService<ScriptingOptions>(), Log.Logger, sp.GetService<ScriptRootLogger>()?.Logger));
// The Runtime's scripted-alarm host resolves its runner through this (it cannot reference the Host).
builder.Services.AddSingleton<IScriptRunnerProvider>(sp =>
sp.GetRequiredService<ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection>());
// lmxopcua#777: the ONE operator-write timeout (DriverWrites:Timeout) every write bound is derived from — the
// driver child's cancel + abandon, its queue budget, DriverHostActor's Ask and the inbound-write gateway's Ask.
@@ -206,6 +206,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <summary>Node-wide script run-guard options for the scripted-alarm engine; null ⇒ compiled defaults.</summary>
private readonly ScriptingOptions? _scripting;
// lmxopcua#780 — the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm; null keeps the engine's
// in-process guard path.
private readonly IScriptRunner? _scriptedAlarmRunner;
/// <summary>
/// Driver children this host has stopped (<see cref="StopChild"/>, an operator Restart) and is watching
/// until they terminate. A value burst from one of them is DROPPED in <see cref="ForwardBatchToMux"/>:
@@ -771,7 +775,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
ScriptingOptions? scripting = null,
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null) =>
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -785,7 +790,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
scripting, ackPreflightView, writeBounds, writeClock));
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -916,8 +921,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
ScriptingOptions? scripting = null,
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null)
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null)
{
_scriptedAlarmRunner = scriptedAlarmRunner;
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
_writeClock = writeClock ?? TimeProvider.System;
_ackPreflightView = ackPreflightView;
@@ -1242,13 +1249,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
var store = _alarmStateStore;
var scriptLogger = _scriptRootLogger.Logger;
var scripting = _scripting;
var alarmRunner = _scriptedAlarmRunner;
_scriptedAlarmHost = Context.ActorOf(
ScriptedAlarmHostActor.Props(
_opcUaPublishActor, _dependencyMux,
() =>
{
var upstream = new DependencyMuxTagUpstreamSource();
return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting));
return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting, alarmRunner));
},
_localNode, telemetryHub: _telemetryHub),
"scripted-alarm-host");
@@ -1265,15 +1273,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="store">The condition-state store.</param>
/// <param name="scriptLogger">The root script logger.</param>
/// <param name="scripting">The node-wide <c>Scripting</c> options; null ⇒ defaults.</param>
/// <param name="scriptRunner">The out-of-process runner for the ScriptedAlarm seam; null ⇒ in process.</param>
/// <returns>A new, unloaded engine.</returns>
internal static ScriptedAlarmEngine CreateScriptedAlarmEngine(
ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.ITagUpstreamSource upstream, IAlarmStateStore store,
Serilog.ILogger scriptLogger, ScriptingOptions? scripting)
Serilog.ILogger scriptLogger, ScriptingOptions? scripting, IScriptRunner? scriptRunner = null)
{
var budget = (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
return new ScriptedAlarmEngine(
upstream, store, new ScriptLoggerFactory(scriptLogger), scriptLogger,
scriptTimeout: budget, scripting: scripting);
scriptTimeout: budget, scripting: scripting, scriptRunner: scriptRunner);
}
private void Bootstrap()
@@ -664,7 +664,11 @@ public static class ServiceCollectionExtensions
// The one configured write timeout (DriverWrites:Timeout, bound + validated in Program.cs) the
// children's write bounds AND this host's write Ask are derived from (lmxopcua#777). Unbound in
// test graphs ⇒ null ⇒ DriverWriteBounds.Default.
writeBounds: resolver.GetService<DriverWriteBounds>()),
writeBounds: resolver.GetService<DriverWriteBounds>(),
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
DriverHostActorName);
registry.Register<DriverHostActorKey>(driverHost);
@@ -34,8 +34,22 @@ internal sealed class WorkerHost
private readonly int _parentPid;
private readonly ConcurrentDictionary<string, LoadedScript> _scripts = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<long, RunningEval> _running = new();
/// <summary>
/// Hand-off to an IDLE evaluation thread: an evaluation is put here only when a thread is known to be waiting
/// for it (<see cref="_idle"/>), so whatever is put here starts at once. Everything else waits in
/// <see cref="_backlog"/> and is reported to the host as deferred (lmxopcua#780 batch submit: per-item deadlines
/// stay exact, see the proto's <c>EvalResults</c>).
/// </summary>
private readonly BlockingCollection<EvalRequest> _queue = new(new ConcurrentQueue<EvalRequest>());
/// <summary>Evaluations that arrived while every evaluation thread was busy, in arrival order. Under <see cref="_schedLock"/>.</summary>
private readonly Queue<EvalRequest> _backlog = new();
private readonly object _schedLock = new();
/// <summary>Evaluation threads committed to taking from <see cref="_queue"/>, minus hand-offs not yet taken. Under <see cref="_schedLock"/>.</summary>
private int _idle;
/// <summary>
/// Loads waiting for a compile thread. Compiles are long, synchronous and CPU-bound, so they run on
/// <see cref="CompileThreads"/> dedicated threads, never on the thread pool: a burst of cold loads (a node start
@@ -225,8 +239,20 @@ internal sealed class WorkerHost
if (_scripts.TryRemove(key, out var gone)) gone.Dispose();
return null;
case Envelope.BodyOneofCase.Eval:
foreach (var item in frame.Eval.Items) Admit(item);
{
List<ulong>? deferred = null;
foreach (var item in frame.Eval.Items)
if (!Admit(item)) (deferred ??= []).Add(item.EvalId);
// One notice per incoming frame. It can reach the host after the deferred run's own start notice (an
// evaluation thread may pick the run up before this is queued); the host treats the two as unordered.
if (deferred is not null)
{
var notice = new EvalResults();
notice.DeferredEvalIds.AddRange(deferred);
Send(new Envelope { Results = notice });
}
return null;
}
case Envelope.BodyOneofCase.Cancel:
if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
{
@@ -288,38 +314,72 @@ internal sealed class WorkerHost
Send(new Envelope { LoadResult = result });
}
private void Admit(EvalRequest request)
/// <summary>
/// Schedules one evaluation: straight to an idle evaluation thread when there is one (it starts now, so the
/// host's clock, started at submit, is right), otherwise into the backlog.
/// </summary>
/// <param name="request">The evaluation.</param>
/// <returns>False when it was deferred to the backlog (the host must stop its clock until it starts).</returns>
private bool Admit(EvalRequest request)
{
if (_queue.Count >= _queueCapacity)
lock (_schedLock)
{
SendResult(new EvalResult
if (_backlog.Count + _queue.Count < _queueCapacity)
{
EvalId = request.EvalId,
Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
});
return;
if (_idle > 0 && _backlog.Count == 0)
{
_idle--;
_queue.Add(request);
return true;
}
_backlog.Enqueue(request);
return false;
}
}
_queue.Add(request);
// Full: answered at once, outside the lock (a send may write to the pipe on this thread).
SendResult(new EvalResult
{
EvalId = request.EvalId,
Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
});
return true;
}
private void EvalThreadLoop()
{
foreach (var request in _queue.GetConsumingEnumerable())
EvalResult? finished = null;
while (true)
{
EvalResult result;
// A thread that just finished a run takes the oldest deferred evaluation, and its start notice rides on
// the finished run's result frame. With nothing deferred it goes idle and waits for a hand-off.
EvalRequest? request = null;
lock (_schedLock)
{
if (finished is not null && _backlog.TryDequeue(out var next)) request = next;
else _idle++;
}
if (finished is not null)
{
var frame = new EvalResults();
frame.Items.Add(finished);
if (request is not null) frame.StartedEvalIds.Add(request.EvalId);
Send(new Envelope { Results = frame });
finished = null;
}
request ??= _queue.Take();
try
{
result = Evaluate(request);
finished = Evaluate(request);
}
catch (Exception ex)
{
result = new EvalResult
finished = new EvalResult
{
EvalId = request.EvalId,
Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = ex.Message },
};
}
SendResult(result);
}
}
@@ -431,7 +491,7 @@ internal sealed class WorkerHost
Heartbeat = new Heartbeat
{
Inflight = (uint)_running.Count,
Queued = (uint)_queue.Count,
Queued = (uint)QueuedCount(),
HeapBytes = GC.GetTotalMemory(false),
Gen2Count = (uint)GC.CollectionCount(2),
},
@@ -489,6 +549,11 @@ internal sealed class WorkerHost
Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
}
private int QueuedCount()
{
lock (_schedLock) return _backlog.Count + _queue.Count;
}
private void SendResult(EvalResult result)
{
var results = new EvalResults();