feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results. The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged. - Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill. - Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider carries the runner to DriverHostActor. - Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial); design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms; batched p99 <= 53 ms at 1,000. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -81,6 +81,17 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// </summary>
|
||||
private readonly ScriptRunGuard _guard;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — the out-of-process runner when <c>Scripting:Runner=OutOfProcess</c> lists the ScriptedAlarm seam;
|
||||
/// null (the default) keeps the in-process guard path, unchanged. With a runner the predicates compile and run in
|
||||
/// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch.
|
||||
/// </summary>
|
||||
private readonly IScriptRunner? _runner;
|
||||
|
||||
// Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the
|
||||
// runner's Recovery to re-evaluate them once. Guarded by _dirtyLock.
|
||||
private readonly HashSet<string> _recoveryArmed = new(StringComparer.Ordinal);
|
||||
|
||||
// ConcurrentDictionary, not a plain Dictionary: every mutation happens under
|
||||
// _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds,
|
||||
// RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request
|
||||
@@ -243,6 +254,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// default used to be <c>TimedScriptEvaluator.DefaultTimeout</c> (250 ms), which is what every production engine
|
||||
/// silently ran with because the host never passed a value.</param>
|
||||
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
|
||||
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of
|
||||
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
|
||||
public ScriptedAlarmEngine(
|
||||
ITagUpstreamSource upstream,
|
||||
IAlarmStateStore store,
|
||||
@@ -250,8 +263,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
ILogger engineLogger,
|
||||
Func<DateTime>? clock = null,
|
||||
TimeSpan? scriptTimeout = null,
|
||||
ScriptingOptions? scripting = null)
|
||||
ScriptingOptions? scripting = null,
|
||||
IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
_runner = scriptRunner;
|
||||
_upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
@@ -268,6 +283,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// <summary>The wall-clock budget each predicate run is given (diagnostics / tests).</summary>
|
||||
public TimeSpan ScriptTimeout => _scriptTimeout;
|
||||
|
||||
/// <summary>The out-of-process runner predicates run through, or null in process (diagnostics / tests).</summary>
|
||||
public IScriptRunner? ScriptRunner => _runner;
|
||||
|
||||
/// <summary>
|
||||
/// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when
|
||||
/// one is in use (diagnostics / tests).
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string> QuarantinedAlarmIds => _runner is null
|
||||
? _guard.QuarantinedKeys
|
||||
: [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.AlarmPredicate)];
|
||||
|
||||
/// <summary>
|
||||
/// OPC UA <c>BadResourceUnavailable</c> — the annotation for a predicate the out-of-process RUNNER could not run
|
||||
/// (its worker restarted under it, or no worker was ready): the condition is frozen exactly as for a script
|
||||
/// fault, but the quality tells the operator the node is sick, not the script (design §9, §14 #3; the same split
|
||||
/// the virtual-tag and Calculation seams make).
|
||||
/// </summary>
|
||||
private const uint StatusBadResourceUnavailable = 0x80040000u;
|
||||
|
||||
/// <summary>
|
||||
/// OPC UA <c>BadInternalError</c> — the quality a condition is annotated with while its predicate
|
||||
/// cannot be evaluated (timed out, refused because the previous run is still executing, or
|
||||
@@ -331,6 +365,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
// A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts
|
||||
// its own through the guard's source identity; this covers "redeployed to retry").
|
||||
_guard.ResetAll();
|
||||
_runner?.Reset(ScriptContextKind.AlarmPredicate);
|
||||
lock (_dirtyLock) _recoveryArmed.Clear();
|
||||
|
||||
var compileFailures = new List<string>();
|
||||
foreach (var def in definitions)
|
||||
@@ -350,7 +386,13 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
// The compiled script is run through _guard (timeout + quarantine) at evaluation time —
|
||||
// the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not
|
||||
// see that the run it abandoned was still executing.
|
||||
var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
|
||||
//
|
||||
// Out of process (lmxopcua#780) the worker compiles and runs the predicate; here the source is only
|
||||
// compiled to PE bytes and dropped — same Roslyn, sandbox and diagnostics, so a broken predicate still
|
||||
// fails the load exactly as in process — and no assembly is loaded into the host.
|
||||
ScriptEvaluator<AlarmPredicateContext, bool>? evaluator = null;
|
||||
if (_runner is null) evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
|
||||
else _ = ScriptEvaluator<AlarmPredicateContext, bool>.CompileToImage(def.PredicateScriptSource);
|
||||
var logger = _loggerFactory.Create(def.AlarmId);
|
||||
|
||||
var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate);
|
||||
@@ -396,14 +438,32 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
// Any predicate emissions queue into `pending` and fire after the gate
|
||||
// is released — so a startup-recovery activation event can call back into
|
||||
// the engine without deadlocking.
|
||||
foreach (var (alarmId, state) in _alarms)
|
||||
if (_runner is null)
|
||||
{
|
||||
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
|
||||
var seed = persisted ?? state.Condition;
|
||||
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
|
||||
.ConfigureAwait(false);
|
||||
_alarms[alarmId] = state with { Condition = afterPredicate };
|
||||
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
|
||||
foreach (var (alarmId, state) in _alarms)
|
||||
{
|
||||
var persisted = await _store.LoadAsync(alarmId, ct).ConfigureAwait(false);
|
||||
var seed = persisted ?? state.Condition;
|
||||
var afterPredicate = await EvaluatePredicateToStateAsync(state, seed, nowUtc: _clock(), ct, pending)
|
||||
.ConfigureAwait(false);
|
||||
_alarms[alarmId] = state with { Condition = afterPredicate };
|
||||
await _store.SaveAsync(afterPredicate, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Out of process: every predicate goes in one batch, then each alarm is applied and saved in the
|
||||
// same order the serial loop above uses.
|
||||
var seeds = new List<(AlarmState State, AlarmConditionState Seed)>(_alarms.Count);
|
||||
foreach (var (alarmId, state) in _alarms)
|
||||
seeds.Add((state, await _store.LoadAsync(alarmId, ct).ConfigureAwait(false) ?? state.Condition));
|
||||
var after = await EvaluatePredicateBatchToStatesAsync(seeds, _clock(), ct, pending).ConfigureAwait(false);
|
||||
for (var i = 0; i < seeds.Count; i++)
|
||||
{
|
||||
var state = seeds[i].State;
|
||||
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
|
||||
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
_loaded = true;
|
||||
@@ -614,7 +674,12 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
|
||||
_valueCache[path] = value;
|
||||
if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
|
||||
MarkDirty(alarmIds);
|
||||
}
|
||||
|
||||
/// <summary>Marks <paramref name="alarmIds"/> dirty and starts the drainer if none is running.</summary>
|
||||
private void MarkDirty(IEnumerable<string> alarmIds)
|
||||
{
|
||||
bool startDrain;
|
||||
lock (_dirtyLock)
|
||||
{
|
||||
@@ -754,17 +819,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
// disposing store or mutating _alarms after clear is unsafe.
|
||||
if (_disposed) return;
|
||||
|
||||
foreach (var id in alarmIds)
|
||||
if (_runner is not null)
|
||||
{
|
||||
if (!_alarms.TryGetValue(id, out var state)) continue;
|
||||
var newState = await EvaluatePredicateToStateAsync(
|
||||
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
|
||||
if (!ReferenceEquals(newState, state.Condition))
|
||||
// Out of process (lmxopcua#780, design §4.5): the whole dirty set in one batch, applied in order.
|
||||
await ReevaluateBatchAsync(alarmIds, ct, pending).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var id in alarmIds)
|
||||
{
|
||||
// Persist before updating in-memory so a store failure leaves
|
||||
// both sides at the prior state.
|
||||
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
|
||||
_alarms[id] = state with { Condition = newState };
|
||||
if (!_alarms.TryGetValue(id, out var state)) continue;
|
||||
var newState = await EvaluatePredicateToStateAsync(
|
||||
state, state.Condition, _clock(), ct, pending).ConfigureAwait(false);
|
||||
if (!ReferenceEquals(newState, state.Condition))
|
||||
{
|
||||
// Persist before updating in-memory so a store failure leaves
|
||||
// both sides at the prior state.
|
||||
await _store.SaveAsync(newState, ct).ConfigureAwait(false);
|
||||
_alarms[id] = state with { Condition = newState };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -837,7 +910,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
try
|
||||
{
|
||||
predicateTrue = await _guard.RunAsync(
|
||||
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context,
|
||||
state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator!, context,
|
||||
_scriptTimeout, ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
@@ -890,6 +963,162 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
return result.State;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the
|
||||
/// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in
|
||||
/// the original order, under <c>_evalGate</c>. Predicates are pure functions of the input snapshot taken here,
|
||||
/// under the gate, so the state machine sees exactly the sequence the serial loop produces. The batch costs one
|
||||
/// round trip for the whole set instead of one per alarm, which is what keeps a large engine inside its budget.
|
||||
/// </summary>
|
||||
private async Task ReevaluateBatchAsync(IReadOnlyList<string> alarmIds, CancellationToken ct, List<ScriptedAlarmEvent> pending)
|
||||
{
|
||||
var items = new List<(AlarmState State, AlarmConditionState Seed)>(alarmIds.Count);
|
||||
foreach (var id in alarmIds)
|
||||
if (_alarms.TryGetValue(id, out var state)) items.Add((state, state.Condition));
|
||||
if (items.Count == 0) return;
|
||||
|
||||
var after = await EvaluatePredicateBatchToStatesAsync(items, _clock(), ct, pending).ConfigureAwait(false);
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var (state, seed) = items[i];
|
||||
if (ReferenceEquals(after[i], seed)) continue;
|
||||
// Persist before updating in-memory so a store failure leaves both sides at the prior state.
|
||||
await _store.SaveAsync(after[i], ct).ConfigureAwait(false);
|
||||
_alarms[state.Definition.AlarmId] = state with { Condition = after[i] };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="EvaluatePredicateToStateAsync"/> for a batch, through <see cref="_runner"/>: the same readiness gate,
|
||||
/// quality tracking and Part 9 application per alarm, in order, with every predicate that has to run submitted
|
||||
/// together. Failure semantics (design §9): a script fault, timeout, kill or quarantine freezes the condition with a
|
||||
/// <c>BadInternalError</c> annotation, exactly as in process; a runner failure (worker restarted, none ready) freezes
|
||||
/// it with <c>BadResourceUnavailable</c> and re-evaluates it once the runner can run it again. A dead worker never
|
||||
/// clears an alarm: no outcome but a completed <c>bool</c> reaches the state machine.
|
||||
/// </summary>
|
||||
private async Task<AlarmConditionState[]> EvaluatePredicateBatchToStatesAsync(
|
||||
IReadOnlyList<(AlarmState State, AlarmConditionState Seed)> items, DateTime nowUtc, CancellationToken ct,
|
||||
List<ScriptedAlarmEvent> pendingEmissions)
|
||||
{
|
||||
var runner = _runner!;
|
||||
var after = new AlarmConditionState[items.Count];
|
||||
var scratches = new AlarmScratch[items.Count];
|
||||
var worst = new uint[items.Count];
|
||||
var slot = new int[items.Count];
|
||||
var invocations = new List<ScriptInvocation>(items.Count);
|
||||
|
||||
// Pass 1, no side effects: snapshot each alarm's inputs and decide whether its predicate runs.
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var state = items[i].State;
|
||||
var scratch = scratches[i] = _scratchByAlarmId.GetOrAdd(
|
||||
state.Definition.AlarmId, _ => new AlarmScratch(state.Inputs, state.Logger, _clock));
|
||||
RefillReadCache(scratch.ReadCache, state.Inputs);
|
||||
worst[i] = WorstInputStatus(scratch.ReadCache);
|
||||
if (!AreInputsReady(scratch.ReadCache))
|
||||
{
|
||||
slot[i] = -1;
|
||||
continue;
|
||||
}
|
||||
slot[i] = invocations.Count;
|
||||
// A copy: the runner may still read the inputs after the caller has its answer (a soft timeout), and the
|
||||
// scratch is refilled by the next pass.
|
||||
invocations.Add(new ScriptInvocation(state.Definition.AlarmId, ScriptContextKind.AlarmPredicate,
|
||||
state.Definition.PredicateScriptSource,
|
||||
new Dictionary<string, DataValueSnapshot>(scratch.ReadCache, StringComparer.Ordinal), _scriptTimeout, state.Logger));
|
||||
}
|
||||
|
||||
var outcomes = invocations.Count == 0 ? [] : runner.RunBatch(invocations, ct);
|
||||
PredicateBatchSubmittedForTest?.Invoke(invocations.Count);
|
||||
|
||||
// Pass 2, in order: exactly what the serial path does with each alarm.
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var (state, seed) = items[i];
|
||||
var scratch = scratches[i];
|
||||
if (slot[i] < 0)
|
||||
{
|
||||
if (TrackQualityBucket(state.Definition.AlarmId, worst[i]))
|
||||
pendingEmissions.Add(BuildQualityEmission(state, seed, worst[i]));
|
||||
after[i] = seed;
|
||||
continue;
|
||||
}
|
||||
|
||||
var outcome = await outcomes[slot[i]].ConfigureAwait(false);
|
||||
after[i] = ApplyPredicateOutcome(state, seed, scratch, worst[i], outcome, nowUtc, ct, pendingEmissions);
|
||||
}
|
||||
return after;
|
||||
}
|
||||
|
||||
/// <summary>One runner outcome onto one alarm (the second half of <see cref="EvaluatePredicateToStateAsync"/>).</summary>
|
||||
private AlarmConditionState ApplyPredicateOutcome(AlarmState state, AlarmConditionState seed, AlarmScratch scratch,
|
||||
uint worstStatus, ScriptRunOutcome outcome, DateTime nowUtc, CancellationToken ct, List<ScriptedAlarmEvent> pendingEmissions)
|
||||
{
|
||||
switch (outcome.Status)
|
||||
{
|
||||
case ScriptRunStatus.Unavailable:
|
||||
// Not the script's fault: frozen, annotated BadResourceUnavailable, and evaluated again (against the
|
||||
// inputs current then) when its worker is back.
|
||||
ArmRecovery(state.Definition.AlarmId, outcome.Recovery);
|
||||
return PredicateDidNotComplete(state, seed, scratch, outcome.Reason ?? ScriptRunOutcome.RunnerUnavailableReason,
|
||||
pendingEmissions, StatusBadResourceUnavailable);
|
||||
case ScriptRunStatus.Failed:
|
||||
var reason = outcome.Reason ?? "evaluator failure";
|
||||
if (ct.IsCancellationRequested) throw new OperationCanceledException(ct);
|
||||
if (reason.StartsWith(ScriptThrewPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
// As in process: a predicate that threw logs an Error and keeps its state, with no quality annotation.
|
||||
scratch.LastNonCompletion = null;
|
||||
state.Logger.Error("Alarm predicate threw — state unchanged: {Reason}", reason[ScriptThrewPrefix.Length..]);
|
||||
return seed;
|
||||
}
|
||||
// Timed out, killed, refused, quarantined, unsupported type, did not compile: frozen + BadInternalError.
|
||||
return PredicateDidNotComplete(state, seed, scratch, reason, pendingEmissions);
|
||||
}
|
||||
|
||||
if (outcome.Value is not bool predicateTrue)
|
||||
{
|
||||
scratch.LastNonCompletion = null;
|
||||
state.Logger.Error("Alarm predicate returned {Type} instead of a bool — state unchanged", outcome.Value?.GetType().Name ?? "null");
|
||||
return seed;
|
||||
}
|
||||
scratch.LastNonCompletion = null;
|
||||
|
||||
// From here on, identical to the serial path: the bucket is tracked only once the predicate completed (I3).
|
||||
var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus);
|
||||
var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc);
|
||||
var transition = result.Emission != EmissionKind.None
|
||||
? BuildEmission(state, result.State, result.Emission, worstStatus)
|
||||
: null;
|
||||
if (transition is not null) pendingEmissions.Add(transition);
|
||||
else if (qualityBucketChanged) pendingEmissions.Add(BuildQualityEmission(state, result.State, worstStatus));
|
||||
return result.State;
|
||||
}
|
||||
|
||||
/// <summary>The prefix the runners put on a predicate that threw (the in-process path catches the exception instead).</summary>
|
||||
private const string ScriptThrewPrefix = "script threw: ";
|
||||
|
||||
/// <summary>
|
||||
/// Re-evaluate <paramref name="alarmId"/> once when <paramref name="recovery"/> completes — the runner's worker is
|
||||
/// back (design §9 "re-evaluated on return"). At most one wait per alarm.
|
||||
/// </summary>
|
||||
private void ArmRecovery(string alarmId, Task? recovery)
|
||||
{
|
||||
if (recovery is null) return;
|
||||
lock (_dirtyLock)
|
||||
{
|
||||
if (_disposed || !_recoveryArmed.Add(alarmId)) return;
|
||||
}
|
||||
_ = recovery.ContinueWith(_ =>
|
||||
{
|
||||
lock (_dirtyLock) _recoveryArmed.Remove(alarmId);
|
||||
if (!_disposed) MarkDirty([alarmId]);
|
||||
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
/// <summary>Test hook: the number of predicates each out-of-process batch pass submitted.</summary>
|
||||
internal Action<int>? PredicateBatchSubmittedForTest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The predicate did not return (timed out, refused because the previous run is still executing, or
|
||||
/// quarantined): hold the condition at <paramref name="seed"/>, log the reason <b>once per change of
|
||||
@@ -900,15 +1129,15 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// </summary>
|
||||
private AlarmConditionState PredicateDidNotComplete(
|
||||
AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason,
|
||||
List<ScriptedAlarmEvent> pendingEmissions)
|
||||
List<ScriptedAlarmEvent> pendingEmissions, uint status = StatusBadInternalError)
|
||||
{
|
||||
if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal))
|
||||
{
|
||||
scratch.LastNonCompletion = reason;
|
||||
state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason);
|
||||
}
|
||||
if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError))
|
||||
pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError));
|
||||
if (TrackQualityBucket(state.Definition.AlarmId, status))
|
||||
pendingEmissions.Add(BuildQualityEmission(state, seed, status));
|
||||
return seed;
|
||||
}
|
||||
|
||||
@@ -1339,7 +1568,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
|
||||
private sealed record AlarmState(
|
||||
ScriptedAlarmDefinition Definition,
|
||||
ScriptEvaluator<AlarmPredicateContext, bool> Evaluator,
|
||||
ScriptEvaluator<AlarmPredicateContext, bool>? Evaluator, // null out of process: the worker holds the compiled form
|
||||
IReadOnlySet<string> Inputs,
|
||||
IReadOnlyList<string> TemplateTokens,
|
||||
ILogger Logger,
|
||||
|
||||
+1
@@ -22,6 +22,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests"/>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
+159
-26
@@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
|
||||
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
|
||||
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
|
||||
int compileThreads = 0)
|
||||
int compileThreads = 0, int evalThreads = 0)
|
||||
{
|
||||
_compileThreads = compileThreads;
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
@@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
|
||||
if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
|
||||
|
||||
var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
|
||||
// Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
|
||||
// queue-behind-a-busy-thread case deterministic).
|
||||
if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
|
||||
_shards = new WorkerShard[_worker.Shards];
|
||||
for (var i = 0; i < _shards.Length; i++)
|
||||
{
|
||||
@@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
/// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
|
||||
internal Action<string>? SlotReleasingForTest { get; set; }
|
||||
|
||||
/// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
|
||||
internal Action<int>? BatchSubmittedForTest { get; set; }
|
||||
|
||||
/// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
|
||||
internal IReadOnlyCollection<string> SuspectKeys =>
|
||||
_keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
|
||||
@@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
|
||||
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
|
||||
// run by then.
|
||||
var heldGate = gate;
|
||||
var released = 0;
|
||||
void ReleaseSlot()
|
||||
{
|
||||
if (Interlocked.Exchange(ref released, 1) != 0) return;
|
||||
SlotReleasingForTest?.Invoke(key);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
heldGate?.Release();
|
||||
}
|
||||
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
|
||||
_worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
|
||||
_worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
|
||||
handedOff = true;
|
||||
return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return ScriptRunOutcome.Failed("evaluation cancelled");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!handedOff)
|
||||
{
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
gate?.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
||||
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
||||
// shard has no live worker (or for the isolation gate).
|
||||
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
|
||||
/// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
|
||||
/// <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
|
||||
/// bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
|
||||
/// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
|
||||
/// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
|
||||
/// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
|
||||
/// <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
|
||||
/// </remarks>
|
||||
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invocations);
|
||||
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
|
||||
Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
|
||||
for (var i = 0; i < invocations.Count; i++)
|
||||
{
|
||||
var invocation = invocations[i];
|
||||
ArgumentNullException.ThrowIfNull(invocation);
|
||||
switch (TryPrepareBatchItem(invocation, out var item, out var answered))
|
||||
{
|
||||
case BatchPreparation.Submit:
|
||||
groups ??= [];
|
||||
if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
|
||||
group.Add((i, item));
|
||||
break;
|
||||
case BatchPreparation.Answered:
|
||||
outcomes[i] = Task.FromResult(answered!);
|
||||
break;
|
||||
default:
|
||||
outcomes[i] = RunAsync(invocation, ct);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (groups is null) return outcomes;
|
||||
|
||||
foreach (var (connection, group) in groups)
|
||||
{
|
||||
var submissions = new BatchSubmission[group.Count];
|
||||
for (var j = 0; j < group.Count; j++)
|
||||
{
|
||||
var item = group[j].Item;
|
||||
submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
|
||||
item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
|
||||
}
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
BatchSubmittedForTest?.Invoke(submissions.Length);
|
||||
var pendings = connection.SubmitBatch(submissions);
|
||||
for (var j = 0; j < group.Count; j++)
|
||||
{
|
||||
var (index, item) = group[j];
|
||||
outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
|
||||
/// where nothing has to be waited for. Takes the key's single-flight slot when it returns
|
||||
/// <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
|
||||
/// </summary>
|
||||
private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
|
||||
{
|
||||
item = null;
|
||||
answered = null;
|
||||
if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
|
||||
|
||||
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
||||
var state = Admit(key, invocation.Kind, invocation.Source);
|
||||
if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
|
||||
|
||||
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
|
||||
if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
|
||||
if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
|
||||
|
||||
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
|
||||
{
|
||||
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
|
||||
answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
return BatchPreparation.Answered;
|
||||
}
|
||||
state.ScriptLogger = invocation.ScriptLogger;
|
||||
NoteLoad(entry);
|
||||
item = new BatchItem(key, state, invocation, shard, connection, entry);
|
||||
return BatchPreparation.Submit;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
|
||||
/// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
|
||||
/// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
|
||||
/// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
|
||||
/// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
|
||||
/// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
|
||||
/// quarantine, suspect) has already run by then.
|
||||
/// </summary>
|
||||
private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
|
||||
{
|
||||
var released = 0;
|
||||
return () =>
|
||||
{
|
||||
if (Interlocked.Exchange(ref released, 1) != 0) return;
|
||||
SlotReleasingForTest?.Invoke(key);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
heldGate?.Release();
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
|
||||
private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
|
||||
WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = ct.CanBeCanceled
|
||||
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
|
||||
: await pending.Outcome.ConfigureAwait(false);
|
||||
@@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
{
|
||||
return ScriptRunOutcome.Failed("evaluation cancelled");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!handedOff)
|
||||
{
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
gate?.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
||||
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
||||
// shard has no live worker (or for the isolation gate).
|
||||
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
|
||||
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
|
||||
|
||||
private enum BatchPreparation
|
||||
{
|
||||
/// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
|
||||
Full,
|
||||
|
||||
/// <summary>Answered without running (refused).</summary>
|
||||
Answered,
|
||||
|
||||
/// <summary>Submit it with the batch; its slot is taken.</summary>
|
||||
Submit,
|
||||
}
|
||||
|
||||
private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
|
||||
WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
|
||||
|
||||
private sealed class KeyState(string source, ScriptContextKind kind)
|
||||
{
|
||||
private byte[]? _sha;
|
||||
|
||||
@@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One evaluation of a <see cref="WorkerConnection.SubmitBatch"/>.</summary>
|
||||
/// <param name="Key">The scoped key.</param>
|
||||
/// <param name="Request">The request (its id is assigned on submit).</param>
|
||||
/// <param name="SoftTimeout">T.</param>
|
||||
/// <param name="HardGrace">G.</param>
|
||||
/// <param name="OnFinished">Runs once when the worker has answered or ended, before the caller resumes (the slot release).</param>
|
||||
internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished);
|
||||
|
||||
/// <summary>What the host needs to launch one worker.</summary>
|
||||
/// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
|
||||
/// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
|
||||
@@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
return pending;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Submits several evaluations as ONE <see cref="EvalBatch"/> frame (lmxopcua#780 batch submit, design §4.5). Each
|
||||
/// item gets its own deadlines and slot release exactly as through <see cref="Submit(string, EvalRequest, TimeSpan?, TimeSpan, Action?)"/>;
|
||||
/// an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts.
|
||||
/// </summary>
|
||||
/// <param name="items">The evaluations (their ids are assigned here).</param>
|
||||
/// <returns>One pending evaluation per item, in order.</returns>
|
||||
public PendingEval[] SubmitBatch(IReadOnlyList<BatchSubmission> items)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
var pendings = new PendingEval[items.Count];
|
||||
var batch = new EvalBatch();
|
||||
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
|
||||
var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher);
|
||||
pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline);
|
||||
_pending[item.Request.EvalId] = pending;
|
||||
batch.Items.Add(item.Request);
|
||||
pendings[i] = pending;
|
||||
}
|
||||
if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted))
|
||||
{
|
||||
foreach (var pending in pendings)
|
||||
if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null);
|
||||
}
|
||||
return pendings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The compiled form of <paramref name="key"/> in this worker when it is loaded at <paramref name="sourceHashHex"/>
|
||||
/// with <paramref name="inputPaths"/> and compiled cleanly — the batch fast path, which never waits for a load.
|
||||
/// </summary>
|
||||
/// <param name="key">The scoped key.</param>
|
||||
/// <param name="sourceHashHex">The source hash (hex).</param>
|
||||
/// <param name="inputPaths">The slot order.</param>
|
||||
/// <returns>The entry, or null when the key needs a (re)load or its load failed.</returns>
|
||||
public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList<string> inputPaths)
|
||||
=> _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths)
|
||||
&& existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut
|
||||
? existing
|
||||
: null;
|
||||
|
||||
private void OnHardDeadline(PendingEval pending)
|
||||
{
|
||||
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
|
||||
@@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
|
||||
// trusts"), except for one that was already completed by a kill race — ignored then.
|
||||
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
|
||||
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
|
||||
else CheckIssued(result.EvalId, "result");
|
||||
}
|
||||
// Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines
|
||||
// restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time.
|
||||
// The two notices are unordered with respect to each other (see PendingEval.Start).
|
||||
foreach (var id in frame.Results.StartedEvalIds)
|
||||
{
|
||||
if (_pending.TryGetValue(id, out var started)) started.Start();
|
||||
else CheckIssued(id, "start notice");
|
||||
}
|
||||
foreach (var id in frame.Results.DeferredEvalIds)
|
||||
{
|
||||
if (_pending.TryGetValue(id, out var deferred)) deferred.Defer();
|
||||
else CheckIssued(id, "deferral notice");
|
||||
}
|
||||
break;
|
||||
case Envelope.BodyOneofCase.LoadStarted:
|
||||
@@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>An id the host never issued is a protocol violation; one it issued and already completed is a benign race.</summary>
|
||||
private void CheckIssued(ulong evalId, string what)
|
||||
{
|
||||
if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}");
|
||||
}
|
||||
|
||||
private async Task PipeEndedAsync(WorkerExitReason? fault)
|
||||
{
|
||||
// The pipe is gone: the worker is dead or dying. Make sure, then report.
|
||||
@@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
|
||||
/// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind
|
||||
/// busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full
|
||||
/// T, then G, from the start — without a stale tick firing early.
|
||||
/// </remarks>
|
||||
internal sealed class PendingEval
|
||||
{
|
||||
/// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
|
||||
@@ -926,9 +1003,15 @@ internal sealed class PendingEval
|
||||
private readonly TaskCompletionSource<EvalResult?> _outcome;
|
||||
private readonly ResultDispatcher? _dispatcher;
|
||||
private readonly Action? _onFinished;
|
||||
private readonly object _clock = new();
|
||||
private Timer? _timer;
|
||||
private Action<PendingEval>? _onHardDeadline;
|
||||
private TimeSpan _soft;
|
||||
private TimeSpan _grace;
|
||||
private long _softDue;
|
||||
private long _hardDue;
|
||||
private bool _deferred;
|
||||
private bool _started;
|
||||
private int _phase;
|
||||
private int _done;
|
||||
|
||||
@@ -954,6 +1037,12 @@ internal sealed class PendingEval
|
||||
/// <summary>T + G, for the kill log line.</summary>
|
||||
public TimeSpan HardDeadline { get; private set; }
|
||||
|
||||
/// <summary>Whether the worker reported this run as queued and has not reported it started (tests / diagnostics).</summary>
|
||||
internal bool IsDeferred
|
||||
{
|
||||
get { lock (_clock) return _deferred; }
|
||||
}
|
||||
|
||||
/// <summary>The worker's result, or null when the worker ended first.</summary>
|
||||
public Task<EvalResult?> Completion => _completion.Task;
|
||||
|
||||
@@ -966,27 +1055,96 @@ internal sealed class PendingEval
|
||||
/// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
|
||||
internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
|
||||
{
|
||||
_soft = soft;
|
||||
_grace = grace;
|
||||
_onHardDeadline = onHardDeadline;
|
||||
HardDeadline = soft + grace;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
_softDue = now + Ticks(soft);
|
||||
_hardDue = _softDue + Ticks(grace);
|
||||
// Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
|
||||
_timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
|
||||
_timer.Change(soft, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running,
|
||||
/// so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire.
|
||||
/// </summary>
|
||||
internal void Defer()
|
||||
{
|
||||
lock (_clock)
|
||||
{
|
||||
if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return;
|
||||
_deferred = true;
|
||||
TryChange(Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline
|
||||
/// had already been reported). A run that was never deferred keeps the clock it has had since submit.
|
||||
/// </summary>
|
||||
internal void Start()
|
||||
{
|
||||
lock (_clock)
|
||||
{
|
||||
_started = true;
|
||||
if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return;
|
||||
_deferred = false;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
if (Volatile.Read(ref _phase) == 0)
|
||||
{
|
||||
_softDue = now + Ticks(_soft);
|
||||
_hardDue = _softDue + Ticks(_grace);
|
||||
TryChange(_soft);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets
|
||||
// the whole T + G from its start before it can be blamed for a kill.
|
||||
_hardDue = now + Ticks(_soft + _grace);
|
||||
TryChange(_soft + _grace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnTimer()
|
||||
{
|
||||
if (Volatile.Read(ref _done) != 0) return;
|
||||
if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
|
||||
bool soft = false, hard = false;
|
||||
lock (_clock)
|
||||
{
|
||||
_outcome.TrySetResult(SoftTimedOut);
|
||||
try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* answered meanwhile */ }
|
||||
return;
|
||||
if (_deferred || Volatile.Read(ref _done) != 0) return;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
if (Volatile.Read(ref _phase) == 0)
|
||||
{
|
||||
// A tick armed before a deferral/start can land early: re-arm for what is left.
|
||||
if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; }
|
||||
Interlocked.Exchange(ref _phase, 1);
|
||||
soft = true;
|
||||
TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero);
|
||||
}
|
||||
else if (Volatile.Read(ref _phase) == 1)
|
||||
{
|
||||
if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; }
|
||||
Interlocked.Exchange(ref _phase, 2);
|
||||
hard = true;
|
||||
}
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
|
||||
// Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
|
||||
if (soft) _outcome.TrySetResult(SoftTimedOut);
|
||||
if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
|
||||
}
|
||||
|
||||
private void TryChange(TimeSpan due)
|
||||
{
|
||||
try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* answered meanwhile */ }
|
||||
}
|
||||
|
||||
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
|
||||
|
||||
/// <summary>
|
||||
/// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
|
||||
/// reader, then hand the caller's resumption to the dispatcher (or the pool).
|
||||
|
||||
@@ -134,8 +134,17 @@ message EvalResult {
|
||||
uint32 run_micros = 6;
|
||||
}
|
||||
|
||||
// Per-item deadlines are exact for queued work too (lmxopcua#780 batch submit): the host arms an evaluation's soft and
|
||||
// hard deadlines when it submits, which is when the run starts as long as the worker has an idle evaluation thread. An
|
||||
// evaluation that arrives while every thread is busy (a batch larger than the scheduler, or work queued behind a
|
||||
// runaway) is reported in `deferred_eval_ids`; the host then stops its clock, and restarts it — the full T, then G —
|
||||
// when the evaluation appears in `started_eval_ids`, which an evaluation thread sends alongside the result of the run it
|
||||
// just finished. So a run waiting behind a sibling is never timed out, struck or killed for the sibling's time.
|
||||
// Either list may be the only content of a frame. A host that predates these fields ignores them.
|
||||
message EvalResults {
|
||||
repeated EvalResult items = 1;
|
||||
repeated uint64 started_eval_ids = 2;
|
||||
repeated uint64 deferred_eval_ids = 3;
|
||||
}
|
||||
|
||||
message ScriptError {
|
||||
|
||||
@@ -233,7 +233,10 @@ public sealed class ScriptWorkerFrameWriter
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Results:
|
||||
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
|
||||
(results ??= new EvalResults()).Items.AddRange(envelope.Results.Items);
|
||||
results ??= new EvalResults();
|
||||
results.Items.AddRange(envelope.Results.Items);
|
||||
results.StartedEvalIds.AddRange(envelope.Results.StartedEvalIds);
|
||||
results.DeferredEvalIds.AddRange(envelope.Results.DeferredEvalIds);
|
||||
break;
|
||||
default:
|
||||
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
|
||||
|
||||
@@ -28,6 +28,31 @@ public interface IScriptRunner
|
||||
/// <returns>The outcome; never faults for a script failure.</returns>
|
||||
Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Submits several evaluations at once (design §4.5): the out-of-process runner sends every one that is ready to
|
||||
/// run as ONE request frame per worker, instead of one frame and one submission per evaluation. Results are
|
||||
/// per item and streamed — each task completes when its own evaluation does, so a slow item never holds a fast
|
||||
/// sibling's result — and every item keeps exactly the semantics of <see cref="RunAsync"/>: its own budget, its
|
||||
/// own single-flight slot, and its own blame (a runaway item is killed and struck alone; an item queued behind
|
||||
/// it is neither timed out nor struck for its time).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default implementation — what <see cref="InProcessScriptRunner"/> uses — is <see cref="RunAsync"/> per
|
||||
/// item, started together. A key that appears twice in one batch is refused for its second item exactly as two
|
||||
/// concurrent <see cref="RunAsync"/> calls would be; callers that may repeat a key split the batch there.
|
||||
/// Never throws for script faults; the returned tasks never fault.
|
||||
/// </remarks>
|
||||
/// <param name="invocations">What to run, in order.</param>
|
||||
/// <param name="ct">Caller cancellation (shutdown); wins over every item's budget.</param>
|
||||
/// <returns>One task per invocation, in the same order.</returns>
|
||||
IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invocations);
|
||||
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
|
||||
for (var i = 0; i < outcomes.Length; i++) outcomes[i] = RunAsync(invocations[i], ct);
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The deploy boundary for one seam: lifts every quarantine and forgets every remembered compile failure of
|
||||
/// <paramref name="kind"/>, and drops its compiled scripts.
|
||||
@@ -50,7 +75,7 @@ public enum ScriptContextKind
|
||||
/// <summary>A Calculation tag: <c>VirtualTagContext</c> → <c>object?</c>.</summary>
|
||||
Calculation = 2,
|
||||
|
||||
/// <summary>A scripted-alarm predicate: <c>AlarmPredicateContext</c> → <c>bool</c> (not run out of process yet).</summary>
|
||||
/// <summary>A scripted-alarm predicate: <c>AlarmPredicateContext</c> → <c>bool</c>.</summary>
|
||||
AlarmPredicate = 3,
|
||||
}
|
||||
|
||||
@@ -142,3 +167,16 @@ public static class ScriptRunOutcomeExtensions
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Which <see cref="IScriptRunner"/> each script seam uses on this node (lmxopcua#780), for components that cannot
|
||||
/// reference the Host's selection directly (the Runtime's scripted-alarm host). Null for a seam means its in-process
|
||||
/// guard path.
|
||||
/// </summary>
|
||||
public interface IScriptRunnerProvider
|
||||
{
|
||||
/// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
|
||||
/// <param name="seam">The seam.</param>
|
||||
/// <returns>The runner, or null.</returns>
|
||||
IScriptRunner? ForSeam(ScriptSeam seam);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
/// <summary>
|
||||
/// Opportunistic batching for a seam whose evaluations are submitted independently — one virtual-tag actor each —
|
||||
/// but often become ready together (one upstream burst fans out to many tags): evaluations submitted while another
|
||||
/// submission is in progress are handed to <see cref="IScriptRunner.RunBatch"/> together, in arrival order, by the
|
||||
/// submitter already holding the flush (lmxopcua#780, design §4.5).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>There is no timer: an evaluation on an idle seam is submitted at once by its own caller, so batching never
|
||||
/// adds latency. Per-item semantics are <see cref="IScriptRunner.RunBatch"/>'s — each item keeps its own budget, slot
|
||||
/// and blame.</para>
|
||||
/// <para>Each caller's task completes synchronously on the thread that completes its run (the runner's result
|
||||
/// dispatcher), exactly as awaiting <see cref="IScriptRunner.RunAsync"/> directly does, so there is no extra thread
|
||||
/// hop.</para>
|
||||
/// <para>A key submitted twice concurrently is refused for the second exactly as two concurrent
|
||||
/// <see cref="IScriptRunner.RunAsync"/> calls would be; the virtual-tag actor never has two evaluations in flight.</para>
|
||||
/// </remarks>
|
||||
public sealed class ScriptBatchCoalescer
|
||||
{
|
||||
private readonly IScriptRunner _runner;
|
||||
private readonly object _lock = new();
|
||||
private List<(ScriptInvocation Invocation, TaskCompletionSource<ScriptRunOutcome> Done)> _queue = [];
|
||||
private bool _flushing;
|
||||
|
||||
/// <summary>Initializes the coalescer.</summary>
|
||||
/// <param name="runner">The runner every batch goes to.</param>
|
||||
public ScriptBatchCoalescer(IScriptRunner runner)
|
||||
{
|
||||
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
|
||||
}
|
||||
|
||||
/// <summary>Test hook: runs once per <see cref="IScriptRunner.RunBatch"/> call, with the batch size.</summary>
|
||||
internal Action<int>? BatchSubmittedForTest { get; set; }
|
||||
|
||||
/// <summary>Submits one evaluation, together with any submitted concurrently.</summary>
|
||||
/// <param name="invocation">What to run.</param>
|
||||
/// <returns>The outcome; never faults.</returns>
|
||||
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invocation);
|
||||
// Synchronous continuations: the caller resumes where its run completed, as with a direct await of RunAsync.
|
||||
var done = new TaskCompletionSource<ScriptRunOutcome>(TaskCreationOptions.None);
|
||||
lock (_lock)
|
||||
{
|
||||
_queue.Add((invocation, done));
|
||||
if (_flushing) return done.Task;
|
||||
_flushing = true;
|
||||
}
|
||||
Flush();
|
||||
return done.Task;
|
||||
}
|
||||
|
||||
private void Flush()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
List<(ScriptInvocation Invocation, TaskCompletionSource<ScriptRunOutcome> Done)> batch;
|
||||
lock (_lock)
|
||||
{
|
||||
if (_queue.Count == 0)
|
||||
{
|
||||
_flushing = false;
|
||||
return;
|
||||
}
|
||||
batch = _queue;
|
||||
_queue = [];
|
||||
}
|
||||
|
||||
IReadOnlyList<Task<ScriptRunOutcome>> outcomes;
|
||||
try
|
||||
{
|
||||
var invocations = new ScriptInvocation[batch.Count];
|
||||
for (var i = 0; i < invocations.Length; i++) invocations[i] = batch[i].Invocation;
|
||||
BatchSubmittedForTest?.Invoke(invocations.Length);
|
||||
outcomes = _runner.RunBatch(invocations);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A defect in the runner must not strand the callers (nor the flush flag).
|
||||
foreach (var (_, done) in batch) done.TrySetResult(ScriptRunOutcome.Failed($"script runner threw: {ex.Message}"));
|
||||
continue;
|
||||
}
|
||||
|
||||
for (var i = 0; i < batch.Count; i++) Link(outcomes[i], batch[i].Done);
|
||||
}
|
||||
}
|
||||
|
||||
private static void Link(Task<ScriptRunOutcome> outcome, TaskCompletionSource<ScriptRunOutcome> done)
|
||||
{
|
||||
if (outcome.IsCompleted)
|
||||
{
|
||||
Settle(outcome, done);
|
||||
return;
|
||||
}
|
||||
outcome.ContinueWith(static (t, s) => Settle(t, (TaskCompletionSource<ScriptRunOutcome>)s!), done,
|
||||
CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private static void Settle(Task<ScriptRunOutcome> outcome, TaskCompletionSource<ScriptRunOutcome> done)
|
||||
{
|
||||
if (outcome.IsCompletedSuccessfully) done.TrySetResult(outcome.Result);
|
||||
else done.TrySetResult(ScriptRunOutcome.Failed(
|
||||
$"script runner threw: {outcome.Exception?.GetBaseException().Message ?? "cancelled"}"));
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,10 @@ public enum ScriptSeam
|
||||
/// <summary>Calculation tags (<c>CalculationEvaluator</c>).</summary>
|
||||
Calculation = 1,
|
||||
|
||||
/// <summary>Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10).</summary>
|
||||
/// <summary>
|
||||
/// Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10): out of process only
|
||||
/// when <c>Scripting:Runner:Seams</c> lists it — an unset list does not select it.
|
||||
/// </summary>
|
||||
ScriptedAlarm = 2,
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,8 @@ public sealed class ScriptingOptions
|
||||
|
||||
/// <summary>
|
||||
/// The seams <see cref="Runner"/> applies to when it is not <see cref="ScriptRunnerMode.InProcess"/>
|
||||
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means all three. Bound by
|
||||
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means VirtualTag and Calculation:
|
||||
/// scripted alarms run out of process only when listed explicitly (design §14 #10, alarms last). Bound by
|
||||
/// <c>ScriptingOptionsBinding</c> from the documented key, because <c>Scripting:Runner</c> is itself a value.
|
||||
/// </summary>
|
||||
public IReadOnlyList<ScriptSeam>? RunnerSeams { get; set; }
|
||||
@@ -91,10 +92,10 @@ public sealed class ScriptingOptions
|
||||
/// <summary>Whether <paramref name="seam"/> is selected for out-of-process execution by these options.</summary>
|
||||
/// <param name="seam">The seam.</param>
|
||||
/// <returns><c>true</c> when <see cref="Runner"/> is <see cref="ScriptRunnerMode.OutOfProcess"/> and the seam is in
|
||||
/// <see cref="RunnerSeams"/> (or that list is empty).</returns>
|
||||
/// <see cref="RunnerSeams"/> (or that list is empty and the seam is not <see cref="ScriptSeam.ScriptedAlarm"/>).</returns>
|
||||
public bool IsOutOfProcess(ScriptSeam seam)
|
||||
=> Runner == ScriptRunnerMode.OutOfProcess
|
||||
&& (RunnerSeams is not { Count: > 0 } seams || seams.Contains(seam));
|
||||
&& (RunnerSeams is { Count: > 0 } seams ? seams.Contains(seam) : seam != ScriptSeam.ScriptedAlarm);
|
||||
|
||||
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="StuckOrphanQuarantineFactor"/>.</summary>
|
||||
public const double MaxStuckOrphanQuarantineFactor = 1000;
|
||||
|
||||
@@ -26,6 +26,8 @@
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests"/>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests"/>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Host.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -522,6 +522,14 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// false timeouts under pool starvation).</summary>
|
||||
private async Task RunEvaluationPumpAsync()
|
||||
{
|
||||
// lmxopcua#780 (design §4.5): with an out-of-process runner a run costs a pipe round trip, so the pump submits
|
||||
// everything queued as one batch and publishes in dequeue order. In process it stays one at a time, unchanged.
|
||||
if (_evaluator.SubmitsBatches)
|
||||
{
|
||||
await RunEvaluationBatchPumpAsync().ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
PendingEvaluation next;
|
||||
@@ -557,6 +565,73 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pump on the out-of-process path (lmxopcua#780, design §4.5): dequeue everything pending — each entry's
|
||||
/// dependency snapshot was captured at enqueue — submit it as ONE batch, and publish in dequeue order. A calc tag
|
||||
/// queued more than once (a burst of changes) ends the batch at its second entry, so every queued evaluation still
|
||||
/// runs, in order, against the snapshot that triggered it, exactly as the one-at-a-time pump would run them.
|
||||
/// </summary>
|
||||
private async Task RunEvaluationBatchPumpAsync()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
List<PendingEvaluation> batch;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_running) _pending.Clear();
|
||||
if (_pending.Count == 0)
|
||||
{
|
||||
_pumpScheduled = false;
|
||||
return;
|
||||
}
|
||||
batch = [];
|
||||
var inBatch = new HashSet<string>(StringComparer.Ordinal);
|
||||
while (_pending.Count > 0 && inBatch.Add(_pending.Peek().Definition.RawPath)) batch.Add(_pending.Dequeue());
|
||||
}
|
||||
|
||||
if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
|
||||
try
|
||||
{
|
||||
await RunEvaluationBatchAsync(batch).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Defect path only (the evaluator never throws for script faults); must not strand the queue.
|
||||
_logger.LogError(ex, "Calculation {Driver}: batch evaluation of {Count} tag(s) failed", _driverInstanceId, batch.Count);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_evalGate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs one batch and publishes every outcome in dequeue order. Under <see cref="_evalGate"/>, not <see cref="_lock"/>.</summary>
|
||||
/// <param name="batch">The prepared evaluations; no calc tag twice.</param>
|
||||
private async Task RunEvaluationBatchAsync(IReadOnlyList<PendingEvaluation> batch)
|
||||
{
|
||||
// Hold-on-Bad entries never reach the runner; the rest are submitted together.
|
||||
var requests = new List<(string, string, IReadOnlyDictionary<string, DataValueSnapshot>)>(batch.Count);
|
||||
var slot = new int[batch.Count];
|
||||
for (var i = 0; i < batch.Count; i++)
|
||||
{
|
||||
if (!InputQuality.IsUsable(batch[i].WorstInputStatus)) { slot[i] = -1; continue; }
|
||||
slot[i] = requests.Count;
|
||||
requests.Add((batch[i].Definition.RawPath, batch[i].Definition.ScriptSource, batch[i].Dependencies));
|
||||
}
|
||||
var results = _evaluator.EvaluateBatch(requests, _evalCts.Token);
|
||||
BatchSubmittedForTest?.Invoke(requests.Count);
|
||||
|
||||
for (var i = 0; i < batch.Count; i++)
|
||||
{
|
||||
if (slot[i] < 0) HoldOnBadInput(batch[i]);
|
||||
else ApplyEvaluationResult(batch[i], await results[slot[i]].ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hook: the number of compiled evaluations each batch pass submitted.</summary>
|
||||
internal Action<int>? BatchSubmittedForTest { get; set; }
|
||||
|
||||
/// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
|
||||
/// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
|
||||
/// completes).</summary>
|
||||
@@ -610,16 +685,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
// sub-code. Uncertain IS usable: the script runs and the output inherits the Uncertain code.
|
||||
if (!InputQuality.IsUsable(pending.WorstInputStatus))
|
||||
{
|
||||
(object? Value, uint StatusCode)? badPublish;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
|
||||
badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
|
||||
}
|
||||
|
||||
// No script log: an input dying is not a script fault, and the script log is the per-failure
|
||||
// SCRIPT diagnostic. Here the status code is the diagnostic.
|
||||
if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
|
||||
HoldOnBadInput(pending);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -627,7 +693,32 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
// then the input's real ones rather than a fabricated Good stamped at evaluation time.
|
||||
var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
|
||||
.ConfigureAwait(false);
|
||||
ApplyEvaluationResult(pending, result);
|
||||
}
|
||||
|
||||
/// <summary>A Bad input holds the script: publish the input's own status (once per transition), no script log.</summary>
|
||||
/// <param name="pending">The prepared evaluation whose worst input is Bad.</param>
|
||||
private void HoldOnBadInput(PendingEvaluation pending)
|
||||
{
|
||||
var def = pending.Definition;
|
||||
(object? Value, uint StatusCode)? badPublish;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
|
||||
badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
|
||||
}
|
||||
|
||||
// No script log: an input dying is not a script fault, and the script log is the per-failure
|
||||
// SCRIPT diagnostic. Here the status code is the diagnostic.
|
||||
if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
|
||||
}
|
||||
|
||||
/// <summary>Record one evaluation's outcome and publish it. Under <see cref="_evalGate"/>, not <see cref="_lock"/>.</summary>
|
||||
/// <param name="pending">The prepared evaluation.</param>
|
||||
/// <param name="result">Its outcome.</param>
|
||||
private void ApplyEvaluationResult(PendingEvaluation pending, VirtualTagEvalResult result)
|
||||
{
|
||||
var def = pending.Definition;
|
||||
string? failureToLog = null;
|
||||
(object? Value, uint StatusCode)? toPublish;
|
||||
lock (_lock)
|
||||
|
||||
@@ -283,6 +283,62 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether compiled runs go to an <see cref="IScriptRunner"/> (the out-of-process seam), so the driver's pump can
|
||||
/// submit everything it has dequeued as one batch (lmxopcua#780, design §4.5). False on the in-process path, whose
|
||||
/// pump stays one evaluation at a time, unchanged.
|
||||
/// </summary>
|
||||
public bool SubmitsBatches => _runner is not null && !_disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates several calc tags at once (design §4.5): the passthrough fast path and the trivial failures are answered
|
||||
/// inline exactly as <see cref="EvaluateAsync"/> answers them, and every compiled run is handed to the runner in ONE
|
||||
/// <see cref="IScriptRunner.RunBatch"/>. Results are per item; each item keeps <see cref="EvaluateAsync"/>'s
|
||||
/// outcome, budget and blame. Without a runner it is <see cref="EvaluateAsync"/> per item, in order.
|
||||
/// </summary>
|
||||
/// <param name="requests">The evaluations: calc-tag id, script source, dependency snapshots. A calc tag must not appear twice.</param>
|
||||
/// <param name="ct">Caller cancellation (driver shutdown).</param>
|
||||
/// <returns>One task per request, in order; none faults.</returns>
|
||||
public IReadOnlyList<Task<VirtualTagEvalResult>> EvaluateBatch(
|
||||
IReadOnlyList<(string CalcTagId, string ScriptSource, IReadOnlyDictionary<string, DataValueSnapshot> Dependencies)> requests,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(requests);
|
||||
var results = new Task<VirtualTagEvalResult>[requests.Count];
|
||||
if (_runner is null || _disposed)
|
||||
{
|
||||
for (var i = 0; i < requests.Count; i++)
|
||||
results[i] = EvaluateAsync(requests[i].CalcTagId, requests[i].ScriptSource, requests[i].Dependencies, ct);
|
||||
return results;
|
||||
}
|
||||
|
||||
List<ScriptInvocation>? invocations = null;
|
||||
List<int>? indexes = null;
|
||||
for (var i = 0; i < requests.Count; i++)
|
||||
{
|
||||
var (id, source, deps) = requests[i];
|
||||
// Exactly the fast answers EvaluateAsync gives before it reaches the runner.
|
||||
if (string.IsNullOrWhiteSpace(source) || PassthroughScript.TryMatch(source, out _))
|
||||
{
|
||||
results[i] = EvaluateAsync(id, source, deps, ct);
|
||||
continue;
|
||||
}
|
||||
var scriptLog = _scriptRoot
|
||||
.ForContext(ScriptLoggerFactory.ScriptIdProperty, id)
|
||||
.ForContext(ScriptLoggerFactory.VirtualTagIdProperty, id);
|
||||
(invocations ??= []).Add(new ScriptInvocation(id, ScriptContextKind.Calculation, source, deps, _runTimeout, scriptLog));
|
||||
(indexes ??= []).Add(i);
|
||||
}
|
||||
if (invocations is null) return results;
|
||||
|
||||
var outcomes = _runner.RunBatch(invocations, ct);
|
||||
for (var j = 0; j < outcomes.Count; j++) results[indexes![j]] = ToResultAsync(outcomes[j]);
|
||||
return results;
|
||||
|
||||
static async Task<VirtualTagEvalResult> ToResultAsync(Task<ScriptRunOutcome> outcome)
|
||||
=> (await outcome.ConfigureAwait(false)).ToVirtualTagEvalResult();
|
||||
}
|
||||
|
||||
/// <summary>Record a compile failure for <paramref name="sourceKey"/> and log it ONCE — the next trigger
|
||||
/// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning.</summary>
|
||||
private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what)
|
||||
|
||||
@@ -12,10 +12,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
/// no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
|
||||
/// workers.
|
||||
/// </summary>
|
||||
public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
|
||||
public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
|
||||
{
|
||||
/// <summary>Seams the out-of-process runner can serve in this build. Scripted alarms go last (design §14 #10).</summary>
|
||||
public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
|
||||
/// <summary>Seams the out-of-process runner can serve in this build.</summary>
|
||||
public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams =
|
||||
[ScriptSeam.VirtualTag, ScriptSeam.Calculation, ScriptSeam.ScriptedAlarm];
|
||||
|
||||
/// <summary>
|
||||
/// The seams an unset or empty <c>Scripting:Runner:Seams</c> selects. Scripted alarms are not among them: they go
|
||||
/// out of process only when listed explicitly, last in the rollout (design §14 #10 — the alarm budget and the
|
||||
/// engine refactor).
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
|
||||
|
||||
private readonly OutOfProcessScriptRunner? _runner;
|
||||
private readonly HashSet<ScriptSeam> _seams;
|
||||
@@ -94,5 +102,5 @@ public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
|
||||
public void Dispose() => _runner?.Dispose();
|
||||
|
||||
private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
|
||||
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : Enum.GetValues<ScriptSeam>();
|
||||
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
private readonly TimeSpan _runTimeout;
|
||||
private readonly ScriptRunGuard _guard;
|
||||
private readonly IScriptRunner? _runner;
|
||||
private readonly ScriptBatchCoalescer? _batches;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
|
||||
@@ -79,6 +80,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
_runTimeout = runTimeout ?? (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
|
||||
_guard = new ScriptRunGuard(scripting, _scriptRoot);
|
||||
_runner = scriptRunner;
|
||||
_batches = scriptRunner is null ? null : new ScriptBatchCoalescer(scriptRunner);
|
||||
}
|
||||
|
||||
/// <summary>The out-of-process runner this evaluator delegates compiled runs to, or null (in process).</summary>
|
||||
@@ -87,6 +89,9 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
/// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
|
||||
public ScriptRunGuard RunGuard => _guard;
|
||||
|
||||
/// <summary>The coalescer the out-of-process path submits through, or null in process (tests).</summary>
|
||||
internal ScriptBatchCoalescer? Batches => _batches;
|
||||
|
||||
// Value-only overload. Every dependency reaches the script stamped Good with an
|
||||
// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
|
||||
// overload instead; this one remains for callers that genuinely hold only
|
||||
@@ -161,9 +166,11 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
// lmxopcua#780 — OutOfProcess: the compiled run goes to the pre-emptible worker pool, which compiles in the
|
||||
// worker and owns timeouts, kills and quarantine for this seam. An Unavailable outcome (worker restarted, no
|
||||
// worker ready) publishes BadResourceUnavailable; everything else keeps today's Bad contract.
|
||||
if (_runner is not null)
|
||||
// Every tag has its own actor, so tags an upstream burst made ready together arrive here concurrently; the
|
||||
// coalescer submits those as one batch (design §4.5) and an evaluation on an idle seam at once.
|
||||
if (_batches is not null)
|
||||
{
|
||||
var outcome = await _runner.RunAsync(new ScriptInvocation(
|
||||
var outcome = await _batches.RunAsync(new ScriptInvocation(
|
||||
virtualTagId, ScriptContextKind.VirtualTag, expression, readCache, _runTimeout, scriptLog)).ConfigureAwait(false);
|
||||
return outcome.ToVirtualTagEvalResult();
|
||||
}
|
||||
|
||||
@@ -385,6 +385,9 @@ if (hasDriver)
|
||||
Log.Information("Scripting runner: {RunnerMode}", scriptingOptions.Runner);
|
||||
builder.Services.AddSingleton(sp => ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection.Create(
|
||||
sp.GetRequiredService<ScriptingOptions>(), Log.Logger, sp.GetService<ScriptRootLogger>()?.Logger));
|
||||
// The Runtime's scripted-alarm host resolves its runner through this (it cannot reference the Host).
|
||||
builder.Services.AddSingleton<IScriptRunnerProvider>(sp =>
|
||||
sp.GetRequiredService<ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection>());
|
||||
|
||||
// lmxopcua#777: the ONE operator-write timeout (DriverWrites:Timeout) every write bound is derived from — the
|
||||
// driver child's cancel + abandon, its queue budget, DriverHostActor's Ask and the inbound-write gateway's Ask.
|
||||
|
||||
@@ -206,6 +206,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <summary>Node-wide script run-guard options for the scripted-alarm engine; null ⇒ compiled defaults.</summary>
|
||||
private readonly ScriptingOptions? _scripting;
|
||||
|
||||
// lmxopcua#780 — the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm; null keeps the engine's
|
||||
// in-process guard path.
|
||||
private readonly IScriptRunner? _scriptedAlarmRunner;
|
||||
|
||||
/// <summary>
|
||||
/// Driver children this host has stopped (<see cref="StopChild"/>, an operator Restart) and is watching
|
||||
/// until they terminate. A value burst from one of them is DROPPED in <see cref="ForwardBatchToMux"/>:
|
||||
@@ -771,7 +775,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
ScriptingOptions? scripting = null,
|
||||
NativeAlarmAckPreflightView? ackPreflightView = null,
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null) =>
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -785,7 +790,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
|
||||
scripting, ackPreflightView, writeBounds, writeClock));
|
||||
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -916,8 +921,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
ScriptingOptions? scripting = null,
|
||||
NativeAlarmAckPreflightView? ackPreflightView = null,
|
||||
DriverWriteBounds? writeBounds = null,
|
||||
TimeProvider? writeClock = null)
|
||||
TimeProvider? writeClock = null,
|
||||
IScriptRunner? scriptedAlarmRunner = null)
|
||||
{
|
||||
_scriptedAlarmRunner = scriptedAlarmRunner;
|
||||
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
|
||||
_writeClock = writeClock ?? TimeProvider.System;
|
||||
_ackPreflightView = ackPreflightView;
|
||||
@@ -1242,13 +1249,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
var store = _alarmStateStore;
|
||||
var scriptLogger = _scriptRootLogger.Logger;
|
||||
var scripting = _scripting;
|
||||
var alarmRunner = _scriptedAlarmRunner;
|
||||
_scriptedAlarmHost = Context.ActorOf(
|
||||
ScriptedAlarmHostActor.Props(
|
||||
_opcUaPublishActor, _dependencyMux,
|
||||
() =>
|
||||
{
|
||||
var upstream = new DependencyMuxTagUpstreamSource();
|
||||
return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting));
|
||||
return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting, alarmRunner));
|
||||
},
|
||||
_localNode, telemetryHub: _telemetryHub),
|
||||
"scripted-alarm-host");
|
||||
@@ -1265,15 +1273,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <param name="store">The condition-state store.</param>
|
||||
/// <param name="scriptLogger">The root script logger.</param>
|
||||
/// <param name="scripting">The node-wide <c>Scripting</c> options; null ⇒ defaults.</param>
|
||||
/// <param name="scriptRunner">The out-of-process runner for the ScriptedAlarm seam; null ⇒ in process.</param>
|
||||
/// <returns>A new, unloaded engine.</returns>
|
||||
internal static ScriptedAlarmEngine CreateScriptedAlarmEngine(
|
||||
ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.ITagUpstreamSource upstream, IAlarmStateStore store,
|
||||
Serilog.ILogger scriptLogger, ScriptingOptions? scripting)
|
||||
Serilog.ILogger scriptLogger, ScriptingOptions? scripting, IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
var budget = (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
|
||||
return new ScriptedAlarmEngine(
|
||||
upstream, store, new ScriptLoggerFactory(scriptLogger), scriptLogger,
|
||||
scriptTimeout: budget, scripting: scripting);
|
||||
scriptTimeout: budget, scripting: scripting, scriptRunner: scriptRunner);
|
||||
}
|
||||
|
||||
private void Bootstrap()
|
||||
|
||||
@@ -664,7 +664,11 @@ public static class ServiceCollectionExtensions
|
||||
// The one configured write timeout (DriverWrites:Timeout, bound + validated in Program.cs) the
|
||||
// children's write bounds AND this host's write Ask are derived from (lmxopcua#777). Unbound in
|
||||
// test graphs ⇒ null ⇒ DriverWriteBounds.Default.
|
||||
writeBounds: resolver.GetService<DriverWriteBounds>()),
|
||||
writeBounds: resolver.GetService<DriverWriteBounds>(),
|
||||
// lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
|
||||
// the InProcess default, when the seam is not listed, and in test graphs that register no provider.
|
||||
scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
|
||||
?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
|
||||
@@ -34,8 +34,22 @@ internal sealed class WorkerHost
|
||||
private readonly int _parentPid;
|
||||
private readonly ConcurrentDictionary<string, LoadedScript> _scripts = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<long, RunningEval> _running = new();
|
||||
/// <summary>
|
||||
/// Hand-off to an IDLE evaluation thread: an evaluation is put here only when a thread is known to be waiting
|
||||
/// for it (<see cref="_idle"/>), so whatever is put here starts at once. Everything else waits in
|
||||
/// <see cref="_backlog"/> and is reported to the host as deferred (lmxopcua#780 batch submit: per-item deadlines
|
||||
/// stay exact, see the proto's <c>EvalResults</c>).
|
||||
/// </summary>
|
||||
private readonly BlockingCollection<EvalRequest> _queue = new(new ConcurrentQueue<EvalRequest>());
|
||||
|
||||
/// <summary>Evaluations that arrived while every evaluation thread was busy, in arrival order. Under <see cref="_schedLock"/>.</summary>
|
||||
private readonly Queue<EvalRequest> _backlog = new();
|
||||
|
||||
private readonly object _schedLock = new();
|
||||
|
||||
/// <summary>Evaluation threads committed to taking from <see cref="_queue"/>, minus hand-offs not yet taken. Under <see cref="_schedLock"/>.</summary>
|
||||
private int _idle;
|
||||
|
||||
/// <summary>
|
||||
/// Loads waiting for a compile thread. Compiles are long, synchronous and CPU-bound, so they run on
|
||||
/// <see cref="CompileThreads"/> dedicated threads, never on the thread pool: a burst of cold loads (a node start
|
||||
@@ -225,8 +239,20 @@ internal sealed class WorkerHost
|
||||
if (_scripts.TryRemove(key, out var gone)) gone.Dispose();
|
||||
return null;
|
||||
case Envelope.BodyOneofCase.Eval:
|
||||
foreach (var item in frame.Eval.Items) Admit(item);
|
||||
{
|
||||
List<ulong>? deferred = null;
|
||||
foreach (var item in frame.Eval.Items)
|
||||
if (!Admit(item)) (deferred ??= []).Add(item.EvalId);
|
||||
// One notice per incoming frame. It can reach the host after the deferred run's own start notice (an
|
||||
// evaluation thread may pick the run up before this is queued); the host treats the two as unordered.
|
||||
if (deferred is not null)
|
||||
{
|
||||
var notice = new EvalResults();
|
||||
notice.DeferredEvalIds.AddRange(deferred);
|
||||
Send(new Envelope { Results = notice });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
case Envelope.BodyOneofCase.Cancel:
|
||||
if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
|
||||
{
|
||||
@@ -288,38 +314,72 @@ internal sealed class WorkerHost
|
||||
Send(new Envelope { LoadResult = result });
|
||||
}
|
||||
|
||||
private void Admit(EvalRequest request)
|
||||
/// <summary>
|
||||
/// Schedules one evaluation: straight to an idle evaluation thread when there is one (it starts now, so the
|
||||
/// host's clock, started at submit, is right), otherwise into the backlog.
|
||||
/// </summary>
|
||||
/// <param name="request">The evaluation.</param>
|
||||
/// <returns>False when it was deferred to the backlog (the host must stop its clock until it starts).</returns>
|
||||
private bool Admit(EvalRequest request)
|
||||
{
|
||||
if (_queue.Count >= _queueCapacity)
|
||||
lock (_schedLock)
|
||||
{
|
||||
SendResult(new EvalResult
|
||||
if (_backlog.Count + _queue.Count < _queueCapacity)
|
||||
{
|
||||
EvalId = request.EvalId,
|
||||
Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
|
||||
});
|
||||
return;
|
||||
if (_idle > 0 && _backlog.Count == 0)
|
||||
{
|
||||
_idle--;
|
||||
_queue.Add(request);
|
||||
return true;
|
||||
}
|
||||
_backlog.Enqueue(request);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_queue.Add(request);
|
||||
// Full: answered at once, outside the lock (a send may write to the pipe on this thread).
|
||||
SendResult(new EvalResult
|
||||
{
|
||||
EvalId = request.EvalId,
|
||||
Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private void EvalThreadLoop()
|
||||
{
|
||||
foreach (var request in _queue.GetConsumingEnumerable())
|
||||
EvalResult? finished = null;
|
||||
while (true)
|
||||
{
|
||||
EvalResult result;
|
||||
// A thread that just finished a run takes the oldest deferred evaluation, and its start notice rides on
|
||||
// the finished run's result frame. With nothing deferred it goes idle and waits for a hand-off.
|
||||
EvalRequest? request = null;
|
||||
lock (_schedLock)
|
||||
{
|
||||
if (finished is not null && _backlog.TryDequeue(out var next)) request = next;
|
||||
else _idle++;
|
||||
}
|
||||
if (finished is not null)
|
||||
{
|
||||
var frame = new EvalResults();
|
||||
frame.Items.Add(finished);
|
||||
if (request is not null) frame.StartedEvalIds.Add(request.EvalId);
|
||||
Send(new Envelope { Results = frame });
|
||||
finished = null;
|
||||
}
|
||||
request ??= _queue.Take();
|
||||
|
||||
try
|
||||
{
|
||||
result = Evaluate(request);
|
||||
finished = Evaluate(request);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result = new EvalResult
|
||||
finished = new EvalResult
|
||||
{
|
||||
EvalId = request.EvalId,
|
||||
Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = ex.Message },
|
||||
};
|
||||
}
|
||||
SendResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,7 +491,7 @@ internal sealed class WorkerHost
|
||||
Heartbeat = new Heartbeat
|
||||
{
|
||||
Inflight = (uint)_running.Count,
|
||||
Queued = (uint)_queue.Count,
|
||||
Queued = (uint)QueuedCount(),
|
||||
HeapBytes = GC.GetTotalMemory(false),
|
||||
Gen2Count = (uint)GC.CollectionCount(2),
|
||||
},
|
||||
@@ -489,6 +549,11 @@ internal sealed class WorkerHost
|
||||
Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
|
||||
}
|
||||
|
||||
private int QueuedCount()
|
||||
{
|
||||
lock (_schedLock) return _backlog.Count + _queue.Count;
|
||||
}
|
||||
|
||||
private void SendResult(EvalResult result)
|
||||
{
|
||||
var results = new EvalResults();
|
||||
|
||||
Reference in New Issue
Block a user