feat(scripting): out-of-process pre-emptible script runner behind Scripting:Runner (lmxopcua#780 P1)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
Opt-in (Scripting:Runner=OutOfProcess; InProcess stays the default and its code path is unchanged). Implements the P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md with the owner's §14 decisions: - Options: Scripting:Runner (InProcess | Shadow | OutOfProcess), Scripting:Runner:Seams, Scripting:Worker:* (Shards 2, HardGrace max(1 s,T), HeapLimitBytes 512 MiB, QuarantineAfterKills 2, UnavailableGrace 5 s, IsolationProbationRuns 20, PeCacheDir validated absolute + outside the content root (fail closed), ExecutablePath). Shadow is not implemented yet: runs InProcess with a startup Warning. - Core.Scripting: IScriptRunner seam + InProcessScriptRunner (today's guard, the parity baseline). - Core.Scripting.Protocol: protobuf envelopes, 4-byte LE framing (zero/oversize/truncated/garbage refused), Variant carrying the exact CLR type; unsupported types are explicit failures. - ScriptWorker app: compiles in the worker, P-thread scheduler, host-driven soft cancel, fail-fast watchdog, heartbeats, ctx.Logger forwarding, exits on pipe EOF / parent gone, forbidden-assembly self-check; Akka and Grpc.Core.Api excluded from its folder. Shipped in script-worker/ (Build + Publish). - Core.Scripting.OutOfProcess: K shards + isolation worker, soft T / hard T+G kill, attribution (deadline kill = one key; crash = suspects -> isolation; isolation crash = strike), kill-strike quarantine, backoff 250 ms -> 30 s, fail closed, cleared allow-listed env, nonce + pid handshake on a current-user pipe, Windows Job Object. - VirtualTag + Calculation seams use the runner when selected; a runner-side failure (worker restarted / runner unavailable) publishes BadResourceUnavailable (0x80040000) — ScadaBridge#112. Script faults keep BadInternalError. The scripted-alarm seam stays in process (alarms last). Tests: new ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests (Category=ScriptWorker, real processes; green on macOS and on windev/Windows), seam tests in Host/Runtime/Calculation suites; mutation-checked. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -137,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
// _logger (ILogger<CalculationDriver>) implements the non-generic ILogger, so it is passed straight
|
||||
// through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution
|
||||
// throws) land on the SAME logger the driver itself was constructed with, not a separate one.
|
||||
_evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting);
|
||||
_evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting, _options.ScriptRunner);
|
||||
|
||||
RebuildTagTable(_options);
|
||||
}
|
||||
@@ -639,7 +639,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
if (!result.Success)
|
||||
{
|
||||
var reason = result.Reason ?? "evaluator failure";
|
||||
toPublish = MarkEvaluationFailed(st, pending.TimestampUtc);
|
||||
toPublish = MarkEvaluationFailed(st, pending.TimestampUtc, result.StatusCode ?? StatusBadInternalError);
|
||||
// The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes
|
||||
// (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which
|
||||
// flooded the script log at the input's cadence. The Bad publish was already once per
|
||||
@@ -725,16 +725,19 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// returns before the evaluator ever runs), so there is no separate "inputs ready" condition here.</summary>
|
||||
/// <param name="st">The tag's published state.</param>
|
||||
/// <param name="timestampUtc">The trigger's timestamp.</param>
|
||||
/// <param name="badStatus">The Bad code to publish (see the body).</param>
|
||||
/// <returns>The Bad snapshot to publish on a fresh Good→Bad transition, else null.</returns>
|
||||
private static (object? Value, uint StatusCode)? MarkEvaluationFailed(TagState st, DateTime timestampUtc)
|
||||
private static (object? Value, uint StatusCode)? MarkEvaluationFailed(TagState st, DateTime timestampUtc, uint badStatus)
|
||||
{
|
||||
if (st.HasPublished && st.LastPublishedStatus == StatusBadInternalError) return null;
|
||||
// badStatus is BadInternalError for every script failure; BadResourceUnavailable when the out-of-process
|
||||
// runner could not run the script (lmxopcua#780). Once per transition, keyed on that code.
|
||||
if (st.HasPublished && st.LastPublishedStatus == badStatus) return null;
|
||||
|
||||
st.HasPublished = true;
|
||||
st.LastPublishedStatus = StatusBadInternalError;
|
||||
st.LastPublishedStatus = badStatus;
|
||||
st.LastTimestampUtc = timestampUtc;
|
||||
// Carry the last-known value (null if none) with Bad quality.
|
||||
return (st.HasValue ? st.LastValue : null, StatusBadInternalError);
|
||||
return (st.HasValue ? st.LastValue : null, badStatus);
|
||||
}
|
||||
|
||||
private void Publish(string rawPath, object? value, uint statusCode, DateTime timestampUtc)
|
||||
|
||||
+9
-3
@@ -40,14 +40,17 @@ public static class CalculationDriverFactoryExtensions
|
||||
/// <param name="scriptRoot">Optional root script logger (fans failure entries onto the <c>script-logs</c> topic).</param>
|
||||
/// <param name="scripting">The node-wide <c>Scripting</c> options every Calculation driver inherits; a key
|
||||
/// authored in a driver's own config overrides that one knob. Null ⇒ the compiled defaults.</param>
|
||||
/// <param name="scriptRunner">lmxopcua#780 — the node's out-of-process runner when the Calculation seam runs out
|
||||
/// of process; null (the default) keeps every driver on the in-process guard.</param>
|
||||
public static void Register(
|
||||
DriverFactoryRegistry registry,
|
||||
ILoggerFactory? loggerFactory = null,
|
||||
ScriptRootLogger? scriptRoot = null,
|
||||
ScriptingOptions? scripting = null)
|
||||
ScriptingOptions? scripting = null,
|
||||
IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(registry);
|
||||
registry.Register(DriverTypeName, (id, json) => CreateInstance(id, json, loggerFactory, scriptRoot, scripting));
|
||||
registry.Register(DriverTypeName, (id, json) => CreateInstance(id, json, loggerFactory, scriptRoot, scripting, scriptRunner));
|
||||
}
|
||||
|
||||
/// <summary>Public overload for the Server-side bootstrapper + test consumers.</summary>
|
||||
@@ -74,10 +77,12 @@ public static class CalculationDriverFactoryExtensions
|
||||
/// <param name="loggerFactory">Optional logger factory for per-instance diagnostics loggers.</param>
|
||||
/// <param name="scriptRoot">Optional root script logger for failure fan-out.</param>
|
||||
/// <param name="nodeScripting">The node-wide run-guard options; null ⇒ the compiled defaults.</param>
|
||||
/// <param name="scriptRunner">The node's out-of-process runner, or null for in process.</param>
|
||||
/// <returns>The constructed <see cref="CalculationDriver"/>.</returns>
|
||||
public static CalculationDriver CreateInstance(
|
||||
string driverInstanceId, string driverConfigJson,
|
||||
ILoggerFactory? loggerFactory, ScriptRootLogger? scriptRoot, ScriptingOptions? nodeScripting)
|
||||
ILoggerFactory? loggerFactory, ScriptRootLogger? scriptRoot, ScriptingOptions? nodeScripting,
|
||||
IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(driverInstanceId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(driverConfigJson);
|
||||
@@ -101,6 +106,7 @@ public static class CalculationDriverFactoryExtensions
|
||||
RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0
|
||||
? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2),
|
||||
Scripting = scripting,
|
||||
ScriptRunner = scriptRunner,
|
||||
};
|
||||
|
||||
return new CalculationDriver(options, driverInstanceId, scriptRoot, logger: logger);
|
||||
|
||||
@@ -18,4 +18,11 @@ public sealed class CalculationDriverOptions
|
||||
/// <see cref="ScriptingOptions"/> defaults (64 MB, 3).
|
||||
/// </summary>
|
||||
public ScriptingOptions? Scripting { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — the node's out-of-process runner when <c>Scripting:Runner=OutOfProcess</c> selects the
|
||||
/// Calculation seam; null (the default) keeps the in-process run guard. Shared by every Calculation driver on the
|
||||
/// node; keys are the calc tags' RawPaths.
|
||||
/// </summary>
|
||||
public IScriptRunner? ScriptRunner { get; init; }
|
||||
}
|
||||
|
||||
@@ -59,6 +59,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
|
||||
private readonly SerilogLogger _scriptRoot;
|
||||
private readonly TimeSpan _runTimeout;
|
||||
private readonly ScriptRunGuard _guard;
|
||||
private readonly IScriptRunner? _runner;
|
||||
|
||||
// Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE,
|
||||
// which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a
|
||||
@@ -76,12 +77,16 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
|
||||
/// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
|
||||
/// <param name="runTimeout">Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator).</param>
|
||||
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
|
||||
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the Calculation seam runs out of
|
||||
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
|
||||
public CalculationEvaluator(
|
||||
ILogger logger,
|
||||
ScriptRootLogger scriptRoot,
|
||||
TimeSpan? runTimeout = null,
|
||||
ScriptingOptions? scripting = null)
|
||||
ScriptingOptions? scripting = null,
|
||||
IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
_runner = scriptRunner;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
|
||||
_runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
|
||||
@@ -188,6 +193,15 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
|
||||
calcTagId, path),
|
||||
logger: scriptLog);
|
||||
|
||||
// lmxopcua#780 — OutOfProcess: the pre-emptible worker pool compiles and runs it, and owns timeouts, kills,
|
||||
// quarantine and the negative compile cache for this seam. Unavailable publishes BadResourceUnavailable.
|
||||
if (_runner is not null)
|
||||
{
|
||||
var outcome = await _runner.RunAsync(new ScriptInvocation(
|
||||
calcTagId, ScriptContextKind.Calculation, scriptSource, readCache, _runTimeout, scriptLog), ct).ConfigureAwait(false);
|
||||
return outcome.ToVirtualTagEvalResult();
|
||||
}
|
||||
|
||||
// A source that already failed to compile fails the same way without touching Roslyn (R07:P4).
|
||||
var sourceKey = HashSource(scriptSource);
|
||||
if (_compileFailures.TryGetValue(sourceKey, out var knownFailure))
|
||||
@@ -295,13 +309,20 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
|
||||
// The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine
|
||||
// (an edited source already lifts its own through the guard's source identity).
|
||||
_guard.ResetAll();
|
||||
_runner?.Reset(ScriptContextKind.Calculation);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ResetQuarantines() => _guard.ResetAll();
|
||||
public void ResetQuarantines()
|
||||
{
|
||||
_guard.ResetAll();
|
||||
_runner?.Reset(ScriptContextKind.Calculation);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
|
||||
public IReadOnlyCollection<string> QuarantinedScripts => _runner is null
|
||||
? _guard.QuarantinedKeys
|
||||
: [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.Calculation)];
|
||||
|
||||
/// <summary>Disposes the evaluator, draining + disposing every cached compile's collectible ALC.</summary>
|
||||
public void Dispose()
|
||||
|
||||
+2
@@ -30,6 +30,8 @@
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests"/>
|
||||
<!-- lmxopcua#780: the worker suite drives a real driver over the real ScriptWorker (IsEvaluationIdle). -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user