feat(scripting): out-of-process pre-emptible script runner behind Scripting:Runner (lmxopcua#780 P1)
v2-ci / unit-tests (push) Waiting to run

Opt-in (Scripting:Runner=OutOfProcess; InProcess stays the default and its code path is unchanged). Implements
the P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md with the owner's §14 decisions:

- Options: Scripting:Runner (InProcess | Shadow | OutOfProcess), Scripting:Runner:Seams, Scripting:Worker:*
  (Shards 2, HardGrace max(1 s,T), HeapLimitBytes 512 MiB, QuarantineAfterKills 2, UnavailableGrace 5 s,
  IsolationProbationRuns 20, PeCacheDir validated absolute + outside the content root (fail closed),
  ExecutablePath). Shadow is not implemented yet: runs InProcess with a startup Warning.
- Core.Scripting: IScriptRunner seam + InProcessScriptRunner (today's guard, the parity baseline).
- Core.Scripting.Protocol: protobuf envelopes, 4-byte LE framing (zero/oversize/truncated/garbage refused),
  Variant carrying the exact CLR type; unsupported types are explicit failures.
- ScriptWorker app: compiles in the worker, P-thread scheduler, host-driven soft cancel, fail-fast
  watchdog, heartbeats, ctx.Logger forwarding, exits on pipe EOF / parent gone, forbidden-assembly
  self-check; Akka and Grpc.Core.Api excluded from its folder. Shipped in script-worker/ (Build + Publish).
- Core.Scripting.OutOfProcess: K shards + isolation worker, soft T / hard T+G kill, attribution
  (deadline kill = one key; crash = suspects -> isolation; isolation crash = strike), kill-strike
  quarantine, backoff 250 ms -> 30 s, fail closed, cleared allow-listed env, nonce + pid handshake on a
  current-user pipe, Windows Job Object.
- VirtualTag + Calculation seams use the runner when selected; a runner-side failure (worker restarted /
  runner unavailable) publishes BadResourceUnavailable (0x80040000) — ScadaBridge#112. Script faults keep
  BadInternalError. The scripted-alarm seam stays in process (alarms last).

Tests: new ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests (Category=ScriptWorker, real processes; green on macOS and
on windev/Windows), seam tests in Host/Runtime/Calculation suites; mutation-checked.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 23:31:51 -05:00
parent f5c30ed0ef
commit 69a6e0b4c8
49 changed files with 5107 additions and 25 deletions
@@ -0,0 +1,213 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// <summary>
/// Where compiled scripts run (<c>Scripting:Runner</c>, lmxopcua#780). <see cref="InProcess"/> is the default
/// and the documented rollback: it is today's <see cref="ScriptRunGuard"/> path, unchanged.
/// </summary>
public enum ScriptRunnerMode
{
/// <summary>Compiled scripts run in the host process under <see cref="ScriptRunGuard"/> (the default).</summary>
InProcess = 0,
/// <summary>
/// Run in process AND submit to the worker, comparing the two (design P2). Not implemented yet: a node
/// configured for it runs <see cref="InProcess"/> and logs a startup Warning.
/// </summary>
Shadow = 1,
/// <summary>
/// Compiled scripts of the selected seams run in the pre-emptible <c>ScriptWorker</c> process pool. The
/// runner fails closed: a worker that cannot start makes the script Bad
/// (<c>BadResourceUnavailable</c>), never an in-process run.
/// </summary>
OutOfProcess = 2,
}
/// <summary>The three host seams that run operator-authored C# (<c>Scripting:Runner:Seams</c>).</summary>
public enum ScriptSeam
{
/// <summary>Virtual tags (<c>RoslynVirtualTagEvaluator</c>).</summary>
VirtualTag = 0,
/// <summary>Calculation tags (<c>CalculationEvaluator</c>).</summary>
Calculation = 1,
/// <summary>Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10).</summary>
ScriptedAlarm = 2,
}
/// <summary>
/// The out-of-process worker pool's knobs (the <c>Scripting:Worker</c> section). Every value is clamped by
/// <see cref="Sanitized"/>, which reports each out-of-range key as a startup Warning, like the rest of the
/// <c>Scripting</c> section. Only consulted when <see cref="ScriptingOptions.Runner"/> is not
/// <see cref="ScriptRunnerMode.InProcess"/>.
/// </summary>
public sealed class ScriptWorkerOptions
{
/// <summary>Default <see cref="Shards"/>: 2 (design §14 #1).</summary>
public const int DefaultShards = 2;
/// <summary>Upper bound of <see cref="Shards"/>.</summary>
public const int MaxShards = 8;
/// <summary>Default <see cref="HeapLimitBytes"/>: 512 MiB.</summary>
public const long DefaultHeapLimitBytes = 512L * 1024 * 1024;
/// <summary>Lower bound of <see cref="HeapLimitBytes"/>: 64 MiB (a worker below this cannot host Roslyn).</summary>
public const long MinHeapLimitBytes = 64L * 1024 * 1024;
/// <summary>Default <see cref="QuarantineAfterKills"/>: 2.</summary>
public const int DefaultQuarantineAfterKills = 2;
/// <summary>Default <see cref="UnavailableGrace"/>: 5 s.</summary>
public static readonly TimeSpan DefaultUnavailableGrace = TimeSpan.FromSeconds(5);
/// <summary>Lower bound of an explicit <see cref="HardGrace"/>: 250 ms.</summary>
public static readonly TimeSpan MinHardGrace = TimeSpan.FromMilliseconds(250);
/// <summary>Upper bound of an explicit <see cref="HardGrace"/>: 30 s.</summary>
public static readonly TimeSpan MaxHardGrace = TimeSpan.FromSeconds(30);
/// <summary>Upper bound of <see cref="UnavailableGrace"/>: 60 s.</summary>
public static readonly TimeSpan MaxUnavailableGrace = TimeSpan.FromSeconds(60);
/// <summary>Default <see cref="IsolationProbationRuns"/>: 20.</summary>
public const int DefaultIsolationProbationRuns = 20;
/// <summary>Shard workers per node; a script is placed by a stable hash of its key. Clamped to 1–8.</summary>
public int Shards { get; set; } = DefaultShards;
/// <summary>
/// Grace after the soft deadline <c>T</c> before the worker running the script is killed. Null (the default)
/// means <c>max(1 s, T)</c> per run (design §14 #2); an explicit value is clamped to 250 ms – 30 s.
/// </summary>
public TimeSpan? HardGrace { get; set; }
/// <summary>The worker's <c>DOTNET_GCHeapHardLimit</c>. Default 512 MiB, minimum 64 MiB.</summary>
public long HeapLimitBytes { get; set; } = DefaultHeapLimitBytes;
/// <summary>Hard-deadline kills (within one hour) after which a script is quarantined. Default 2; 1–100.</summary>
public int QuarantineAfterKills { get; set; } = DefaultQuarantineAfterKills;
/// <summary>
/// How long an evaluation waits for its shard to become ready (a restart, a backoff) before it reports
/// <c>script runner unavailable</c>. Default 5 s; 0–60 s.
/// </summary>
public TimeSpan UnavailableGrace { get; set; } = DefaultUnavailableGrace;
/// <summary>
/// Successful runs a suspect script must complete in the isolation worker before it returns to its shard.
/// Default 20; 1–10 000.
/// </summary>
public int IsolationProbationRuns { get; set; } = DefaultIsolationProbationRuns;
/// <summary>
/// Directory of the node-local compiled-PE cache (design §7). Must be absolute and outside the content root
/// when set; startup fails otherwise (the family's fail-closed path rule). Reserved: the cache itself has
/// not landed yet (lmxopcua#780 P1 remainder), so a worker restart recompiles.
/// </summary>
public string? PeCacheDir { get; set; }
/// <summary>
/// Path of the worker entry assembly (<c>ZB.MOM.WW.OtOpcUa.ScriptWorker.dll</c>). Null (the default) means
/// <c>script-worker/</c> under the host's base directory, where the Host build places it.
/// </summary>
public string? ExecutablePath { get; set; }
/// <summary>The hard grace that applies to a run with soft budget <paramref name="softTimeout"/>.</summary>
/// <param name="softTimeout">The run's soft deadline <c>T</c>.</param>
/// <returns><see cref="HardGrace"/> when set, otherwise <c>max(1 s, T)</c>.</returns>
public TimeSpan EffectiveHardGrace(TimeSpan softTimeout)
=> HardGrace ?? (softTimeout > TimeSpan.FromSeconds(1) ? softTimeout : TimeSpan.FromSeconds(1));
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
/// <param name="adjustments">The clamps applied, naming each key.</param>
/// <returns>A new, clamped instance.</returns>
public ScriptWorkerOptions Sanitized(out IReadOnlyList<string> adjustments)
{
const string prefix = ScriptingOptions.SectionName + ":Worker:";
var notes = new List<string>();
var shards = Shards;
if (shards < 1 || shards > MaxShards)
{
var clamped = Math.Clamp(shards, 1, MaxShards);
notes.Add($"{prefix}{nameof(Shards)} {shards} is outside 1–{MaxShards}; clamped to {clamped}.");
shards = clamped;
}
var grace = HardGrace;
if (grace is { } g && (g < MinHardGrace || g > MaxHardGrace))
{
var clamped = g < MinHardGrace ? MinHardGrace : MaxHardGrace;
notes.Add($"{prefix}{nameof(HardGrace)} {g} is outside {MinHardGrace}–{MaxHardGrace}; clamped to {clamped}.");
grace = clamped;
}
var heap = HeapLimitBytes;
if (heap < MinHeapLimitBytes)
{
notes.Add($"{prefix}{nameof(HeapLimitBytes)} {heap} is below the minimum; clamped to {MinHeapLimitBytes}.");
heap = MinHeapLimitBytes;
}
var kills = QuarantineAfterKills;
if (kills < 1 || kills > 100)
{
var clamped = Math.Clamp(kills, 1, 100);
notes.Add($"{prefix}{nameof(QuarantineAfterKills)} {kills} is outside 1–100; clamped to {clamped}.");
kills = clamped;
}
var unavailable = UnavailableGrace;
if (unavailable < TimeSpan.Zero || unavailable > MaxUnavailableGrace)
{
var clamped = unavailable < TimeSpan.Zero ? TimeSpan.Zero : MaxUnavailableGrace;
notes.Add($"{prefix}{nameof(UnavailableGrace)} {unavailable} is outside 0–{MaxUnavailableGrace}; clamped to {clamped}.");
unavailable = clamped;
}
var probation = IsolationProbationRuns;
if (probation < 1 || probation > 10_000)
{
var clamped = Math.Clamp(probation, 1, 10_000);
notes.Add($"{prefix}{nameof(IsolationProbationRuns)} {probation} is outside 1–10000; clamped to {clamped}.");
probation = clamped;
}
adjustments = notes;
return new ScriptWorkerOptions
{
Shards = shards,
HardGrace = grace,
HeapLimitBytes = heap,
QuarantineAfterKills = kills,
UnavailableGrace = unavailable,
IsolationProbationRuns = probation,
PeCacheDir = PeCacheDir,
ExecutablePath = ExecutablePath,
};
}
/// <summary>
/// Fail-closed path validation (design §7): <see cref="PeCacheDir"/>, when set, must be absolute and must not
/// sit inside <paramref name="contentRootPath"/>. Same rule as the family's <c>Secrets:SqlitePath</c>.
/// </summary>
/// <param name="contentRootPath">The host's content root.</param>
/// <returns>The violation, or null when the options are acceptable.</returns>
public string? ValidatePaths(string contentRootPath)
{
if (string.IsNullOrWhiteSpace(PeCacheDir)) return null;
const string key = ScriptingOptions.SectionName + ":Worker:" + nameof(PeCacheDir);
if (!Path.IsPathFullyQualified(PeCacheDir))
return $"{key} '{PeCacheDir}' must be an absolute path.";
var dir = Path.TrimEndingDirectorySeparator(Path.GetFullPath(PeCacheDir));
var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(contentRootPath));
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
if (string.Equals(dir, root, comparison)
|| dir.StartsWith(root + Path.DirectorySeparatorChar, comparison)
|| dir.StartsWith(root + Path.AltDirectorySeparatorChar, comparison))
return $"{key} '{PeCacheDir}' is inside the content root '{contentRootPath}'; it must live outside it.";
return null;
}
}