feat(scripting): out-of-process pre-emptible script runner behind Scripting:Runner (lmxopcua#780 P1)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
Opt-in (Scripting:Runner=OutOfProcess; InProcess stays the default and its code path is unchanged). Implements the P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md with the owner's §14 decisions: - Options: Scripting:Runner (InProcess | Shadow | OutOfProcess), Scripting:Runner:Seams, Scripting:Worker:* (Shards 2, HardGrace max(1 s,T), HeapLimitBytes 512 MiB, QuarantineAfterKills 2, UnavailableGrace 5 s, IsolationProbationRuns 20, PeCacheDir validated absolute + outside the content root (fail closed), ExecutablePath). Shadow is not implemented yet: runs InProcess with a startup Warning. - Core.Scripting: IScriptRunner seam + InProcessScriptRunner (today's guard, the parity baseline). - Core.Scripting.Protocol: protobuf envelopes, 4-byte LE framing (zero/oversize/truncated/garbage refused), Variant carrying the exact CLR type; unsupported types are explicit failures. - ScriptWorker app: compiles in the worker, P-thread scheduler, host-driven soft cancel, fail-fast watchdog, heartbeats, ctx.Logger forwarding, exits on pipe EOF / parent gone, forbidden-assembly self-check; Akka and Grpc.Core.Api excluded from its folder. Shipped in script-worker/ (Build + Publish). - Core.Scripting.OutOfProcess: K shards + isolation worker, soft T / hard T+G kill, attribution (deadline kill = one key; crash = suspects -> isolation; isolation crash = strike), kill-strike quarantine, backoff 250 ms -> 30 s, fail closed, cleared allow-listed env, nonce + pid handshake on a current-user pipe, Windows Job Object. - VirtualTag + Calculation seams use the runner when selected; a runner-side failure (worker restarted / runner unavailable) publishes BadResourceUnavailable (0x80040000) — ScadaBridge#112. Script faults keep BadInternalError. The scripted-alarm seam stays in process (alarms last). Tests: new ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests (Category=ScriptWorker, real processes; green on macOS and on windev/Windows), seam tests in Host/Runtime/Calculation suites; mutation-checked. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,213 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
/// <summary>
|
||||
/// Where compiled scripts run (<c>Scripting:Runner</c>, lmxopcua#780). <see cref="InProcess"/> is the default
|
||||
/// and the documented rollback: it is today's <see cref="ScriptRunGuard"/> path, unchanged.
|
||||
/// </summary>
|
||||
public enum ScriptRunnerMode
|
||||
{
|
||||
/// <summary>Compiled scripts run in the host process under <see cref="ScriptRunGuard"/> (the default).</summary>
|
||||
InProcess = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Run in process AND submit to the worker, comparing the two (design P2). Not implemented yet: a node
|
||||
/// configured for it runs <see cref="InProcess"/> and logs a startup Warning.
|
||||
/// </summary>
|
||||
Shadow = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Compiled scripts of the selected seams run in the pre-emptible <c>ScriptWorker</c> process pool. The
|
||||
/// runner fails closed: a worker that cannot start makes the script Bad
|
||||
/// (<c>BadResourceUnavailable</c>), never an in-process run.
|
||||
/// </summary>
|
||||
OutOfProcess = 2,
|
||||
}
|
||||
|
||||
/// <summary>The three host seams that run operator-authored C# (<c>Scripting:Runner:Seams</c>).</summary>
|
||||
public enum ScriptSeam
|
||||
{
|
||||
/// <summary>Virtual tags (<c>RoslynVirtualTagEvaluator</c>).</summary>
|
||||
VirtualTag = 0,
|
||||
|
||||
/// <summary>Calculation tags (<c>CalculationEvaluator</c>).</summary>
|
||||
Calculation = 1,
|
||||
|
||||
/// <summary>Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10).</summary>
|
||||
ScriptedAlarm = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The out-of-process worker pool's knobs (the <c>Scripting:Worker</c> section). Every value is clamped by
|
||||
/// <see cref="Sanitized"/>, which reports each out-of-range key as a startup Warning, like the rest of the
|
||||
/// <c>Scripting</c> section. Only consulted when <see cref="ScriptingOptions.Runner"/> is not
|
||||
/// <see cref="ScriptRunnerMode.InProcess"/>.
|
||||
/// </summary>
|
||||
public sealed class ScriptWorkerOptions
|
||||
{
|
||||
/// <summary>Default <see cref="Shards"/>: 2 (design §14 #1).</summary>
|
||||
public const int DefaultShards = 2;
|
||||
|
||||
/// <summary>Upper bound of <see cref="Shards"/>.</summary>
|
||||
public const int MaxShards = 8;
|
||||
|
||||
/// <summary>Default <see cref="HeapLimitBytes"/>: 512 MiB.</summary>
|
||||
public const long DefaultHeapLimitBytes = 512L * 1024 * 1024;
|
||||
|
||||
/// <summary>Lower bound of <see cref="HeapLimitBytes"/>: 64 MiB (a worker below this cannot host Roslyn).</summary>
|
||||
public const long MinHeapLimitBytes = 64L * 1024 * 1024;
|
||||
|
||||
/// <summary>Default <see cref="QuarantineAfterKills"/>: 2.</summary>
|
||||
public const int DefaultQuarantineAfterKills = 2;
|
||||
|
||||
/// <summary>Default <see cref="UnavailableGrace"/>: 5 s.</summary>
|
||||
public static readonly TimeSpan DefaultUnavailableGrace = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Lower bound of an explicit <see cref="HardGrace"/>: 250 ms.</summary>
|
||||
public static readonly TimeSpan MinHardGrace = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>Upper bound of an explicit <see cref="HardGrace"/>: 30 s.</summary>
|
||||
public static readonly TimeSpan MaxHardGrace = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Upper bound of <see cref="UnavailableGrace"/>: 60 s.</summary>
|
||||
public static readonly TimeSpan MaxUnavailableGrace = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>Default <see cref="IsolationProbationRuns"/>: 20.</summary>
|
||||
public const int DefaultIsolationProbationRuns = 20;
|
||||
|
||||
/// <summary>Shard workers per node; a script is placed by a stable hash of its key. Clamped to 1–8.</summary>
|
||||
public int Shards { get; set; } = DefaultShards;
|
||||
|
||||
/// <summary>
|
||||
/// Grace after the soft deadline <c>T</c> before the worker running the script is killed. Null (the default)
|
||||
/// means <c>max(1 s, T)</c> per run (design §14 #2); an explicit value is clamped to 250 ms – 30 s.
|
||||
/// </summary>
|
||||
public TimeSpan? HardGrace { get; set; }
|
||||
|
||||
/// <summary>The worker's <c>DOTNET_GCHeapHardLimit</c>. Default 512 MiB, minimum 64 MiB.</summary>
|
||||
public long HeapLimitBytes { get; set; } = DefaultHeapLimitBytes;
|
||||
|
||||
/// <summary>Hard-deadline kills (within one hour) after which a script is quarantined. Default 2; 1–100.</summary>
|
||||
public int QuarantineAfterKills { get; set; } = DefaultQuarantineAfterKills;
|
||||
|
||||
/// <summary>
|
||||
/// How long an evaluation waits for its shard to become ready (a restart, a backoff) before it reports
|
||||
/// <c>script runner unavailable</c>. Default 5 s; 0–60 s.
|
||||
/// </summary>
|
||||
public TimeSpan UnavailableGrace { get; set; } = DefaultUnavailableGrace;
|
||||
|
||||
/// <summary>
|
||||
/// Successful runs a suspect script must complete in the isolation worker before it returns to its shard.
|
||||
/// Default 20; 1–10 000.
|
||||
/// </summary>
|
||||
public int IsolationProbationRuns { get; set; } = DefaultIsolationProbationRuns;
|
||||
|
||||
/// <summary>
|
||||
/// Directory of the node-local compiled-PE cache (design §7). Must be absolute and outside the content root
|
||||
/// when set; startup fails otherwise (the family's fail-closed path rule). Reserved: the cache itself has
|
||||
/// not landed yet (lmxopcua#780 P1 remainder), so a worker restart recompiles.
|
||||
/// </summary>
|
||||
public string? PeCacheDir { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Path of the worker entry assembly (<c>ZB.MOM.WW.OtOpcUa.ScriptWorker.dll</c>). Null (the default) means
|
||||
/// <c>script-worker/</c> under the host's base directory, where the Host build places it.
|
||||
/// </summary>
|
||||
public string? ExecutablePath { get; set; }
|
||||
|
||||
/// <summary>The hard grace that applies to a run with soft budget <paramref name="softTimeout"/>.</summary>
|
||||
/// <param name="softTimeout">The run's soft deadline <c>T</c>.</param>
|
||||
/// <returns><see cref="HardGrace"/> when set, otherwise <c>max(1 s, T)</c>.</returns>
|
||||
public TimeSpan EffectiveHardGrace(TimeSpan softTimeout)
|
||||
=> HardGrace ?? (softTimeout > TimeSpan.FromSeconds(1) ? softTimeout : TimeSpan.FromSeconds(1));
|
||||
|
||||
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
|
||||
/// <param name="adjustments">The clamps applied, naming each key.</param>
|
||||
/// <returns>A new, clamped instance.</returns>
|
||||
public ScriptWorkerOptions Sanitized(out IReadOnlyList<string> adjustments)
|
||||
{
|
||||
const string prefix = ScriptingOptions.SectionName + ":Worker:";
|
||||
var notes = new List<string>();
|
||||
|
||||
var shards = Shards;
|
||||
if (shards < 1 || shards > MaxShards)
|
||||
{
|
||||
var clamped = Math.Clamp(shards, 1, MaxShards);
|
||||
notes.Add($"{prefix}{nameof(Shards)} {shards} is outside 1–{MaxShards}; clamped to {clamped}.");
|
||||
shards = clamped;
|
||||
}
|
||||
|
||||
var grace = HardGrace;
|
||||
if (grace is { } g && (g < MinHardGrace || g > MaxHardGrace))
|
||||
{
|
||||
var clamped = g < MinHardGrace ? MinHardGrace : MaxHardGrace;
|
||||
notes.Add($"{prefix}{nameof(HardGrace)} {g} is outside {MinHardGrace}–{MaxHardGrace}; clamped to {clamped}.");
|
||||
grace = clamped;
|
||||
}
|
||||
|
||||
var heap = HeapLimitBytes;
|
||||
if (heap < MinHeapLimitBytes)
|
||||
{
|
||||
notes.Add($"{prefix}{nameof(HeapLimitBytes)} {heap} is below the minimum; clamped to {MinHeapLimitBytes}.");
|
||||
heap = MinHeapLimitBytes;
|
||||
}
|
||||
|
||||
var kills = QuarantineAfterKills;
|
||||
if (kills < 1 || kills > 100)
|
||||
{
|
||||
var clamped = Math.Clamp(kills, 1, 100);
|
||||
notes.Add($"{prefix}{nameof(QuarantineAfterKills)} {kills} is outside 1–100; clamped to {clamped}.");
|
||||
kills = clamped;
|
||||
}
|
||||
|
||||
var unavailable = UnavailableGrace;
|
||||
if (unavailable < TimeSpan.Zero || unavailable > MaxUnavailableGrace)
|
||||
{
|
||||
var clamped = unavailable < TimeSpan.Zero ? TimeSpan.Zero : MaxUnavailableGrace;
|
||||
notes.Add($"{prefix}{nameof(UnavailableGrace)} {unavailable} is outside 0–{MaxUnavailableGrace}; clamped to {clamped}.");
|
||||
unavailable = clamped;
|
||||
}
|
||||
|
||||
var probation = IsolationProbationRuns;
|
||||
if (probation < 1 || probation > 10_000)
|
||||
{
|
||||
var clamped = Math.Clamp(probation, 1, 10_000);
|
||||
notes.Add($"{prefix}{nameof(IsolationProbationRuns)} {probation} is outside 1–10000; clamped to {clamped}.");
|
||||
probation = clamped;
|
||||
}
|
||||
|
||||
adjustments = notes;
|
||||
return new ScriptWorkerOptions
|
||||
{
|
||||
Shards = shards,
|
||||
HardGrace = grace,
|
||||
HeapLimitBytes = heap,
|
||||
QuarantineAfterKills = kills,
|
||||
UnavailableGrace = unavailable,
|
||||
IsolationProbationRuns = probation,
|
||||
PeCacheDir = PeCacheDir,
|
||||
ExecutablePath = ExecutablePath,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fail-closed path validation (design §7): <see cref="PeCacheDir"/>, when set, must be absolute and must not
|
||||
/// sit inside <paramref name="contentRootPath"/>. Same rule as the family's <c>Secrets:SqlitePath</c>.
|
||||
/// </summary>
|
||||
/// <param name="contentRootPath">The host's content root.</param>
|
||||
/// <returns>The violation, or null when the options are acceptable.</returns>
|
||||
public string? ValidatePaths(string contentRootPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(PeCacheDir)) return null;
|
||||
const string key = ScriptingOptions.SectionName + ":Worker:" + nameof(PeCacheDir);
|
||||
if (!Path.IsPathFullyQualified(PeCacheDir))
|
||||
return $"{key} '{PeCacheDir}' must be an absolute path.";
|
||||
var dir = Path.TrimEndingDirectorySeparator(Path.GetFullPath(PeCacheDir));
|
||||
var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(contentRootPath));
|
||||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
if (string.Equals(dir, root, comparison)
|
||||
|| dir.StartsWith(root + Path.DirectorySeparatorChar, comparison)
|
||||
|| dir.StartsWith(root + Path.AltDirectorySeparatorChar, comparison))
|
||||
return $"{key} '{PeCacheDir}' is inside the content root '{contentRootPath}'; it must live outside it.";
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user