2a8e2981fe
v2-ci / unit-tests (push) Has been cancelled
Owner decisions §14 #5/#6 as built (design §12.5): - Scripting:Worker:Identity (default unset, Windows only, fail-closed validation): each worker starts under a restricted token derived from the host's. Every privilege but SeChangeNotify is removed; the restricting SIDs are NT SERVICE\OtOpcUaScriptWorker's SID + Everyone + Users + RESTRICTED; the token and default DACLs are host + SYSTEM + identity + RESTRICTED. The worker is created suspended, put in its Job Object before it runs, inherits only its stdio pipes, gets an ACL'd work dir as TEMP, and runs with DOTNET_EnableDiagnostics=0. The identity is granted on the host's window station and desktop, because USER32 init made coreclr.dll fail with ERROR_DLL_INIT_FAILED on windev. The logon SID is not used because the host's process DACL gives it PROCESS_VM_READ. - The worker pipe on Windows always has an explicit protected DACL: host account + identity only. There is no owner ACE, so an elevated host's Administrators gets no access. - The worker treats access denied on its parent-pid poll as alive; the Job Object covers that case. - scripts/install/Install-ScriptWorkerIsolation.ps1 + ScriptWorkerIsolation.psm1 are idempotent and support -WhatIf, -Uninstall and -PlanOnly. They create the disabled anchor service, the PeCacheDir ACL (protected, no Users, key and markers host-only, pe\ Modify for the identity), read+execute on script-worker\, a ReadData deny on the install root, the outbound firewall block on the worker exe, and optionally the host-service env. - Tests: the dry-run harness scripts/install/tests/Test-ScriptWorkerIsolation.ps1 (46 assertions, mutation-checked); ScriptWorkerIdentityTests; WorkerIsolationTests, which cover the pipe ACL, token access and a real restricted worker running a script, all green on windev. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
334 lines
16 KiB
C#
334 lines
16 KiB
C#
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||
|
||
/// <summary>
|
||
/// Where compiled scripts run (<c>Scripting:Runner</c>, lmxopcua#780). <see cref="InProcess"/> is the default
|
||
/// and the documented rollback: it is today's <see cref="ScriptRunGuard"/> path, unchanged.
|
||
/// </summary>
|
||
public enum ScriptRunnerMode
|
||
{
|
||
/// <summary>Compiled scripts run in the host process under <see cref="ScriptRunGuard"/> (the default).</summary>
|
||
InProcess = 0,
|
||
|
||
/// <summary>
|
||
/// Run in process — that result is the one published — AND mirror each evaluation of the selected seams to the
|
||
/// worker pool, comparing the two (design P2, <see cref="ShadowScriptRunner"/>). A time-boxed diagnostic: it
|
||
/// doubles script CPU and keeps the in-process risk. Nothing on the shadow side reaches a published value or
|
||
/// readiness.
|
||
/// </summary>
|
||
Shadow = 1,
|
||
|
||
/// <summary>
|
||
/// Compiled scripts of the selected seams run in the pre-emptible <c>ScriptWorker</c> process pool. The
|
||
/// runner fails closed: a worker that cannot start makes the script Bad
|
||
/// (<c>BadResourceUnavailable</c>), never an in-process run.
|
||
/// </summary>
|
||
OutOfProcess = 2,
|
||
}
|
||
|
||
/// <summary>The three host seams that run operator-authored C# (<c>Scripting:Runner:Seams</c>).</summary>
|
||
public enum ScriptSeam
|
||
{
|
||
/// <summary>Virtual tags (<c>RoslynVirtualTagEvaluator</c>).</summary>
|
||
VirtualTag = 0,
|
||
|
||
/// <summary>Calculation tags (<c>CalculationEvaluator</c>).</summary>
|
||
Calculation = 1,
|
||
|
||
/// <summary>
|
||
/// Scripted-alarm predicates (<c>ScriptedAlarmEngine</c>). Last in the rollout (design §14 #10): out of process only
|
||
/// when <c>Scripting:Runner:Seams</c> lists it — an unset list does not select it.
|
||
/// </summary>
|
||
ScriptedAlarm = 2,
|
||
}
|
||
|
||
/// <summary>
|
||
/// The out-of-process worker pool's knobs (the <c>Scripting:Worker</c> section). Every value is clamped by
|
||
/// <see cref="Sanitized"/>, which reports each out-of-range key as a startup Warning, like the rest of the
|
||
/// <c>Scripting</c> section. Only consulted when <see cref="ScriptingOptions.Runner"/> is not
|
||
/// <see cref="ScriptRunnerMode.InProcess"/>.
|
||
/// </summary>
|
||
public sealed class ScriptWorkerOptions
|
||
{
|
||
/// <summary>Default <see cref="Shards"/>: 2 (design §14 #1).</summary>
|
||
public const int DefaultShards = 2;
|
||
|
||
/// <summary>Upper bound of <see cref="Shards"/>.</summary>
|
||
public const int MaxShards = 8;
|
||
|
||
/// <summary>Default <see cref="HeapLimitBytes"/>: 512 MiB.</summary>
|
||
public const long DefaultHeapLimitBytes = 512L * 1024 * 1024;
|
||
|
||
/// <summary>Lower bound of <see cref="HeapLimitBytes"/>: 64 MiB (a worker below this cannot host Roslyn).</summary>
|
||
public const long MinHeapLimitBytes = 64L * 1024 * 1024;
|
||
|
||
/// <summary>Default <see cref="QuarantineAfterKills"/>: 2.</summary>
|
||
public const int DefaultQuarantineAfterKills = 2;
|
||
|
||
/// <summary>Default <see cref="UnavailableGrace"/>: 5 s.</summary>
|
||
public static readonly TimeSpan DefaultUnavailableGrace = TimeSpan.FromSeconds(5);
|
||
|
||
/// <summary>Lower bound of an explicit <see cref="HardGrace"/>: 250 ms.</summary>
|
||
public static readonly TimeSpan MinHardGrace = TimeSpan.FromMilliseconds(250);
|
||
|
||
/// <summary>Upper bound of an explicit <see cref="HardGrace"/>: 30 s.</summary>
|
||
public static readonly TimeSpan MaxHardGrace = TimeSpan.FromSeconds(30);
|
||
|
||
/// <summary>Upper bound of <see cref="UnavailableGrace"/>: 60 s.</summary>
|
||
public static readonly TimeSpan MaxUnavailableGrace = TimeSpan.FromSeconds(60);
|
||
|
||
/// <summary>Default <see cref="IsolationProbationRuns"/>: 20.</summary>
|
||
public const int DefaultIsolationProbationRuns = 20;
|
||
|
||
/// <summary>Default <see cref="IsolationIdleTimeout"/>: 10 min (design §3.3).</summary>
|
||
public static readonly TimeSpan DefaultIsolationIdleTimeout = TimeSpan.FromMinutes(10);
|
||
|
||
/// <summary>Default <see cref="CircuitOpenAfter"/>: 2 min (design §5.3.6).</summary>
|
||
public static readonly TimeSpan DefaultCircuitOpenAfter = TimeSpan.FromMinutes(2);
|
||
|
||
/// <summary>Shard workers per node; a script is placed by a stable hash of its key. Clamped to 1–8.</summary>
|
||
public int Shards { get; set; } = DefaultShards;
|
||
|
||
/// <summary>
|
||
/// Grace after the soft deadline <c>T</c> before the worker running the script is killed. Null (the default)
|
||
/// means <c>max(1 s, T)</c> per run (design §14 #2); an explicit value is clamped to 250 ms – 30 s.
|
||
/// </summary>
|
||
public TimeSpan? HardGrace { get; set; }
|
||
|
||
/// <summary>The worker's <c>DOTNET_GCHeapHardLimit</c>. Default 512 MiB, minimum 64 MiB.</summary>
|
||
public long HeapLimitBytes { get; set; } = DefaultHeapLimitBytes;
|
||
|
||
/// <summary>Hard-deadline kills (within one hour) after which a script is quarantined. Default 2; 1–100.</summary>
|
||
public int QuarantineAfterKills { get; set; } = DefaultQuarantineAfterKills;
|
||
|
||
/// <summary>
|
||
/// How long an evaluation waits for its shard to become ready (a restart, a backoff) before it reports
|
||
/// <c>script runner unavailable</c>. Default 5 s; 0–60 s.
|
||
/// </summary>
|
||
public TimeSpan UnavailableGrace { get; set; } = DefaultUnavailableGrace;
|
||
|
||
/// <summary>
|
||
/// Successful runs a suspect script must complete in the isolation worker before it returns to its shard.
|
||
/// Default 20; 1–10 000.
|
||
/// </summary>
|
||
public int IsolationProbationRuns { get; set; } = DefaultIsolationProbationRuns;
|
||
|
||
/// <summary>
|
||
/// How long the isolation worker may sit with nothing to run before it is stopped (it is started again by the
|
||
/// next suspect). Default 10 min; 1 s – 24 h.
|
||
/// </summary>
|
||
public TimeSpan IsolationIdleTimeout { get; set; } = DefaultIsolationIdleTimeout;
|
||
|
||
/// <summary>
|
||
/// The crash-loop circuit (design §5.3.6): when every shard worker has been down for this long, the runner
|
||
/// reports <c>Unavailable</c>, logs one Error and answers evaluations at once (no <see cref="UnavailableGrace"/>
|
||
/// wait) while it keeps retrying at the backoff cap. Default 2 min; 1 s – 1 h.
|
||
/// </summary>
|
||
public TimeSpan CircuitOpenAfter { get; set; } = DefaultCircuitOpenAfter;
|
||
|
||
/// <summary>
|
||
/// Directory of the node-local compiled-PE cache (design §7): HMAC-authenticated PE images, so a worker restart
|
||
/// does not re-enter Roslyn. Must be absolute and outside the content root when set; startup fails otherwise
|
||
/// (the family's fail-closed path rule). Null (the default) runs without a cache: every worker start recompiles.
|
||
/// </summary>
|
||
public string? PeCacheDir { get; set; }
|
||
|
||
/// <summary>
|
||
/// Path of the worker entry assembly (<c>ZB.MOM.WW.OtOpcUa.ScriptWorker.dll</c>). Null (the default) means
|
||
/// <c>script-worker/</c> under the host's base directory, where the Host build places it.
|
||
/// </summary>
|
||
public string? ExecutablePath { get; set; }
|
||
|
||
/// <summary>
|
||
/// The worker's identity on Windows (design §6, owner decision §14 #5): <c>NT SERVICE\<name></c> — the virtual
|
||
/// account <c>scripts/install/Install-ScriptWorkerIsolation.ps1</c> registers, by default
|
||
/// <c>NT SERVICE\OtOpcUaScriptWorker</c> — or a SID string. When set, each worker starts under a restricted token
|
||
/// derived from the host's (see <see cref="ScriptWorkerIdentity"/>): every privilege removed, and every access check
|
||
/// must ALSO pass for this identity, <c>Everyone</c>, <c>BUILTIN\Users</c> or <c>RESTRICTED</c>. The pipe ACL then
|
||
/// admits the host account and this identity only. Null (the default) keeps the host's own token. Windows only:
|
||
/// startup refuses it elsewhere.
|
||
/// </summary>
|
||
public string? Identity { get; set; }
|
||
|
||
/// <summary>The hard grace that applies to a run with soft budget <paramref name="softTimeout"/>.</summary>
|
||
/// <param name="softTimeout">The run's soft deadline <c>T</c>.</param>
|
||
/// <returns><see cref="HardGrace"/> when set, otherwise <c>max(1 s, T)</c>.</returns>
|
||
public TimeSpan EffectiveHardGrace(TimeSpan softTimeout)
|
||
=> HardGrace ?? (softTimeout > TimeSpan.FromSeconds(1) ? softTimeout : TimeSpan.FromSeconds(1));
|
||
|
||
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
|
||
/// <param name="adjustments">The clamps applied, naming each key.</param>
|
||
/// <returns>A new, clamped instance.</returns>
|
||
public ScriptWorkerOptions Sanitized(out IReadOnlyList<string> adjustments)
|
||
{
|
||
const string prefix = ScriptingOptions.SectionName + ":Worker:";
|
||
var notes = new List<string>();
|
||
|
||
var shards = Shards;
|
||
if (shards < 1 || shards > MaxShards)
|
||
{
|
||
var clamped = Math.Clamp(shards, 1, MaxShards);
|
||
notes.Add($"{prefix}{nameof(Shards)} {shards} is outside 1–{MaxShards}; clamped to {clamped}.");
|
||
shards = clamped;
|
||
}
|
||
|
||
var grace = HardGrace;
|
||
if (grace is { } g && (g < MinHardGrace || g > MaxHardGrace))
|
||
{
|
||
var clamped = g < MinHardGrace ? MinHardGrace : MaxHardGrace;
|
||
notes.Add($"{prefix}{nameof(HardGrace)} {g} is outside {MinHardGrace}–{MaxHardGrace}; clamped to {clamped}.");
|
||
grace = clamped;
|
||
}
|
||
|
||
var heap = HeapLimitBytes;
|
||
if (heap < MinHeapLimitBytes)
|
||
{
|
||
notes.Add($"{prefix}{nameof(HeapLimitBytes)} {heap} is below the minimum; clamped to {MinHeapLimitBytes}.");
|
||
heap = MinHeapLimitBytes;
|
||
}
|
||
|
||
var kills = QuarantineAfterKills;
|
||
if (kills < 1 || kills > 100)
|
||
{
|
||
var clamped = Math.Clamp(kills, 1, 100);
|
||
notes.Add($"{prefix}{nameof(QuarantineAfterKills)} {kills} is outside 1–100; clamped to {clamped}.");
|
||
kills = clamped;
|
||
}
|
||
|
||
var unavailable = UnavailableGrace;
|
||
if (unavailable < TimeSpan.Zero || unavailable > MaxUnavailableGrace)
|
||
{
|
||
var clamped = unavailable < TimeSpan.Zero ? TimeSpan.Zero : MaxUnavailableGrace;
|
||
notes.Add($"{prefix}{nameof(UnavailableGrace)} {unavailable} is outside 0–{MaxUnavailableGrace}; clamped to {clamped}.");
|
||
unavailable = clamped;
|
||
}
|
||
|
||
var probation = IsolationProbationRuns;
|
||
if (probation < 1 || probation > 10_000)
|
||
{
|
||
var clamped = Math.Clamp(probation, 1, 10_000);
|
||
notes.Add($"{prefix}{nameof(IsolationProbationRuns)} {probation} is outside 1–10000; clamped to {clamped}.");
|
||
probation = clamped;
|
||
}
|
||
|
||
var idle = IsolationIdleTimeout;
|
||
if (idle < TimeSpan.FromSeconds(1) || idle > TimeSpan.FromHours(24))
|
||
{
|
||
var clamped = idle < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : TimeSpan.FromHours(24);
|
||
notes.Add($"{prefix}{nameof(IsolationIdleTimeout)} {idle} is outside 00:00:01–1.00:00:00; clamped to {clamped}.");
|
||
idle = clamped;
|
||
}
|
||
|
||
var circuit = CircuitOpenAfter;
|
||
if (circuit < TimeSpan.FromSeconds(1) || circuit > TimeSpan.FromHours(1))
|
||
{
|
||
var clamped = circuit < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : TimeSpan.FromHours(1);
|
||
notes.Add($"{prefix}{nameof(CircuitOpenAfter)} {circuit} is outside 00:00:01–01:00:00; clamped to {clamped}.");
|
||
circuit = clamped;
|
||
}
|
||
|
||
adjustments = notes;
|
||
return new ScriptWorkerOptions
|
||
{
|
||
IsolationIdleTimeout = idle,
|
||
CircuitOpenAfter = circuit,
|
||
Shards = shards,
|
||
HardGrace = grace,
|
||
HeapLimitBytes = heap,
|
||
QuarantineAfterKills = kills,
|
||
UnavailableGrace = unavailable,
|
||
IsolationProbationRuns = probation,
|
||
PeCacheDir = PeCacheDir,
|
||
ExecutablePath = ExecutablePath,
|
||
Identity = Identity,
|
||
};
|
||
}
|
||
|
||
/// <summary>
|
||
/// Fail-closed path validation (design §7): <see cref="PeCacheDir"/>, when set, must be absolute and must not
|
||
/// sit inside <paramref name="contentRootPath"/>. Same rule as the family's <c>Secrets:SqlitePath</c>.
|
||
/// </summary>
|
||
/// <param name="contentRootPath">The host's content root.</param>
|
||
/// <returns>The violation, or null when the options are acceptable.</returns>
|
||
public string? ValidatePaths(string contentRootPath)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(PeCacheDir)) return null;
|
||
const string key = ScriptingOptions.SectionName + ":Worker:" + nameof(PeCacheDir);
|
||
if (!Path.IsPathFullyQualified(PeCacheDir))
|
||
return $"{key} '{PeCacheDir}' must be an absolute path.";
|
||
var dir = Path.TrimEndingDirectorySeparator(Path.GetFullPath(PeCacheDir));
|
||
var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(contentRootPath));
|
||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||
if (string.Equals(dir, root, comparison)
|
||
|| dir.StartsWith(root + Path.DirectorySeparatorChar, comparison)
|
||
|| dir.StartsWith(root + Path.AltDirectorySeparatorChar, comparison))
|
||
return $"{key} '{PeCacheDir}' is inside the content root '{contentRootPath}'; it must live outside it.";
|
||
return null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Fail-closed identity validation: <see cref="Identity"/>, when set, must be <c>NT SERVICE\<name></c> or a SID
|
||
/// string, and the host must run on Windows.
|
||
/// </summary>
|
||
/// <param name="isWindows">Whether the host runs on Windows (callers pass <see cref="OperatingSystem.IsWindows"/>).</param>
|
||
/// <returns>The violation, or null when the identity is unset or acceptable.</returns>
|
||
public string? ValidateIdentity(bool isWindows)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(Identity)) return null;
|
||
const string key = ScriptingOptions.SectionName + ":Worker:" + nameof(Identity);
|
||
if (!isWindows)
|
||
return $"{key} '{Identity}' is only supported on Windows (it is a restricted token); unset it on this host.";
|
||
return ScriptWorkerIdentity.TryResolveSid(Identity, out _, out var error) ? null : $"{key} '{Identity}': {error}";
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The shadow comparison's knobs (the <c>Scripting:Shadow</c> section, lmxopcua#780 P2). Only consulted when
|
||
/// <see cref="ScriptingOptions.Runner"/> is <see cref="ScriptRunnerMode.Shadow"/>.
|
||
/// </summary>
|
||
public sealed class ScriptShadowOptions
|
||
{
|
||
/// <summary>Default <see cref="MaxInFlight"/>: 256.</summary>
|
||
public const int DefaultMaxInFlight = 256;
|
||
|
||
/// <summary>Default <see cref="MismatchLogsPerMinute"/>: 20.</summary>
|
||
public const int DefaultMismatchLogsPerMinute = 20;
|
||
|
||
/// <summary>
|
||
/// Shadow runs outstanding at once (all seams). An evaluation that would exceed it is not mirrored and is counted
|
||
/// as dropped (<c>reason = behind</c>): the shadow is best effort and never queues without bound. Default 256;
|
||
/// 1–100 000.
|
||
/// </summary>
|
||
public int MaxInFlight { get; set; } = DefaultMaxInFlight;
|
||
|
||
/// <summary>
|
||
/// Mismatch Warning lines per minute (each script logs once per mismatch reason until its seam redeploys; this
|
||
/// caps a burst of many scripts). Default 20; 0 (counted, never logged) – 10 000.
|
||
/// </summary>
|
||
public int MismatchLogsPerMinute { get; set; } = DefaultMismatchLogsPerMinute;
|
||
|
||
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
|
||
/// <param name="adjustments">The clamps applied, naming each key.</param>
|
||
/// <returns>A new, clamped instance.</returns>
|
||
public ScriptShadowOptions Sanitized(out IReadOnlyList<string> adjustments)
|
||
{
|
||
const string prefix = ScriptingOptions.SectionName + ":Shadow:";
|
||
var notes = new List<string>();
|
||
var inFlight = MaxInFlight;
|
||
if (inFlight < 1 || inFlight > 100_000)
|
||
{
|
||
var clamped = Math.Clamp(inFlight, 1, 100_000);
|
||
notes.Add($"{prefix}{nameof(MaxInFlight)} {inFlight} is outside 1–100000; clamped to {clamped}.");
|
||
inFlight = clamped;
|
||
}
|
||
var logs = MismatchLogsPerMinute;
|
||
if (logs < 0 || logs > 10_000)
|
||
{
|
||
var clamped = Math.Clamp(logs, 0, 10_000);
|
||
notes.Add($"{prefix}{nameof(MismatchLogsPerMinute)} {logs} is outside 0–10000; clamped to {clamped}.");
|
||
logs = clamped;
|
||
}
|
||
adjustments = notes;
|
||
return new ScriptShadowOptions { MaxInFlight = inFlight, MismatchLogsPerMinute = logs };
|
||
}
|
||
}
|