feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -9,8 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
/// the <see cref="ScriptRunnerMode.InProcess"/> default every seam gets null and keeps its in-process guard path,
|
||||
/// unchanged. Under <see cref="ScriptRunnerMode.OutOfProcess"/> the selected seams share one
|
||||
/// <see cref="OutOfProcessScriptRunner"/>, which starts no worker until the first compiled evaluation, so a node with
|
||||
/// no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
|
||||
/// workers.
|
||||
/// no scripts (or an admin-only node) never launches one. Under <see cref="ScriptRunnerMode.Shadow"/> (P2) the selected
|
||||
/// seams get a <see cref="ShadowScriptRunner"/> over such a runner: they keep their in-process path, which is what they
|
||||
/// publish, and mirror each compiled evaluation to the worker pool for comparison. Disposed with the host's service
|
||||
/// provider, which stops the workers.
|
||||
/// </summary>
|
||||
public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
|
||||
{
|
||||
@@ -26,25 +28,43 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
|
||||
public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
|
||||
|
||||
private readonly OutOfProcessScriptRunner? _runner;
|
||||
private readonly ShadowScriptRunner? _shadow;
|
||||
private readonly HashSet<ScriptSeam> _seams;
|
||||
|
||||
private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, IEnumerable<ScriptSeam> seams)
|
||||
private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, ShadowScriptRunner? shadow, IEnumerable<ScriptSeam> seams)
|
||||
{
|
||||
_runner = runner;
|
||||
_shadow = shadow;
|
||||
_seams = [.. seams];
|
||||
}
|
||||
|
||||
/// <summary>The out-of-process runner, when one is configured.</summary>
|
||||
/// <summary>The out-of-process runner, when one is configured (under Shadow, the one shadow runs go to).</summary>
|
||||
public IScriptRunner? OutOfProcessRunner => _runner;
|
||||
|
||||
/// <summary>The out-of-process runner's status (the readiness check), or null when every seam runs in process.</summary>
|
||||
/// <summary>The shadow comparison, under <see cref="ScriptRunnerMode.Shadow"/>; otherwise null.</summary>
|
||||
public ShadowScriptRunner? Shadow => _shadow;
|
||||
|
||||
/// <summary>
|
||||
/// The out-of-process runner's status (the readiness check), or null when every seam publishes in-process
|
||||
/// results — <see cref="ScriptRunnerMode.InProcess"/>, and also <see cref="ScriptRunnerMode.Shadow"/>, whose worker
|
||||
/// pool must never move readiness (see <see cref="ShadowStatus"/>).
|
||||
/// </summary>
|
||||
/// <returns>The status, or null.</returns>
|
||||
public ScriptRunnerStatus? Status() => _runner?.Status();
|
||||
public ScriptRunnerStatus? Status() => _shadow is null ? _runner?.Status() : null;
|
||||
|
||||
/// <summary>Under <see cref="ScriptRunnerMode.Shadow"/>: the shadow worker pool's status and the comparison counters.</summary>
|
||||
/// <returns>The shadow status, or null in any other mode.</returns>
|
||||
public (ScriptRunnerStatus Runner, ScriptShadowStats Stats)? ShadowStatus()
|
||||
=> _shadow is not null && _runner is not null ? (_runner.Status(), _shadow.Stats()) : null;
|
||||
|
||||
/// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
|
||||
/// <param name="seam">The seam.</param>
|
||||
/// <returns>The runner, or null.</returns>
|
||||
public IScriptRunner? ForSeam(ScriptSeam seam) => _runner is not null && _seams.Contains(seam) ? _runner : null;
|
||||
/// <returns>The out-of-process runner; under Shadow the <see cref="ShadowScriptRunner"/>; or null.</returns>
|
||||
public IScriptRunner? ForSeam(ScriptSeam seam)
|
||||
{
|
||||
if (_runner is null || !_seams.Contains(seam)) return null;
|
||||
return _shadow ?? (IScriptRunner)_runner;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Startup checks for the runner options: returns the Warnings to log, and throws when the configuration must
|
||||
@@ -63,21 +83,27 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
|
||||
if (options.Worker.ValidatePaths(contentRootPath) is { } violation)
|
||||
throw new InvalidOperationException($"Scripting misconfiguration: {violation}");
|
||||
|
||||
if (options.Runner == ScriptRunnerMode.Shadow)
|
||||
var mode = $"Scripting:Runner={options.Runner}";
|
||||
var shadow = options.Runner == ScriptRunnerMode.Shadow;
|
||||
if (shadow)
|
||||
{
|
||||
warnings.Add("Scripting:Runner=Shadow is not implemented yet (lmxopcua#780 P2); every seam runs InProcess.");
|
||||
return warnings;
|
||||
warnings.Add($"{mode}: every selected script runs twice — in process (published) and in the ScriptWorker " +
|
||||
$"(compared only: {string.Join(", ", RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains))}). " +
|
||||
"It doubles script CPU and keeps the in-process risk: a time-boxed diagnostic, not a production mode.");
|
||||
}
|
||||
|
||||
foreach (var seam in RequestedSeams(options))
|
||||
{
|
||||
if (!SupportedOutOfProcessSeams.Contains(seam))
|
||||
warnings.Add($"Scripting:Runner=OutOfProcess: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
|
||||
warnings.Add($"{mode}: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
|
||||
}
|
||||
var workerPath = OutOfProcessScriptRunner.ResolveWorkerPath(options.Worker);
|
||||
if (!File.Exists(workerPath))
|
||||
warnings.Add($"Scripting:Runner=OutOfProcess: the script worker '{workerPath}' does not exist; every out-of-process " +
|
||||
"script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
|
||||
warnings.Add(shadow
|
||||
? $"{mode}: the script worker '{workerPath}' does not exist; every shadow run is skipped as shadow_unavailable " +
|
||||
"(published values are unaffected)."
|
||||
: $"{mode}: the script worker '{workerPath}' does not exist; every out-of-process " +
|
||||
"script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
|
||||
return warnings;
|
||||
}
|
||||
|
||||
@@ -89,17 +115,34 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
|
||||
public static ScriptRunnerSelection Create(ScriptingOptions options, SerilogLogger log, SerilogLogger? scriptRoot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
if (options.Runner != ScriptRunnerMode.OutOfProcess) return new ScriptRunnerSelection(null, []);
|
||||
ArgumentNullException.ThrowIfNull(log);
|
||||
if (options.Runner == ScriptRunnerMode.InProcess) return new ScriptRunnerSelection(null, null, []);
|
||||
var seams = RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains).ToArray();
|
||||
if (seams.Length == 0) return new ScriptRunnerSelection(null, []);
|
||||
return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), seams);
|
||||
if (seams.Length == 0) return new ScriptRunnerSelection(null, null, []);
|
||||
if (options.Runner == ScriptRunnerMode.OutOfProcess)
|
||||
return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), null, seams);
|
||||
|
||||
// Shadow: the worker pool's own lines (kills, quarantines, restarts) are about shadow runs, so they carry
|
||||
// ScriptRunnerRole=Shadow and go to the host log, never to the operator's script log — the published script was
|
||||
// not killed or quarantined. ctx.Logger lines of shadow runs are dropped (the in-process run wrote them).
|
||||
var shadowLog = log.ForContext("ScriptRunnerRole", "Shadow");
|
||||
var runner = new OutOfProcessScriptRunner(options, shadowLog, shadowLog);
|
||||
return new ScriptRunnerSelection(runner, new ShadowScriptRunner(runner, options, log), seams);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync() => _runner?.DisposeAsync() ?? ValueTask.CompletedTask;
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_shadow?.Dispose();
|
||||
if (_runner is not null) await _runner.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => _runner?.Dispose();
|
||||
public void Dispose()
|
||||
{
|
||||
_shadow?.Dispose();
|
||||
_runner?.Dispose();
|
||||
}
|
||||
|
||||
private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
|
||||
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;
|
||||
|
||||
Reference in New Issue
Block a user