feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -87,6 +87,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch.
|
||||
/// </summary>
|
||||
private readonly IScriptRunner? _runner;
|
||||
private readonly IScriptShadow? _shadow;
|
||||
|
||||
// Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the
|
||||
// runner's Recovery to re-evaluate them once. Guarded by _dirtyLock.
|
||||
@@ -255,7 +256,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// silently ran with because the host never passed a value.</param>
|
||||
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
|
||||
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of
|
||||
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
|
||||
/// process; null (the default) keeps the in-process guard path, unchanged. Under <c>Scripting:Runner=Shadow</c> it is an
|
||||
/// <see cref="IScriptShadow"/>: the in-process path stays the one that drives the state machine and each predicate run
|
||||
/// is mirrored to it for comparison (P2).</param>
|
||||
public ScriptedAlarmEngine(
|
||||
ITagUpstreamSource upstream,
|
||||
IAlarmStateStore store,
|
||||
@@ -266,7 +269,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
ScriptingOptions? scripting = null,
|
||||
IScriptRunner? scriptRunner = null)
|
||||
{
|
||||
_runner = scriptRunner;
|
||||
_shadow = scriptRunner as IScriptShadow;
|
||||
_runner = _shadow is null ? scriptRunner : null;
|
||||
_upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
@@ -286,6 +290,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// <summary>The out-of-process runner predicates run through, or null in process (diagnostics / tests).</summary>
|
||||
public IScriptRunner? ScriptRunner => _runner;
|
||||
|
||||
/// <summary>The shadow predicate runs are mirrored to (<c>Scripting:Runner=Shadow</c>), or null.</summary>
|
||||
public IScriptShadow? ScriptShadow => _shadow;
|
||||
|
||||
/// <summary>
|
||||
/// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when
|
||||
/// one is in use (diagnostics / tests).
|
||||
@@ -366,6 +373,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
// its own through the guard's source identity; this covers "redeployed to retry").
|
||||
_guard.ResetAll();
|
||||
_runner?.Reset(ScriptContextKind.AlarmPredicate);
|
||||
_shadow?.Reset(ScriptContextKind.AlarmPredicate);
|
||||
lock (_dirtyLock) _recoveryArmed.Clear();
|
||||
|
||||
var compileFailures = new List<string>();
|
||||
@@ -920,23 +928,29 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
}
|
||||
catch (ScriptTimeoutException tex)
|
||||
{
|
||||
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(
|
||||
$"{ZB.MOM.WW.OtOpcUa.Commons.Engines.ScriptEvalFailureReasons.TimedOutPrefix}{tex.Timeout.TotalSeconds:F1}s"));
|
||||
return PredicateDidNotComplete(state, seed, scratch,
|
||||
$"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions);
|
||||
}
|
||||
catch (ScriptRunRefusedException ex)
|
||||
{
|
||||
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(ex.Message));
|
||||
return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
|
||||
}
|
||||
catch (ScriptQuarantinedException ex)
|
||||
{
|
||||
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(ex.Message));
|
||||
return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed($"script threw: {ex.Message}"));
|
||||
scratch.LastNonCompletion = null;
|
||||
state.Logger.Error(ex, "Alarm predicate threw — state unchanged");
|
||||
return seed;
|
||||
}
|
||||
MirrorPredicate(state, scratch, ScriptRunOutcome.Ok(predicateTrue));
|
||||
scratch.LastNonCompletion = null;
|
||||
|
||||
// Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last
|
||||
@@ -963,6 +977,19 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
return result.State;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 P2 — Shadow: mirrors one in-process predicate run, after it has driven the state machine's input,
|
||||
/// to the shadow runner for comparison. Never blocks; the inputs are copied before it returns, so the alarm's
|
||||
/// reused read cache can be refilled at once.
|
||||
/// </summary>
|
||||
private void MirrorPredicate(AlarmState state, AlarmScratch scratch, ScriptRunOutcome published)
|
||||
{
|
||||
if (_shadow is null) return;
|
||||
_shadow.Mirror(new ScriptInvocation(
|
||||
state.Definition.AlarmId, ScriptContextKind.AlarmPredicate, state.Definition.PredicateScriptSource,
|
||||
scratch.ReadCache, _scriptTimeout), published);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the
|
||||
/// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in
|
||||
|
||||
@@ -76,7 +76,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
public const string WorkerFolderName = "script-worker";
|
||||
|
||||
/// <summary>Failure reason when the worker could not take more work.</summary>
|
||||
public const string WorkerBusyReason = "script worker busy";
|
||||
public const string WorkerBusyReason = ScriptRunOutcome.WorkerBusyReason;
|
||||
|
||||
/// <summary>PE cache entries unused for this long are pruned at startup.</summary>
|
||||
internal static readonly TimeSpan PeCacheMaxAge = TimeSpan.FromDays(30);
|
||||
|
||||
@@ -122,6 +122,9 @@ public sealed record ScriptRunOutcome(ScriptRunStatus Status, object? Value, str
|
||||
/// <summary>Reason when no worker became ready within <see cref="ScriptWorkerOptions.UnavailableGrace"/>.</summary>
|
||||
public const string RunnerUnavailableReason = "script runner unavailable";
|
||||
|
||||
/// <summary>Reason when the worker could not take more work (the out-of-process runner's back-pressure answer).</summary>
|
||||
public const string WorkerBusyReason = "script worker busy";
|
||||
|
||||
/// <summary>
|
||||
/// For <see cref="ScriptRunStatus.Unavailable"/>: completes when the runner can run this script again (its worker
|
||||
/// is back), so the seam re-evaluates it then with its latest inputs (design §9, "re-evaluated on the shard's
|
||||
@@ -175,7 +178,11 @@ public static class ScriptRunOutcomeExtensions
|
||||
/// </summary>
|
||||
public interface IScriptRunnerProvider
|
||||
{
|
||||
/// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
|
||||
/// <summary>
|
||||
/// The runner <paramref name="seam"/> must use, or null for its in-process path. Under
|
||||
/// <c>Scripting:Runner=Shadow</c> this is a <see cref="ShadowScriptRunner"/>: the seam recognises it as an
|
||||
/// <see cref="IScriptShadow"/>, keeps its in-process path and mirrors each evaluation to it.
|
||||
/// </summary>
|
||||
/// <param name="seam">The seam.</param>
|
||||
/// <returns>The runner, or null.</returns>
|
||||
IScriptRunner? ForSeam(ScriptSeam seam);
|
||||
|
||||
@@ -10,8 +10,10 @@ public enum ScriptRunnerMode
|
||||
InProcess = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Run in process AND submit to the worker, comparing the two (design P2). Not implemented yet: a node
|
||||
/// configured for it runs <see cref="InProcess"/> and logs a startup Warning.
|
||||
/// Run in process — that result is the one published — AND mirror each evaluation of the selected seams to the
|
||||
/// worker pool, comparing the two (design P2, <see cref="ShadowScriptRunner"/>). A time-boxed diagnostic: it
|
||||
/// doubles script CPU and keeps the in-process risk. Nothing on the shadow side reaches a published value or
|
||||
/// readiness.
|
||||
/// </summary>
|
||||
Shadow = 1,
|
||||
|
||||
@@ -251,3 +253,54 @@ public sealed class ScriptWorkerOptions
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The shadow comparison's knobs (the <c>Scripting:Shadow</c> section, lmxopcua#780 P2). Only consulted when
|
||||
/// <see cref="ScriptingOptions.Runner"/> is <see cref="ScriptRunnerMode.Shadow"/>.
|
||||
/// </summary>
|
||||
public sealed class ScriptShadowOptions
|
||||
{
|
||||
/// <summary>Default <see cref="MaxInFlight"/>: 256.</summary>
|
||||
public const int DefaultMaxInFlight = 256;
|
||||
|
||||
/// <summary>Default <see cref="MismatchLogsPerMinute"/>: 20.</summary>
|
||||
public const int DefaultMismatchLogsPerMinute = 20;
|
||||
|
||||
/// <summary>
|
||||
/// Shadow runs outstanding at once (all seams). An evaluation that would exceed it is not mirrored and is counted
|
||||
/// as dropped (<c>reason = behind</c>): the shadow is best effort and never queues without bound. Default 256;
|
||||
/// 1–100 000.
|
||||
/// </summary>
|
||||
public int MaxInFlight { get; set; } = DefaultMaxInFlight;
|
||||
|
||||
/// <summary>
|
||||
/// Mismatch Warning lines per minute (each script logs once per mismatch reason until its seam redeploys; this
|
||||
/// caps a burst of many scripts). Default 20; 0 (counted, never logged) – 10 000.
|
||||
/// </summary>
|
||||
public int MismatchLogsPerMinute { get; set; } = DefaultMismatchLogsPerMinute;
|
||||
|
||||
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
|
||||
/// <param name="adjustments">The clamps applied, naming each key.</param>
|
||||
/// <returns>A new, clamped instance.</returns>
|
||||
public ScriptShadowOptions Sanitized(out IReadOnlyList<string> adjustments)
|
||||
{
|
||||
const string prefix = ScriptingOptions.SectionName + ":Shadow:";
|
||||
var notes = new List<string>();
|
||||
var inFlight = MaxInFlight;
|
||||
if (inFlight < 1 || inFlight > 100_000)
|
||||
{
|
||||
var clamped = Math.Clamp(inFlight, 1, 100_000);
|
||||
notes.Add($"{prefix}{nameof(MaxInFlight)} {inFlight} is outside 1–100000; clamped to {clamped}.");
|
||||
inFlight = clamped;
|
||||
}
|
||||
var logs = MismatchLogsPerMinute;
|
||||
if (logs < 0 || logs > 10_000)
|
||||
{
|
||||
var clamped = Math.Clamp(logs, 0, 10_000);
|
||||
notes.Add($"{prefix}{nameof(MismatchLogsPerMinute)} {logs} is outside 0–10000; clamped to {clamped}.");
|
||||
logs = clamped;
|
||||
}
|
||||
adjustments = notes;
|
||||
return new ScriptShadowOptions { MaxInFlight = inFlight, MismatchLogsPerMinute = logs };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +89,9 @@ public sealed class ScriptingOptions
|
||||
/// <summary>The out-of-process worker pool's knobs (<c>Scripting:Worker</c>).</summary>
|
||||
public ScriptWorkerOptions Worker { get; set; } = new();
|
||||
|
||||
/// <summary>The shadow comparison's knobs (<c>Scripting:Shadow</c>), for <see cref="ScriptRunnerMode.Shadow"/>.</summary>
|
||||
public ScriptShadowOptions Shadow { get; set; } = new();
|
||||
|
||||
/// <summary>Whether <paramref name="seam"/> is selected for out-of-process execution by these options.</summary>
|
||||
/// <param name="seam">The seam.</param>
|
||||
/// <returns><c>true</c> when <see cref="Runner"/> is <see cref="ScriptRunnerMode.OutOfProcess"/> and the seam is in
|
||||
@@ -185,6 +188,8 @@ public sealed class ScriptingOptions
|
||||
|
||||
var worker = (Worker ?? new ScriptWorkerOptions()).Sanitized(out var workerNotes);
|
||||
notes.AddRange(workerNotes);
|
||||
var shadow = (Shadow ?? new ScriptShadowOptions()).Sanitized(out var shadowNotes);
|
||||
notes.AddRange(shadowNotes);
|
||||
|
||||
adjustments = notes;
|
||||
return new ScriptingOptions
|
||||
@@ -196,6 +201,7 @@ public sealed class ScriptingOptions
|
||||
Runner = Runner,
|
||||
RunnerSeams = RunnerSeams is null ? null : [.. RunnerSeams],
|
||||
Worker = worker,
|
||||
Shadow = shadow,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,510 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Globalization;
|
||||
using Serilog;
|
||||
using Serilog.Core;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
/// <summary>
|
||||
/// The mirror half of <c>Scripting:Runner=Shadow</c> (lmxopcua#780 P2, design §12): a seam that has already produced
|
||||
/// — and published — its in-process outcome hands the same evaluation here, and it is run again on the shadow
|
||||
/// runner (the out-of-process worker pool) and compared.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A seam receives its <see cref="IScriptShadow"/> as the <see cref="IScriptRunner"/> it is constructed with (the
|
||||
/// node's runner selection hands out <see cref="ShadowScriptRunner"/>, which is both). A seam that recognises it keeps
|
||||
/// its own in-process path, unchanged, and calls <see cref="Mirror"/> after each compiled evaluation.
|
||||
/// </remarks>
|
||||
public interface IScriptShadow
|
||||
{
|
||||
/// <summary>
|
||||
/// Mirrors one evaluation whose outcome <paramref name="published"/> the seam has already published. Never
|
||||
/// blocks, never throws, and nothing it does reaches the seam: the shadow run is queued on the pool, or dropped
|
||||
/// (and counted) when the shadow side is behind.
|
||||
/// </summary>
|
||||
/// <param name="invocation">The evaluation as the seam ran it. Its inputs are copied before this returns, so the
|
||||
/// caller may reuse its read cache at once.</param>
|
||||
/// <param name="published">The in-process outcome — the source of truth.</param>
|
||||
void Mirror(ScriptInvocation invocation, ScriptRunOutcome published);
|
||||
|
||||
/// <summary>The seam's deploy boundary: lifts the shadow runner's quarantines for <paramref name="kind"/> and
|
||||
/// re-arms its once-per-script mismatch log lines.</summary>
|
||||
/// <param name="kind">The seam's context kind.</param>
|
||||
void Reset(ScriptContextKind kind);
|
||||
}
|
||||
|
||||
/// <summary>How one shadow comparison came out.</summary>
|
||||
public enum ScriptShadowVerdict
|
||||
{
|
||||
/// <summary>The shadow run agreed with the published outcome.</summary>
|
||||
Match = 0,
|
||||
|
||||
/// <summary>The shadow run disagreed; <see cref="ScriptShadowComparison.Reason"/> names how.</summary>
|
||||
Mismatch = 1,
|
||||
|
||||
/// <summary>The shadow runner could not run the script (worker restarted, none ready, worker busy): not compared.</summary>
|
||||
ShadowUnavailable = 2,
|
||||
}
|
||||
|
||||
/// <summary>The result of comparing a published outcome with its shadow.</summary>
|
||||
/// <param name="Verdict">Match, mismatch, or not compared.</param>
|
||||
/// <param name="Reason">For a mismatch, the metric reason: <c>value</c>, <c>type</c>, <c>status</c>,
|
||||
/// <c>failure_kind</c>, <c>exception</c> or <c>unsupported_type</c>.</param>
|
||||
/// <param name="Published">A one-line description of the published outcome.</param>
|
||||
/// <param name="Shadow">A one-line description of the shadow outcome.</param>
|
||||
public sealed record ScriptShadowComparison(ScriptShadowVerdict Verdict, string? Reason, string Published, string Shadow)
|
||||
{
|
||||
/// <summary>The diff summary a mismatch log line carries.</summary>
|
||||
public string Summary => $"published {Published}; shadow {Shadow}";
|
||||
}
|
||||
|
||||
/// <summary>Counters of one <see cref="ShadowScriptRunner"/> since it started.</summary>
|
||||
/// <param name="Compared">Shadow runs that completed and were compared (match or mismatch).</param>
|
||||
/// <param name="Matched">Comparisons that agreed.</param>
|
||||
/// <param name="Mismatched">Comparisons that disagreed.</param>
|
||||
/// <param name="ShadowUnavailable">Shadow runs the runner could not run (not compared).</param>
|
||||
/// <param name="DroppedBehind">Evaluations not mirrored because <see cref="ScriptShadowOptions.MaxInFlight"/> shadow
|
||||
/// runs were already outstanding.</param>
|
||||
/// <param name="DroppedKeyBusy">Evaluations not mirrored because the same script's previous shadow run was still
|
||||
/// outstanding.</param>
|
||||
/// <param name="InFlight">Shadow runs outstanding now.</param>
|
||||
public sealed record ScriptShadowStats(
|
||||
long Compared, long Matched, long Mismatched, long ShadowUnavailable, long DroppedBehind, long DroppedKeyBusy, int InFlight)
|
||||
{
|
||||
/// <summary>Every evaluation that was not mirrored.</summary>
|
||||
public long Dropped => DroppedBehind + DroppedKeyBusy;
|
||||
}
|
||||
|
||||
/// <summary>One script's mismatch record.</summary>
|
||||
/// <param name="Kind">The seam's context kind.</param>
|
||||
/// <param name="Key">The script key (un-namespaced).</param>
|
||||
/// <param name="Count">Mismatches counted for it.</param>
|
||||
/// <param name="LastReason">The reason of the latest one.</param>
|
||||
/// <param name="LastSummary">The diff summary of the latest one.</param>
|
||||
public sealed record ScriptShadowMismatch(ScriptContextKind Kind, string Key, long Count, string LastReason, string LastSummary);
|
||||
|
||||
/// <summary>
|
||||
/// <c>Scripting:Runner=Shadow</c> (lmxopcua#780 P2, design §12): every evaluation of a shadowed seam runs IN PROCESS —
|
||||
/// that result is the one published — and is mirrored, best effort, to a shadow <see cref="IScriptRunner"/> (the
|
||||
/// out-of-process worker pool), whose result is only compared: value, CLR type, status and failure kind.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <list type="bullet">
|
||||
/// <item><b>The published value never depends on the shadow.</b> <see cref="Mirror"/> copies the inputs and queues
|
||||
/// the shadow run on the pool; a shadow-side timeout, kill, crash, quarantine, a runner that hangs or throws, all
|
||||
/// stay here.</item>
|
||||
/// <item><b>Bounded.</b> At most <see cref="ScriptShadowOptions.MaxInFlight"/> shadow runs are outstanding and at
|
||||
/// most one per script; anything beyond that is dropped and counted (<c>otopcua.script.shadow.dropped</c>,
|
||||
/// <c>reason</c> = <c>behind</c> | <c>key_busy</c>). One per script also keeps the out-of-process runner's
|
||||
/// single-flight rule from refusing a shadow run and reporting a false mismatch.</item>
|
||||
/// <item><b>Counted per script.</b> <c>otopcua.script.shadow.compared{kind, result}</c> and
|
||||
/// <c>otopcua.script.shadow.mismatch{kind, reason, script}</c>; <see cref="Mismatches"/> keeps the latest diff per
|
||||
/// script. A mismatch logs one Warning with the diff summary the first time each script mismatches for each
|
||||
/// reason (re-armed at the seam's deploy boundary), and at most <see cref="ScriptShadowOptions.MismatchLogsPerMinute"/>
|
||||
/// such lines a minute.</item>
|
||||
/// <item><b>Not compared</b>: a shadow outcome that is the runner's fault (<see cref="ScriptRunStatus.Unavailable"/>,
|
||||
/// or the worker busy) — that is node health, not a script divergence.</item>
|
||||
/// <item><b>Unsupported types</b> (design §14 #11): a value the protocol cannot carry makes the shadow answer
|
||||
/// <c>unsupported type</c>, which counts as a mismatch with reason <c>unsupported_type</c> — the census the flip to
|
||||
/// OutOfProcess needs, since out of process that evaluation would publish Bad.</item>
|
||||
/// </list>
|
||||
/// As an <see cref="IScriptRunner"/>, <see cref="RunAsync"/> runs an <see cref="InProcessScriptRunner"/> and mirrors
|
||||
/// its outcome, for callers that only speak the runner interface. The three production seams recognise
|
||||
/// <see cref="IScriptShadow"/> instead and keep their own in-process code, so what they publish is byte-for-byte
|
||||
/// what <c>InProcess</c> publishes.
|
||||
/// </remarks>
|
||||
public sealed class ShadowScriptRunner : IScriptRunner, IScriptShadow, IDisposable
|
||||
{
|
||||
private static readonly Counter<long> ComparedCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
||||
"otopcua.script.shadow.compared", unit: "{eval}",
|
||||
description: "Shadow-mode evaluations whose shadow run completed (tags: kind; result = match | mismatch | shadow_unavailable).");
|
||||
|
||||
private static readonly Counter<long> MismatchCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
||||
"otopcua.script.shadow.mismatch", unit: "{eval}",
|
||||
description: "Shadow-mode evaluations whose out-of-process result differed from the published in-process one " +
|
||||
"(tags: kind; reason = value | type | status | failure_kind | exception | unsupported_type; script).");
|
||||
|
||||
private static readonly Counter<long> DroppedCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
||||
"otopcua.script.shadow.dropped", unit: "{eval}",
|
||||
description: "Shadow-mode evaluations not mirrored because the shadow side was behind (tags: kind; reason = behind | key_busy).");
|
||||
|
||||
private static readonly ConcurrentDictionary<ShadowScriptRunner, byte> LiveRunners = new();
|
||||
|
||||
static ShadowScriptRunner()
|
||||
{
|
||||
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.shadow.inflight",
|
||||
static () => LiveRunners.Keys.Select(r => new Measurement<int>(Volatile.Read(ref r._inFlight))),
|
||||
unit: "{eval}", description: "Shadow-mode runs outstanding on the shadow runner.");
|
||||
}
|
||||
|
||||
private readonly IScriptRunner _shadow;
|
||||
private readonly InProcessScriptRunner _primary;
|
||||
private readonly ScriptShadowOptions _options;
|
||||
private readonly ILogger _log;
|
||||
private readonly Func<DateTime> _clock;
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private readonly ConcurrentDictionary<string, byte> _busy = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, byte> _logged = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, ScriptShadowMismatch> _mismatches = new(StringComparer.Ordinal);
|
||||
private readonly object _logLock = new();
|
||||
private DateTime _logWindowStart;
|
||||
private int _logsInWindow;
|
||||
private long _logsSuppressed;
|
||||
private int _inFlight;
|
||||
private long _compared, _matched, _mismatched, _unavailable, _droppedBehind, _droppedBusy;
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>Initializes the shadow runner.</summary>
|
||||
/// <param name="shadow">The runner every mirrored evaluation is run on again (the out-of-process pool). Not owned.</param>
|
||||
/// <param name="scripting">The node's <c>Scripting</c> options (the in-process guard's knobs and <c>Scripting:Shadow</c>);
|
||||
/// null ⇒ defaults.</param>
|
||||
/// <param name="log">Where mismatch lines go; null ⇒ Serilog's global logger.</param>
|
||||
/// <param name="clock">The mismatch-log rate limiter's clock (tests); null ⇒ <see cref="DateTime.UtcNow"/>.</param>
|
||||
public ShadowScriptRunner(IScriptRunner shadow, ScriptingOptions? scripting = null, ILogger? log = null, Func<DateTime>? clock = null)
|
||||
{
|
||||
_shadow = shadow ?? throw new ArgumentNullException(nameof(shadow));
|
||||
var options = (scripting ?? new ScriptingOptions()).Sanitized(out _);
|
||||
_options = options.Shadow;
|
||||
_log = (log ?? Log.Logger).ForContext<ShadowScriptRunner>();
|
||||
_clock = clock ?? (() => DateTime.UtcNow);
|
||||
_primary = new InProcessScriptRunner(options, log);
|
||||
_logWindowStart = _clock();
|
||||
LiveRunners.TryAdd(this, 0);
|
||||
}
|
||||
|
||||
/// <summary>The runner shadow runs go to.</summary>
|
||||
public IScriptRunner Shadow => _shadow;
|
||||
|
||||
/// <summary>The counters since start.</summary>
|
||||
/// <returns>A snapshot.</returns>
|
||||
public ScriptShadowStats Stats() => new(
|
||||
Interlocked.Read(ref _compared), Interlocked.Read(ref _matched), Interlocked.Read(ref _mismatched),
|
||||
Interlocked.Read(ref _unavailable), Interlocked.Read(ref _droppedBehind), Interlocked.Read(ref _droppedBusy),
|
||||
Volatile.Read(ref _inFlight));
|
||||
|
||||
/// <summary>Every script that has mismatched, with its count and latest diff.</summary>
|
||||
public IReadOnlyCollection<ScriptShadowMismatch> Mismatches => _mismatches.Values.ToArray();
|
||||
|
||||
/// <summary>
|
||||
/// Waits until no shadow run is outstanding (tests and diagnostics: a shadow run is asynchronous to the
|
||||
/// evaluation that triggered it).
|
||||
/// </summary>
|
||||
/// <param name="timeout">How long to wait.</param>
|
||||
/// <returns><c>true</c> when idle within <paramref name="timeout"/>.</returns>
|
||||
public async Task<bool> WaitIdleAsync(TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (Volatile.Read(ref _inFlight) > 0)
|
||||
{
|
||||
if (DateTime.UtcNow >= deadline) return false;
|
||||
await Task.Delay(10).ConfigureAwait(false);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default)
|
||||
{
|
||||
var published = await _primary.RunAsync(invocation, ct).ConfigureAwait(false);
|
||||
if (!ct.IsCancellationRequested) Mirror(invocation, published);
|
||||
return published;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Mirror(ScriptInvocation invocation, ScriptRunOutcome published)
|
||||
{
|
||||
if (invocation is null || published is null || Volatile.Read(ref _disposed) != 0) return;
|
||||
var scoped = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
||||
if (!_busy.TryAdd(scoped, 0))
|
||||
{
|
||||
Drop(invocation.Kind, "key_busy", ref _droppedBusy);
|
||||
return;
|
||||
}
|
||||
if (Interlocked.Increment(ref _inFlight) > _options.MaxInFlight)
|
||||
{
|
||||
Release(scoped);
|
||||
Drop(invocation.Kind, "behind", ref _droppedBehind);
|
||||
return;
|
||||
}
|
||||
|
||||
ScriptInvocation copy;
|
||||
try
|
||||
{
|
||||
// The caller may refill its read cache as soon as this returns (the alarm engine reuses one per alarm).
|
||||
copy = invocation with
|
||||
{
|
||||
Inputs = new Dictionary<string, DataValueSnapshot>(invocation.Inputs, StringComparer.Ordinal),
|
||||
// The in-process run already wrote this evaluation's ctx.Logger lines; the shadow's are not repeated.
|
||||
ScriptLogger = Logger.None,
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Release(scoped);
|
||||
_log.Debug(ex, "Script shadow: could not copy the inputs of {ScriptKey}; not mirrored", invocation.Key);
|
||||
return;
|
||||
}
|
||||
|
||||
// Never on the caller's thread: a shadow runner may do synchronous work (encode, write) before its first await.
|
||||
_ = Task.Run(() => RunShadowAsync(scoped, copy, published));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Reset(ScriptContextKind kind)
|
||||
{
|
||||
_primary.Reset(kind);
|
||||
try
|
||||
{
|
||||
_shadow.Reset(kind);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "Script shadow: the shadow runner's reset failed for {ScriptKind}", kind);
|
||||
}
|
||||
var prefix = InProcessScriptRunner.ScopedKey(kind, string.Empty);
|
||||
foreach (var key in _logged.Keys)
|
||||
if (key.StartsWith(prefix, StringComparison.Ordinal)) _logged.TryRemove(key, out _);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => _primary.QuarantinedKeys(kind);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
|
||||
LiveRunners.TryRemove(this, out _);
|
||||
_stopping.Cancel();
|
||||
_primary.Dispose();
|
||||
_stopping.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares a published outcome with its shadow (pure; the table the mismatch reasons are tested from).
|
||||
/// </summary>
|
||||
/// <param name="published">The in-process outcome.</param>
|
||||
/// <param name="shadow">The shadow runner's outcome.</param>
|
||||
/// <returns>The comparison.</returns>
|
||||
public static ScriptShadowComparison Compare(ScriptRunOutcome published, ScriptRunOutcome shadow)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(published);
|
||||
ArgumentNullException.ThrowIfNull(shadow);
|
||||
var p = Describe(published);
|
||||
var s = Describe(shadow);
|
||||
|
||||
if (shadow.Status == ScriptRunStatus.Unavailable
|
||||
|| (shadow.Status == ScriptRunStatus.Failed && shadow.Reason == ScriptRunOutcome.WorkerBusyReason))
|
||||
return new ScriptShadowComparison(ScriptShadowVerdict.ShadowUnavailable, null, p, s);
|
||||
|
||||
ScriptShadowComparison Mismatch(string reason) => new(ScriptShadowVerdict.Mismatch, reason, p, s);
|
||||
var match = new ScriptShadowComparison(ScriptShadowVerdict.Match, null, p, s);
|
||||
|
||||
if (published.Status == ScriptRunStatus.Ok && shadow.Status == ScriptRunStatus.Ok)
|
||||
{
|
||||
if (published.Value is null && shadow.Value is null) return match;
|
||||
if (published.Value is null || shadow.Value is null) return Mismatch("value");
|
||||
if (published.Value.GetType() != shadow.Value.GetType()) return Mismatch("type");
|
||||
return ValuesEqual(published.Value, shadow.Value) ? match : Mismatch("value");
|
||||
}
|
||||
|
||||
var publishedClass = published.Status == ScriptRunStatus.Ok ? null : FailureClass(published.Reason);
|
||||
var shadowClass = shadow.Status == ScriptRunStatus.Ok ? null : FailureClass(shadow.Reason);
|
||||
if (shadowClass == UnsupportedTypeClass && publishedClass != UnsupportedTypeClass) return Mismatch("unsupported_type");
|
||||
if (publishedClass is null || shadowClass is null) return Mismatch("status");
|
||||
if (publishedClass != shadowClass) return Mismatch("failure_kind");
|
||||
if (publishedClass == ThrewClass && !string.Equals(published.Reason, shadow.Reason, StringComparison.Ordinal))
|
||||
return Mismatch("exception");
|
||||
return match;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The failure class a reason belongs to, for comparison: <c>compile</c>, <c>budget</c> (timed out, killed,
|
||||
/// refused behind a run still going, quarantined — the two runners reach these by different routes for the same
|
||||
/// runaway), <c>threw</c>, <c>unsupported_type</c>, <c>cancelled</c> or <c>other</c>.
|
||||
/// </summary>
|
||||
/// <param name="reason">A failure reason.</param>
|
||||
/// <returns>The class.</returns>
|
||||
public static string FailureClass(string? reason)
|
||||
{
|
||||
var r = reason ?? string.Empty;
|
||||
if (r.StartsWith("compile error:", StringComparison.Ordinal)
|
||||
|| r.StartsWith("sandbox violation:", StringComparison.Ordinal)
|
||||
|| r.StartsWith("compile failure:", StringComparison.Ordinal))
|
||||
return "compile";
|
||||
if (r.StartsWith(ScriptEvalFailureReasons.TimedOutPrefix, StringComparison.Ordinal)
|
||||
|| r.StartsWith("timed out after ", StringComparison.Ordinal)
|
||||
|| r == ScriptRunGuard.PreviousRunStillRunningReason
|
||||
|| r.StartsWith(ScriptRunGuard.QuarantinedReasonPrefix, StringComparison.Ordinal))
|
||||
return "budget";
|
||||
if (r.StartsWith("script threw:", StringComparison.Ordinal)) return ThrewClass;
|
||||
if (r.StartsWith("unsupported type:", StringComparison.Ordinal)) return UnsupportedTypeClass;
|
||||
if (r.StartsWith("evaluation cancelled", StringComparison.Ordinal)) return "cancelled";
|
||||
return "other";
|
||||
}
|
||||
|
||||
private const string ThrewClass = "threw";
|
||||
private const string UnsupportedTypeClass = "unsupported_type";
|
||||
private const int MaxDescribedLength = 160;
|
||||
|
||||
private async Task RunShadowAsync(string scoped, ScriptInvocation invocation, ScriptRunOutcome published)
|
||||
{
|
||||
try
|
||||
{
|
||||
ScriptRunOutcome shadow;
|
||||
try
|
||||
{
|
||||
shadow = await _shadow.RunAsync(invocation, _stopping.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A runner must not throw for a script fault; one that does is the runner's problem, not a divergence.
|
||||
shadow = ScriptRunOutcome.Unavailable($"shadow runner threw: {ex.Message}");
|
||||
}
|
||||
if (Volatile.Read(ref _disposed) != 0) return;
|
||||
Record(invocation, published, shadow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "Script shadow: comparing {ScriptKey} failed", invocation.Key);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Release(scoped);
|
||||
}
|
||||
}
|
||||
|
||||
private void Record(ScriptInvocation invocation, ScriptRunOutcome published, ScriptRunOutcome shadow)
|
||||
{
|
||||
var comparison = Compare(published, shadow);
|
||||
var kindTag = new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind));
|
||||
switch (comparison.Verdict)
|
||||
{
|
||||
case ScriptShadowVerdict.ShadowUnavailable:
|
||||
Interlocked.Increment(ref _unavailable);
|
||||
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "shadow_unavailable"));
|
||||
return;
|
||||
case ScriptShadowVerdict.Match:
|
||||
Interlocked.Increment(ref _compared);
|
||||
Interlocked.Increment(ref _matched);
|
||||
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "match"));
|
||||
return;
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _compared);
|
||||
Interlocked.Increment(ref _mismatched);
|
||||
var reason = comparison.Reason!;
|
||||
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "mismatch"));
|
||||
MismatchCounter.Add(1, kindTag,
|
||||
new KeyValuePair<string, object?>("reason", reason),
|
||||
new KeyValuePair<string, object?>("script", invocation.Key));
|
||||
|
||||
var scoped = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
||||
_mismatches.AddOrUpdate(scoped,
|
||||
_ => new ScriptShadowMismatch(invocation.Kind, invocation.Key, 1, reason, comparison.Summary),
|
||||
(_, prior) => prior with { Count = prior.Count + 1, LastReason = reason, LastSummary = comparison.Summary });
|
||||
|
||||
if (!_logged.TryAdd($"{scoped}|{reason}", 0)) return;
|
||||
long suppressed;
|
||||
lock (_logLock)
|
||||
{
|
||||
var now = _clock();
|
||||
if (now - _logWindowStart >= TimeSpan.FromMinutes(1))
|
||||
{
|
||||
_logWindowStart = now;
|
||||
_logsInWindow = 0;
|
||||
}
|
||||
if (_logsInWindow >= _options.MismatchLogsPerMinute)
|
||||
{
|
||||
_logsSuppressed++;
|
||||
return;
|
||||
}
|
||||
_logsInWindow++;
|
||||
suppressed = _logsSuppressed;
|
||||
_logsSuppressed = 0;
|
||||
}
|
||||
_log.Warning(
|
||||
"Script shadow mismatch: {ScriptKind} {ScriptKey} ({MismatchReason}) — {MismatchSummary}{Suppressed}",
|
||||
invocation.Kind, invocation.Key, reason, comparison.Summary,
|
||||
suppressed > 0 ? $" ({suppressed} earlier mismatch line(s) suppressed by the rate limit)" : string.Empty);
|
||||
}
|
||||
|
||||
private void Release(string scoped)
|
||||
{
|
||||
_busy.TryRemove(scoped, out _);
|
||||
Interlocked.Decrement(ref _inFlight);
|
||||
}
|
||||
|
||||
private static void Drop(ScriptContextKind kind, string reason, ref long counter)
|
||||
{
|
||||
Interlocked.Increment(ref counter);
|
||||
DroppedCounter.Add(1,
|
||||
new KeyValuePair<string, object?>("kind", KindTag(kind)),
|
||||
new KeyValuePair<string, object?>("reason", reason));
|
||||
}
|
||||
|
||||
private static bool ValuesEqual(object a, object b)
|
||||
=> a is Array || b is Array
|
||||
? StructuralComparisons.StructuralEqualityComparer.Equals(a, b)
|
||||
: a.Equals(b);
|
||||
|
||||
/// <summary>A one-line description of an outcome for a mismatch summary.</summary>
|
||||
/// <param name="outcome">The outcome.</param>
|
||||
/// <returns>e.g. <c>Ok Int32 42</c>, <c>Failed [threw] script threw: …</c>.</returns>
|
||||
public static string Describe(ScriptRunOutcome outcome)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(outcome);
|
||||
var text = outcome.Status switch
|
||||
{
|
||||
ScriptRunStatus.Ok => outcome.Value is null ? "Ok null" : $"Ok {outcome.Value.GetType().Name} {Format(outcome.Value)}",
|
||||
ScriptRunStatus.Unavailable => $"Unavailable {outcome.Reason}",
|
||||
_ => $"Failed [{FailureClass(outcome.Reason)}] {outcome.Reason}",
|
||||
};
|
||||
return text.Length <= MaxDescribedLength ? text : string.Concat(text.AsSpan(0, MaxDescribedLength), "…");
|
||||
}
|
||||
|
||||
private static string Format(object value) => value switch
|
||||
{
|
||||
string s => $"\"{s}\"",
|
||||
DateTime d => $"{d.ToString("O", CultureInfo.InvariantCulture)} ({d.Kind})",
|
||||
Array a => $"[{string.Join(", ", a.Cast<object?>().Take(8).Select(v => v is null ? "null" : Convert.ToString(v, CultureInfo.InvariantCulture)))}{(a.Length > 8 ? ", …" : string.Empty)}]",
|
||||
IFormattable f => f.ToString(null, CultureInfo.InvariantCulture),
|
||||
_ => value.ToString() ?? string.Empty,
|
||||
};
|
||||
|
||||
private static string KindTag(ScriptContextKind kind) => kind switch
|
||||
{
|
||||
ScriptContextKind.VirtualTag => "virtual_tag",
|
||||
ScriptContextKind.Calculation => "calculation",
|
||||
ScriptContextKind.AlarmPredicate => "alarm",
|
||||
_ => "other",
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Maps a value seam's in-process result onto the shadow comparison's outcome type.</summary>
|
||||
public static class ScriptShadowOutcomes
|
||||
{
|
||||
/// <summary>
|
||||
/// The <see cref="ScriptRunOutcome"/> equivalent of an in-process <see cref="VirtualTagEvalResult"/>, or null when
|
||||
/// the result is the host's own lifecycle (an evaluator disposed by a redeploy, a shutdown cancellation) rather
|
||||
/// than the script's — those are not mirrored.
|
||||
/// </summary>
|
||||
/// <param name="result">The seam's in-process result.</param>
|
||||
/// <returns>The outcome to mirror, or null.</returns>
|
||||
public static ScriptRunOutcome? ToMirroredOutcome(this VirtualTagEvalResult result)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
if (result.Success) return ScriptRunOutcome.Ok(result.Value);
|
||||
var reason = result.Reason ?? "evaluator failure";
|
||||
if (reason is "evaluator disposed" or ScriptEvalFailureReasons.EvaluatorDisposedDuringEvaluation
|
||||
|| reason.StartsWith("evaluation cancelled", StringComparison.Ordinal))
|
||||
return null;
|
||||
return result.StatusCode == Commons.OpcUa.OpcUaStatus.BadResourceUnavailable
|
||||
? ScriptRunOutcome.Unavailable(reason)
|
||||
: ScriptRunOutcome.Failed(reason);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user