|
|
|
@@ -0,0 +1,510 @@
|
|
|
|
|
using System.Collections;
|
|
|
|
|
using System.Collections.Concurrent;
|
|
|
|
|
using System.Diagnostics.Metrics;
|
|
|
|
|
using System.Globalization;
|
|
|
|
|
using Serilog;
|
|
|
|
|
using Serilog.Core;
|
|
|
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
|
|
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
|
|
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
|
|
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The mirror half of <c>Scripting:Runner=Shadow</c> (lmxopcua#780 P2, design §12): a seam that has already produced
|
|
|
|
|
/// — and published — its in-process outcome hands the same evaluation here, and it is run again on the shadow
|
|
|
|
|
/// runner (the out-of-process worker pool) and compared.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <remarks>
|
|
|
|
|
/// A seam receives its <see cref="IScriptShadow"/> as the <see cref="IScriptRunner"/> it is constructed with (the
|
|
|
|
|
/// node's runner selection hands out <see cref="ShadowScriptRunner"/>, which is both). A seam that recognises it keeps
|
|
|
|
|
/// its own in-process path, unchanged, and calls <see cref="Mirror"/> after each compiled evaluation.
|
|
|
|
|
/// </remarks>
|
|
|
|
|
public interface IScriptShadow
|
|
|
|
|
{
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Mirrors one evaluation whose outcome <paramref name="published"/> the seam has already published. Never
|
|
|
|
|
/// blocks, never throws, and nothing it does reaches the seam: the shadow run is queued on the pool, or dropped
|
|
|
|
|
/// (and counted) when the shadow side is behind.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="invocation">The evaluation as the seam ran it. Its inputs are copied before this returns, so the
|
|
|
|
|
/// caller may reuse its read cache at once.</param>
|
|
|
|
|
/// <param name="published">The in-process outcome — the source of truth.</param>
|
|
|
|
|
void Mirror(ScriptInvocation invocation, ScriptRunOutcome published);
|
|
|
|
|
|
|
|
|
|
/// <summary>The seam's deploy boundary: lifts the shadow runner's quarantines for <paramref name="kind"/> and
|
|
|
|
|
/// re-arms its once-per-script mismatch log lines.</summary>
|
|
|
|
|
/// <param name="kind">The seam's context kind.</param>
|
|
|
|
|
void Reset(ScriptContextKind kind);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>How one shadow comparison came out.</summary>
|
|
|
|
|
public enum ScriptShadowVerdict
|
|
|
|
|
{
|
|
|
|
|
/// <summary>The shadow run agreed with the published outcome.</summary>
|
|
|
|
|
Match = 0,
|
|
|
|
|
|
|
|
|
|
/// <summary>The shadow run disagreed; <see cref="ScriptShadowComparison.Reason"/> names how.</summary>
|
|
|
|
|
Mismatch = 1,
|
|
|
|
|
|
|
|
|
|
/// <summary>The shadow runner could not run the script (worker restarted, none ready, worker busy): not compared.</summary>
|
|
|
|
|
ShadowUnavailable = 2,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The result of comparing a published outcome with its shadow.</summary>
|
|
|
|
|
/// <param name="Verdict">Match, mismatch, or not compared.</param>
|
|
|
|
|
/// <param name="Reason">For a mismatch, the metric reason: <c>value</c>, <c>type</c>, <c>status</c>,
|
|
|
|
|
/// <c>failure_kind</c>, <c>exception</c> or <c>unsupported_type</c>.</param>
|
|
|
|
|
/// <param name="Published">A one-line description of the published outcome.</param>
|
|
|
|
|
/// <param name="Shadow">A one-line description of the shadow outcome.</param>
|
|
|
|
|
public sealed record ScriptShadowComparison(ScriptShadowVerdict Verdict, string? Reason, string Published, string Shadow)
|
|
|
|
|
{
|
|
|
|
|
/// <summary>The diff summary a mismatch log line carries.</summary>
|
|
|
|
|
public string Summary => $"published {Published}; shadow {Shadow}";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Counters of one <see cref="ShadowScriptRunner"/> since it started.</summary>
|
|
|
|
|
/// <param name="Compared">Shadow runs that completed and were compared (match or mismatch).</param>
|
|
|
|
|
/// <param name="Matched">Comparisons that agreed.</param>
|
|
|
|
|
/// <param name="Mismatched">Comparisons that disagreed.</param>
|
|
|
|
|
/// <param name="ShadowUnavailable">Shadow runs the runner could not run (not compared).</param>
|
|
|
|
|
/// <param name="DroppedBehind">Evaluations not mirrored because <see cref="ScriptShadowOptions.MaxInFlight"/> shadow
|
|
|
|
|
/// runs were already outstanding.</param>
|
|
|
|
|
/// <param name="DroppedKeyBusy">Evaluations not mirrored because the same script's previous shadow run was still
|
|
|
|
|
/// outstanding.</param>
|
|
|
|
|
/// <param name="InFlight">Shadow runs outstanding now.</param>
|
|
|
|
|
public sealed record ScriptShadowStats(
|
|
|
|
|
long Compared, long Matched, long Mismatched, long ShadowUnavailable, long DroppedBehind, long DroppedKeyBusy, int InFlight)
|
|
|
|
|
{
|
|
|
|
|
/// <summary>Every evaluation that was not mirrored.</summary>
|
|
|
|
|
public long Dropped => DroppedBehind + DroppedKeyBusy;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>One script's mismatch record.</summary>
|
|
|
|
|
/// <param name="Kind">The seam's context kind.</param>
|
|
|
|
|
/// <param name="Key">The script key (un-namespaced).</param>
|
|
|
|
|
/// <param name="Count">Mismatches counted for it.</param>
|
|
|
|
|
/// <param name="LastReason">The reason of the latest one.</param>
|
|
|
|
|
/// <param name="LastSummary">The diff summary of the latest one.</param>
|
|
|
|
|
public sealed record ScriptShadowMismatch(ScriptContextKind Kind, string Key, long Count, string LastReason, string LastSummary);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// <c>Scripting:Runner=Shadow</c> (lmxopcua#780 P2, design §12): every evaluation of a shadowed seam runs IN PROCESS —
|
|
|
|
|
/// that result is the one published — and is mirrored, best effort, to a shadow <see cref="IScriptRunner"/> (the
|
|
|
|
|
/// out-of-process worker pool), whose result is only compared: value, CLR type, status and failure kind.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <remarks>
|
|
|
|
|
/// <list type="bullet">
|
|
|
|
|
/// <item><b>The published value never depends on the shadow.</b> <see cref="Mirror"/> copies the inputs and queues
|
|
|
|
|
/// the shadow run on the pool; a shadow-side timeout, kill, crash, quarantine, a runner that hangs or throws, all
|
|
|
|
|
/// stay here.</item>
|
|
|
|
|
/// <item><b>Bounded.</b> At most <see cref="ScriptShadowOptions.MaxInFlight"/> shadow runs are outstanding and at
|
|
|
|
|
/// most one per script; anything beyond that is dropped and counted (<c>otopcua.script.shadow.dropped</c>,
|
|
|
|
|
/// <c>reason</c> = <c>behind</c> | <c>key_busy</c>). One per script also keeps the out-of-process runner's
|
|
|
|
|
/// single-flight rule from refusing a shadow run and reporting a false mismatch.</item>
|
|
|
|
|
/// <item><b>Counted per script.</b> <c>otopcua.script.shadow.compared{kind, result}</c> and
|
|
|
|
|
/// <c>otopcua.script.shadow.mismatch{kind, reason, script}</c>; <see cref="Mismatches"/> keeps the latest diff per
|
|
|
|
|
/// script. A mismatch logs one Warning with the diff summary the first time each script mismatches for each
|
|
|
|
|
/// reason (re-armed at the seam's deploy boundary), and at most <see cref="ScriptShadowOptions.MismatchLogsPerMinute"/>
|
|
|
|
|
/// such lines a minute.</item>
|
|
|
|
|
/// <item><b>Not compared</b>: a shadow outcome that is the runner's fault (<see cref="ScriptRunStatus.Unavailable"/>,
|
|
|
|
|
/// or the worker busy) — that is node health, not a script divergence.</item>
|
|
|
|
|
/// <item><b>Unsupported types</b> (design §14 #11): a value the protocol cannot carry makes the shadow answer
|
|
|
|
|
/// <c>unsupported type</c>, which counts as a mismatch with reason <c>unsupported_type</c> — the census the flip to
|
|
|
|
|
/// OutOfProcess needs, since out of process that evaluation would publish Bad.</item>
|
|
|
|
|
/// </list>
|
|
|
|
|
/// As an <see cref="IScriptRunner"/>, <see cref="RunAsync"/> runs an <see cref="InProcessScriptRunner"/> and mirrors
|
|
|
|
|
/// its outcome, for callers that only speak the runner interface. The three production seams recognise
|
|
|
|
|
/// <see cref="IScriptShadow"/> instead and keep their own in-process code, so what they publish is byte-for-byte
|
|
|
|
|
/// what <c>InProcess</c> publishes.
|
|
|
|
|
/// </remarks>
|
|
|
|
|
public sealed class ShadowScriptRunner : IScriptRunner, IScriptShadow, IDisposable
|
|
|
|
|
{
|
|
|
|
|
private static readonly Counter<long> ComparedCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
|
|
|
|
"otopcua.script.shadow.compared", unit: "{eval}",
|
|
|
|
|
description: "Shadow-mode evaluations whose shadow run completed (tags: kind; result = match | mismatch | shadow_unavailable).");
|
|
|
|
|
|
|
|
|
|
private static readonly Counter<long> MismatchCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
|
|
|
|
"otopcua.script.shadow.mismatch", unit: "{eval}",
|
|
|
|
|
description: "Shadow-mode evaluations whose out-of-process result differed from the published in-process one " +
|
|
|
|
|
"(tags: kind; reason = value | type | status | failure_kind | exception | unsupported_type; script).");
|
|
|
|
|
|
|
|
|
|
private static readonly Counter<long> DroppedCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
|
|
|
|
"otopcua.script.shadow.dropped", unit: "{eval}",
|
|
|
|
|
description: "Shadow-mode evaluations not mirrored because the shadow side was behind (tags: kind; reason = behind | key_busy).");
|
|
|
|
|
|
|
|
|
|
private static readonly ConcurrentDictionary<ShadowScriptRunner, byte> LiveRunners = new();
|
|
|
|
|
|
|
|
|
|
static ShadowScriptRunner()
|
|
|
|
|
{
|
|
|
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.shadow.inflight",
|
|
|
|
|
static () => LiveRunners.Keys.Select(r => new Measurement<int>(Volatile.Read(ref r._inFlight))),
|
|
|
|
|
unit: "{eval}", description: "Shadow-mode runs outstanding on the shadow runner.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private readonly IScriptRunner _shadow;
|
|
|
|
|
private readonly InProcessScriptRunner _primary;
|
|
|
|
|
private readonly ScriptShadowOptions _options;
|
|
|
|
|
private readonly ILogger _log;
|
|
|
|
|
private readonly Func<DateTime> _clock;
|
|
|
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
|
|
|
private readonly ConcurrentDictionary<string, byte> _busy = new(StringComparer.Ordinal);
|
|
|
|
|
private readonly ConcurrentDictionary<string, byte> _logged = new(StringComparer.Ordinal);
|
|
|
|
|
private readonly ConcurrentDictionary<string, ScriptShadowMismatch> _mismatches = new(StringComparer.Ordinal);
|
|
|
|
|
private readonly object _logLock = new();
|
|
|
|
|
private DateTime _logWindowStart;
|
|
|
|
|
private int _logsInWindow;
|
|
|
|
|
private long _logsSuppressed;
|
|
|
|
|
private int _inFlight;
|
|
|
|
|
private long _compared, _matched, _mismatched, _unavailable, _droppedBehind, _droppedBusy;
|
|
|
|
|
private int _disposed;
|
|
|
|
|
|
|
|
|
|
/// <summary>Initializes the shadow runner.</summary>
|
|
|
|
|
/// <param name="shadow">The runner every mirrored evaluation is run on again (the out-of-process pool). Not owned.</param>
|
|
|
|
|
/// <param name="scripting">The node's <c>Scripting</c> options (the in-process guard's knobs and <c>Scripting:Shadow</c>);
|
|
|
|
|
/// null ⇒ defaults.</param>
|
|
|
|
|
/// <param name="log">Where mismatch lines go; null ⇒ Serilog's global logger.</param>
|
|
|
|
|
/// <param name="clock">The mismatch-log rate limiter's clock (tests); null ⇒ <see cref="DateTime.UtcNow"/>.</param>
|
|
|
|
|
public ShadowScriptRunner(IScriptRunner shadow, ScriptingOptions? scripting = null, ILogger? log = null, Func<DateTime>? clock = null)
|
|
|
|
|
{
|
|
|
|
|
_shadow = shadow ?? throw new ArgumentNullException(nameof(shadow));
|
|
|
|
|
var options = (scripting ?? new ScriptingOptions()).Sanitized(out _);
|
|
|
|
|
_options = options.Shadow;
|
|
|
|
|
_log = (log ?? Log.Logger).ForContext<ShadowScriptRunner>();
|
|
|
|
|
_clock = clock ?? (() => DateTime.UtcNow);
|
|
|
|
|
_primary = new InProcessScriptRunner(options, log);
|
|
|
|
|
_logWindowStart = _clock();
|
|
|
|
|
LiveRunners.TryAdd(this, 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>The runner shadow runs go to.</summary>
|
|
|
|
|
public IScriptRunner Shadow => _shadow;
|
|
|
|
|
|
|
|
|
|
/// <summary>The counters since start.</summary>
|
|
|
|
|
/// <returns>A snapshot.</returns>
|
|
|
|
|
public ScriptShadowStats Stats() => new(
|
|
|
|
|
Interlocked.Read(ref _compared), Interlocked.Read(ref _matched), Interlocked.Read(ref _mismatched),
|
|
|
|
|
Interlocked.Read(ref _unavailable), Interlocked.Read(ref _droppedBehind), Interlocked.Read(ref _droppedBusy),
|
|
|
|
|
Volatile.Read(ref _inFlight));
|
|
|
|
|
|
|
|
|
|
/// <summary>Every script that has mismatched, with its count and latest diff.</summary>
|
|
|
|
|
public IReadOnlyCollection<ScriptShadowMismatch> Mismatches => _mismatches.Values.ToArray();
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Waits until no shadow run is outstanding (tests and diagnostics: a shadow run is asynchronous to the
|
|
|
|
|
/// evaluation that triggered it).
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="timeout">How long to wait.</param>
|
|
|
|
|
/// <returns><c>true</c> when idle within <paramref name="timeout"/>.</returns>
|
|
|
|
|
public async Task<bool> WaitIdleAsync(TimeSpan timeout)
|
|
|
|
|
{
|
|
|
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
|
|
|
while (Volatile.Read(ref _inFlight) > 0)
|
|
|
|
|
{
|
|
|
|
|
if (DateTime.UtcNow >= deadline) return false;
|
|
|
|
|
await Task.Delay(10).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public async Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default)
|
|
|
|
|
{
|
|
|
|
|
var published = await _primary.RunAsync(invocation, ct).ConfigureAwait(false);
|
|
|
|
|
if (!ct.IsCancellationRequested) Mirror(invocation, published);
|
|
|
|
|
return published;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public void Mirror(ScriptInvocation invocation, ScriptRunOutcome published)
|
|
|
|
|
{
|
|
|
|
|
if (invocation is null || published is null || Volatile.Read(ref _disposed) != 0) return;
|
|
|
|
|
var scoped = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
|
|
|
|
if (!_busy.TryAdd(scoped, 0))
|
|
|
|
|
{
|
|
|
|
|
Drop(invocation.Kind, "key_busy", ref _droppedBusy);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (Interlocked.Increment(ref _inFlight) > _options.MaxInFlight)
|
|
|
|
|
{
|
|
|
|
|
Release(scoped);
|
|
|
|
|
Drop(invocation.Kind, "behind", ref _droppedBehind);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ScriptInvocation copy;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
// The caller may refill its read cache as soon as this returns (the alarm engine reuses one per alarm).
|
|
|
|
|
copy = invocation with
|
|
|
|
|
{
|
|
|
|
|
Inputs = new Dictionary<string, DataValueSnapshot>(invocation.Inputs, StringComparer.Ordinal),
|
|
|
|
|
// The in-process run already wrote this evaluation's ctx.Logger lines; the shadow's are not repeated.
|
|
|
|
|
ScriptLogger = Logger.None,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
Release(scoped);
|
|
|
|
|
_log.Debug(ex, "Script shadow: could not copy the inputs of {ScriptKey}; not mirrored", invocation.Key);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Never on the caller's thread: a shadow runner may do synchronous work (encode, write) before its first await.
|
|
|
|
|
_ = Task.Run(() => RunShadowAsync(scoped, copy, published));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public void Reset(ScriptContextKind kind)
|
|
|
|
|
{
|
|
|
|
|
_primary.Reset(kind);
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
_shadow.Reset(kind);
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
_log.Warning(ex, "Script shadow: the shadow runner's reset failed for {ScriptKind}", kind);
|
|
|
|
|
}
|
|
|
|
|
var prefix = InProcessScriptRunner.ScopedKey(kind, string.Empty);
|
|
|
|
|
foreach (var key in _logged.Keys)
|
|
|
|
|
if (key.StartsWith(prefix, StringComparison.Ordinal)) _logged.TryRemove(key, out _);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => _primary.QuarantinedKeys(kind);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public void Dispose()
|
|
|
|
|
{
|
|
|
|
|
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
|
|
|
|
|
LiveRunners.TryRemove(this, out _);
|
|
|
|
|
_stopping.Cancel();
|
|
|
|
|
_primary.Dispose();
|
|
|
|
|
_stopping.Dispose();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Compares a published outcome with its shadow (pure; the table the mismatch reasons are tested from).
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="published">The in-process outcome.</param>
|
|
|
|
|
/// <param name="shadow">The shadow runner's outcome.</param>
|
|
|
|
|
/// <returns>The comparison.</returns>
|
|
|
|
|
public static ScriptShadowComparison Compare(ScriptRunOutcome published, ScriptRunOutcome shadow)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(published);
|
|
|
|
|
ArgumentNullException.ThrowIfNull(shadow);
|
|
|
|
|
var p = Describe(published);
|
|
|
|
|
var s = Describe(shadow);
|
|
|
|
|
|
|
|
|
|
if (shadow.Status == ScriptRunStatus.Unavailable
|
|
|
|
|
|| (shadow.Status == ScriptRunStatus.Failed && shadow.Reason == ScriptRunOutcome.WorkerBusyReason))
|
|
|
|
|
return new ScriptShadowComparison(ScriptShadowVerdict.ShadowUnavailable, null, p, s);
|
|
|
|
|
|
|
|
|
|
ScriptShadowComparison Mismatch(string reason) => new(ScriptShadowVerdict.Mismatch, reason, p, s);
|
|
|
|
|
var match = new ScriptShadowComparison(ScriptShadowVerdict.Match, null, p, s);
|
|
|
|
|
|
|
|
|
|
if (published.Status == ScriptRunStatus.Ok && shadow.Status == ScriptRunStatus.Ok)
|
|
|
|
|
{
|
|
|
|
|
if (published.Value is null && shadow.Value is null) return match;
|
|
|
|
|
if (published.Value is null || shadow.Value is null) return Mismatch("value");
|
|
|
|
|
if (published.Value.GetType() != shadow.Value.GetType()) return Mismatch("type");
|
|
|
|
|
return ValuesEqual(published.Value, shadow.Value) ? match : Mismatch("value");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var publishedClass = published.Status == ScriptRunStatus.Ok ? null : FailureClass(published.Reason);
|
|
|
|
|
var shadowClass = shadow.Status == ScriptRunStatus.Ok ? null : FailureClass(shadow.Reason);
|
|
|
|
|
if (shadowClass == UnsupportedTypeClass && publishedClass != UnsupportedTypeClass) return Mismatch("unsupported_type");
|
|
|
|
|
if (publishedClass is null || shadowClass is null) return Mismatch("status");
|
|
|
|
|
if (publishedClass != shadowClass) return Mismatch("failure_kind");
|
|
|
|
|
if (publishedClass == ThrewClass && !string.Equals(published.Reason, shadow.Reason, StringComparison.Ordinal))
|
|
|
|
|
return Mismatch("exception");
|
|
|
|
|
return match;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The failure class a reason belongs to, for comparison: <c>compile</c>, <c>budget</c> (timed out, killed,
|
|
|
|
|
/// refused behind a run still going, quarantined — the two runners reach these by different routes for the same
|
|
|
|
|
/// runaway), <c>threw</c>, <c>unsupported_type</c>, <c>cancelled</c> or <c>other</c>.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="reason">A failure reason.</param>
|
|
|
|
|
/// <returns>The class.</returns>
|
|
|
|
|
public static string FailureClass(string? reason)
|
|
|
|
|
{
|
|
|
|
|
var r = reason ?? string.Empty;
|
|
|
|
|
if (r.StartsWith("compile error:", StringComparison.Ordinal)
|
|
|
|
|
|| r.StartsWith("sandbox violation:", StringComparison.Ordinal)
|
|
|
|
|
|| r.StartsWith("compile failure:", StringComparison.Ordinal))
|
|
|
|
|
return "compile";
|
|
|
|
|
if (r.StartsWith(ScriptEvalFailureReasons.TimedOutPrefix, StringComparison.Ordinal)
|
|
|
|
|
|| r.StartsWith("timed out after ", StringComparison.Ordinal)
|
|
|
|
|
|| r == ScriptRunGuard.PreviousRunStillRunningReason
|
|
|
|
|
|| r.StartsWith(ScriptRunGuard.QuarantinedReasonPrefix, StringComparison.Ordinal))
|
|
|
|
|
return "budget";
|
|
|
|
|
if (r.StartsWith("script threw:", StringComparison.Ordinal)) return ThrewClass;
|
|
|
|
|
if (r.StartsWith("unsupported type:", StringComparison.Ordinal)) return UnsupportedTypeClass;
|
|
|
|
|
if (r.StartsWith("evaluation cancelled", StringComparison.Ordinal)) return "cancelled";
|
|
|
|
|
return "other";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private const string ThrewClass = "threw";
|
|
|
|
|
private const string UnsupportedTypeClass = "unsupported_type";
|
|
|
|
|
private const int MaxDescribedLength = 160;
|
|
|
|
|
|
|
|
|
|
private async Task RunShadowAsync(string scoped, ScriptInvocation invocation, ScriptRunOutcome published)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
ScriptRunOutcome shadow;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
shadow = await _shadow.RunAsync(invocation, _stopping.Token).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
// A runner must not throw for a script fault; one that does is the runner's problem, not a divergence.
|
|
|
|
|
shadow = ScriptRunOutcome.Unavailable($"shadow runner threw: {ex.Message}");
|
|
|
|
|
}
|
|
|
|
|
if (Volatile.Read(ref _disposed) != 0) return;
|
|
|
|
|
Record(invocation, published, shadow);
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
_log.Debug(ex, "Script shadow: comparing {ScriptKey} failed", invocation.Key);
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
|
Release(scoped);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void Record(ScriptInvocation invocation, ScriptRunOutcome published, ScriptRunOutcome shadow)
|
|
|
|
|
{
|
|
|
|
|
var comparison = Compare(published, shadow);
|
|
|
|
|
var kindTag = new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind));
|
|
|
|
|
switch (comparison.Verdict)
|
|
|
|
|
{
|
|
|
|
|
case ScriptShadowVerdict.ShadowUnavailable:
|
|
|
|
|
Interlocked.Increment(ref _unavailable);
|
|
|
|
|
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "shadow_unavailable"));
|
|
|
|
|
return;
|
|
|
|
|
case ScriptShadowVerdict.Match:
|
|
|
|
|
Interlocked.Increment(ref _compared);
|
|
|
|
|
Interlocked.Increment(ref _matched);
|
|
|
|
|
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "match"));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Interlocked.Increment(ref _compared);
|
|
|
|
|
Interlocked.Increment(ref _mismatched);
|
|
|
|
|
var reason = comparison.Reason!;
|
|
|
|
|
ComparedCounter.Add(1, kindTag, new KeyValuePair<string, object?>("result", "mismatch"));
|
|
|
|
|
MismatchCounter.Add(1, kindTag,
|
|
|
|
|
new KeyValuePair<string, object?>("reason", reason),
|
|
|
|
|
new KeyValuePair<string, object?>("script", invocation.Key));
|
|
|
|
|
|
|
|
|
|
var scoped = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
|
|
|
|
_mismatches.AddOrUpdate(scoped,
|
|
|
|
|
_ => new ScriptShadowMismatch(invocation.Kind, invocation.Key, 1, reason, comparison.Summary),
|
|
|
|
|
(_, prior) => prior with { Count = prior.Count + 1, LastReason = reason, LastSummary = comparison.Summary });
|
|
|
|
|
|
|
|
|
|
if (!_logged.TryAdd($"{scoped}|{reason}", 0)) return;
|
|
|
|
|
long suppressed;
|
|
|
|
|
lock (_logLock)
|
|
|
|
|
{
|
|
|
|
|
var now = _clock();
|
|
|
|
|
if (now - _logWindowStart >= TimeSpan.FromMinutes(1))
|
|
|
|
|
{
|
|
|
|
|
_logWindowStart = now;
|
|
|
|
|
_logsInWindow = 0;
|
|
|
|
|
}
|
|
|
|
|
if (_logsInWindow >= _options.MismatchLogsPerMinute)
|
|
|
|
|
{
|
|
|
|
|
_logsSuppressed++;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
_logsInWindow++;
|
|
|
|
|
suppressed = _logsSuppressed;
|
|
|
|
|
_logsSuppressed = 0;
|
|
|
|
|
}
|
|
|
|
|
_log.Warning(
|
|
|
|
|
"Script shadow mismatch: {ScriptKind} {ScriptKey} ({MismatchReason}) — {MismatchSummary}{Suppressed}",
|
|
|
|
|
invocation.Kind, invocation.Key, reason, comparison.Summary,
|
|
|
|
|
suppressed > 0 ? $" ({suppressed} earlier mismatch line(s) suppressed by the rate limit)" : string.Empty);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void Release(string scoped)
|
|
|
|
|
{
|
|
|
|
|
_busy.TryRemove(scoped, out _);
|
|
|
|
|
Interlocked.Decrement(ref _inFlight);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void Drop(ScriptContextKind kind, string reason, ref long counter)
|
|
|
|
|
{
|
|
|
|
|
Interlocked.Increment(ref counter);
|
|
|
|
|
DroppedCounter.Add(1,
|
|
|
|
|
new KeyValuePair<string, object?>("kind", KindTag(kind)),
|
|
|
|
|
new KeyValuePair<string, object?>("reason", reason));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static bool ValuesEqual(object a, object b)
|
|
|
|
|
=> a is Array || b is Array
|
|
|
|
|
? StructuralComparisons.StructuralEqualityComparer.Equals(a, b)
|
|
|
|
|
: a.Equals(b);
|
|
|
|
|
|
|
|
|
|
/// <summary>A one-line description of an outcome for a mismatch summary.</summary>
|
|
|
|
|
/// <param name="outcome">The outcome.</param>
|
|
|
|
|
/// <returns>e.g. <c>Ok Int32 42</c>, <c>Failed [threw] script threw: …</c>.</returns>
|
|
|
|
|
public static string Describe(ScriptRunOutcome outcome)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(outcome);
|
|
|
|
|
var text = outcome.Status switch
|
|
|
|
|
{
|
|
|
|
|
ScriptRunStatus.Ok => outcome.Value is null ? "Ok null" : $"Ok {outcome.Value.GetType().Name} {Format(outcome.Value)}",
|
|
|
|
|
ScriptRunStatus.Unavailable => $"Unavailable {outcome.Reason}",
|
|
|
|
|
_ => $"Failed [{FailureClass(outcome.Reason)}] {outcome.Reason}",
|
|
|
|
|
};
|
|
|
|
|
return text.Length <= MaxDescribedLength ? text : string.Concat(text.AsSpan(0, MaxDescribedLength), "…");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string Format(object value) => value switch
|
|
|
|
|
{
|
|
|
|
|
string s => $"\"{s}\"",
|
|
|
|
|
DateTime d => $"{d.ToString("O", CultureInfo.InvariantCulture)} ({d.Kind})",
|
|
|
|
|
Array a => $"[{string.Join(", ", a.Cast<object?>().Take(8).Select(v => v is null ? "null" : Convert.ToString(v, CultureInfo.InvariantCulture)))}{(a.Length > 8 ? ", …" : string.Empty)}]",
|
|
|
|
|
IFormattable f => f.ToString(null, CultureInfo.InvariantCulture),
|
|
|
|
|
_ => value.ToString() ?? string.Empty,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
private static string KindTag(ScriptContextKind kind) => kind switch
|
|
|
|
|
{
|
|
|
|
|
ScriptContextKind.VirtualTag => "virtual_tag",
|
|
|
|
|
ScriptContextKind.Calculation => "calculation",
|
|
|
|
|
ScriptContextKind.AlarmPredicate => "alarm",
|
|
|
|
|
_ => "other",
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Maps a value seam's in-process result onto the shadow comparison's outcome type.</summary>
|
|
|
|
|
public static class ScriptShadowOutcomes
|
|
|
|
|
{
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The <see cref="ScriptRunOutcome"/> equivalent of an in-process <see cref="VirtualTagEvalResult"/>, or null when
|
|
|
|
|
/// the result is the host's own lifecycle (an evaluator disposed by a redeploy, a shutdown cancellation) rather
|
|
|
|
|
/// than the script's — those are not mirrored.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="result">The seam's in-process result.</param>
|
|
|
|
|
/// <returns>The outcome to mirror, or null.</returns>
|
|
|
|
|
public static ScriptRunOutcome? ToMirroredOutcome(this VirtualTagEvalResult result)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(result);
|
|
|
|
|
if (result.Success) return ScriptRunOutcome.Ok(result.Value);
|
|
|
|
|
var reason = result.Reason ?? "evaluator failure";
|
|
|
|
|
if (reason is "evaluator disposed" or ScriptEvalFailureReasons.EvaluatorDisposedDuringEvaluation
|
|
|
|
|
|| reason.StartsWith("evaluation cancelled", StringComparison.Ordinal))
|
|
|
|
|
return null;
|
|
|
|
|
return result.StatusCode == Commons.OpcUa.OpcUaStatus.BadResourceUnavailable
|
|
|
|
|
? ScriptRunOutcome.Unavailable(reason)
|
|
|
|
|
: ScriptRunOutcome.Failed(reason);
|
|
|
|
|
}
|
|
|
|
|
}
|
Joseph Doherty