f50209082e
otopcua.script.ipc.timeouts gains the tag answer = stalled | retrigger: a later trigger refused while the key's stalled run is still unanswered was counted nowhere (not a script refusal, so not ScriptRunGuard's counter). KeyState carries its Kind for the tag. Test: the stalled-worker test listens for the instrument and sees one stalled and one retrigger (before: only stalled).
1051 lines
56 KiB
C#
1051 lines
56 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using System.Diagnostics.Metrics;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using Serilog;
|
|
using Serilog.Events;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
|
|
/// <summary>The out-of-process runner's health (the <c>otopcua.script.runner.state</c> gauge: 0 / 1 / 2).</summary>
|
|
public enum ScriptRunnerState
|
|
{
|
|
/// <summary>Every shard that has been needed is up.</summary>
|
|
Ready = 0,
|
|
|
|
/// <summary>At least one shard (or the isolation worker) is down after a failure; the rest serve.</summary>
|
|
Degraded = 1,
|
|
|
|
/// <summary>The crash-loop circuit is open: every shard has been down for <c>CircuitOpenAfter</c> (design §5.3.6).</summary>
|
|
Unavailable = 2,
|
|
}
|
|
|
|
/// <summary>A point-in-time view of the runner (the readiness check and diagnostics).</summary>
|
|
/// <param name="State">The overall state.</param>
|
|
/// <param name="Shards">Every shard, the isolation worker last.</param>
|
|
/// <param name="SuspectCount">Keys currently routed to the isolation worker.</param>
|
|
/// <param name="QuarantinedCount">Keys currently quarantined (all seams).</param>
|
|
/// <param name="PeCacheEnabled">Whether the compiled-PE cache is in use.</param>
|
|
public sealed record ScriptRunnerStatus(
|
|
ScriptRunnerState State,
|
|
IReadOnlyList<ScriptWorkerShardStatus> Shards,
|
|
int SuspectCount,
|
|
int QuarantinedCount,
|
|
bool PeCacheEnabled);
|
|
|
|
/// <summary>
|
|
/// The pre-emptible <see cref="IScriptRunner"/> (lmxopcua#780, design §3–§10): compiled scripts run in a pool of
|
|
/// <c>ScriptWorker</c> processes — K shards placed by a stable hash of the script key, plus an isolation worker
|
|
/// for suspects — so a script that overruns its hard deadline is stopped by killing its worker, and an OOM, a
|
|
/// native crash or a fail-fast takes down a worker instead of the server.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <list type="bullet">
|
|
/// <item><b>Two deadlines.</b> At the soft deadline <c>T</c> the caller gets today's timeout failure (and the
|
|
/// run token is cancelled inside the worker). At <c>T + G</c> (<see cref="ScriptWorkerOptions.EffectiveHardGrace"/>)
|
|
/// the worker is killed and the key takes a kill strike.</item>
|
|
/// <item><b>Attribution</b> runs synchronously in the worker's exit path, before any evaluation it held is
|
|
/// completed — so a quarantine earned by a kill is in place before the key's slot is released and the next
|
|
/// trigger is admitted. A deadline kill blames exactly one key; the other evaluations in flight on that worker
|
|
/// are victims and report <see cref="ScriptRunOutcome.WorkerRestartedReason"/> (no strike). A crash with no
|
|
/// deadline crossed makes every key in flight — evaluating or compiling — a suspect, and suspects run (and
|
|
/// compile) in the isolation worker one at a time, where a crash names exactly one key; that is also how a
|
|
/// script whose compile keeps crashing its worker is found. A suspect returns to its shard after
|
|
/// <see cref="ScriptWorkerOptions.IsolationProbationRuns"/> clean runs.</item>
|
|
/// <item><b>Quarantine</b> lives here, in the host, so it survives worker restarts: kill strikes (within an
|
|
/// hour), consecutive soft timeouts, or one allocation-budget breach. Reasons keep the
|
|
/// <see cref="ScriptRunGuard.QuarantinedReasonPrefix"/>, and a source change or <see cref="Reset"/> lifts them.</item>
|
|
/// <item><b>Fail closed.</b> When no worker is ready within <see cref="ScriptWorkerOptions.UnavailableGrace"/>
|
|
/// the result is <see cref="ScriptRunStatus.Unavailable"/> — never an in-process run — carrying a
|
|
/// <see cref="ScriptRunOutcome.Recovery"/> task the seam re-evaluates on.</item>
|
|
/// <item><b>PE cache</b> (design §7): with <see cref="ScriptWorkerOptions.PeCacheDir"/> set, workers load
|
|
/// HMAC-verified compiled images instead of re-entering Roslyn after a restart.</item>
|
|
/// </list>
|
|
/// </remarks>
|
|
public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable, IDisposable
|
|
{
|
|
/// <summary>The worker entry assembly's file name.</summary>
|
|
public const string WorkerAssemblyFileName = "ZB.MOM.WW.OtOpcUa.ScriptWorker.dll";
|
|
|
|
/// <summary>The folder under the host's base directory the Host build places the worker in.</summary>
|
|
public const string WorkerFolderName = "script-worker";
|
|
|
|
/// <summary>Failure reason when the worker could not take more work.</summary>
|
|
public const string WorkerBusyReason = ScriptRunOutcome.WorkerBusyReason;
|
|
|
|
/// <summary>PE cache entries unused for this long are pruned at startup.</summary>
|
|
internal static readonly TimeSpan PeCacheMaxAge = TimeSpan.FromDays(30);
|
|
|
|
/// <summary>At most this many PE cache entries are kept.</summary>
|
|
internal const int PeCacheMaxEntries = 50_000;
|
|
|
|
private static readonly TimeSpan KillStrikeWindow = TimeSpan.FromHours(1);
|
|
|
|
private static readonly ConcurrentDictionary<OutOfProcessScriptRunner, byte> LiveRunners = new();
|
|
|
|
private static readonly Counter<long> CompileCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
|
"otopcua.script.compile", unit: "{compile}",
|
|
description: "Out-of-process script loads (tag: cache = hit | miss | rejected | uncached | negative).");
|
|
|
|
private static readonly Histogram<double> RoundTripHistogram = OtOpcUaTelemetry.Meter.CreateHistogram<double>(
|
|
"otopcua.script.ipc.roundtrip", unit: "us",
|
|
description: "Host-measured submit-to-result time of one out-of-process evaluation (tag: kind).");
|
|
|
|
private static readonly Histogram<double> EvalDurationHistogram = OtOpcUaTelemetry.Meter.CreateHistogram<double>(
|
|
"otopcua.script.eval.duration", unit: "us",
|
|
description: "Worker-measured run time of one out-of-process evaluation (tag: kind).");
|
|
|
|
private static readonly Counter<long> TransportTimeoutCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
|
"otopcua.script.ipc.timeouts", unit: "{eval}",
|
|
description: "Out-of-process evaluations answered 'script worker unresponsive' (lmxopcua#827; tags: kind, answer = stalled | retrigger). stalled: no result and no soft-deadline notice from the worker within T + G of submit — a starved or stalled worker, not a slow script; retrigger: a later trigger of the same script refused while that stalled run is still unanswered.");
|
|
|
|
static OutOfProcessScriptRunner()
|
|
{
|
|
// Gauges observe every live runner (one per node in production; tests make several).
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.runner.state",
|
|
static () => LiveRunners.Keys.Select(r => new Measurement<int>((int)r.State)),
|
|
unit: "{state}", description: "Out-of-process script runner state: 0 ready, 1 degraded, 2 unavailable (circuit open).");
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.worker.up",
|
|
static () => ShardMeasurements(s => s.Up ? 1 : 0),
|
|
unit: "{process}", description: "1 when the shard's ScriptWorker is live (tag: shard).");
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.worker.inflight",
|
|
static () => ShardMeasurements(s => s.InFlight),
|
|
unit: "{eval}", description: "Evaluations outstanding on the shard's ScriptWorker (tag: shard).");
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.worker.queue_depth",
|
|
static () => ShardMeasurements(s => s.Queued),
|
|
unit: "{eval}", description: "Evaluations queued inside the shard's ScriptWorker, from its heartbeat (tag: shard).");
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.worker.heap_bytes",
|
|
static () => LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
|
|
.Select(s => new Measurement<long>(s.HeapBytes, new KeyValuePair<string, object?>("shard", s.Label))),
|
|
unit: "By", description: "The shard's ScriptWorker managed heap, from its heartbeat (tag: shard).");
|
|
OtOpcUaTelemetry.Meter.CreateObservableGauge("otopcua.script.isolation.suspects",
|
|
static () => LiveRunners.Keys.Select(r => new Measurement<int>(r.SuspectKeys.Count)),
|
|
unit: "{script}", description: "Scripts currently routed to the isolation worker.");
|
|
}
|
|
|
|
private readonly ScriptingOptions _options;
|
|
private readonly ScriptWorkerOptions _worker;
|
|
private readonly ILogger _log;
|
|
private readonly ILogger _scriptRoot;
|
|
private readonly WorkerShard[] _shards;
|
|
private readonly WorkerShard _isolation;
|
|
private readonly ConcurrentDictionary<string, KeyState> _keys = new(StringComparer.Ordinal);
|
|
private readonly ConcurrentDictionary<string, string> _compileFailures = new(StringComparer.Ordinal);
|
|
private readonly Timer _maintenance;
|
|
private readonly string _markerDirectory;
|
|
private readonly string? _peCacheDirectory;
|
|
private readonly byte[]? _peCacheKey;
|
|
private readonly int _compileThreads;
|
|
|
|
/// <summary>Test seam: the launched workers' initial silence limit (null = three heartbeat intervals).</summary>
|
|
private readonly TimeSpan? _heartbeatSilenceLimit;
|
|
private long _peCacheHits;
|
|
private long _peCacheMisses;
|
|
private long _peCacheRejected;
|
|
private int _maintaining;
|
|
private int _circuitLogged;
|
|
private int _disposed;
|
|
|
|
/// <summary>Initializes the runner. No process starts until the first evaluation.</summary>
|
|
/// <param name="options">The node's <c>Scripting</c> options (sanitized again here).</param>
|
|
/// <param name="log">Host diagnostics: starts, kills, quarantines; null ⇒ Serilog's global logger.</param>
|
|
/// <param name="scriptRoot">Fallback logger for forwarded <c>ctx.Logger</c> lines; null ⇒ <paramref name="log"/>.</param>
|
|
public OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log = null, ILogger? scriptRoot = null)
|
|
: this(options, log, scriptRoot, heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60))
|
|
{
|
|
}
|
|
|
|
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
|
|
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
|
|
int compileThreads = 0, int evalThreads = 0, TimeSpan? heartbeatSilenceLimit = null)
|
|
{
|
|
_compileThreads = compileThreads;
|
|
_heartbeatSilenceLimit = heartbeatSilenceLimit;
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
_options = options.Sanitized(out _);
|
|
_worker = _options.Worker;
|
|
_log = log ?? Log.Logger;
|
|
_scriptRoot = scriptRoot ?? _log;
|
|
LoadTimeout = loadTimeout;
|
|
WorkerPath = ResolveWorkerPath(_worker);
|
|
// Fail closed (startup also refuses it, ScriptRunnerSelection.Validate): a configured identity that cannot be
|
|
// applied must never silently fall back to the host's own token.
|
|
if (_worker.ValidateIdentity(OperatingSystem.IsWindows()) is { } identityViolation)
|
|
throw new InvalidOperationException($"Scripting misconfiguration: {identityViolation}");
|
|
WorkerSid = string.IsNullOrWhiteSpace(_worker.Identity)
|
|
? null
|
|
: ScriptWorkerIdentity.TryResolveSid(_worker.Identity, out var sid, out _) ? sid : null;
|
|
|
|
// The PE cache is best effort: a directory or key that cannot be used is a Warning and a cache-less runner,
|
|
// never a startup failure (the path RULE is enforced at startup by ScriptRunnerSelection.Validate).
|
|
if (!string.IsNullOrWhiteSpace(_worker.PeCacheDir))
|
|
{
|
|
try
|
|
{
|
|
_peCacheKey = ScriptPeCacheStore.LoadOrCreateKey(_worker.PeCacheDir);
|
|
_peCacheDirectory = _worker.PeCacheDir;
|
|
var pruned = new ScriptPeCacheStore(_peCacheDirectory, _peCacheKey).Prune(PeCacheMaxAge, PeCacheMaxEntries);
|
|
if (pruned > 0) _log.Information("Script PE cache {Directory}: pruned {Count} stale entr(ies)", _peCacheDirectory, pruned);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
_peCacheKey = null;
|
|
_peCacheDirectory = null;
|
|
_log.Warning(ex, "Script PE cache {Directory} is unusable; ScriptWorkers will recompile every script on each start",
|
|
_worker.PeCacheDir);
|
|
}
|
|
}
|
|
|
|
// Design §3.3: workers a dead predecessor of THIS host instance left behind, identified by their launch
|
|
// markers — never by process name.
|
|
_markerDirectory = markerDirectory ?? WorkerLaunchMarker.DirectoryFor(_worker.PeCacheDir);
|
|
var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
|
|
if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
|
|
|
|
// Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
|
|
// queue-behind-a-busy-thread case deterministic).
|
|
if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
|
|
_shards = new WorkerShard[_worker.Shards];
|
|
for (var i = 0; i < _shards.Length; i++)
|
|
{
|
|
var label = i.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
|
_shards[i] = NewShard(label, isolation: false, evalThreads, heartbeatInterval);
|
|
}
|
|
_isolation = NewShard("iso", isolation: true, evalThreads: 1, heartbeatInterval);
|
|
|
|
var tick = maintenanceInterval ?? TimeSpan.FromSeconds(1);
|
|
_maintenance = new Timer(_ => _ = MaintainAsync(), null, tick, tick);
|
|
LiveRunners.TryAdd(this, 0);
|
|
}
|
|
|
|
/// <summary>The worker entry assembly this runner launches.</summary>
|
|
public string WorkerPath { get; }
|
|
|
|
/// <summary>The worker identity's SID (<c>Scripting:Worker:Identity</c>), or null when workers keep the host's token.</summary>
|
|
public string? WorkerSid { get; }
|
|
|
|
/// <summary>The overall state (the readiness check, the state gauge).</summary>
|
|
public ScriptRunnerState State => ComputeState(_shards.Select(s => s.DownFor).ToArray(), _isolation.DownFor, _worker.CircuitOpenAfter);
|
|
|
|
/// <summary>A snapshot of the runner for diagnostics and health.</summary>
|
|
/// <returns>The status.</returns>
|
|
public ScriptRunnerStatus Status() => new(
|
|
State,
|
|
AllShards.Select(s => s.Status()).ToArray(),
|
|
SuspectKeys.Count,
|
|
_keys.Count(kv => kv.Value.QuarantineReason is not null),
|
|
_peCacheDirectory is not null);
|
|
|
|
/// <summary>
|
|
/// How long a compile in the worker may run before the worker is killed, counted from the moment a compile thread
|
|
/// starts it — never the time it spent queued behind other compiles (lmxopcua#790).
|
|
/// </summary>
|
|
internal TimeSpan LoadTimeout { get; }
|
|
|
|
/// <summary>The shard workers (tests).</summary>
|
|
internal IReadOnlyList<WorkerShard> Shards => _shards;
|
|
|
|
/// <summary>The isolation worker (tests).</summary>
|
|
internal WorkerShard Isolation => _isolation;
|
|
|
|
/// <summary>The marker directory this runner writes launch markers to and sweeps at startup.</summary>
|
|
internal string MarkerDirectory => _markerDirectory;
|
|
|
|
/// <summary>PE cache outcomes reported by this runner's workers (tests / diagnostics).</summary>
|
|
internal (long Hits, long Misses, long Rejected) PeCacheCounts =>
|
|
(Interlocked.Read(ref _peCacheHits), Interlocked.Read(ref _peCacheMisses), Interlocked.Read(ref _peCacheRejected));
|
|
|
|
/// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
|
|
internal Action<string>? SlotReleasingForTest { get; set; }
|
|
|
|
/// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
|
|
internal Action<int>? BatchSubmittedForTest { get; set; }
|
|
|
|
/// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
|
|
internal IReadOnlyCollection<string> SuspectKeys =>
|
|
_keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
|
|
|
|
private IEnumerable<WorkerShard> AllShards => _shards.Append(_isolation);
|
|
|
|
/// <summary>The shard a key is placed on when it is not a suspect: a stable FNV-1a hash, so placement survives restarts.</summary>
|
|
/// <param name="scopedKey">The scoped key.</param>
|
|
/// <param name="shardCount">K.</param>
|
|
/// <returns>The shard index.</returns>
|
|
internal static int ShardIndex(string scopedKey, int shardCount)
|
|
{
|
|
var hash = 2166136261u;
|
|
foreach (var b in Encoding.UTF8.GetBytes(scopedKey))
|
|
{
|
|
hash ^= b;
|
|
hash *= 16777619u;
|
|
}
|
|
return (int)(hash % (uint)shardCount);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The state rule: Unavailable when every shard has been down for at least <paramref name="circuitOpenAfter"/>,
|
|
/// Degraded when any shard or the isolation worker is down, Ready otherwise.
|
|
/// </summary>
|
|
/// <param name="shardsDownFor">Each regular shard's down time (null = up / never needed).</param>
|
|
/// <param name="isolationDownFor">The isolation worker's.</param>
|
|
/// <param name="circuitOpenAfter">The circuit threshold.</param>
|
|
/// <returns>The state.</returns>
|
|
internal static ScriptRunnerState ComputeState(IReadOnlyList<TimeSpan?> shardsDownFor, TimeSpan? isolationDownFor, TimeSpan circuitOpenAfter)
|
|
{
|
|
if (shardsDownFor.Count > 0 && shardsDownFor.All(d => d is { } down && down >= circuitOpenAfter))
|
|
return ScriptRunnerState.Unavailable;
|
|
return shardsDownFor.Any(d => d is not null) || isolationDownFor is not null
|
|
? ScriptRunnerState.Degraded
|
|
: ScriptRunnerState.Ready;
|
|
}
|
|
|
|
/// <summary>The default worker path, or the configured one.</summary>
|
|
/// <param name="worker">The worker options.</param>
|
|
/// <returns>The absolute worker path.</returns>
|
|
/// <remarks>
|
|
/// With no <see cref="ScriptWorkerOptions.ExecutablePath"/>, the worker is looked up in <c>script-worker/</c>
|
|
/// under the host's base directory by <see cref="ResolveWorkerEntry"/>: a self-contained worker (the Windows
|
|
/// installs, which have no .NET runtime) is started through its apphost, a framework-dependent one (docker-dev)
|
|
/// through the dotnet muxer.
|
|
/// </remarks>
|
|
public static string ResolveWorkerPath(ScriptWorkerOptions worker)
|
|
=> string.IsNullOrWhiteSpace(worker.ExecutablePath)
|
|
? ResolveWorkerEntry(Path.Combine(AppContext.BaseDirectory, WorkerFolderName))
|
|
: Path.GetFullPath(worker.ExecutablePath);
|
|
|
|
/// <summary>The worker's apphost file name on this OS.</summary>
|
|
public static string WorkerAppHostFileName
|
|
=> OperatingSystem.IsWindows() ? "ZB.MOM.WW.OtOpcUa.ScriptWorker.exe" : "ZB.MOM.WW.OtOpcUa.ScriptWorker";
|
|
|
|
/// <summary>
|
|
/// The entry a worker folder should be launched through: its apphost when the folder holds a
|
|
/// <b>self-contained</b> worker, otherwise the entry assembly (run through the dotnet muxer).
|
|
/// </summary>
|
|
/// <param name="folder">The worker folder.</param>
|
|
/// <returns>The absolute path of the apphost or of the entry assembly (which may not exist).</returns>
|
|
/// <remarks>
|
|
/// A framework-dependent build also emits an apphost, but that apphost needs an installed runtime the
|
|
/// self-contained Windows hosts do not have, and the muxer path is what docker-dev and the tests have always
|
|
/// used, so the apphost is chosen only for a self-contained worker. Self-contained is read from the worker's
|
|
/// <c>runtimeconfig.json</c>: the SDK writes <c>includedFrameworks</c> for a self-contained app and
|
|
/// <c>framework</c>/<c>frameworks</c> for a framework-dependent one.
|
|
/// </remarks>
|
|
public static string ResolveWorkerEntry(string folder)
|
|
{
|
|
var full = Path.GetFullPath(folder);
|
|
var appHost = Path.Combine(full, WorkerAppHostFileName);
|
|
return File.Exists(appHost) && IsSelfContainedWorker(full) ? appHost : Path.Combine(full, WorkerAssemblyFileName);
|
|
}
|
|
|
|
/// <summary>Whether the worker in <paramref name="folder"/> was published self-contained.</summary>
|
|
/// <param name="folder">The worker folder.</param>
|
|
/// <returns>True when its <c>runtimeconfig.json</c> lists <c>includedFrameworks</c>.</returns>
|
|
public static bool IsSelfContainedWorker(string folder)
|
|
{
|
|
var config = Path.Combine(folder, Path.GetFileNameWithoutExtension(WorkerAssemblyFileName) + ".runtimeconfig.json");
|
|
if (!File.Exists(config)) return false;
|
|
try
|
|
{
|
|
using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(config),
|
|
new System.Text.Json.JsonDocumentOptions { CommentHandling = System.Text.Json.JsonCommentHandling.Skip, AllowTrailingCommas = true });
|
|
return doc.RootElement.TryGetProperty("runtimeOptions", out var options)
|
|
&& options.TryGetProperty("includedFrameworks", out _);
|
|
}
|
|
catch (Exception ex) when (ex is System.Text.Json.JsonException or IOException or UnauthorizedAccessException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(invocation);
|
|
if (Volatile.Read(ref _disposed) != 0) return ScriptRunOutcome.Unavailable(ScriptRunOutcome.RunnerUnavailableReason);
|
|
|
|
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
|
var state = Admit(key, invocation.Kind, invocation.Source);
|
|
if (state.QuarantineReason is { } quarantined) return ScriptRunOutcome.Failed(quarantined);
|
|
|
|
// The source hash and its negative-cache key are per source, so they live on the key's state (computed once per
|
|
// source, not per evaluation, lmxopcua#789).
|
|
if (_compileFailures.TryGetValue(state.FailureKey, out var knownFailure))
|
|
{
|
|
CompileCounter.Add(1, new KeyValuePair<string, object?>("cache", "negative"));
|
|
return ScriptRunOutcome.Failed(knownFailure);
|
|
}
|
|
|
|
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) return Refused(key, state);
|
|
|
|
var handedOff = false;
|
|
SemaphoreSlim? gate = null;
|
|
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
|
|
WorkerConnection? connection = null;
|
|
try
|
|
{
|
|
state.ScriptLogger = invocation.ScriptLogger;
|
|
if (shard.IsolationGate is { } isolationGate)
|
|
{
|
|
if (!await isolationGate.WaitAsync(Grace(), ct).ConfigureAwait(false))
|
|
return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
|
|
gate = isolationGate;
|
|
}
|
|
|
|
// Fast path: the shard's worker is up (no state computation, no async hop).
|
|
connection = shard.Current ?? await shard.GetReadyAsync(Grace(), ct).ConfigureAwait(false);
|
|
if (connection is null) return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
|
|
|
|
var paths = SlotOrder(state, invocation.Inputs);
|
|
var entry = await connection
|
|
.EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, state.Sha, state.ShaHex, paths)
|
|
.WaitAsync(ct).ConfigureAwait(false);
|
|
if (entry.TimedOut)
|
|
{
|
|
// A compile that kept a compile thread busy for LoadTimeout after it STARTED is the script's fault (a
|
|
// pathological source): the connection killed the worker and the exit handler struck the key. Time
|
|
// spent queued behind other compiles is not counted (lmxopcua#790).
|
|
return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {connection.LoadTimeout.TotalSeconds:F0} s");
|
|
}
|
|
if (entry.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
|
|
NoteLoad(entry);
|
|
if (!outcome.Ok)
|
|
{
|
|
var reason = outcome.Error?.Kind switch
|
|
{
|
|
ScriptError.Types.Kind.Compile => $"compile error: {outcome.Error.Message}",
|
|
ScriptError.Types.Kind.Sandbox => $"sandbox violation: {outcome.Error.Message}",
|
|
_ => $"compile failure: {outcome.Error?.Message}",
|
|
};
|
|
// Remembered until the next deploy boundary, so a broken script never re-enters Roslyn (design §7).
|
|
_compileFailures.TryAdd(state.FailureKey, reason);
|
|
return ScriptRunOutcome.Failed(reason);
|
|
}
|
|
|
|
// From here the in-flight slot (and the isolation gate) belong to the evaluation itself: they are released
|
|
// when the worker answers or dies — not when this caller stops waiting at the soft deadline. That keeps a
|
|
// key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. The release runs
|
|
// inside the connection BEFORE the caller's await resumes, so a caller that awaited one run and starts the
|
|
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
|
|
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
|
|
// run by then.
|
|
var submitted = Stopwatch.GetTimestamp();
|
|
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
|
|
_worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
|
|
handedOff = true;
|
|
return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
return ScriptRunOutcome.Failed("evaluation cancelled");
|
|
}
|
|
finally
|
|
{
|
|
if (!handedOff)
|
|
{
|
|
Interlocked.Exchange(ref state.InFlight, 0);
|
|
gate?.Release();
|
|
}
|
|
}
|
|
|
|
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
|
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
|
// shard has no live worker (or for the isolation gate).
|
|
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
/// <remarks>
|
|
/// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
|
|
/// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
|
|
/// <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
|
|
/// bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
|
|
/// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
|
|
/// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
|
|
/// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
|
|
/// <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
|
|
/// </remarks>
|
|
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(invocations);
|
|
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
|
|
Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
|
|
for (var i = 0; i < invocations.Count; i++)
|
|
{
|
|
var invocation = invocations[i];
|
|
ArgumentNullException.ThrowIfNull(invocation);
|
|
switch (TryPrepareBatchItem(invocation, out var item, out var answered))
|
|
{
|
|
case BatchPreparation.Submit:
|
|
groups ??= [];
|
|
if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
|
|
group.Add((i, item));
|
|
break;
|
|
case BatchPreparation.Answered:
|
|
outcomes[i] = Task.FromResult(answered!);
|
|
break;
|
|
default:
|
|
outcomes[i] = RunAsync(invocation, ct);
|
|
break;
|
|
}
|
|
}
|
|
if (groups is null) return outcomes;
|
|
|
|
foreach (var (connection, group) in groups)
|
|
{
|
|
var submissions = new BatchSubmission[group.Count];
|
|
for (var j = 0; j < group.Count; j++)
|
|
{
|
|
var item = group[j].Item;
|
|
submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
|
|
item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
|
|
}
|
|
var submitted = Stopwatch.GetTimestamp();
|
|
BatchSubmittedForTest?.Invoke(submissions.Length);
|
|
var pendings = connection.SubmitBatch(submissions);
|
|
for (var j = 0; j < group.Count; j++)
|
|
{
|
|
var (index, item) = group[j];
|
|
outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
|
|
}
|
|
}
|
|
return outcomes;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
|
|
/// where nothing has to be waited for. Takes the key's single-flight slot when it returns
|
|
/// <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
|
|
/// </summary>
|
|
private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
|
|
{
|
|
item = null;
|
|
answered = null;
|
|
if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
|
|
|
|
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
|
var state = Admit(key, invocation.Kind, invocation.Source);
|
|
if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
|
|
|
|
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
|
|
if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
|
|
if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
|
|
|
|
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
|
|
{
|
|
answered = Refused(key, state);
|
|
return BatchPreparation.Answered;
|
|
}
|
|
state.ScriptLogger = invocation.ScriptLogger;
|
|
NoteLoad(entry);
|
|
item = new BatchItem(key, state, invocation, shard, connection, entry);
|
|
return BatchPreparation.Submit;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
|
|
/// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
|
|
/// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
|
|
/// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
|
|
/// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
|
|
/// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
|
|
/// quarantine, suspect) has already run by then.
|
|
/// </summary>
|
|
private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
|
|
{
|
|
var released = 0;
|
|
return () =>
|
|
{
|
|
if (Interlocked.Exchange(ref released, 1) != 0) return;
|
|
SlotReleasingForTest?.Invoke(key);
|
|
Volatile.Write(ref state.StalledRun, null);
|
|
Interlocked.Exchange(ref state.InFlight, 0);
|
|
heldGate?.Release();
|
|
};
|
|
}
|
|
|
|
/// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
|
|
private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
|
|
WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
var result = ct.CanBeCanceled
|
|
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
|
|
: await pending.Outcome.ConfigureAwait(false);
|
|
if (ReferenceEquals(result, PendingEval.SoftTimedOut))
|
|
{
|
|
// As in process: the run token is cancelled only once the caller has its timeout, so a polling script
|
|
// ends itself (and frees its slot) without the worker being killed.
|
|
connection.Cancel(pending);
|
|
if (ct.IsCancellationRequested) return ScriptRunOutcome.Failed("evaluation cancelled");
|
|
if (NoteTimeout(key, state, invocation.Timeout)) return ScriptRunOutcome.Failed(state.QuarantineReason!);
|
|
return ScriptRunOutcome.Failed($"{ScriptEvalFailureReasons.TimedOutPrefix}{invocation.Timeout.TotalSeconds:F1}s");
|
|
}
|
|
|
|
if (ReferenceEquals(result, PendingEval.TransportTimedOut))
|
|
{
|
|
// lmxopcua#827: T + G passed with neither a result nor a soft-deadline notice. The worker runs the soft
|
|
// clock from the run's start, so this is the pipe or a starved / stalled worker — the runner's fault,
|
|
// not (yet) the script's: no timeout counted, no Cancel (the run may not even have started), and the
|
|
// slot stays with the run until the worker answers. The seam re-evaluates when it does (or when the
|
|
// worker is replaced, if the heartbeat monitor kills it).
|
|
TransportTimeoutCounter.Add(1, new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind)),
|
|
new KeyValuePair<string, object?>("answer", "stalled"));
|
|
// …unless the worker then reports the script itself over T (slow AND late — a notice delayed past
|
|
// T + G by the watchdog poll and the pipe lands here too): it still gets its Cancel, so a polling
|
|
// script ends itself instead of costing the whole worker at notice + G, and it counts as a timeout
|
|
// (#827 review F2). Off the reader: the completion source runs this asynchronously.
|
|
var timeout = invocation.Timeout;
|
|
pending.LateSoftExpiry?.ContinueWith(_ =>
|
|
{
|
|
connection.Cancel(pending);
|
|
NoteTimeout(key, state, timeout);
|
|
}, CancellationToken.None, TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously,
|
|
TaskScheduler.Default);
|
|
// Read by a refusal only while it is incomplete, so a write that loses a race with the slot release
|
|
// is harmless.
|
|
Volatile.Write(ref state.StalledRun, pending.Completion);
|
|
if (ct.IsCancellationRequested) return ScriptRunOutcome.Failed("evaluation cancelled");
|
|
return ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerUnresponsiveReason) with { Recovery = pending.Completion };
|
|
}
|
|
|
|
if (result is null) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
|
|
var kindTag = new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind));
|
|
RoundTripHistogram.Record(Stopwatch.GetElapsedTime(submitted).TotalMicroseconds, kindTag);
|
|
EvalDurationHistogram.Record(result.RunMicros, kindTag);
|
|
return Complete(key, state, result, shard, connection);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
return ScriptRunOutcome.Failed("evaluation cancelled");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The slot order for this evaluation's inputs: the key's previous order when the input set is the same (the
|
|
/// steady state — no sort, no allocation, lmxopcua#789), otherwise the ordinal sort of the new set.
|
|
/// </summary>
|
|
private static string[] SlotOrder(KeyState state, IReadOnlyDictionary<string, Core.Abstractions.DataValueSnapshot> inputs)
|
|
{
|
|
var cached = state.Paths;
|
|
if (cached is not null && cached.Length == inputs.Count)
|
|
{
|
|
var same = true;
|
|
foreach (var path in cached)
|
|
{
|
|
if (!inputs.ContainsKey(path)) { same = false; break; }
|
|
}
|
|
if (same) return cached;
|
|
}
|
|
var fresh = inputs.Keys.Order(StringComparer.Ordinal).ToArray();
|
|
state.Paths = fresh;
|
|
return fresh;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void Reset(ScriptContextKind kind)
|
|
{
|
|
var prefix = InProcessScriptRunner.ScopedKey(kind, string.Empty);
|
|
var dropped = new List<string>();
|
|
foreach (var key in _keys.Keys)
|
|
{
|
|
if (key.StartsWith(prefix, StringComparison.Ordinal) && _keys.TryRemove(key, out _)) dropped.Add(key);
|
|
}
|
|
foreach (var key in _compileFailures.Keys)
|
|
{
|
|
if (key.StartsWith(prefix, StringComparison.Ordinal)) _compileFailures.TryRemove(key, out _);
|
|
}
|
|
foreach (var shard in AllShards)
|
|
shard.Current?.Unload(dropped);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind)
|
|
{
|
|
var prefix = InProcessScriptRunner.ScopedKey(kind, string.Empty);
|
|
return _keys
|
|
.Where(kv => kv.Value.QuarantineReason is not null && kv.Key.StartsWith(prefix, StringComparison.Ordinal))
|
|
.Select(kv => kv.Key[prefix.Length..])
|
|
.ToArray();
|
|
}
|
|
|
|
/// <summary>Stops every worker.</summary>
|
|
/// <returns>A task that completes when they have ended.</returns>
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
|
|
LiveRunners.TryRemove(this, out _);
|
|
await _maintenance.DisposeAsync().ConfigureAwait(false);
|
|
await Task.WhenAll(AllShards.Select(s => s.DisposeAsync().AsTask())).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Stops every worker (blocking).</summary>
|
|
public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
|
|
/// <summary>
|
|
/// The periodic pass: stop the isolation worker once it has been idle for
|
|
/// <see cref="ScriptWorkerOptions.IsolationIdleTimeout"/>, and log the circuit opening / closing once each.
|
|
/// </summary>
|
|
/// <returns>A task for the pass.</returns>
|
|
internal async Task MaintainAsync()
|
|
{
|
|
if (Volatile.Read(ref _disposed) != 0 || Interlocked.Exchange(ref _maintaining, 1) != 0) return;
|
|
try
|
|
{
|
|
await _isolation.StopIfIdleAsync(_worker.IsolationIdleTimeout).ConfigureAwait(false);
|
|
|
|
var open = State == ScriptRunnerState.Unavailable;
|
|
if (open && Interlocked.Exchange(ref _circuitLogged, 1) == 0)
|
|
{
|
|
_log.Error("Script runner circuit OPEN: every ScriptWorker shard has been down for at least {After}; every compiled script on this node reports {Reason} until a worker starts (retrying at the backoff cap)",
|
|
_worker.CircuitOpenAfter, ScriptRunOutcome.RunnerUnavailableReason);
|
|
}
|
|
else if (!open && Interlocked.Exchange(ref _circuitLogged, 0) == 1)
|
|
{
|
|
_log.Information("Script runner circuit closed: a ScriptWorker shard is serving again");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_log.Warning(ex, "Script runner maintenance pass failed");
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Exchange(ref _maintaining, 0);
|
|
}
|
|
}
|
|
|
|
private WorkerShard NewShard(string label, bool isolation, int evalThreads, TimeSpan heartbeatInterval)
|
|
{
|
|
var spec = new WorkerLaunchSpec(
|
|
Shard: label,
|
|
WorkerPath: WorkerPath,
|
|
HeapLimitBytes: _worker.HeapLimitBytes,
|
|
EvalThreads: evalThreads,
|
|
HardGrace: _worker.HardGrace ?? TimeSpan.FromSeconds(Math.Max(1, _options.RunTimeout.TotalSeconds)),
|
|
HeartbeatInterval: heartbeatInterval,
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15),
|
|
PeCacheDirectory: _peCacheDirectory,
|
|
PeCacheKey: _peCacheKey,
|
|
MarkerDirectory: _markerDirectory,
|
|
LoadTimeout: LoadTimeout,
|
|
CompileThreads: _compileThreads,
|
|
WorkerSid: WorkerSid,
|
|
HeartbeatSilenceLimit: _heartbeatSilenceLimit);
|
|
return new WorkerShard(label, isolation,
|
|
(onExiting, ct) => WorkerConnection.StartAsync(spec, _log, OnLogs, onExiting, ct),
|
|
OnWorkerExit, _log);
|
|
}
|
|
|
|
/// <summary>The shard a key's NEXT evaluation goes to (attribution may just have made it a suspect).</summary>
|
|
private WorkerShard RouteFor(string key, KeyState state)
|
|
=> state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
|
|
|
|
private static ScriptRunOutcome Unavailable(string reason, WorkerShard shard, WorkerConnection? failed)
|
|
=> ScriptRunOutcome.Unavailable(reason) with { Recovery = shard.WhenRecoveredAsync(failed) };
|
|
|
|
/// <summary>
|
|
/// The answer to an evaluation whose key's previous run still holds its single-flight slot: the usual
|
|
/// script-attributable refusal — unless that run was answered "worker unresponsive" (lmxopcua#827) and the
|
|
/// worker still has not answered it, in which case nothing says the script is slow, and the refusal is the
|
|
/// runner's fault too (re-evaluated when the stalled run completes).
|
|
/// </summary>
|
|
private static ScriptRunOutcome Refused(string key, KeyState state)
|
|
{
|
|
if (Volatile.Read(ref state.StalledRun) is { IsCompleted: false } stalled)
|
|
{
|
|
// Counted, like the run it waits on (#827 review F5): it is not a script refusal, so not ScriptRunGuard's.
|
|
TransportTimeoutCounter.Add(1, new KeyValuePair<string, object?>("kind", KindTag(state.Kind)),
|
|
new KeyValuePair<string, object?>("answer", "retrigger"));
|
|
return ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerUnresponsiveReason) with { Recovery = stalled };
|
|
}
|
|
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
|
|
return ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
|
|
}
|
|
|
|
private void NoteLoad(WorkerConnection.LoadEntry entry)
|
|
{
|
|
// Counted once per compile, by the caller that waited for it (an already-loaded key is not a load).
|
|
if (Interlocked.Exchange(ref entry.Counted, 1) != 0 || entry.Outcome is not { Ok: true } outcome) return;
|
|
var tag = outcome.Cache switch
|
|
{
|
|
PeCache.Hit => "hit",
|
|
PeCache.Miss => "miss",
|
|
PeCache.Rejected => "rejected",
|
|
_ => "uncached",
|
|
};
|
|
CompileCounter.Add(1, new KeyValuePair<string, object?>("cache", tag));
|
|
switch (outcome.Cache)
|
|
{
|
|
case PeCache.Hit: Interlocked.Increment(ref _peCacheHits); break;
|
|
case PeCache.Miss: Interlocked.Increment(ref _peCacheMisses); break;
|
|
case PeCache.Rejected:
|
|
Interlocked.Increment(ref _peCacheRejected);
|
|
_log.Warning("Script {ScriptKey}: its PE cache entry failed verification and was deleted; recompiled", outcome.Key);
|
|
break;
|
|
}
|
|
}
|
|
|
|
private KeyState Admit(string key, ScriptContextKind kind, string source)
|
|
{
|
|
while (true)
|
|
{
|
|
var state = _keys.GetOrAdd(key, static (_, arg) => new KeyState(arg.source, arg.kind), (source, kind));
|
|
if (ReferenceEquals(state.Source, source) || string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
|
|
// The script changed under this key: a clean slate, exactly as the in-process guard does. An evaluation
|
|
// of the old source still in flight releases the OLD state's slot when it ends.
|
|
if (_keys.TryUpdate(key, new KeyState(source, kind), state)) continue;
|
|
}
|
|
}
|
|
|
|
private EvalRequest BuildRequest(string key, WorkerConnection.LoadEntry entry, ScriptInvocation invocation)
|
|
{
|
|
var request = new EvalRequest
|
|
{
|
|
Key = key,
|
|
NowUtcTicks = DateTime.UtcNow.Ticks,
|
|
SoftTimeoutMs = (uint)Math.Clamp(invocation.Timeout.TotalMilliseconds, 1, uint.MaxValue),
|
|
HardGraceMs = (uint)Math.Clamp(_worker.EffectiveHardGrace(invocation.Timeout).TotalMilliseconds, 1, uint.MaxValue),
|
|
};
|
|
foreach (var (path, snapshot) in invocation.Inputs)
|
|
{
|
|
if (snapshot is null) continue;
|
|
var input = new Input
|
|
{
|
|
StatusCode = snapshot.StatusCode,
|
|
SourceTsTicks = snapshot.SourceTimestampUtc is { } source ? VariantCodec.ToUtc(source).Ticks : 0,
|
|
ServerTsTicks = VariantCodec.ToUtc(snapshot.ServerTimestampUtc).Ticks,
|
|
};
|
|
if (entry.Slots.TryGetValue(path, out var slot)) input.Slot = slot;
|
|
else input.Path = path;
|
|
if (VariantCodec.TryEncode(snapshot.Value, out var variant, out var unsupported))
|
|
{
|
|
input.Value = variant;
|
|
}
|
|
else
|
|
{
|
|
// Never silently converted (design §14 #11): the worker answers UNSUPPORTED_TYPE for this input.
|
|
input.Unsupported = true;
|
|
input.UnsupportedType = unsupported ?? "?";
|
|
}
|
|
request.Inputs.Add(input);
|
|
}
|
|
return request;
|
|
}
|
|
|
|
private ScriptRunOutcome Complete(string key, KeyState state, EvalResult result, WorkerShard shard, WorkerConnection connection)
|
|
{
|
|
// The script returned control (a value or a throw): the soft-timeout streak resets, as in process.
|
|
Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0);
|
|
|
|
if (_options.MaxAllocBytesPerRun > 0 && (long)result.AllocatedBytes > _options.MaxAllocBytesPerRun)
|
|
{
|
|
Quarantine(key, state,
|
|
$"one evaluation allocated {result.AllocatedBytes / (1024.0 * 1024.0):F1} MB, over the " +
|
|
$"{_options.MaxAllocBytesPerRun / (1024.0 * 1024.0):F0} MB budget (Scripting:MaxAllocBytesPerRun); " +
|
|
"the script is not evaluated again until its source changes or it is redeployed",
|
|
"allocation");
|
|
return ScriptRunOutcome.Failed(state.QuarantineReason!);
|
|
}
|
|
|
|
if (state.Suspect && Interlocked.Increment(ref state.ProbationRuns) >= _worker.IsolationProbationRuns)
|
|
{
|
|
state.Suspect = false;
|
|
_log.Information("Script {ScriptKey} completed {Runs} clean run(s) in the isolation worker; returning it to its shard",
|
|
key, _worker.IsolationProbationRuns);
|
|
}
|
|
|
|
switch (result.OutcomeCase)
|
|
{
|
|
case EvalResult.OutcomeOneofCase.Value:
|
|
try
|
|
{
|
|
return ScriptRunOutcome.Ok(VariantCodec.Decode(result.Value));
|
|
}
|
|
catch (ScriptWorkerProtocolException ex)
|
|
{
|
|
return ScriptRunOutcome.Failed($"script worker returned an undecodable value: {ex.Message}");
|
|
}
|
|
case EvalResult.OutcomeOneofCase.Predicate:
|
|
return ScriptRunOutcome.Ok(result.Predicate);
|
|
case EvalResult.OutcomeOneofCase.Error:
|
|
return result.Error.Kind switch
|
|
{
|
|
ScriptError.Types.Kind.Threw => ScriptRunOutcome.Failed($"script threw: {result.Error.Message}"),
|
|
ScriptError.Types.Kind.UnsupportedType => ScriptRunOutcome.Failed($"unsupported type: {result.Error.Message}"),
|
|
ScriptError.Types.Kind.Busy => ScriptRunOutcome.Failed(WorkerBusyReason),
|
|
ScriptError.Types.Kind.NotLoaded => Unavailable(ScriptRunOutcome.WorkerRestartedReason, shard, connection),
|
|
_ => ScriptRunOutcome.Failed($"script failed: {result.Error.Message}"),
|
|
};
|
|
default:
|
|
return ScriptRunOutcome.Ok(null);
|
|
}
|
|
}
|
|
|
|
private bool NoteTimeout(string key, KeyState state, TimeSpan timeout)
|
|
{
|
|
var count = Interlocked.Increment(ref state.ConsecutiveTimeouts);
|
|
if (count < _options.QuarantineAfterConsecutiveTimeouts) return false;
|
|
Quarantine(key, state,
|
|
$"{count} consecutive evaluation(s) timed out (budget {timeout.TotalMilliseconds:F0} ms); " +
|
|
"the script is not evaluated again until its source changes or it is redeployed",
|
|
"timeouts");
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Attribution for one worker exit. Called synchronously from the worker's exit path BEFORE any evaluation or
|
|
/// compile it held is completed (see <see cref="WorkerConnection.StartAsync"/>), so its verdict is in place
|
|
/// before the affected keys' single-flight slots are released.
|
|
/// </summary>
|
|
/// <param name="shard">The shard whose worker ended.</param>
|
|
/// <param name="exit">How it ended.</param>
|
|
internal void OnWorkerExit(WorkerShard shard, WorkerExit exit)
|
|
{
|
|
switch (exit.Reason)
|
|
{
|
|
case WorkerExitReason.KilledDeadline when exit.CulpritKey is { } culprit:
|
|
KillStrike(culprit, "it exceeded its hard deadline and its worker was killed", "killed");
|
|
break;
|
|
case WorkerExitReason.KilledLoadTimeout when exit.CulpritKey is { } culprit:
|
|
KillStrike(culprit, "its compile did not finish and its worker was killed", "killed");
|
|
break;
|
|
case WorkerExitReason.Crash or WorkerExitReason.KilledHeartbeat or WorkerExitReason.KilledUnresponsive or WorkerExitReason.ProtocolFault:
|
|
// Compiles count as well as evaluations: a source that crashes Roslyn (or the load) must be found too.
|
|
var implicated = exit.InFlightKeys.Concat(exit.LoadingKeys).Distinct(StringComparer.Ordinal).ToArray();
|
|
if (shard.IsIsolation)
|
|
{
|
|
// One evaluation (or compile) at a time here: the crash is this key's (design §5.3.3).
|
|
foreach (var key in implicated)
|
|
KillStrike(key, exit.InFlightKeys.Contains(key) ? "it crashed the isolation worker" : "its compile crashed the isolation worker",
|
|
"isolated_crash");
|
|
}
|
|
else if (implicated.Length > 0)
|
|
{
|
|
foreach (var key in implicated)
|
|
{
|
|
if (!_keys.TryGetValue(key, out var state)) continue;
|
|
state.Suspect = true;
|
|
Interlocked.Exchange(ref state.ProbationRuns, 0);
|
|
}
|
|
_log.Warning("ScriptWorker[{Shard}] ended ({Reason}) with no deadline crossed; {Count} script(s) in flight move to the isolation worker: {Suspects}",
|
|
shard.Label, exit.Reason, implicated.Length, string.Join(", ", implicated));
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
private void KillStrike(string key, string why, string metricReason)
|
|
{
|
|
if (!_keys.TryGetValue(key, out var state)) return;
|
|
int strikes;
|
|
lock (state.KillStrikes)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
state.KillStrikes.Add(now);
|
|
state.KillStrikes.RemoveAll(t => now - t > KillStrikeWindow);
|
|
strikes = state.KillStrikes.Count;
|
|
}
|
|
if (strikes >= _worker.QuarantineAfterKills)
|
|
{
|
|
Quarantine(key, state,
|
|
$"{why} ({strikes} time(s) in the last hour); the script is not evaluated again until its source changes or it is redeployed",
|
|
metricReason);
|
|
}
|
|
}
|
|
|
|
private void Quarantine(string key, KeyState state, string why, string metricReason)
|
|
{
|
|
var reason = ScriptRunGuard.QuarantinedReasonPrefix + why;
|
|
if (Interlocked.CompareExchange(ref state.QuarantineReason, reason, null) is not null) return;
|
|
ScriptRunGuard.QuarantinedCounter.Add(1,
|
|
new KeyValuePair<string, object?>("script", key),
|
|
new KeyValuePair<string, object?>("reason", metricReason));
|
|
_scriptRoot.ForContext(ScriptLoggerFactory.ScriptNameProperty, key).Error("Script {ScriptKey} quarantined: {Reason}", key, why);
|
|
}
|
|
|
|
private void OnLogs(LogBatch batch)
|
|
{
|
|
foreach (var line in batch.Lines)
|
|
{
|
|
var logger = _keys.TryGetValue(line.Key, out var state) && state.ScriptLogger is { } scoped ? scoped : _scriptRoot;
|
|
var level = line.Level <= (uint)LogEventLevel.Fatal ? (LogEventLevel)line.Level : LogEventLevel.Information;
|
|
if (string.IsNullOrEmpty(line.Exception))
|
|
logger.Write(level, "{ScriptMessage:l}", line.Rendered);
|
|
else
|
|
logger.Write(level, "{ScriptMessage:l}{NewLine}{ScriptException:l}", line.Rendered, Environment.NewLine, line.Exception);
|
|
}
|
|
if (batch.Dropped > 0)
|
|
_log.Warning("ScriptWorker dropped {Dropped} script log line(s): a script is logging faster than the pipe drains", batch.Dropped);
|
|
}
|
|
|
|
private static string KindTag(ScriptContextKind kind) => kind switch
|
|
{
|
|
ScriptContextKind.VirtualTag => "virtual_tag",
|
|
ScriptContextKind.Calculation => "calculation",
|
|
ScriptContextKind.AlarmPredicate => "alarm",
|
|
_ => "other",
|
|
};
|
|
|
|
private static IEnumerable<Measurement<int>> ShardMeasurements(Func<ScriptWorkerShardStatus, int> value)
|
|
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
|
|
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
|
|
|
|
private enum BatchPreparation
|
|
{
|
|
/// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
|
|
Full,
|
|
|
|
/// <summary>Answered without running (refused).</summary>
|
|
Answered,
|
|
|
|
/// <summary>Submit it with the batch; its slot is taken.</summary>
|
|
Submit,
|
|
}
|
|
|
|
private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
|
|
WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
|
|
|
|
private sealed class KeyState(string source, ScriptContextKind kind)
|
|
{
|
|
private byte[]? _sha;
|
|
private string? _shaHex;
|
|
private string? _failureKey;
|
|
|
|
public readonly string Source = source;
|
|
|
|
/// <summary>The seam (metric tags).</summary>
|
|
public readonly ScriptContextKind Kind = kind;
|
|
|
|
/// <summary>SHA-256 of <see cref="Source"/> (computed on first use; a benign race computes the same bytes).</summary>
|
|
public byte[] Sha => _sha ??= SHA256.HashData(Encoding.UTF8.GetBytes(Source));
|
|
|
|
/// <summary>Its upper-case hex form.</summary>
|
|
public string ShaHex => _shaHex ??= Convert.ToHexString(Sha);
|
|
|
|
/// <summary>The negative compile cache's key for this source (kind-scoped).</summary>
|
|
public string FailureKey => _failureKey ??= InProcessScriptRunner.ScopedKey(Kind, ShaHex);
|
|
|
|
/// <summary>The slot order last used for this key (see <see cref="SlotOrder"/>).</summary>
|
|
public string[]? Paths;
|
|
public readonly List<DateTime> KillStrikes = [];
|
|
public int InFlight;
|
|
public int ConsecutiveTimeouts;
|
|
public int ProbationRuns;
|
|
public volatile bool Suspect;
|
|
public string? QuarantineReason;
|
|
public ILogger? ScriptLogger;
|
|
|
|
/// <summary>
|
|
/// The completion of this key's in-flight run while it is answered "worker unresponsive" and not yet answered
|
|
/// by the worker (lmxopcua#827); null otherwise. Cleared by the slot release.
|
|
/// </summary>
|
|
public Task? StalledRun;
|
|
}
|
|
}
|