Joseph Doherty
f50b29abae
v2-ci / unit-tests (push) Failing after 1h12m10s
Second P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md; still opt-in behind Scripting:Runner=OutOfProcess (InProcess unchanged). - Compiled-PE cache (design §7, §14 #8): Scripting:Worker:PeCacheDir holds <dir>/pe/<id>.pe entries authenticated with HMAC-SHA256 under a per-node random key (<dir>/pe-cache.key, created by the host, 0600/0700 on Unix), handed to the worker in Hello. Entry id = SHA-256(compile fingerprint + kind + source); the fingerprint covers the Roslyn version, the Core.Scripting build (analyzer, wrapper, stack guard), the sandbox reference set and the runtime. Edited, truncated, other-key or renamed entries are rejected, deleted and recompiled; the worker never loads unverified bytes. Pruned by recency at startup (30 d / 50 000). ScriptEvaluator splits into CompileToImage + FromImage. Measured on 40 scripts, restart to every first result: macOS 820 ms no cache / 840 ms empty / 85 ms warm; windev (Windows) 9.5 s / 9.4 s / 1.0 s. - Quarantine race: WorkerConnection runs exit attribution synchronously BEFORE completing the evaluations/loads it held (and before Exited), and the shard passes it straight to the runner — a kill strike is in place before the killed key's slot is released. - Supervision: a dead worker restarts by itself after its backoff; Unavailable outcomes carry ScriptRunOutcome.Recovery (-> VirtualTagEvalResult.RetryWhen) and the VT actor / Calculation driver re-evaluate once against current inputs when it completes. Isolation worker stops after Scripting:Worker:IsolationIdleTimeout (10 min). Keys compiling when a worker dies are suspects too (isolation finds a poison compile and quarantines it). Startup orphan sweep by launch marker: kills only a process whose marker's host is gone and whose pid + start time + image match. - Crash-loop circuit (Scripting:Worker:CircuitOpenAfter, 2 min): runner Unavailable, one Error, evaluations answered at once while shards keep retrying at the cap. - Observability (§10): otopcua.script.runner.state, worker.up/inflight/queue_depth/heap_bytes (shard), ipc.roundtrip + eval.duration (kind), compile{cache}, isolation.suspects, exits reason=idle. `script-runner` health check on the ready tier, at worst Degraded (§14 #12: degraded but still ready). Left on #780: self-contained worker publish, installer (account, pipe ACL, firewall), docker-dev live drill, scripted-alarm seam, the §4.5 per-seam batch submit, the §8 perf harness. Tests: SupervisionTests, PeCacheStoreTests/PeCacheWorkerTests (real workers), ScriptRunnerHealthCheckTests, VT + Calc recovery seam tests; 14 mutations each turned their test red. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
778 lines
37 KiB
C#
778 lines
37 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using System.IO.Pipes;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Threading.Channels;
|
|
using Serilog;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
|
|
/// <summary>How a worker process ended.</summary>
|
|
internal enum WorkerExitReason
|
|
{
|
|
/// <summary>It exited by itself with code 0 (a Shutdown).</summary>
|
|
Clean,
|
|
|
|
/// <summary>The host killed it: a script crossed its hard deadline (<see cref="WorkerExit.CulpritKey"/>).</summary>
|
|
KilledDeadline,
|
|
|
|
/// <summary>The host killed it: no frame for three heartbeat intervals.</summary>
|
|
KilledHeartbeat,
|
|
|
|
/// <summary>The host killed it: a compile did not finish in time (<see cref="WorkerExit.CulpritKey"/>).</summary>
|
|
KilledLoadTimeout,
|
|
|
|
/// <summary>It died by itself (OOM, heap limit, fail-fast, external kill).</summary>
|
|
Crash,
|
|
|
|
/// <summary>A framing or handshake violation.</summary>
|
|
ProtocolFault,
|
|
|
|
/// <summary>The runner is shutting down.</summary>
|
|
Disposed,
|
|
|
|
/// <summary>The host stopped an idle worker (the isolation worker after <c>IsolationIdleTimeout</c>); not a failure.</summary>
|
|
Idle,
|
|
}
|
|
|
|
/// <summary>The end of one worker process.</summary>
|
|
/// <param name="Reason">Why it ended.</param>
|
|
/// <param name="CulpritKey">The scoped key the host blamed, for a deadline or load kill.</param>
|
|
/// <param name="ExitCode">The OS exit code, when known.</param>
|
|
/// <param name="InFlightKeys">The scoped keys that had an evaluation outstanding when it ended.</param>
|
|
/// <param name="LoadingKeys">The scoped keys whose compile (load) was outstanding when it ended.</param>
|
|
internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, int? ExitCode, IReadOnlyList<string> InFlightKeys,
|
|
IReadOnlyList<string> LoadingKeys)
|
|
{
|
|
/// <summary>An exit with no load in flight.</summary>
|
|
public WorkerExit(WorkerExitReason reason, string? culpritKey, int? exitCode, IReadOnlyList<string> inFlightKeys)
|
|
: this(reason, culpritKey, exitCode, inFlightKeys, [])
|
|
{
|
|
}
|
|
}
|
|
|
|
/// <summary>What the host needs to launch one worker.</summary>
|
|
/// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
|
|
/// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
|
|
/// <param name="HeapLimitBytes">The worker's <c>DOTNET_GCHeapHardLimit</c>.</param>
|
|
/// <param name="EvalThreads">The worker scheduler's width (1 for the isolation worker).</param>
|
|
/// <param name="HardGrace">The worker watchdog's grace past the soft deadline.</param>
|
|
/// <param name="HeartbeatInterval">How often the worker sends a heartbeat.</param>
|
|
/// <param name="HandshakeTimeout">How long a launch may take to reach Ready.</param>
|
|
/// <param name="PeCacheDirectory">The compiled-PE cache directory handed to the worker; null = no cache.</param>
|
|
/// <param name="PeCacheKey">The cache's HMAC key (with <paramref name="PeCacheDirectory"/>).</param>
|
|
/// <param name="MarkerDirectory">Where the launch marker for the orphan sweep is written; null = none.</param>
|
|
internal sealed record WorkerLaunchSpec(
|
|
string Shard,
|
|
string WorkerPath,
|
|
long HeapLimitBytes,
|
|
int EvalThreads,
|
|
TimeSpan HardGrace,
|
|
TimeSpan HeartbeatInterval,
|
|
TimeSpan HandshakeTimeout,
|
|
string? PeCacheDirectory = null,
|
|
byte[]? PeCacheKey = null,
|
|
string? MarkerDirectory = null);
|
|
|
|
/// <summary>
|
|
/// One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed
|
|
/// environment, a random pipe that accepts exactly one client, a nonce handshake, a frame reader and a coalescing
|
|
/// writer, heartbeat supervision, and the kill. It never restarts itself; <see cref="WorkerShard"/> does that.
|
|
/// </summary>
|
|
internal sealed class WorkerConnection : IAsyncDisposable
|
|
{
|
|
/// <summary>
|
|
/// Environment variables a worker inherits from the host (design §6): the runtime's own knobs, the temp
|
|
/// directories and the minimum a Windows process needs. Everything else — notably <c>ServerHistorian__ApiKey</c>,
|
|
/// ConfigDb connection strings and every other <c>Section__Key</c> override — is dropped.
|
|
/// </summary>
|
|
internal static readonly IReadOnlyList<string> InheritedEnvironment =
|
|
[
|
|
"PATH", "TEMP", "TMP", "TMPDIR", "HOME", "SystemRoot", "SYSTEMROOT", "windir", "SystemDrive", "ComSpec",
|
|
"PATHEXT", "NUMBER_OF_PROCESSORS", "PROCESSOR_ARCHITECTURE", "LOCALAPPDATA",
|
|
];
|
|
|
|
private readonly Process _process;
|
|
private readonly NamedPipeServerStream _pipe;
|
|
private readonly IDisposable? _job;
|
|
private readonly string _workDir;
|
|
private readonly ILogger _log;
|
|
private readonly Action<LogBatch> _onLogs;
|
|
private readonly Channel<Envelope> _outbound = Channel.CreateUnbounded<Envelope>(new UnboundedChannelOptions { SingleReader = true });
|
|
private readonly ConcurrentDictionary<ulong, PendingEval> _pending = new();
|
|
private readonly ConcurrentDictionary<ulong, (string Key, TaskCompletionSource<LoadResult> Completion)> _loads = new();
|
|
private readonly ConcurrentDictionary<string, LoadEntry> _loaded = new(StringComparer.Ordinal);
|
|
private readonly TaskCompletionSource<WorkerExit> _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
private readonly TimeSpan _heartbeatInterval;
|
|
private readonly Action<WorkerExit> _onExiting;
|
|
private readonly string? _markerPath;
|
|
private TaskCompletionSource<DiagnosticsReply>? _diag;
|
|
private Heartbeat? _lastHeartbeat;
|
|
private long _lastUsedTimestamp;
|
|
private int _finished;
|
|
private (WorkerExitReason Reason, string? Culprit)? _killedFor;
|
|
private long _lastFrameTimestamp;
|
|
private long _seq;
|
|
private long _nextEvalId;
|
|
private long _nextLoadId;
|
|
private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
|
|
private int _disposed;
|
|
|
|
private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir,
|
|
ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, Action<WorkerExit> onExiting, string? markerPath)
|
|
{
|
|
_onExiting = onExiting;
|
|
_markerPath = markerPath;
|
|
Shard = shard;
|
|
_process = process;
|
|
_pipe = pipe;
|
|
_job = job;
|
|
_workDir = workDir;
|
|
_log = log;
|
|
_onLogs = onLogs;
|
|
_heartbeatInterval = heartbeatInterval;
|
|
Pid = process.Id;
|
|
}
|
|
|
|
/// <summary>The shard label.</summary>
|
|
public string Shard { get; }
|
|
|
|
/// <summary>The worker's process id.</summary>
|
|
public int Pid { get; }
|
|
|
|
/// <summary>Completes when the worker has ended, with why.</summary>
|
|
public Task<WorkerExit> Exited => _exited.Task;
|
|
|
|
/// <summary>Whether the worker is still serving.</summary>
|
|
public bool IsAlive => !_exited.Task.IsCompleted && !_stopping.IsCancellationRequested;
|
|
|
|
/// <summary><see cref="Stopwatch.GetTimestamp"/> when the handshake completed.</summary>
|
|
public long ReadyTimestamp { get; private set; }
|
|
|
|
/// <summary>Evaluations currently outstanding on this worker.</summary>
|
|
public int InFlightCount => _pending.Count;
|
|
|
|
/// <summary>Compiles currently outstanding on this worker.</summary>
|
|
public int LoadingCount => _loads.Count;
|
|
|
|
/// <summary>The worker's last heartbeat (its in-flight / queued counts and heap), or null before the first.</summary>
|
|
public Heartbeat? LastHeartbeat => Volatile.Read(ref _lastHeartbeat);
|
|
|
|
/// <summary><see cref="Stopwatch.GetTimestamp"/> of the last load or evaluation submitted (or of Ready).</summary>
|
|
public long LastUsedTimestamp => Interlocked.Read(ref _lastUsedTimestamp);
|
|
|
|
/// <summary>Launches a worker and completes the handshake.</summary>
|
|
/// <param name="spec">What to launch.</param>
|
|
/// <param name="log">Host diagnostics (stderr lines, exits).</param>
|
|
/// <param name="onLogs">Receives the worker's forwarded <c>ctx.Logger</c> batches.</param>
|
|
/// <param name="onExiting">
|
|
/// Attribution, called synchronously when the worker has ended and BEFORE any outstanding evaluation or load is
|
|
/// completed or <see cref="Exited"/> is signalled. A kill strike or suspect mark recorded here is therefore in
|
|
/// place before any caller can see its evaluation end and trigger the script again (the quarantine race,
|
|
/// lmxopcua#780).
|
|
/// </param>
|
|
/// <param name="ct">Cancellation.</param>
|
|
/// <returns>A ready connection.</returns>
|
|
public static async Task<WorkerConnection> StartAsync(WorkerLaunchSpec spec, ILogger log, Action<LogBatch> onLogs,
|
|
Action<WorkerExit> onExiting, CancellationToken ct)
|
|
{
|
|
var pipeName = "otopcua-sw-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(12)).ToLowerInvariant();
|
|
var nonce = Convert.ToHexString(RandomNumberGenerator.GetBytes(32));
|
|
var workDir = Path.Combine(Path.GetTempPath(), "otopcua-scriptworker", pipeName);
|
|
Directory.CreateDirectory(workDir);
|
|
|
|
// One instance, current user only: the pipe accepts exactly one connection, from this account.
|
|
var pipe = new NamedPipeServerStream(pipeName, PipeDirection.InOut, maxNumberOfServerInstances: 1,
|
|
PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
|
|
Process? process = null;
|
|
IDisposable? job = null;
|
|
string? markerPath = null;
|
|
try
|
|
{
|
|
var psi = BuildStartInfo(spec, pipeName, nonce, workDir);
|
|
process = new Process { StartInfo = psi, EnableRaisingEvents = true };
|
|
if (!process.Start()) throw new InvalidOperationException("the worker process did not start");
|
|
// Before anything else can fail: a host that dies from here on leaves a marker the next start's orphan
|
|
// sweep can match to exactly this process (design §3.3).
|
|
if (spec.MarkerDirectory is { } markerDir) markerPath = WorkerLaunchMarker.Write(markerDir, process, pipeName, log);
|
|
if (OperatingSystem.IsWindows())
|
|
{
|
|
// Native and JIT memory backstop above the managed heap limit.
|
|
job = WindowsJobObject.Assign(process, spec.HeapLimitBytes + 256L * 1024 * 1024);
|
|
}
|
|
var shardLabel = spec.Shard;
|
|
process.ErrorDataReceived += (_, e) =>
|
|
{
|
|
if (e.Data is { Length: > 0 } line) log.Warning("ScriptWorker[{Shard}] {WorkerStderr}", shardLabel, line);
|
|
};
|
|
process.OutputDataReceived += (_, e) =>
|
|
{
|
|
if (e.Data is { Length: > 0 } line) log.Debug("ScriptWorker[{Shard}] stdout {WorkerStdout}", shardLabel, line);
|
|
};
|
|
process.BeginErrorReadLine();
|
|
process.BeginOutputReadLine();
|
|
|
|
using var handshakeCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
handshakeCts.CancelAfter(spec.HandshakeTimeout);
|
|
var exitedEarly = process.WaitForExitAsync(handshakeCts.Token);
|
|
var accept = pipe.WaitForConnectionAsync(handshakeCts.Token);
|
|
if (await Task.WhenAny(accept, exitedEarly).ConfigureAwait(false) != accept)
|
|
throw new InvalidOperationException(
|
|
$"the worker exited (code {SafeExitCode(process)}) before connecting to its pipe");
|
|
await accept.ConfigureAwait(false);
|
|
|
|
var hello = new Hello
|
|
{
|
|
Nonce = nonce,
|
|
MaxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes,
|
|
EvalThreads = (uint)spec.EvalThreads,
|
|
QueueCapacity = 4096,
|
|
HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue),
|
|
HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds,
|
|
MaxLogLines = 1000,
|
|
};
|
|
if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey)
|
|
{
|
|
hello.PeCacheDir = cacheDir;
|
|
hello.PeCacheKey = Google.Protobuf.ByteString.CopyFrom(cacheKey);
|
|
}
|
|
await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
|
|
|
|
var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval,
|
|
onExiting, markerPath);
|
|
connection.Begin();
|
|
return connection;
|
|
}
|
|
catch
|
|
{
|
|
try { if (process is { HasExited: false }) process.Kill(entireProcessTree: true); } catch { /* best effort */ }
|
|
job?.Dispose();
|
|
process?.Dispose();
|
|
await pipe.DisposeAsync().ConfigureAwait(false);
|
|
TryDeleteDirectory(workDir);
|
|
WorkerLaunchMarker.TryDelete(markerPath);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The host half of the handshake: send <paramref name="hello"/>, require a <c>Ready</c> as the first frame,
|
|
/// echoing <paramref name="expectedNonce"/> (constant-time compare) from <paramref name="expectedPid"/>.
|
|
/// </summary>
|
|
/// <param name="stream">The accepted pipe.</param>
|
|
/// <param name="hello">The Hello to send.</param>
|
|
/// <param name="expectedNonce">The nonce put in the worker's environment.</param>
|
|
/// <param name="expectedPid">The launched process id; null skips the check.</param>
|
|
/// <param name="ct">Cancellation (the handshake timeout).</param>
|
|
/// <returns>The worker's Ready.</returns>
|
|
/// <exception cref="ScriptWorkerProtocolException">Anything but a matching Ready.</exception>
|
|
internal static async Task<Ready> HandshakeAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid, CancellationToken ct)
|
|
{
|
|
await ScriptWorkerFrameCodec.WriteAsync(stream, new Envelope
|
|
{
|
|
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
|
Seq = 1,
|
|
Hello = hello,
|
|
}, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
|
|
await stream.FlushAsync(ct).ConfigureAwait(false);
|
|
|
|
var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false)
|
|
?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
|
|
if (reply.BodyCase != Envelope.BodyOneofCase.Ready)
|
|
throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}");
|
|
var expected = Encoding.UTF8.GetBytes(expectedNonce);
|
|
var actual = Encoding.UTF8.GetBytes(reply.Ready.Nonce ?? string.Empty);
|
|
if (!CryptographicOperations.FixedTimeEquals(expected, actual))
|
|
throw new ScriptWorkerProtocolException("the worker's nonce does not match the launch nonce");
|
|
if (expectedPid is { } pid && reply.Ready.Pid != pid)
|
|
throw new ScriptWorkerProtocolException($"the pipe client reports pid {reply.Ready.Pid}, not the launched {pid}");
|
|
return reply.Ready;
|
|
}
|
|
|
|
private static ProcessStartInfo BuildStartInfo(WorkerLaunchSpec spec, string pipeName, string nonce, string workDir)
|
|
{
|
|
var psi = new ProcessStartInfo
|
|
{
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true,
|
|
RedirectStandardError = true,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardInput = false,
|
|
WorkingDirectory = workDir,
|
|
};
|
|
if (spec.WorkerPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
psi.FileName = ResolveMuxer();
|
|
psi.ArgumentList.Add("exec");
|
|
psi.ArgumentList.Add(spec.WorkerPath);
|
|
}
|
|
else
|
|
{
|
|
psi.FileName = spec.WorkerPath;
|
|
}
|
|
|
|
// Cleared, then repopulated from an allow-list (design §6). ProcessStartInfo.Environment starts as a copy of
|
|
// the host's environment; ScadaBridge's sandbox inherits it whole, which this design rejects.
|
|
var inherited = psi.Environment.ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal);
|
|
psi.Environment.Clear();
|
|
foreach (var (name, value) in inherited)
|
|
{
|
|
if (value is null) continue;
|
|
if (InheritedEnvironment.Contains(name, StringComparer.OrdinalIgnoreCase)
|
|
|| (name.StartsWith("DOTNET_", StringComparison.OrdinalIgnoreCase)
|
|
&& !name.StartsWith("DOTNET_GCHeap", StringComparison.OrdinalIgnoreCase)))
|
|
psi.Environment[name] = value;
|
|
}
|
|
psi.Environment["DOTNET_GCHeapHardLimit"] = "0x" + spec.HeapLimitBytes.ToString("X", System.Globalization.CultureInfo.InvariantCulture);
|
|
psi.Environment["DOTNET_gcServer"] = "0";
|
|
psi.Environment["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1";
|
|
psi.Environment[ScriptWorkerEnvironment.PipeName] = pipeName;
|
|
psi.Environment[ScriptWorkerEnvironment.Nonce] = nonce;
|
|
psi.Environment[ScriptWorkerEnvironment.ParentPid] = Environment.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
|
psi.Environment[ScriptWorkerEnvironment.Shard] = spec.Shard;
|
|
return psi;
|
|
}
|
|
|
|
/// <summary>The dotnet muxer that runs a framework-dependent worker <c>.dll</c>.</summary>
|
|
/// <returns>The muxer path, or plain <c>dotnet</c> (resolved on PATH) as a last resort.</returns>
|
|
internal static string ResolveMuxer()
|
|
{
|
|
var hostPath = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH");
|
|
if (!string.IsNullOrEmpty(hostPath) && File.Exists(hostPath)) return hostPath;
|
|
var processPath = Environment.ProcessPath;
|
|
if (processPath is not null
|
|
&& string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase))
|
|
return processPath;
|
|
var root = Environment.GetEnvironmentVariable("DOTNET_ROOT");
|
|
if (!string.IsNullOrEmpty(root))
|
|
{
|
|
var candidate = Path.Combine(root, OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet");
|
|
if (File.Exists(candidate)) return candidate;
|
|
}
|
|
return "dotnet";
|
|
}
|
|
|
|
private void Begin()
|
|
{
|
|
ReadyTimestamp = Stopwatch.GetTimestamp();
|
|
Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
|
|
Interlocked.Exchange(ref _lastUsedTimestamp, ReadyTimestamp);
|
|
_ = Task.Run(ReadLoopAsync);
|
|
_ = Task.Run(WriteLoopAsync);
|
|
_ = Task.Run(HeartbeatMonitorAsync);
|
|
_ = Task.Run(WatchProcessAsync);
|
|
}
|
|
|
|
/// <summary>Ensures <paramref name="key"/> is compiled in this worker at <paramref name="sourceHash"/>.</summary>
|
|
/// <param name="key">The scoped key.</param>
|
|
/// <param name="kind">The context kind.</param>
|
|
/// <param name="source">The source.</param>
|
|
/// <param name="sourceHash">SHA-256 of the source.</param>
|
|
/// <param name="inputPaths">The slot order.</param>
|
|
/// <returns>The load outcome and the slot map; null when the worker ended first.</returns>
|
|
public Task<LoadEntry?> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
|
|
{
|
|
var hashHex = Convert.ToHexString(sourceHash);
|
|
if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths))
|
|
return existing.Ready;
|
|
|
|
var entry = new LoadEntry(hashHex, inputPaths);
|
|
_loaded[key] = entry;
|
|
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
|
var loadId = (ulong)Interlocked.Increment(ref _nextLoadId);
|
|
var tcs = new TaskCompletionSource<LoadResult>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_loads[loadId] = (key, tcs);
|
|
var load = new LoadScripts { LoadId = loadId };
|
|
var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) };
|
|
script.InputPaths.AddRange(inputPaths);
|
|
load.Scripts.Add(script);
|
|
if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; }
|
|
|
|
_ = tcs.Task.ContinueWith(t =>
|
|
{
|
|
if (t.IsCompletedSuccessfully)
|
|
{
|
|
var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key);
|
|
entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } });
|
|
if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
|
|
}
|
|
else
|
|
{
|
|
entry.Complete(null);
|
|
_loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
|
|
}
|
|
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
|
return entry.Ready;
|
|
}
|
|
|
|
/// <summary>Forgets a key's compiled form here and in the worker.</summary>
|
|
/// <param name="keys">The scoped keys.</param>
|
|
public void Unload(IReadOnlyCollection<string> keys)
|
|
{
|
|
var unload = new Unload();
|
|
foreach (var key in keys)
|
|
if (_loaded.TryRemove(key, out _)) unload.Keys.Add(key);
|
|
if (unload.Keys.Count > 0) Send(new Envelope { Unload = unload });
|
|
}
|
|
|
|
/// <summary>Submits one evaluation. Its <see cref="PendingEval.Completion"/> yields null when the worker ends first.</summary>
|
|
/// <param name="key">The scoped key.</param>
|
|
/// <param name="request">The request (its id is assigned here).</param>
|
|
/// <returns>The pending evaluation.</returns>
|
|
public PendingEval Submit(string key, EvalRequest request)
|
|
{
|
|
request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
|
|
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
|
var pending = new PendingEval(key, request.EvalId);
|
|
_pending[request.EvalId] = pending;
|
|
var batch = new EvalBatch();
|
|
batch.Items.Add(request);
|
|
if (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted)
|
|
{
|
|
if (_pending.TryRemove(request.EvalId, out _)) pending.Complete(null);
|
|
}
|
|
return pending;
|
|
}
|
|
|
|
/// <summary>Cancels a run's <c>ctx.CancellationToken</c> in the worker (sent at the soft deadline).</summary>
|
|
/// <param name="pending">The evaluation.</param>
|
|
public void Cancel(PendingEval pending)
|
|
{
|
|
if (!pending.Completion.IsCompleted) Send(new Envelope { Cancel = new Protocol.V1.Cancel { EvalId = pending.EvalId } });
|
|
}
|
|
|
|
/// <summary>Asks the worker for its process diagnostics (loaded assemblies, environment names).</summary>
|
|
/// <param name="timeout">How long to wait.</param>
|
|
/// <returns>The reply.</returns>
|
|
public async Task<DiagnosticsReply> DiagnosticsAsync(TimeSpan timeout)
|
|
{
|
|
var tcs = new TaskCompletionSource<DiagnosticsReply>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
Interlocked.Exchange(ref _diag, tcs);
|
|
Send(new Envelope { DiagRequest = new DiagnosticsRequest() });
|
|
return await tcs.Task.WaitAsync(timeout).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Kills the worker (process tree). The first reason given is the one recorded.</summary>
|
|
/// <param name="reason">Why.</param>
|
|
/// <param name="culpritKey">The scoped key blamed, if any.</param>
|
|
public void Kill(WorkerExitReason reason, string? culpritKey)
|
|
{
|
|
lock (_pending)
|
|
{
|
|
_killedFor ??= (reason, culpritKey);
|
|
}
|
|
try
|
|
{
|
|
if (!_process.HasExited) _process.Kill(entireProcessTree: true);
|
|
}
|
|
catch (InvalidOperationException) { /* already gone */ }
|
|
catch (System.ComponentModel.Win32Exception ex)
|
|
{
|
|
_log.Warning(ex, "ScriptWorker[{Shard}] kill of pid {Pid} failed", Shard, Pid);
|
|
}
|
|
}
|
|
|
|
/// <summary>Asks the worker to stop, waits briefly, then kills it.</summary>
|
|
/// <returns>A task that completes when the worker has ended.</returns>
|
|
public ValueTask DisposeAsync() => StopAsync(WorkerExitReason.Disposed);
|
|
|
|
/// <summary>As <see cref="DisposeAsync"/>, recording <paramref name="reason"/> as the exit reason.</summary>
|
|
/// <param name="reason">Why the host is stopping it (<see cref="WorkerExitReason.Disposed"/> or <see cref="WorkerExitReason.Idle"/>).</param>
|
|
/// <returns>A task that completes when the worker has ended.</returns>
|
|
public async ValueTask StopAsync(WorkerExitReason reason)
|
|
{
|
|
// Both the shard (on exit) and the runner (on shutdown) dispose a connection; only the first does the work.
|
|
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
|
|
if (!_exited.Task.IsCompleted)
|
|
{
|
|
lock (_pending) { _killedFor ??= (reason, null); }
|
|
Send(new Envelope { Shutdown = new Shutdown() });
|
|
try
|
|
{
|
|
await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Kill(WorkerExitReason.Disposed, null);
|
|
}
|
|
try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
|
|
}
|
|
_stopping.Cancel();
|
|
_outbound.Writer.TryComplete();
|
|
await _pipe.DisposeAsync().ConfigureAwait(false);
|
|
_job?.Dispose();
|
|
_process.Dispose();
|
|
TryDeleteDirectory(_workDir);
|
|
WorkerLaunchMarker.TryDelete(_markerPath);
|
|
}
|
|
|
|
/// <summary>Test hook: closes the host end of the pipe without touching the process, as a dying host would.</summary>
|
|
internal void SeverPipeForTest() => _pipe.Dispose();
|
|
|
|
private bool Send(Envelope envelope)
|
|
{
|
|
envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
|
|
return _outbound.Writer.TryWrite(envelope);
|
|
}
|
|
|
|
private async Task WriteLoopAsync()
|
|
{
|
|
var reader = _outbound.Reader;
|
|
try
|
|
{
|
|
while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false))
|
|
{
|
|
// Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch.
|
|
EvalBatch? batch = null;
|
|
while (reader.TryRead(out var envelope))
|
|
{
|
|
if (envelope.BodyCase == Envelope.BodyOneofCase.Eval)
|
|
{
|
|
(batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
|
|
continue;
|
|
}
|
|
envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
|
|
await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
|
}
|
|
if (batch is not null)
|
|
{
|
|
await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope
|
|
{
|
|
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
|
Seq = (ulong)Interlocked.Increment(ref _seq),
|
|
Eval = batch,
|
|
}, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
|
}
|
|
await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (OperationCanceledException) { }
|
|
catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException)
|
|
{
|
|
if (!_stopping.IsCancellationRequested)
|
|
_log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
|
|
Kill(WorkerExitReason.Crash, null);
|
|
}
|
|
}
|
|
|
|
private async Task ReadLoopAsync()
|
|
{
|
|
WorkerExitReason? fault = null;
|
|
try
|
|
{
|
|
while (true)
|
|
{
|
|
var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
|
if (frame is null) break;
|
|
Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
|
|
switch (frame.BodyCase)
|
|
{
|
|
case Envelope.BodyOneofCase.Results:
|
|
foreach (var result in frame.Results.Items)
|
|
{
|
|
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
|
|
// trusts"), except for one that was already completed by a kill race — ignored then.
|
|
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
|
|
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
|
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
|
|
}
|
|
break;
|
|
case Envelope.BodyOneofCase.LoadResult:
|
|
if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Completion.TrySetResult(frame.LoadResult);
|
|
else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
|
|
break;
|
|
case Envelope.BodyOneofCase.Logs:
|
|
try { _onLogs(frame.Logs); }
|
|
catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
|
|
break;
|
|
case Envelope.BodyOneofCase.Heartbeat:
|
|
Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
|
|
break;
|
|
case Envelope.BodyOneofCase.DiagReply:
|
|
Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
|
|
break;
|
|
case Envelope.BodyOneofCase.Fault:
|
|
_log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
|
|
break;
|
|
default:
|
|
throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) { }
|
|
catch (ScriptWorkerProtocolException ex)
|
|
{
|
|
_log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
|
|
fault = WorkerExitReason.ProtocolFault;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or ObjectDisposedException)
|
|
{
|
|
}
|
|
|
|
// The pipe is gone: the worker is dead or dying. Make sure, then report.
|
|
try
|
|
{
|
|
if (fault is { } f) Kill(f, null);
|
|
try
|
|
{
|
|
await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Kill(WorkerExitReason.Crash, null);
|
|
try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
|
|
}
|
|
}
|
|
catch (InvalidOperationException)
|
|
{
|
|
// The process object was disposed by a concurrent shutdown: the worker is already gone.
|
|
}
|
|
Finish();
|
|
}
|
|
|
|
private async Task WatchProcessAsync()
|
|
{
|
|
try { await _process.WaitForExitAsync(_stopping.Token).ConfigureAwait(false); }
|
|
catch (OperationCanceledException) { return; }
|
|
catch (InvalidOperationException) { return; }
|
|
// The reader normally finishes first on EOF; this covers a worker that died while the pipe stayed open.
|
|
await Task.Delay(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
|
|
Finish();
|
|
}
|
|
|
|
private async Task HeartbeatMonitorAsync()
|
|
{
|
|
using var timer = new PeriodicTimer(_heartbeatInterval);
|
|
try
|
|
{
|
|
while (await timer.WaitForNextTickAsync(_stopping.Token).ConfigureAwait(false))
|
|
{
|
|
if (_exited.Task.IsCompleted) return;
|
|
var silent = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastFrameTimestamp));
|
|
if (silent > _heartbeatInterval * 3)
|
|
{
|
|
_log.Warning("ScriptWorker[{Shard}] pid {Pid} sent nothing for {Silent} ms; killing it", Shard, Pid, (long)silent.TotalMilliseconds);
|
|
Kill(WorkerExitReason.KilledHeartbeat, null);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) { }
|
|
}
|
|
|
|
private void Finish()
|
|
{
|
|
// The reader (on EOF) and the process watcher both get here; only the first reports.
|
|
if (Interlocked.Exchange(ref _finished, 1) != 0) return;
|
|
// No new frame can be queued from here on, so every Submit / load either fails its Send (and completes itself)
|
|
// or was registered before this point and is completed below.
|
|
_outbound.Writer.TryComplete();
|
|
int? code = null;
|
|
try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { }
|
|
|
|
(WorkerExitReason Reason, string? Culprit)? killed;
|
|
lock (_pending) { killed = _killedFor; }
|
|
var reason = killed?.Reason ?? (code == 0 ? WorkerExitReason.Clean : WorkerExitReason.Crash);
|
|
var inFlight = _pending.Values.Select(p => p.Key).Distinct(StringComparer.Ordinal).ToArray();
|
|
var loading = _loads.Values.Select(l => l.Key).Distinct(StringComparer.Ordinal).ToArray();
|
|
var exit = new WorkerExit(reason, killed?.Culprit, code, inFlight, loading);
|
|
|
|
// Attribution FIRST, synchronously: a kill strike / quarantine / suspect mark must be in place before any
|
|
// evaluation below completes, because completing it releases the key's single-flight slot and the very next
|
|
// trigger is admitted against whatever the quarantine table says at that instant (lmxopcua#780 race fix).
|
|
try { _onExiting(exit); }
|
|
catch (Exception ex) { _log.Error(ex, "ScriptWorker[{Shard}] exit attribution threw", Shard); }
|
|
_exited.TrySetResult(exit);
|
|
|
|
// Every outstanding evaluation and load ends now; null tells the waiter "the worker went away".
|
|
foreach (var id in _pending.Keys.ToArray())
|
|
if (_pending.TryRemove(id, out var pending)) pending.Complete(null);
|
|
foreach (var id in _loads.Keys.ToArray())
|
|
if (_loads.TryRemove(id, out var load)) load.Completion.TrySetCanceled();
|
|
Volatile.Read(ref _diag)?.TrySetCanceled();
|
|
_outbound.Writer.TryComplete();
|
|
}
|
|
|
|
private static string SafeExitCode(Process process)
|
|
{
|
|
try { return process.HasExited ? process.ExitCode.ToString(System.Globalization.CultureInfo.InvariantCulture) : "?"; }
|
|
catch (InvalidOperationException) { return "?"; }
|
|
}
|
|
|
|
private static void TryDeleteDirectory(string dir)
|
|
{
|
|
try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ }
|
|
}
|
|
|
|
/// <summary>A key's compiled form in this worker.</summary>
|
|
internal sealed class LoadEntry
|
|
{
|
|
private readonly TaskCompletionSource<LoadEntry?> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
/// <summary>Set once the runner has counted this load in its metrics.</summary>
|
|
public int Counted;
|
|
|
|
public LoadEntry(string sourceHash, IReadOnlyList<string> inputPaths)
|
|
{
|
|
SourceHash = sourceHash;
|
|
InputPaths = inputPaths;
|
|
var slots = new Dictionary<string, uint>(inputPaths.Count, StringComparer.Ordinal);
|
|
for (var i = 0; i < inputPaths.Count; i++) slots[inputPaths[i]] = (uint)i;
|
|
Slots = slots;
|
|
}
|
|
|
|
public string SourceHash { get; }
|
|
|
|
public IReadOnlyList<string> InputPaths { get; }
|
|
|
|
/// <summary>Path → slot in the registered order.</summary>
|
|
public IReadOnlyDictionary<string, uint> Slots { get; }
|
|
|
|
/// <summary>The worker's verdict (null until it arrives).</summary>
|
|
public LoadOutcome? Outcome { get; private set; }
|
|
|
|
public Task<LoadEntry?> Ready => _ready.Task;
|
|
|
|
public bool SameSlots(IReadOnlyList<string> inputPaths)
|
|
{
|
|
if (inputPaths.Count != InputPaths.Count) return false;
|
|
for (var i = 0; i < inputPaths.Count; i++)
|
|
if (!string.Equals(inputPaths[i], InputPaths[i], StringComparison.Ordinal)) return false;
|
|
return true;
|
|
}
|
|
|
|
public void Complete(LoadOutcome? outcome)
|
|
{
|
|
Outcome = outcome;
|
|
_ready.TrySetResult(outcome is null ? null : this);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>One evaluation outstanding on a worker.</summary>
|
|
internal sealed class PendingEval
|
|
{
|
|
private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public PendingEval(string key, ulong evalId)
|
|
{
|
|
Key = key;
|
|
EvalId = evalId;
|
|
}
|
|
|
|
/// <summary>The scoped key.</summary>
|
|
public string Key { get; }
|
|
|
|
/// <summary>The eval id on the wire.</summary>
|
|
public ulong EvalId { get; }
|
|
|
|
/// <summary>The worker's result, or null when the worker ended first.</summary>
|
|
public Task<EvalResult?> Completion => _completion.Task;
|
|
|
|
public void Complete(EvalResult? result) => _completion.TrySetResult(result);
|
|
}
|