using System.Collections.Concurrent; using System.Diagnostics; using System.IO.Pipes; using System.Security.Cryptography; using System.Text; using System.Threading.Channels; using Serilog; using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol; using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; /// How a worker process ended. internal enum WorkerExitReason { /// It exited by itself with code 0 (a Shutdown). Clean, /// The host killed it: a script crossed its hard deadline (). KilledDeadline, /// The host killed it: no frame for three heartbeat intervals. KilledHeartbeat, /// The host killed it: a compile did not finish in time (). KilledLoadTimeout, /// It died by itself (OOM, heap limit, fail-fast, external kill). Crash, /// A framing or handshake violation. ProtocolFault, /// The runner is shutting down. Disposed, } /// The end of one worker process. /// Why it ended. /// The scoped key the host blamed, for a deadline or load kill. /// The OS exit code, when known. /// The scoped keys that had an evaluation outstanding when it ended. internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, int? ExitCode, IReadOnlyList InFlightKeys); /// What the host needs to launch one worker. /// The shard label (0, 1, … or iso). /// The worker entry assembly (.dll, run through the dotnet muxer) or apphost. /// The worker's DOTNET_GCHeapHardLimit. /// The worker scheduler's width (1 for the isolation worker). /// The worker watchdog's grace past the soft deadline. /// How often the worker sends a heartbeat. /// How long a launch may take to reach Ready. internal sealed record WorkerLaunchSpec( string Shard, string WorkerPath, long HeapLimitBytes, int EvalThreads, TimeSpan HardGrace, TimeSpan HeartbeatInterval, TimeSpan HandshakeTimeout); /// /// One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed /// environment, a random pipe that accepts exactly one client, a nonce handshake, a frame reader and a coalescing /// writer, heartbeat supervision, and the kill. It never restarts itself; does that. /// internal sealed class WorkerConnection : IAsyncDisposable { /// /// Environment variables a worker inherits from the host (design §6): the runtime's own knobs, the temp /// directories and the minimum a Windows process needs. Everything else — notably ServerHistorian__ApiKey, /// ConfigDb connection strings and every other Section__Key override — is dropped. /// internal static readonly IReadOnlyList InheritedEnvironment = [ "PATH", "TEMP", "TMP", "TMPDIR", "HOME", "SystemRoot", "SYSTEMROOT", "windir", "SystemDrive", "ComSpec", "PATHEXT", "NUMBER_OF_PROCESSORS", "PROCESSOR_ARCHITECTURE", "LOCALAPPDATA", ]; private readonly Process _process; private readonly NamedPipeServerStream _pipe; private readonly IDisposable? _job; private readonly string _workDir; private readonly ILogger _log; private readonly Action _onLogs; private readonly Channel _outbound = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true }); private readonly ConcurrentDictionary _pending = new(); private readonly ConcurrentDictionary> _loads = new(); private readonly ConcurrentDictionary _loaded = new(StringComparer.Ordinal); private readonly TaskCompletionSource _exited = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly CancellationTokenSource _stopping = new(); private readonly TimeSpan _heartbeatInterval; private TaskCompletionSource? _diag; private (WorkerExitReason Reason, string? Culprit)? _killedFor; private long _lastFrameTimestamp; private long _seq; private long _nextEvalId; private long _nextLoadId; private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes; private int _disposed; private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir, ILogger log, Action onLogs, TimeSpan heartbeatInterval) { Shard = shard; _process = process; _pipe = pipe; _job = job; _workDir = workDir; _log = log; _onLogs = onLogs; _heartbeatInterval = heartbeatInterval; Pid = process.Id; } /// The shard label. public string Shard { get; } /// The worker's process id. public int Pid { get; } /// Completes when the worker has ended, with why. public Task Exited => _exited.Task; /// Whether the worker is still serving. public bool IsAlive => !_exited.Task.IsCompleted && !_stopping.IsCancellationRequested; /// when the handshake completed. public long ReadyTimestamp { get; private set; } /// Evaluations currently outstanding on this worker. public int InFlightCount => _pending.Count; /// Launches a worker and completes the handshake. /// What to launch. /// Host diagnostics (stderr lines, exits). /// Receives the worker's forwarded ctx.Logger batches. /// Cancellation. /// A ready connection. public static async Task StartAsync(WorkerLaunchSpec spec, ILogger log, Action onLogs, CancellationToken ct) { var pipeName = "otopcua-sw-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(12)).ToLowerInvariant(); var nonce = Convert.ToHexString(RandomNumberGenerator.GetBytes(32)); var workDir = Path.Combine(Path.GetTempPath(), "otopcua-scriptworker", pipeName); Directory.CreateDirectory(workDir); // One instance, current user only: the pipe accepts exactly one connection, from this account. var pipe = new NamedPipeServerStream(pipeName, PipeDirection.InOut, maxNumberOfServerInstances: 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly); Process? process = null; IDisposable? job = null; try { var psi = BuildStartInfo(spec, pipeName, nonce, workDir); process = new Process { StartInfo = psi, EnableRaisingEvents = true }; if (!process.Start()) throw new InvalidOperationException("the worker process did not start"); if (OperatingSystem.IsWindows()) { // Native and JIT memory backstop above the managed heap limit. job = WindowsJobObject.Assign(process, spec.HeapLimitBytes + 256L * 1024 * 1024); } var shardLabel = spec.Shard; process.ErrorDataReceived += (_, e) => { if (e.Data is { Length: > 0 } line) log.Warning("ScriptWorker[{Shard}] {WorkerStderr}", shardLabel, line); }; process.OutputDataReceived += (_, e) => { if (e.Data is { Length: > 0 } line) log.Debug("ScriptWorker[{Shard}] stdout {WorkerStdout}", shardLabel, line); }; process.BeginErrorReadLine(); process.BeginOutputReadLine(); using var handshakeCts = CancellationTokenSource.CreateLinkedTokenSource(ct); handshakeCts.CancelAfter(spec.HandshakeTimeout); var exitedEarly = process.WaitForExitAsync(handshakeCts.Token); var accept = pipe.WaitForConnectionAsync(handshakeCts.Token); if (await Task.WhenAny(accept, exitedEarly).ConfigureAwait(false) != accept) throw new InvalidOperationException( $"the worker exited (code {SafeExitCode(process)}) before connecting to its pipe"); await accept.ConfigureAwait(false); var hello = new Hello { Nonce = nonce, MaxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes, EvalThreads = (uint)spec.EvalThreads, QueueCapacity = 4096, HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue), HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds, MaxLogLines = 1000, }; await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false); var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval); connection.Begin(); return connection; } catch { try { if (process is { HasExited: false }) process.Kill(entireProcessTree: true); } catch { /* best effort */ } job?.Dispose(); process?.Dispose(); await pipe.DisposeAsync().ConfigureAwait(false); TryDeleteDirectory(workDir); throw; } } /// /// The host half of the handshake: send , require a Ready as the first frame, /// echoing (constant-time compare) from . /// /// The accepted pipe. /// The Hello to send. /// The nonce put in the worker's environment. /// The launched process id; null skips the check. /// Cancellation (the handshake timeout). /// The worker's Ready. /// Anything but a matching Ready. internal static async Task HandshakeAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid, CancellationToken ct) { await ScriptWorkerFrameCodec.WriteAsync(stream, new Envelope { ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion, Seq = 1, Hello = hello, }, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false); await stream.FlushAsync(ct).ConfigureAwait(false); var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false) ?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake"); if (reply.BodyCase != Envelope.BodyOneofCase.Ready) throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}"); var expected = Encoding.UTF8.GetBytes(expectedNonce); var actual = Encoding.UTF8.GetBytes(reply.Ready.Nonce ?? string.Empty); if (!CryptographicOperations.FixedTimeEquals(expected, actual)) throw new ScriptWorkerProtocolException("the worker's nonce does not match the launch nonce"); if (expectedPid is { } pid && reply.Ready.Pid != pid) throw new ScriptWorkerProtocolException($"the pipe client reports pid {reply.Ready.Pid}, not the launched {pid}"); return reply.Ready; } private static ProcessStartInfo BuildStartInfo(WorkerLaunchSpec spec, string pipeName, string nonce, string workDir) { var psi = new ProcessStartInfo { UseShellExecute = false, CreateNoWindow = true, RedirectStandardError = true, RedirectStandardOutput = true, RedirectStandardInput = false, WorkingDirectory = workDir, }; if (spec.WorkerPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) { psi.FileName = ResolveMuxer(); psi.ArgumentList.Add("exec"); psi.ArgumentList.Add(spec.WorkerPath); } else { psi.FileName = spec.WorkerPath; } // Cleared, then repopulated from an allow-list (design §6). ProcessStartInfo.Environment starts as a copy of // the host's environment; ScadaBridge's sandbox inherits it whole, which this design rejects. var inherited = psi.Environment.ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal); psi.Environment.Clear(); foreach (var (name, value) in inherited) { if (value is null) continue; if (InheritedEnvironment.Contains(name, StringComparer.OrdinalIgnoreCase) || (name.StartsWith("DOTNET_", StringComparison.OrdinalIgnoreCase) && !name.StartsWith("DOTNET_GCHeap", StringComparison.OrdinalIgnoreCase))) psi.Environment[name] = value; } psi.Environment["DOTNET_GCHeapHardLimit"] = "0x" + spec.HeapLimitBytes.ToString("X", System.Globalization.CultureInfo.InvariantCulture); psi.Environment["DOTNET_gcServer"] = "0"; psi.Environment["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1"; psi.Environment[ScriptWorkerEnvironment.PipeName] = pipeName; psi.Environment[ScriptWorkerEnvironment.Nonce] = nonce; psi.Environment[ScriptWorkerEnvironment.ParentPid] = Environment.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture); psi.Environment[ScriptWorkerEnvironment.Shard] = spec.Shard; return psi; } /// The dotnet muxer that runs a framework-dependent worker .dll. /// The muxer path, or plain dotnet (resolved on PATH) as a last resort. internal static string ResolveMuxer() { var hostPath = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH"); if (!string.IsNullOrEmpty(hostPath) && File.Exists(hostPath)) return hostPath; var processPath = Environment.ProcessPath; if (processPath is not null && string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) return processPath; var root = Environment.GetEnvironmentVariable("DOTNET_ROOT"); if (!string.IsNullOrEmpty(root)) { var candidate = Path.Combine(root, OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet"); if (File.Exists(candidate)) return candidate; } return "dotnet"; } private void Begin() { ReadyTimestamp = Stopwatch.GetTimestamp(); Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp); _ = Task.Run(ReadLoopAsync); _ = Task.Run(WriteLoopAsync); _ = Task.Run(HeartbeatMonitorAsync); _ = Task.Run(WatchProcessAsync); } /// Ensures is compiled in this worker at . /// The scoped key. /// The context kind. /// The source. /// SHA-256 of the source. /// The slot order. /// The load outcome and the slot map; null when the worker ended first. public Task EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList inputPaths) { var hashHex = Convert.ToHexString(sourceHash); if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths)) return existing.Ready; var entry = new LoadEntry(hashHex, inputPaths); _loaded[key] = entry; var loadId = (ulong)Interlocked.Increment(ref _nextLoadId); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _loads[loadId] = tcs; var load = new LoadScripts { LoadId = loadId }; var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) }; script.InputPaths.AddRange(inputPaths); load.Scripts.Add(script); if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; } _ = tcs.Task.ContinueWith(t => { if (t.IsCompletedSuccessfully) { var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key); entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } }); if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair(key, entry)); } else { entry.Complete(null); _loaded.TryRemove(new KeyValuePair(key, entry)); } }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); return entry.Ready; } /// Forgets a key's compiled form here and in the worker. /// The scoped keys. public void Unload(IReadOnlyCollection keys) { var unload = new Unload(); foreach (var key in keys) if (_loaded.TryRemove(key, out _)) unload.Keys.Add(key); if (unload.Keys.Count > 0) Send(new Envelope { Unload = unload }); } /// Submits one evaluation. Its yields null when the worker ends first. /// The scoped key. /// The request (its id is assigned here). /// The pending evaluation. public PendingEval Submit(string key, EvalRequest request) { request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId); var pending = new PendingEval(key, request.EvalId); _pending[request.EvalId] = pending; var batch = new EvalBatch(); batch.Items.Add(request); if (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted) { if (_pending.TryRemove(request.EvalId, out _)) pending.Complete(null); } return pending; } /// Cancels a run's ctx.CancellationToken in the worker (sent at the soft deadline). /// The evaluation. public void Cancel(PendingEval pending) { if (!pending.Completion.IsCompleted) Send(new Envelope { Cancel = new Protocol.V1.Cancel { EvalId = pending.EvalId } }); } /// Asks the worker for its process diagnostics (loaded assemblies, environment names). /// How long to wait. /// The reply. public async Task DiagnosticsAsync(TimeSpan timeout) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); Interlocked.Exchange(ref _diag, tcs); Send(new Envelope { DiagRequest = new DiagnosticsRequest() }); return await tcs.Task.WaitAsync(timeout).ConfigureAwait(false); } /// Kills the worker (process tree). The first reason given is the one recorded. /// Why. /// The scoped key blamed, if any. public void Kill(WorkerExitReason reason, string? culpritKey) { lock (_pending) { _killedFor ??= (reason, culpritKey); } try { if (!_process.HasExited) _process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { /* already gone */ } catch (System.ComponentModel.Win32Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] kill of pid {Pid} failed", Shard, Pid); } } /// Asks the worker to stop, waits briefly, then kills it. /// A task that completes when the worker has ended. public async ValueTask DisposeAsync() { // Both the shard (on exit) and the runner (on shutdown) dispose a connection; only the first does the work. if (Interlocked.Exchange(ref _disposed, 1) != 0) return; if (!_exited.Task.IsCompleted) { lock (_pending) { _killedFor ??= (WorkerExitReason.Disposed, null); } Send(new Envelope { Shutdown = new Shutdown() }); try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } catch (TimeoutException) { Kill(WorkerExitReason.Disposed, null); } try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { } } _stopping.Cancel(); _outbound.Writer.TryComplete(); await _pipe.DisposeAsync().ConfigureAwait(false); _job?.Dispose(); _process.Dispose(); TryDeleteDirectory(_workDir); } /// Test hook: closes the host end of the pipe without touching the process, as a dying host would. internal void SeverPipeForTest() => _pipe.Dispose(); private bool Send(Envelope envelope) { envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion; return _outbound.Writer.TryWrite(envelope); } private async Task WriteLoopAsync() { var reader = _outbound.Reader; try { while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false)) { // Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch. EvalBatch? batch = null; while (reader.TryRead(out var envelope)) { if (envelope.BodyCase == Envelope.BodyOneofCase.Eval) { (batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items); continue; } envelope.Seq = (ulong)Interlocked.Increment(ref _seq); await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false); } if (batch is not null) { await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope { ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion, Seq = (ulong)Interlocked.Increment(ref _seq), Eval = batch, }, _maxFrameBytes, _stopping.Token).ConfigureAwait(false); } await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false); } } catch (OperationCanceledException) { } catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException) { if (!_stopping.IsCancellationRequested) _log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard); Kill(WorkerExitReason.Crash, null); } } private async Task ReadLoopAsync() { WorkerExitReason? fault = null; try { while (true) { var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false); if (frame is null) break; Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp()); switch (frame.BodyCase) { case Envelope.BodyOneofCase.Results: foreach (var result in frame.Results.Items) { // An eval id with no outstanding request is a protocol violation (design §6 "what the host // trusts"), except for one that was already completed by a kill race — ignored then. if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result); else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId)) throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}"); } break; case Envelope.BodyOneofCase.LoadResult: if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.TrySetResult(frame.LoadResult); else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}"); break; case Envelope.BodyOneofCase.Logs: try { _onLogs(frame.Logs); } catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); } break; case Envelope.BodyOneofCase.Heartbeat: break; case Envelope.BodyOneofCase.DiagReply: Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply); break; case Envelope.BodyOneofCase.Fault: _log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message); break; default: throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker"); } } } catch (OperationCanceledException) { } catch (ScriptWorkerProtocolException ex) { _log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid); fault = WorkerExitReason.ProtocolFault; } catch (Exception ex) when (ex is IOException or ObjectDisposedException) { } // The pipe is gone: the worker is dead or dying. Make sure, then report. try { if (fault is { } f) Kill(f, null); try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); } catch (TimeoutException) { Kill(WorkerExitReason.Crash, null); try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { } } } catch (InvalidOperationException) { // The process object was disposed by a concurrent shutdown: the worker is already gone. } Finish(); } private async Task WatchProcessAsync() { try { await _process.WaitForExitAsync(_stopping.Token).ConfigureAwait(false); } catch (OperationCanceledException) { return; } catch (InvalidOperationException) { return; } // The reader normally finishes first on EOF; this covers a worker that died while the pipe stayed open. await Task.Delay(TimeSpan.FromSeconds(2)).ConfigureAwait(false); Finish(); } private async Task HeartbeatMonitorAsync() { using var timer = new PeriodicTimer(_heartbeatInterval); try { while (await timer.WaitForNextTickAsync(_stopping.Token).ConfigureAwait(false)) { if (_exited.Task.IsCompleted) return; var silent = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastFrameTimestamp)); if (silent > _heartbeatInterval * 3) { _log.Warning("ScriptWorker[{Shard}] pid {Pid} sent nothing for {Silent} ms; killing it", Shard, Pid, (long)silent.TotalMilliseconds); Kill(WorkerExitReason.KilledHeartbeat, null); return; } } } catch (OperationCanceledException) { } } private void Finish() { int? code = null; try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { } (WorkerExitReason Reason, string? Culprit)? killed; lock (_pending) { killed = _killedFor; } var reason = killed?.Reason ?? (code == 0 ? WorkerExitReason.Clean : WorkerExitReason.Crash); var inFlight = _pending.Values.Select(p => p.Key).Distinct(StringComparer.Ordinal).ToArray(); if (!_exited.TrySetResult(new WorkerExit(reason, killed?.Culprit, code, inFlight))) return; // Every outstanding evaluation and load ends now; null tells the waiter "the worker went away". foreach (var id in _pending.Keys.ToArray()) if (_pending.TryRemove(id, out var pending)) pending.Complete(null); foreach (var id in _loads.Keys.ToArray()) if (_loads.TryRemove(id, out var load)) load.TrySetCanceled(); Volatile.Read(ref _diag)?.TrySetCanceled(); _outbound.Writer.TryComplete(); } private static string SafeExitCode(Process process) { try { return process.HasExited ? process.ExitCode.ToString(System.Globalization.CultureInfo.InvariantCulture) : "?"; } catch (InvalidOperationException) { return "?"; } } private static void TryDeleteDirectory(string dir) { try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ } } /// A key's compiled form in this worker. internal sealed class LoadEntry { private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); public LoadEntry(string sourceHash, IReadOnlyList inputPaths) { SourceHash = sourceHash; InputPaths = inputPaths; var slots = new Dictionary(inputPaths.Count, StringComparer.Ordinal); for (var i = 0; i < inputPaths.Count; i++) slots[inputPaths[i]] = (uint)i; Slots = slots; } public string SourceHash { get; } public IReadOnlyList InputPaths { get; } /// Path → slot in the registered order. public IReadOnlyDictionary Slots { get; } /// The worker's verdict (null until it arrives). public LoadOutcome? Outcome { get; private set; } public Task Ready => _ready.Task; public bool SameSlots(IReadOnlyList inputPaths) { if (inputPaths.Count != InputPaths.Count) return false; for (var i = 0; i < inputPaths.Count; i++) if (!string.Equals(inputPaths[i], InputPaths[i], StringComparison.Ordinal)) return false; return true; } public void Complete(LoadOutcome? outcome) { Outcome = outcome; _ready.TrySetResult(outcome is null ? null : this); } } } /// One evaluation outstanding on a worker. internal sealed class PendingEval { private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously); public PendingEval(string key, ulong evalId) { Key = key; EvalId = evalId; } /// The scoped key. public string Key { get; } /// The eval id on the wire. public ulong EvalId { get; } /// The worker's result, or null when the worker ended first. public Task Completion => _completion.Task; public void Complete(EvalResult? result) => _completion.TrySetResult(result); }