using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO.Pipes;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Channels;
using Serilog;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
/// How a worker process ended.
internal enum WorkerExitReason
{
/// It exited by itself with code 0 (a Shutdown).
Clean,
/// The host killed it: a script crossed its hard deadline ().
KilledDeadline,
/// The host killed it: no frame for three heartbeat intervals.
KilledHeartbeat,
/// The host killed it: a compile did not finish in time ().
KilledLoadTimeout,
/// It died by itself (OOM, heap limit, fail-fast, external kill).
Crash,
/// A framing or handshake violation.
ProtocolFault,
/// The runner is shutting down.
Disposed,
}
/// The end of one worker process.
/// Why it ended.
/// The scoped key the host blamed, for a deadline or load kill.
/// The OS exit code, when known.
/// The scoped keys that had an evaluation outstanding when it ended.
internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, int? ExitCode, IReadOnlyList InFlightKeys);
/// What the host needs to launch one worker.
/// The shard label (0, 1, … or iso).
/// The worker entry assembly (.dll, run through the dotnet muxer) or apphost.
/// The worker's DOTNET_GCHeapHardLimit.
/// The worker scheduler's width (1 for the isolation worker).
/// The worker watchdog's grace past the soft deadline.
/// How often the worker sends a heartbeat.
/// How long a launch may take to reach Ready.
internal sealed record WorkerLaunchSpec(
string Shard,
string WorkerPath,
long HeapLimitBytes,
int EvalThreads,
TimeSpan HardGrace,
TimeSpan HeartbeatInterval,
TimeSpan HandshakeTimeout);
///
/// One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed
/// environment, a random pipe that accepts exactly one client, a nonce handshake, a frame reader and a coalescing
/// writer, heartbeat supervision, and the kill. It never restarts itself; does that.
///
internal sealed class WorkerConnection : IAsyncDisposable
{
///
/// Environment variables a worker inherits from the host (design §6): the runtime's own knobs, the temp
/// directories and the minimum a Windows process needs. Everything else — notably ServerHistorian__ApiKey,
/// ConfigDb connection strings and every other Section__Key override — is dropped.
///
internal static readonly IReadOnlyList InheritedEnvironment =
[
"PATH", "TEMP", "TMP", "TMPDIR", "HOME", "SystemRoot", "SYSTEMROOT", "windir", "SystemDrive", "ComSpec",
"PATHEXT", "NUMBER_OF_PROCESSORS", "PROCESSOR_ARCHITECTURE", "LOCALAPPDATA",
];
private readonly Process _process;
private readonly NamedPipeServerStream _pipe;
private readonly IDisposable? _job;
private readonly string _workDir;
private readonly ILogger _log;
private readonly Action _onLogs;
private readonly Channel _outbound = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true });
private readonly ConcurrentDictionary _pending = new();
private readonly ConcurrentDictionary> _loads = new();
private readonly ConcurrentDictionary _loaded = new(StringComparer.Ordinal);
private readonly TaskCompletionSource _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly CancellationTokenSource _stopping = new();
private readonly TimeSpan _heartbeatInterval;
private TaskCompletionSource? _diag;
private (WorkerExitReason Reason, string? Culprit)? _killedFor;
private long _lastFrameTimestamp;
private long _seq;
private long _nextEvalId;
private long _nextLoadId;
private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
private int _disposed;
private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir,
ILogger log, Action onLogs, TimeSpan heartbeatInterval)
{
Shard = shard;
_process = process;
_pipe = pipe;
_job = job;
_workDir = workDir;
_log = log;
_onLogs = onLogs;
_heartbeatInterval = heartbeatInterval;
Pid = process.Id;
}
/// The shard label.
public string Shard { get; }
/// The worker's process id.
public int Pid { get; }
/// Completes when the worker has ended, with why.
public Task Exited => _exited.Task;
/// Whether the worker is still serving.
public bool IsAlive => !_exited.Task.IsCompleted && !_stopping.IsCancellationRequested;
/// when the handshake completed.
public long ReadyTimestamp { get; private set; }
/// Evaluations currently outstanding on this worker.
public int InFlightCount => _pending.Count;
/// Launches a worker and completes the handshake.
/// What to launch.
/// Host diagnostics (stderr lines, exits).
/// Receives the worker's forwarded ctx.Logger batches.
/// Cancellation.
/// A ready connection.
public static async Task StartAsync(WorkerLaunchSpec spec, ILogger log, Action onLogs, CancellationToken ct)
{
var pipeName = "otopcua-sw-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(12)).ToLowerInvariant();
var nonce = Convert.ToHexString(RandomNumberGenerator.GetBytes(32));
var workDir = Path.Combine(Path.GetTempPath(), "otopcua-scriptworker", pipeName);
Directory.CreateDirectory(workDir);
// One instance, current user only: the pipe accepts exactly one connection, from this account.
var pipe = new NamedPipeServerStream(pipeName, PipeDirection.InOut, maxNumberOfServerInstances: 1,
PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
Process? process = null;
IDisposable? job = null;
try
{
var psi = BuildStartInfo(spec, pipeName, nonce, workDir);
process = new Process { StartInfo = psi, EnableRaisingEvents = true };
if (!process.Start()) throw new InvalidOperationException("the worker process did not start");
if (OperatingSystem.IsWindows())
{
// Native and JIT memory backstop above the managed heap limit.
job = WindowsJobObject.Assign(process, spec.HeapLimitBytes + 256L * 1024 * 1024);
}
var shardLabel = spec.Shard;
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is { Length: > 0 } line) log.Warning("ScriptWorker[{Shard}] {WorkerStderr}", shardLabel, line);
};
process.OutputDataReceived += (_, e) =>
{
if (e.Data is { Length: > 0 } line) log.Debug("ScriptWorker[{Shard}] stdout {WorkerStdout}", shardLabel, line);
};
process.BeginErrorReadLine();
process.BeginOutputReadLine();
using var handshakeCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
handshakeCts.CancelAfter(spec.HandshakeTimeout);
var exitedEarly = process.WaitForExitAsync(handshakeCts.Token);
var accept = pipe.WaitForConnectionAsync(handshakeCts.Token);
if (await Task.WhenAny(accept, exitedEarly).ConfigureAwait(false) != accept)
throw new InvalidOperationException(
$"the worker exited (code {SafeExitCode(process)}) before connecting to its pipe");
await accept.ConfigureAwait(false);
var hello = new Hello
{
Nonce = nonce,
MaxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes,
EvalThreads = (uint)spec.EvalThreads,
QueueCapacity = 4096,
HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue),
HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds,
MaxLogLines = 1000,
};
await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval);
connection.Begin();
return connection;
}
catch
{
try { if (process is { HasExited: false }) process.Kill(entireProcessTree: true); } catch { /* best effort */ }
job?.Dispose();
process?.Dispose();
await pipe.DisposeAsync().ConfigureAwait(false);
TryDeleteDirectory(workDir);
throw;
}
}
///
/// The host half of the handshake: send , require a Ready as the first frame,
/// echoing (constant-time compare) from .
///
/// The accepted pipe.
/// The Hello to send.
/// The nonce put in the worker's environment.
/// The launched process id; null skips the check.
/// Cancellation (the handshake timeout).
/// The worker's Ready.
/// Anything but a matching Ready.
internal static async Task HandshakeAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid, CancellationToken ct)
{
await ScriptWorkerFrameCodec.WriteAsync(stream, new Envelope
{
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
Seq = 1,
Hello = hello,
}, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
await stream.FlushAsync(ct).ConfigureAwait(false);
var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false)
?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
if (reply.BodyCase != Envelope.BodyOneofCase.Ready)
throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}");
var expected = Encoding.UTF8.GetBytes(expectedNonce);
var actual = Encoding.UTF8.GetBytes(reply.Ready.Nonce ?? string.Empty);
if (!CryptographicOperations.FixedTimeEquals(expected, actual))
throw new ScriptWorkerProtocolException("the worker's nonce does not match the launch nonce");
if (expectedPid is { } pid && reply.Ready.Pid != pid)
throw new ScriptWorkerProtocolException($"the pipe client reports pid {reply.Ready.Pid}, not the launched {pid}");
return reply.Ready;
}
private static ProcessStartInfo BuildStartInfo(WorkerLaunchSpec spec, string pipeName, string nonce, string workDir)
{
var psi = new ProcessStartInfo
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
RedirectStandardOutput = true,
RedirectStandardInput = false,
WorkingDirectory = workDir,
};
if (spec.WorkerPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase))
{
psi.FileName = ResolveMuxer();
psi.ArgumentList.Add("exec");
psi.ArgumentList.Add(spec.WorkerPath);
}
else
{
psi.FileName = spec.WorkerPath;
}
// Cleared, then repopulated from an allow-list (design §6). ProcessStartInfo.Environment starts as a copy of
// the host's environment; ScadaBridge's sandbox inherits it whole, which this design rejects.
var inherited = psi.Environment.ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal);
psi.Environment.Clear();
foreach (var (name, value) in inherited)
{
if (value is null) continue;
if (InheritedEnvironment.Contains(name, StringComparer.OrdinalIgnoreCase)
|| (name.StartsWith("DOTNET_", StringComparison.OrdinalIgnoreCase)
&& !name.StartsWith("DOTNET_GCHeap", StringComparison.OrdinalIgnoreCase)))
psi.Environment[name] = value;
}
psi.Environment["DOTNET_GCHeapHardLimit"] = "0x" + spec.HeapLimitBytes.ToString("X", System.Globalization.CultureInfo.InvariantCulture);
psi.Environment["DOTNET_gcServer"] = "0";
psi.Environment["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1";
psi.Environment[ScriptWorkerEnvironment.PipeName] = pipeName;
psi.Environment[ScriptWorkerEnvironment.Nonce] = nonce;
psi.Environment[ScriptWorkerEnvironment.ParentPid] = Environment.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture);
psi.Environment[ScriptWorkerEnvironment.Shard] = spec.Shard;
return psi;
}
/// The dotnet muxer that runs a framework-dependent worker .dll.
/// The muxer path, or plain dotnet (resolved on PATH) as a last resort.
internal static string ResolveMuxer()
{
var hostPath = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH");
if (!string.IsNullOrEmpty(hostPath) && File.Exists(hostPath)) return hostPath;
var processPath = Environment.ProcessPath;
if (processPath is not null
&& string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase))
return processPath;
var root = Environment.GetEnvironmentVariable("DOTNET_ROOT");
if (!string.IsNullOrEmpty(root))
{
var candidate = Path.Combine(root, OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet");
if (File.Exists(candidate)) return candidate;
}
return "dotnet";
}
private void Begin()
{
ReadyTimestamp = Stopwatch.GetTimestamp();
Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
_ = Task.Run(ReadLoopAsync);
_ = Task.Run(WriteLoopAsync);
_ = Task.Run(HeartbeatMonitorAsync);
_ = Task.Run(WatchProcessAsync);
}
/// Ensures is compiled in this worker at .
/// The scoped key.
/// The context kind.
/// The source.
/// SHA-256 of the source.
/// The slot order.
/// The load outcome and the slot map; null when the worker ended first.
public Task EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList inputPaths)
{
var hashHex = Convert.ToHexString(sourceHash);
if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths))
return existing.Ready;
var entry = new LoadEntry(hashHex, inputPaths);
_loaded[key] = entry;
var loadId = (ulong)Interlocked.Increment(ref _nextLoadId);
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_loads[loadId] = tcs;
var load = new LoadScripts { LoadId = loadId };
var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) };
script.InputPaths.AddRange(inputPaths);
load.Scripts.Add(script);
if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; }
_ = tcs.Task.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
{
var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key);
entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } });
if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair(key, entry));
}
else
{
entry.Complete(null);
_loaded.TryRemove(new KeyValuePair(key, entry));
}
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
return entry.Ready;
}
/// Forgets a key's compiled form here and in the worker.
/// The scoped keys.
public void Unload(IReadOnlyCollection keys)
{
var unload = new Unload();
foreach (var key in keys)
if (_loaded.TryRemove(key, out _)) unload.Keys.Add(key);
if (unload.Keys.Count > 0) Send(new Envelope { Unload = unload });
}
/// Submits one evaluation. Its yields null when the worker ends first.
/// The scoped key.
/// The request (its id is assigned here).
/// The pending evaluation.
public PendingEval Submit(string key, EvalRequest request)
{
request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
var pending = new PendingEval(key, request.EvalId);
_pending[request.EvalId] = pending;
var batch = new EvalBatch();
batch.Items.Add(request);
if (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted)
{
if (_pending.TryRemove(request.EvalId, out _)) pending.Complete(null);
}
return pending;
}
/// Cancels a run's ctx.CancellationToken in the worker (sent at the soft deadline).
/// The evaluation.
public void Cancel(PendingEval pending)
{
if (!pending.Completion.IsCompleted) Send(new Envelope { Cancel = new Protocol.V1.Cancel { EvalId = pending.EvalId } });
}
/// Asks the worker for its process diagnostics (loaded assemblies, environment names).
/// How long to wait.
/// The reply.
public async Task DiagnosticsAsync(TimeSpan timeout)
{
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Interlocked.Exchange(ref _diag, tcs);
Send(new Envelope { DiagRequest = new DiagnosticsRequest() });
return await tcs.Task.WaitAsync(timeout).ConfigureAwait(false);
}
/// Kills the worker (process tree). The first reason given is the one recorded.
/// Why.
/// The scoped key blamed, if any.
public void Kill(WorkerExitReason reason, string? culpritKey)
{
lock (_pending)
{
_killedFor ??= (reason, culpritKey);
}
try
{
if (!_process.HasExited) _process.Kill(entireProcessTree: true);
}
catch (InvalidOperationException) { /* already gone */ }
catch (System.ComponentModel.Win32Exception ex)
{
_log.Warning(ex, "ScriptWorker[{Shard}] kill of pid {Pid} failed", Shard, Pid);
}
}
/// Asks the worker to stop, waits briefly, then kills it.
/// A task that completes when the worker has ended.
public async ValueTask DisposeAsync()
{
// Both the shard (on exit) and the runner (on shutdown) dispose a connection; only the first does the work.
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
if (!_exited.Task.IsCompleted)
{
lock (_pending) { _killedFor ??= (WorkerExitReason.Disposed, null); }
Send(new Envelope { Shutdown = new Shutdown() });
try
{
await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
catch (TimeoutException)
{
Kill(WorkerExitReason.Disposed, null);
}
try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
}
_stopping.Cancel();
_outbound.Writer.TryComplete();
await _pipe.DisposeAsync().ConfigureAwait(false);
_job?.Dispose();
_process.Dispose();
TryDeleteDirectory(_workDir);
}
/// Test hook: closes the host end of the pipe without touching the process, as a dying host would.
internal void SeverPipeForTest() => _pipe.Dispose();
private bool Send(Envelope envelope)
{
envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
return _outbound.Writer.TryWrite(envelope);
}
private async Task WriteLoopAsync()
{
var reader = _outbound.Reader;
try
{
while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false))
{
// Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch.
EvalBatch? batch = null;
while (reader.TryRead(out var envelope))
{
if (envelope.BodyCase == Envelope.BodyOneofCase.Eval)
{
(batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
continue;
}
envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
}
if (batch is not null)
{
await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope
{
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
Seq = (ulong)Interlocked.Increment(ref _seq),
Eval = batch,
}, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
}
await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false);
}
}
catch (OperationCanceledException) { }
catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException)
{
if (!_stopping.IsCancellationRequested)
_log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
Kill(WorkerExitReason.Crash, null);
}
}
private async Task ReadLoopAsync()
{
WorkerExitReason? fault = null;
try
{
while (true)
{
var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
if (frame is null) break;
Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
switch (frame.BodyCase)
{
case Envelope.BodyOneofCase.Results:
foreach (var result in frame.Results.Items)
{
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
// trusts"), except for one that was already completed by a kill race — ignored then.
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
}
break;
case Envelope.BodyOneofCase.LoadResult:
if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.TrySetResult(frame.LoadResult);
else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
break;
case Envelope.BodyOneofCase.Logs:
try { _onLogs(frame.Logs); }
catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
break;
case Envelope.BodyOneofCase.Heartbeat:
break;
case Envelope.BodyOneofCase.DiagReply:
Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
break;
case Envelope.BodyOneofCase.Fault:
_log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
break;
default:
throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
}
}
}
catch (OperationCanceledException) { }
catch (ScriptWorkerProtocolException ex)
{
_log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
fault = WorkerExitReason.ProtocolFault;
}
catch (Exception ex) when (ex is IOException or ObjectDisposedException)
{
}
// The pipe is gone: the worker is dead or dying. Make sure, then report.
try
{
if (fault is { } f) Kill(f, null);
try
{
await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
}
catch (TimeoutException)
{
Kill(WorkerExitReason.Crash, null);
try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
}
}
catch (InvalidOperationException)
{
// The process object was disposed by a concurrent shutdown: the worker is already gone.
}
Finish();
}
private async Task WatchProcessAsync()
{
try { await _process.WaitForExitAsync(_stopping.Token).ConfigureAwait(false); }
catch (OperationCanceledException) { return; }
catch (InvalidOperationException) { return; }
// The reader normally finishes first on EOF; this covers a worker that died while the pipe stayed open.
await Task.Delay(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
Finish();
}
private async Task HeartbeatMonitorAsync()
{
using var timer = new PeriodicTimer(_heartbeatInterval);
try
{
while (await timer.WaitForNextTickAsync(_stopping.Token).ConfigureAwait(false))
{
if (_exited.Task.IsCompleted) return;
var silent = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastFrameTimestamp));
if (silent > _heartbeatInterval * 3)
{
_log.Warning("ScriptWorker[{Shard}] pid {Pid} sent nothing for {Silent} ms; killing it", Shard, Pid, (long)silent.TotalMilliseconds);
Kill(WorkerExitReason.KilledHeartbeat, null);
return;
}
}
}
catch (OperationCanceledException) { }
}
private void Finish()
{
int? code = null;
try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { }
(WorkerExitReason Reason, string? Culprit)? killed;
lock (_pending) { killed = _killedFor; }
var reason = killed?.Reason ?? (code == 0 ? WorkerExitReason.Clean : WorkerExitReason.Crash);
var inFlight = _pending.Values.Select(p => p.Key).Distinct(StringComparer.Ordinal).ToArray();
if (!_exited.TrySetResult(new WorkerExit(reason, killed?.Culprit, code, inFlight))) return;
// Every outstanding evaluation and load ends now; null tells the waiter "the worker went away".
foreach (var id in _pending.Keys.ToArray())
if (_pending.TryRemove(id, out var pending)) pending.Complete(null);
foreach (var id in _loads.Keys.ToArray())
if (_loads.TryRemove(id, out var load)) load.TrySetCanceled();
Volatile.Read(ref _diag)?.TrySetCanceled();
_outbound.Writer.TryComplete();
}
private static string SafeExitCode(Process process)
{
try { return process.HasExited ? process.ExitCode.ToString(System.Globalization.CultureInfo.InvariantCulture) : "?"; }
catch (InvalidOperationException) { return "?"; }
}
private static void TryDeleteDirectory(string dir)
{
try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ }
}
/// A key's compiled form in this worker.
internal sealed class LoadEntry
{
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
public LoadEntry(string sourceHash, IReadOnlyList inputPaths)
{
SourceHash = sourceHash;
InputPaths = inputPaths;
var slots = new Dictionary(inputPaths.Count, StringComparer.Ordinal);
for (var i = 0; i < inputPaths.Count; i++) slots[inputPaths[i]] = (uint)i;
Slots = slots;
}
public string SourceHash { get; }
public IReadOnlyList InputPaths { get; }
/// Path → slot in the registered order.
public IReadOnlyDictionary Slots { get; }
/// The worker's verdict (null until it arrives).
public LoadOutcome? Outcome { get; private set; }
public Task Ready => _ready.Task;
public bool SameSlots(IReadOnlyList inputPaths)
{
if (inputPaths.Count != InputPaths.Count) return false;
for (var i = 0; i < inputPaths.Count; i++)
if (!string.Equals(inputPaths[i], InputPaths[i], StringComparison.Ordinal)) return false;
return true;
}
public void Complete(LoadOutcome? outcome)
{
Outcome = outcome;
_ready.TrySetResult(outcome is null ? null : this);
}
}
}
/// One evaluation outstanding on a worker.
internal sealed class PendingEval
{
private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
public PendingEval(string key, ulong evalId)
{
Key = key;
EvalId = evalId;
}
/// The scoped key.
public string Key { get; }
/// The eval id on the wire.
public ulong EvalId { get; }
/// The worker's result, or null when the worker ended first.
public Task Completion => _completion.Task;
public void Complete(EvalResult? result) => _completion.TrySetResult(result);
}