perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)
v2-ci / unit-tests (push) Has been cancelled
v2-ci / unit-tests (push) Has been cancelled
#789 (partial, further tuning deferred): combining inline frame writer, buffered frame reader, one timer per evaluation for T and T+G, slot release on the reader before the caller resumes, a non-spinning per-connection result dispatcher, and blocking pipe I/O on Unix (both ends) so the socket never enters the socket engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2: 228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % -> 123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache worker RSS, and times kill latency from the soft-timeout answer. #790: the worker sends LoadStarted; the compile timeout is armed then, so a load queued behind other compiles is never killed or struck. #788: one Error with stack per down period, rate-limited Warnings with attempt count, one Information on recovery. Design doc §8.2 notes + §12.2 record. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+90
-71
@@ -136,6 +136,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
private readonly string _markerDirectory;
|
||||
private readonly string? _peCacheDirectory;
|
||||
private readonly byte[]? _peCacheKey;
|
||||
private readonly int _compileThreads;
|
||||
private long _peCacheHits;
|
||||
private long _peCacheMisses;
|
||||
private long _peCacheRejected;
|
||||
@@ -153,8 +154,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
}
|
||||
|
||||
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
|
||||
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
|
||||
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
|
||||
int compileThreads = 0)
|
||||
{
|
||||
_compileThreads = compileThreads;
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_options = options.Sanitized(out _);
|
||||
_worker = _options.Worker;
|
||||
@@ -218,7 +221,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
_keys.Count(kv => kv.Value.QuarantineReason is not null),
|
||||
_peCacheDirectory is not null);
|
||||
|
||||
/// <summary>How long a compile in the worker may take before the worker is killed.</summary>
|
||||
/// <summary>
|
||||
/// How long a compile in the worker may run before the worker is killed, counted from the moment a compile thread
|
||||
/// starts it — never the time it spent queued behind other compiles (lmxopcua#790).
|
||||
/// </summary>
|
||||
internal TimeSpan LoadTimeout { get; }
|
||||
|
||||
/// <summary>The shard workers (tests).</summary>
|
||||
@@ -340,12 +346,12 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
if (Volatile.Read(ref _disposed) != 0) return ScriptRunOutcome.Unavailable(ScriptRunOutcome.RunnerUnavailableReason);
|
||||
|
||||
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
||||
var state = Admit(key, invocation.Source);
|
||||
var state = Admit(key, invocation.Kind, invocation.Source);
|
||||
if (state.QuarantineReason is { } quarantined) return ScriptRunOutcome.Failed(quarantined);
|
||||
|
||||
var sha = SHA256.HashData(Encoding.UTF8.GetBytes(invocation.Source));
|
||||
var failureKey = InProcessScriptRunner.ScopedKey(invocation.Kind, Convert.ToHexString(sha));
|
||||
if (_compileFailures.TryGetValue(failureKey, out var knownFailure))
|
||||
// The source hash and its negative-cache key are per source, so they live on the key's state (computed once per
|
||||
// source, not per evaluation, lmxopcua#789).
|
||||
if (_compileFailures.TryGetValue(state.FailureKey, out var knownFailure))
|
||||
{
|
||||
CompileCounter.Add(1, new KeyValuePair<string, object?>("cache", "negative"));
|
||||
return ScriptRunOutcome.Failed(knownFailure);
|
||||
@@ -364,37 +370,29 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
try
|
||||
{
|
||||
state.ScriptLogger = invocation.ScriptLogger;
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero
|
||||
// grace still kicks a start that is due, so the shards keep retrying at the backoff cap.
|
||||
var grace = State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
if (shard.IsolationGate is { } isolationGate)
|
||||
{
|
||||
if (!await isolationGate.WaitAsync(grace, ct).ConfigureAwait(false))
|
||||
if (!await isolationGate.WaitAsync(Grace(), ct).ConfigureAwait(false))
|
||||
return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
|
||||
gate = isolationGate;
|
||||
}
|
||||
|
||||
connection = await shard.GetReadyAsync(grace, ct).ConfigureAwait(false);
|
||||
// Fast path: the shard's worker is up (no state computation, no async hop).
|
||||
connection = shard.Current ?? await shard.GetReadyAsync(Grace(), ct).ConfigureAwait(false);
|
||||
if (connection is null) return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
|
||||
|
||||
var paths = invocation.Inputs.Keys.Order(StringComparer.Ordinal).ToArray();
|
||||
WorkerConnection.LoadEntry? entry;
|
||||
try
|
||||
var paths = SlotOrder(state, invocation.Inputs);
|
||||
var entry = await connection
|
||||
.EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, state.Sha, state.ShaHex, paths)
|
||||
.WaitAsync(ct).ConfigureAwait(false);
|
||||
if (entry.TimedOut)
|
||||
{
|
||||
entry = await connection
|
||||
.EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, sha, paths)
|
||||
.WaitAsync(LoadTimeout, ct).ConfigureAwait(false);
|
||||
// A compile that kept a compile thread busy for LoadTimeout after it STARTED is the script's fault (a
|
||||
// pathological source): the connection killed the worker and the exit handler struck the key. Time
|
||||
// spent queued behind other compiles is not counted (lmxopcua#790).
|
||||
return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {connection.LoadTimeout.TotalSeconds:F0} s");
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// A compile that never finishes is the script's fault (a pathological source): the worker dies, the
|
||||
// key takes a kill strike through the exit handler.
|
||||
_log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] exceeded {Timeout} s; killing the worker",
|
||||
key, shard.Label, LoadTimeout.TotalSeconds);
|
||||
connection.Kill(WorkerExitReason.KilledLoadTimeout, key);
|
||||
return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {LoadTimeout.TotalSeconds:F0} s");
|
||||
}
|
||||
if (entry?.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
|
||||
if (entry.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
|
||||
NoteLoad(entry);
|
||||
if (!outcome.Ok)
|
||||
{
|
||||
@@ -405,19 +403,17 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
_ => $"compile failure: {outcome.Error?.Message}",
|
||||
};
|
||||
// Remembered until the next deploy boundary, so a broken script never re-enters Roslyn (design §7).
|
||||
_compileFailures.TryAdd(failureKey, reason);
|
||||
_compileFailures.TryAdd(state.FailureKey, reason);
|
||||
return ScriptRunOutcome.Failed(reason);
|
||||
}
|
||||
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
var pending = connection.Submit(key, BuildRequest(key, entry, invocation));
|
||||
ArmHardDeadline(connection, pending, invocation.Timeout + _worker.EffectiveHardGrace(invocation.Timeout));
|
||||
|
||||
// From here the in-flight slot (and the isolation gate) belong to the evaluation itself: they are released
|
||||
// when the worker answers or dies — not when this caller stops waiting at the soft deadline. That keeps a
|
||||
// key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. On a worker
|
||||
// death, attribution (strike, quarantine, suspect) has already run when this continuation fires.
|
||||
handedOff = true;
|
||||
// key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. The release runs
|
||||
// inside the connection BEFORE the caller's await resumes, so a caller that awaited one run and starts the
|
||||
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
|
||||
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
|
||||
// run by then.
|
||||
var heldGate = gate;
|
||||
var released = 0;
|
||||
void ReleaseSlot()
|
||||
@@ -427,20 +423,16 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
heldGate?.Release();
|
||||
}
|
||||
_ = pending.Completion.ContinueWith(_ => ReleaseSlot(),
|
||||
CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
|
||||
EvalResult? result;
|
||||
try
|
||||
{
|
||||
result = await pending.Completion.WaitAsync(invocation.Timeout, ct).ConfigureAwait(false);
|
||||
// The worker has answered (or died, with attribution already run): free the slot HERE, before this
|
||||
// caller resumes. The continuation above is not ordered against this await, so relying on it alone
|
||||
// let a caller that awaited one run and then started the next see "previous evaluation still running"
|
||||
// (found by the §8 perf harness: 32 of 200 sequential same-key runs refused). Idempotent.
|
||||
ReleaseSlot();
|
||||
}
|
||||
catch (TimeoutException)
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
|
||||
_worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
|
||||
handedOff = true;
|
||||
|
||||
var result = ct.CanBeCanceled
|
||||
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
|
||||
: await pending.Outcome.ConfigureAwait(false);
|
||||
if (ReferenceEquals(result, PendingEval.SoftTimedOut))
|
||||
{
|
||||
// As in process: the run token is cancelled only once the caller has its timeout, so a polling script
|
||||
// ends itself (and frees its slot) without the worker being killed.
|
||||
@@ -468,6 +460,32 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
gate?.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
||||
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
||||
// shard has no live worker (or for the isolation gate).
|
||||
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The slot order for this evaluation's inputs: the key's previous order when the input set is the same (the
|
||||
/// steady state — no sort, no allocation, lmxopcua#789), otherwise the ordinal sort of the new set.
|
||||
/// </summary>
|
||||
private static string[] SlotOrder(KeyState state, IReadOnlyDictionary<string, Core.Abstractions.DataValueSnapshot> inputs)
|
||||
{
|
||||
var cached = state.Paths;
|
||||
if (cached is not null && cached.Length == inputs.Count)
|
||||
{
|
||||
var same = true;
|
||||
foreach (var path in cached)
|
||||
{
|
||||
if (!inputs.ContainsKey(path)) { same = false; break; }
|
||||
}
|
||||
if (same) return cached;
|
||||
}
|
||||
var fresh = inputs.Keys.Order(StringComparer.Ordinal).ToArray();
|
||||
state.Paths = fresh;
|
||||
return fresh;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -555,7 +573,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
HandshakeTimeout: TimeSpan.FromSeconds(15),
|
||||
PeCacheDirectory: _peCacheDirectory,
|
||||
PeCacheKey: _peCacheKey,
|
||||
MarkerDirectory: _markerDirectory);
|
||||
MarkerDirectory: _markerDirectory,
|
||||
LoadTimeout: LoadTimeout,
|
||||
CompileThreads: _compileThreads);
|
||||
return new WorkerShard(label, isolation,
|
||||
(onExiting, ct) => WorkerConnection.StartAsync(spec, _log, OnLogs, onExiting, ct),
|
||||
OnWorkerExit, _log);
|
||||
@@ -591,15 +611,15 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
}
|
||||
}
|
||||
|
||||
private KeyState Admit(string key, string source)
|
||||
private KeyState Admit(string key, ScriptContextKind kind, string source)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), source);
|
||||
if (string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
|
||||
var state = _keys.GetOrAdd(key, static (_, arg) => new KeyState(arg.source, arg.kind), (source, kind));
|
||||
if (ReferenceEquals(state.Source, source) || string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
|
||||
// The script changed under this key: a clean slate, exactly as the in-process guard does. An evaluation
|
||||
// of the old source still in flight releases the OLD state's slot when it ends.
|
||||
if (_keys.TryUpdate(key, new KeyState(source), state)) continue;
|
||||
if (_keys.TryUpdate(key, new KeyState(source, kind), state)) continue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -638,23 +658,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
return request;
|
||||
}
|
||||
|
||||
private void ArmHardDeadline(WorkerConnection connection, PendingEval pending, TimeSpan after)
|
||||
{
|
||||
var cts = new CancellationTokenSource(after);
|
||||
var registration = cts.Token.Register(() =>
|
||||
{
|
||||
if (pending.Completion.IsCompleted) return;
|
||||
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
|
||||
pending.Key, (long)after.TotalMilliseconds, connection.Shard, connection.Pid);
|
||||
connection.Kill(WorkerExitReason.KilledDeadline, pending.Key);
|
||||
});
|
||||
_ = pending.Completion.ContinueWith(_ =>
|
||||
{
|
||||
registration.Dispose();
|
||||
cts.Dispose();
|
||||
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private ScriptRunOutcome Complete(string key, KeyState state, EvalResult result, WorkerShard shard, WorkerConnection connection)
|
||||
{
|
||||
// The script returned control (a value or a throw): the soft-timeout streak resets, as in process.
|
||||
@@ -813,9 +816,25 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
|
||||
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
|
||||
|
||||
private sealed class KeyState(string source)
|
||||
private sealed class KeyState(string source, ScriptContextKind kind)
|
||||
{
|
||||
private byte[]? _sha;
|
||||
private string? _shaHex;
|
||||
private string? _failureKey;
|
||||
|
||||
public readonly string Source = source;
|
||||
|
||||
/// <summary>SHA-256 of <see cref="Source"/> (computed on first use; a benign race computes the same bytes).</summary>
|
||||
public byte[] Sha => _sha ??= SHA256.HashData(Encoding.UTF8.GetBytes(Source));
|
||||
|
||||
/// <summary>Its upper-case hex form.</summary>
|
||||
public string ShaHex => _shaHex ??= Convert.ToHexString(Sha);
|
||||
|
||||
/// <summary>The negative compile cache's key for this source (kind-scoped).</summary>
|
||||
public string FailureKey => _failureKey ??= InProcessScriptRunner.ScopedKey(kind, ShaHex);
|
||||
|
||||
/// <summary>The slot order last used for this key (see <see cref="SlotOrder"/>).</summary>
|
||||
public string[]? Paths;
|
||||
public readonly List<DateTime> KillStrikes = [];
|
||||
public int InFlight;
|
||||
public int ConsecutiveTimeouts;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
|
||||
/// <summary>
|
||||
/// The thread that resumes the callers of one worker's evaluations (lmxopcua#789).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Why not the thread pool: a result that wakes a pool worker makes that worker spin-wait for more work when it
|
||||
/// is done, and results arrive one at a time, spread out, so every evaluation paid a full spin. The §8 harness
|
||||
/// measured that spin as most of the host's CPU at 10k evals/s (with
|
||||
/// <c>DOTNET_ThreadPool_UnfairSemaphoreSpinLimit=0</c> the out-of-process runner cost only ~10 points of a core more
|
||||
/// than in-process). This thread blocks without spinning (<see cref="Monitor.Wait(object)"/>) and runs the
|
||||
/// continuations of every result that is ready in one wake-up.</para>
|
||||
/// <para>Why not the reader thread: the continuation is the caller's code (the Calculation pump publishes to OPC UA
|
||||
/// subscribers after its await). On the reader it could hold up heartbeats and results, and a slow consumer could
|
||||
/// get a healthy worker killed for silence. Here a slow consumer delays only the resumption of other callers; the
|
||||
/// reader has already settled each result — its deadline timer cancelled, its key's slot released — before handing
|
||||
/// it over, so no deadline can fire on an evaluation that has answered.</para>
|
||||
/// <para>A caller that blocks on another evaluation's result from inside its continuation (sync over async) would
|
||||
/// wait on this very thread; no seam does, and <see cref="IScriptRunner.RunAsync"/> is async-only for that reason.</para>
|
||||
/// </remarks>
|
||||
internal sealed class ResultDispatcher
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly Queue<(PendingEval Pending, EvalResult? Result)> _queue = new();
|
||||
private bool _waiting;
|
||||
private bool _stopped;
|
||||
|
||||
/// <summary>Starts the thread.</summary>
|
||||
/// <param name="name">The thread name.</param>
|
||||
public ResultDispatcher(string name)
|
||||
{
|
||||
new Thread(Run) { IsBackground = true, Name = name }.Start();
|
||||
}
|
||||
|
||||
/// <summary>Queues a caller's resumption. After <see cref="Stop"/> it runs on the calling thread instead.</summary>
|
||||
/// <param name="pending">The evaluation.</param>
|
||||
/// <param name="result">Its result (null: the worker ended).</param>
|
||||
public void Post(PendingEval pending, EvalResult? result)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_stopped)
|
||||
{
|
||||
_queue.Enqueue((pending, result));
|
||||
if (_waiting) Monitor.Pulse(_gate);
|
||||
return;
|
||||
}
|
||||
}
|
||||
pending.SignalOutcome(result);
|
||||
}
|
||||
|
||||
/// <summary>Lets the thread drain what is queued and end.</summary>
|
||||
public void Stop()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_stopped = true;
|
||||
Monitor.Pulse(_gate);
|
||||
}
|
||||
}
|
||||
|
||||
private void Run()
|
||||
{
|
||||
var batch = new List<(PendingEval Pending, EvalResult? Result)>();
|
||||
while (true)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
while (_queue.Count == 0)
|
||||
{
|
||||
if (_stopped) return;
|
||||
_waiting = true;
|
||||
Monitor.Wait(_gate);
|
||||
_waiting = false;
|
||||
}
|
||||
while (_queue.TryDequeue(out var item)) batch.Add(item);
|
||||
}
|
||||
foreach (var (pending, result) in batch)
|
||||
{
|
||||
// TrySetResult runs the caller's continuation here; an await or ContinueWith captures its own
|
||||
// exceptions, so nothing should escape — but this thread must survive a misbehaving one regardless.
|
||||
try { pending.SignalOutcome(result); }
|
||||
catch (Exception) { /* the caller's own task carries its failure */ }
|
||||
}
|
||||
batch.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,14 @@ internal sealed class WindowsJobObject : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Kills every process in the job at once (lmxopcua#789). The job admits exactly one process, so this is the
|
||||
/// worker's whole tree without <c>Process.Kill(entireProcessTree: true)</c>'s system-wide process snapshot,
|
||||
/// which cost about 100–200 ms of the Windows kill latency.
|
||||
/// </summary>
|
||||
/// <returns>False when the job could not be terminated (the caller falls back to a process kill).</returns>
|
||||
public bool Terminate() => !_handle.IsClosed && TerminateJobObject(_handle, unchecked((uint)-1));
|
||||
|
||||
/// <summary>Closes the job handle, which kills every process still in it.</summary>
|
||||
public void Dispose() => _handle.Dispose();
|
||||
|
||||
@@ -98,7 +106,7 @@ internal sealed class WindowsJobObject : IDisposable
|
||||
public UIntPtr PeakJobMemoryUsed;
|
||||
}
|
||||
|
||||
#pragma warning disable SYSLIB1054 // DllImport kept: LibraryImport would need AllowUnsafeBlocks for three calls.
|
||||
#pragma warning disable SYSLIB1054 // DllImport kept: LibraryImport would need AllowUnsafeBlocks for four calls.
|
||||
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
private static extern SafeFileHandle CreateJobObjectW(IntPtr securityAttributes, string? name);
|
||||
|
||||
@@ -110,5 +118,9 @@ internal sealed class WindowsJobObject : IDisposable
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool AssignProcessToJobObject(SafeFileHandle job, IntPtr process);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool TerminateJobObject(SafeFileHandle job, uint exitCode);
|
||||
#pragma warning restore SYSLIB1054
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ using System.Diagnostics;
|
||||
using System.IO.Pipes;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Channels;
|
||||
using Serilog;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
@@ -65,6 +64,11 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
|
||||
/// <param name="PeCacheDirectory">The compiled-PE cache directory handed to the worker; null = no cache.</param>
|
||||
/// <param name="PeCacheKey">The cache's HMAC key (with <paramref name="PeCacheDirectory"/>).</param>
|
||||
/// <param name="MarkerDirectory">Where the launch marker for the orphan sweep is written; null = none.</param>
|
||||
/// <param name="LoadTimeout">
|
||||
/// How long one compile may run, counted from the worker's <c>LoadStarted</c> for it (not from when the host sent
|
||||
/// it, lmxopcua#790), before the worker is killed and the key struck; null = 60 s.
|
||||
/// </param>
|
||||
/// <param name="CompileThreads">The worker's dedicated compile threads; 0 = its default.</param>
|
||||
internal sealed record WorkerLaunchSpec(
|
||||
string Shard,
|
||||
string WorkerPath,
|
||||
@@ -75,7 +79,16 @@ internal sealed record WorkerLaunchSpec(
|
||||
TimeSpan HandshakeTimeout,
|
||||
string? PeCacheDirectory = null,
|
||||
byte[]? PeCacheKey = null,
|
||||
string? MarkerDirectory = null);
|
||||
string? MarkerDirectory = null,
|
||||
TimeSpan? LoadTimeout = null,
|
||||
int CompileThreads = 0)
|
||||
{
|
||||
/// <summary>The default compile timeout.</summary>
|
||||
public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary><see cref="LoadTimeout"/> or its default.</summary>
|
||||
public TimeSpan EffectiveLoadTimeout => LoadTimeout ?? DefaultLoadTimeout;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed
|
||||
@@ -101,13 +114,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
private readonly string _workDir;
|
||||
private readonly ILogger _log;
|
||||
private readonly Action<LogBatch> _onLogs;
|
||||
private readonly Channel<Envelope> _outbound = Channel.CreateUnbounded<Envelope>(new UnboundedChannelOptions { SingleReader = true });
|
||||
private readonly ScriptWorkerFrameWriter _writer;
|
||||
private readonly ResultDispatcher _dispatcher;
|
||||
private readonly ConcurrentDictionary<ulong, PendingEval> _pending = new();
|
||||
private readonly ConcurrentDictionary<ulong, (string Key, TaskCompletionSource<LoadResult> Completion)> _loads = new();
|
||||
private readonly ConcurrentDictionary<ulong, LoadInFlight> _loads = new();
|
||||
private readonly ConcurrentDictionary<string, LoadEntry> _loaded = new(StringComparer.Ordinal);
|
||||
private readonly TaskCompletionSource<WorkerExit> _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private readonly TimeSpan _heartbeatInterval;
|
||||
private TimeSpan _loadTimeout;
|
||||
private readonly Action<WorkerExit> _onExiting;
|
||||
private readonly string? _markerPath;
|
||||
private TaskCompletionSource<DiagnosticsReply>? _diag;
|
||||
@@ -116,15 +131,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
private int _finished;
|
||||
private (WorkerExitReason Reason, string? Culprit)? _killedFor;
|
||||
private long _lastFrameTimestamp;
|
||||
private long _seq;
|
||||
private long _nextEvalId;
|
||||
private long _nextLoadId;
|
||||
private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
|
||||
private int _disposed;
|
||||
|
||||
private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir,
|
||||
ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, Action<WorkerExit> onExiting, string? markerPath)
|
||||
ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, TimeSpan loadTimeout, Action<WorkerExit> onExiting,
|
||||
string? markerPath)
|
||||
{
|
||||
_loadTimeout = loadTimeout;
|
||||
_onExiting = onExiting;
|
||||
_markerPath = markerPath;
|
||||
Shard = shard;
|
||||
@@ -136,6 +151,10 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
_onLogs = onLogs;
|
||||
_heartbeatInterval = heartbeatInterval;
|
||||
Pid = process.Id;
|
||||
// The handshake used seq 1. A failed write means the pipe is gone: the worker is dead or dying.
|
||||
_writer = new ScriptWorkerFrameWriter(pipe, OnWriteFault, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, firstSeq: 1,
|
||||
synchronous: ScriptWorkerFrameCodec.UseBlockingIo);
|
||||
_dispatcher = new ResultDispatcher($"ScriptWorker[{shard}].Results");
|
||||
}
|
||||
|
||||
/// <summary>The shard label.</summary>
|
||||
@@ -156,6 +175,16 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <summary>Evaluations currently outstanding on this worker.</summary>
|
||||
public int InFlightCount => _pending.Count;
|
||||
|
||||
/// <summary>
|
||||
/// The compile timeout applied to loads that START from now on (test hook: a test warms the worker's compiler
|
||||
/// under the launch value, then shortens it).
|
||||
/// </summary>
|
||||
internal TimeSpan LoadTimeout
|
||||
{
|
||||
get => _loadTimeout;
|
||||
set => _loadTimeout = value;
|
||||
}
|
||||
|
||||
/// <summary>Compiles currently outstanding on this worker.</summary>
|
||||
public int LoadingCount => _loads.Count;
|
||||
|
||||
@@ -234,16 +263,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue),
|
||||
HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds,
|
||||
MaxLogLines = 1000,
|
||||
CompileThreads = (uint)Math.Max(0, spec.CompileThreads),
|
||||
};
|
||||
if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey)
|
||||
{
|
||||
hello.PeCacheDir = cacheDir;
|
||||
hello.PeCacheKey = Google.Protobuf.ByteString.CopyFrom(cacheKey);
|
||||
}
|
||||
await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
|
||||
if (ScriptWorkerFrameCodec.UseBlockingIo)
|
||||
await HandshakeBlockingAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
|
||||
else
|
||||
await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
|
||||
|
||||
var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval,
|
||||
onExiting, markerPath);
|
||||
spec.EffectiveLoadTimeout, onExiting, markerPath);
|
||||
connection.Begin();
|
||||
return connection;
|
||||
}
|
||||
@@ -280,8 +313,41 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
}, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
|
||||
await stream.FlushAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false)
|
||||
?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
|
||||
var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
|
||||
return CheckReady(reply, expectedNonce, expectedPid);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// As <see cref="HandshakeAsync"/> with blocking I/O only (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>):
|
||||
/// one asynchronous call would turn the socket under a Unix pipe non-blocking for the connection's whole life.
|
||||
/// The blocking read runs off the caller and is abandoned at <paramref name="ct"/>; the caller then kills the
|
||||
/// worker and disposes the pipe, which ends it.
|
||||
/// </summary>
|
||||
/// <param name="stream">The accepted pipe.</param>
|
||||
/// <param name="hello">The Hello to send.</param>
|
||||
/// <param name="expectedNonce">The nonce put in the worker's environment.</param>
|
||||
/// <param name="expectedPid">The launched process id; null skips the check.</param>
|
||||
/// <param name="ct">Cancellation (the handshake timeout).</param>
|
||||
/// <returns>The worker's Ready.</returns>
|
||||
internal static async Task<Ready> HandshakeBlockingAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var reply = await Task.Run(() =>
|
||||
{
|
||||
ScriptWorkerFrameCodec.Write(stream, new Envelope
|
||||
{
|
||||
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
||||
Seq = 1,
|
||||
Hello = hello,
|
||||
}, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
|
||||
return ScriptWorkerFrameCodec.Read(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
|
||||
}, CancellationToken.None).WaitAsync(ct).ConfigureAwait(false);
|
||||
return CheckReady(reply, expectedNonce, expectedPid);
|
||||
}
|
||||
|
||||
private static Ready CheckReady(Envelope? reply, string expectedNonce, int? expectedPid)
|
||||
{
|
||||
if (reply is null) throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
|
||||
if (reply.BodyCase != Envelope.BodyOneofCase.Ready)
|
||||
throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}");
|
||||
var expected = Encoding.UTF8.GetBytes(expectedNonce);
|
||||
@@ -361,8 +427,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
ReadyTimestamp = Stopwatch.GetTimestamp();
|
||||
Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
|
||||
Interlocked.Exchange(ref _lastUsedTimestamp, ReadyTimestamp);
|
||||
_ = Task.Run(ReadLoopAsync);
|
||||
_ = Task.Run(WriteLoopAsync);
|
||||
if (!ScriptWorkerFrameCodec.UseBlockingIo) _ = Task.Run(ReadLoopAsync);
|
||||
else new Thread(ReadLoopBlocking) { IsBackground = true, Name = $"ScriptWorker[{Shard}].Reader" }.Start();
|
||||
_ = Task.Run(HeartbeatMonitorAsync);
|
||||
_ = Task.Run(WatchProcessAsync);
|
||||
}
|
||||
@@ -373,39 +439,39 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <param name="source">The source.</param>
|
||||
/// <param name="sourceHash">SHA-256 of the source.</param>
|
||||
/// <param name="inputPaths">The slot order.</param>
|
||||
/// <returns>The load outcome and the slot map; null when the worker ended first.</returns>
|
||||
public Task<LoadEntry?> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
|
||||
/// <returns>The entry once the load has an outcome (see <see cref="LoadEntry.Outcome"/>).</returns>
|
||||
public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
|
||||
=> EnsureLoadedAsync(key, kind, source, sourceHash, Convert.ToHexString(sourceHash), inputPaths);
|
||||
|
||||
/// <summary>As the other overload, with the hash's hex form precomputed (the runner caches it per key, lmxopcua#789).</summary>
|
||||
/// <param name="key">The scoped key.</param>
|
||||
/// <param name="kind">The context kind.</param>
|
||||
/// <param name="source">The source.</param>
|
||||
/// <param name="sourceHash">SHA-256 of the source.</param>
|
||||
/// <param name="sourceHashHex">Its upper-case hex form.</param>
|
||||
/// <param name="inputPaths">The slot order.</param>
|
||||
/// <returns>
|
||||
/// The entry once the load has an outcome: <see cref="LoadEntry.Outcome"/> is the worker's verdict, or null when
|
||||
/// the worker ended first; <see cref="LoadEntry.TimedOut"/> marks the compile this connection killed the worker
|
||||
/// for.
|
||||
/// </returns>
|
||||
public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, string sourceHashHex,
|
||||
IReadOnlyList<string> inputPaths)
|
||||
{
|
||||
var hashHex = Convert.ToHexString(sourceHash);
|
||||
if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths))
|
||||
if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths))
|
||||
return existing.Ready;
|
||||
|
||||
var entry = new LoadEntry(hashHex, inputPaths);
|
||||
var entry = new LoadEntry(sourceHashHex, inputPaths);
|
||||
_loaded[key] = entry;
|
||||
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
||||
var loadId = (ulong)Interlocked.Increment(ref _nextLoadId);
|
||||
var tcs = new TaskCompletionSource<LoadResult>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_loads[loadId] = (key, tcs);
|
||||
var load = new LoadScripts { LoadId = loadId };
|
||||
var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) };
|
||||
script.InputPaths.AddRange(inputPaths);
|
||||
load.Scripts.Add(script);
|
||||
if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; }
|
||||
|
||||
_ = tcs.Task.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
{
|
||||
var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key);
|
||||
entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } });
|
||||
if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
|
||||
}
|
||||
else
|
||||
{
|
||||
entry.Complete(null);
|
||||
_loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
|
||||
}
|
||||
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
var inFlight = new LoadInFlight(loadId, key, entry, this);
|
||||
_loads[loadId] = inFlight;
|
||||
if (!Send(new Envelope { Load = load }) && _loads.TryRemove(loadId, out _)) inFlight.Finish(null);
|
||||
return entry.Ready;
|
||||
}
|
||||
|
||||
@@ -423,11 +489,29 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <param name="key">The scoped key.</param>
|
||||
/// <param name="request">The request (its id is assigned here).</param>
|
||||
/// <returns>The pending evaluation.</returns>
|
||||
public PendingEval Submit(string key, EvalRequest request)
|
||||
public PendingEval Submit(string key, EvalRequest request) => Submit(key, request, null, TimeSpan.Zero, null);
|
||||
|
||||
/// <summary>
|
||||
/// Submits one evaluation with its two deadlines (design §5.1) on ONE timer (lmxopcua#789): at
|
||||
/// <paramref name="softTimeout"/> <see cref="PendingEval.Outcome"/> yields <see cref="PendingEval.SoftTimedOut"/>,
|
||||
/// and <paramref name="hardGrace"/> later, if the worker still has not answered, the worker is killed with this
|
||||
/// key as the culprit.
|
||||
/// </summary>
|
||||
/// <param name="key">The scoped key.</param>
|
||||
/// <param name="request">The request (its id is assigned here).</param>
|
||||
/// <param name="softTimeout">T; null = no deadlines.</param>
|
||||
/// <param name="hardGrace">G.</param>
|
||||
/// <param name="onFinished">
|
||||
/// Runs once, synchronously, when the worker has answered or ended — BEFORE <see cref="PendingEval.Outcome"/>
|
||||
/// completes — so a caller that resumes from its await never finds its own run still holding the key's slot.
|
||||
/// </param>
|
||||
/// <returns>The pending evaluation.</returns>
|
||||
public PendingEval Submit(string key, EvalRequest request, TimeSpan? softTimeout, TimeSpan hardGrace, Action? onFinished)
|
||||
{
|
||||
request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
|
||||
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
||||
var pending = new PendingEval(key, request.EvalId);
|
||||
var pending = new PendingEval(key, request.EvalId, onFinished, _dispatcher);
|
||||
if (softTimeout is { } soft) pending.Arm(soft, hardGrace, OnHardDeadline);
|
||||
_pending[request.EvalId] = pending;
|
||||
var batch = new EvalBatch();
|
||||
batch.Items.Add(request);
|
||||
@@ -438,6 +522,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
return pending;
|
||||
}
|
||||
|
||||
private void OnHardDeadline(PendingEval pending)
|
||||
{
|
||||
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
|
||||
pending.Key, (long)pending.HardDeadline.TotalMilliseconds, Shard, Pid);
|
||||
Kill(WorkerExitReason.KilledDeadline, pending.Key);
|
||||
}
|
||||
|
||||
/// <summary>Cancels a run's <c>ctx.CancellationToken</c> in the worker (sent at the soft deadline).</summary>
|
||||
/// <param name="pending">The evaluation.</param>
|
||||
public void Cancel(PendingEval pending)
|
||||
@@ -467,6 +558,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
}
|
||||
try
|
||||
{
|
||||
// The job holds exactly this one process (ACTIVE_PROCESS = 1), so terminating it is the whole tree, at once.
|
||||
if (OperatingSystem.IsWindows() && _job is WindowsJobObject job && job.Terminate()) return;
|
||||
if (!_process.HasExited) _process.Kill(entireProcessTree: true);
|
||||
}
|
||||
catch (InvalidOperationException) { /* already gone */ }
|
||||
@@ -502,7 +595,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
|
||||
}
|
||||
_stopping.Cancel();
|
||||
_outbound.Writer.TryComplete();
|
||||
_writer.Complete();
|
||||
await _pipe.DisposeAsync().ConfigureAwait(false);
|
||||
_job?.Dispose();
|
||||
_process.Dispose();
|
||||
@@ -513,106 +606,112 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <summary>Test hook: closes the host end of the pipe without touching the process, as a dying host would.</summary>
|
||||
internal void SeverPipeForTest() => _pipe.Dispose();
|
||||
|
||||
private bool Send(Envelope envelope)
|
||||
private bool Send(Envelope envelope) => _writer.TryWrite(envelope);
|
||||
|
||||
private void OnWriteFault(Exception ex)
|
||||
{
|
||||
envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
|
||||
return _outbound.Writer.TryWrite(envelope);
|
||||
if (!_stopping.IsCancellationRequested)
|
||||
_log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
|
||||
Kill(WorkerExitReason.Crash, null);
|
||||
}
|
||||
|
||||
private async Task WriteLoopAsync()
|
||||
/// <summary>
|
||||
/// The reader, on a dedicated thread with blocking reads of a socket that never turns non-blocking
|
||||
/// (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, lmxopcua#789): a frame wakes this thread straight from
|
||||
/// the kernel, instead of the socket engine's epoll thread queueing a thread-pool work item per read. Not on
|
||||
/// Windows, where a synchronous read of an overlapped pipe is itself completed through the I/O completion port;
|
||||
/// the async loop is the cheaper path there.
|
||||
/// </summary>
|
||||
private void ReadLoopBlocking()
|
||||
{
|
||||
var reader = _outbound.Reader;
|
||||
WorkerExitReason? fault = null;
|
||||
var reader = new ScriptWorkerFrameReader(_pipe);
|
||||
try
|
||||
{
|
||||
while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false))
|
||||
{
|
||||
// Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch.
|
||||
EvalBatch? batch = null;
|
||||
while (reader.TryRead(out var envelope))
|
||||
{
|
||||
if (envelope.BodyCase == Envelope.BodyOneofCase.Eval)
|
||||
{
|
||||
(batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
|
||||
continue;
|
||||
}
|
||||
envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
|
||||
await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
||||
}
|
||||
if (batch is not null)
|
||||
{
|
||||
await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope
|
||||
{
|
||||
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
||||
Seq = (ulong)Interlocked.Increment(ref _seq),
|
||||
Eval = batch,
|
||||
}, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
||||
}
|
||||
await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false);
|
||||
}
|
||||
while (reader.Read() is { } frame) HandleFrame(frame);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException)
|
||||
catch (ScriptWorkerProtocolException ex)
|
||||
{
|
||||
if (!_stopping.IsCancellationRequested)
|
||||
_log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
|
||||
Kill(WorkerExitReason.Crash, null);
|
||||
fault = ProtocolFault(ex);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException
|
||||
or System.Net.Sockets.SocketException)
|
||||
{
|
||||
}
|
||||
_ = PipeEndedAsync(fault);
|
||||
}
|
||||
|
||||
private async Task ReadLoopAsync()
|
||||
{
|
||||
WorkerExitReason? fault = null;
|
||||
var reader = new ScriptWorkerFrameReader(_pipe);
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
|
||||
if (frame is null) break;
|
||||
Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
|
||||
switch (frame.BodyCase)
|
||||
{
|
||||
case Envelope.BodyOneofCase.Results:
|
||||
foreach (var result in frame.Results.Items)
|
||||
{
|
||||
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
|
||||
// trusts"), except for one that was already completed by a kill race — ignored then.
|
||||
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
|
||||
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
|
||||
}
|
||||
break;
|
||||
case Envelope.BodyOneofCase.LoadResult:
|
||||
if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Completion.TrySetResult(frame.LoadResult);
|
||||
else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Logs:
|
||||
try { _onLogs(frame.Logs); }
|
||||
catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Heartbeat:
|
||||
Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
|
||||
break;
|
||||
case Envelope.BodyOneofCase.DiagReply:
|
||||
Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Fault:
|
||||
_log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
|
||||
break;
|
||||
default:
|
||||
throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
|
||||
}
|
||||
}
|
||||
while (await reader.ReadAsync(_stopping.Token).ConfigureAwait(false) is { } frame) HandleFrame(frame);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
catch (ScriptWorkerProtocolException ex)
|
||||
{
|
||||
_log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
|
||||
fault = WorkerExitReason.ProtocolFault;
|
||||
fault = ProtocolFault(ex);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or ObjectDisposedException)
|
||||
{
|
||||
}
|
||||
await PipeEndedAsync(fault).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private WorkerExitReason ProtocolFault(ScriptWorkerProtocolException ex)
|
||||
{
|
||||
_log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
|
||||
return WorkerExitReason.ProtocolFault;
|
||||
}
|
||||
|
||||
private void HandleFrame(Envelope frame)
|
||||
{
|
||||
Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
|
||||
switch (frame.BodyCase)
|
||||
{
|
||||
case Envelope.BodyOneofCase.Results:
|
||||
foreach (var result in frame.Results.Items)
|
||||
{
|
||||
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
|
||||
// trusts"), except for one that was already completed by a kill race — ignored then.
|
||||
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
|
||||
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
|
||||
}
|
||||
break;
|
||||
case Envelope.BodyOneofCase.LoadStarted:
|
||||
// The compile timeout starts now (lmxopcua#790). An unknown id is a load already completed by a
|
||||
// kill race (ignored) or one never issued (a protocol violation).
|
||||
if (_loads.TryGetValue(frame.LoadStarted.LoadId, out var starting)) starting.Started(_loadTimeout);
|
||||
else if (frame.LoadStarted.LoadId == 0 || frame.LoadStarted.LoadId > (ulong)Interlocked.Read(ref _nextLoadId))
|
||||
throw new ScriptWorkerProtocolException($"load start for unknown load id {frame.LoadStarted.LoadId}");
|
||||
break;
|
||||
case Envelope.BodyOneofCase.LoadResult:
|
||||
if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Finish(frame.LoadResult);
|
||||
else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Logs:
|
||||
try { _onLogs(frame.Logs); }
|
||||
catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Heartbeat:
|
||||
Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
|
||||
break;
|
||||
case Envelope.BodyOneofCase.DiagReply:
|
||||
Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Fault:
|
||||
_log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
|
||||
break;
|
||||
default:
|
||||
throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PipeEndedAsync(WorkerExitReason? fault)
|
||||
{
|
||||
// The pipe is gone: the worker is dead or dying. Make sure, then report.
|
||||
try
|
||||
{
|
||||
@@ -670,7 +769,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
if (Interlocked.Exchange(ref _finished, 1) != 0) return;
|
||||
// No new frame can be queued from here on, so every Submit / load either fails its Send (and completes itself)
|
||||
// or was registered before this point and is completed below.
|
||||
_outbound.Writer.TryComplete();
|
||||
_writer.Complete();
|
||||
int? code = null;
|
||||
try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { }
|
||||
|
||||
@@ -692,9 +791,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
foreach (var id in _pending.Keys.ToArray())
|
||||
if (_pending.TryRemove(id, out var pending)) pending.Complete(null);
|
||||
foreach (var id in _loads.Keys.ToArray())
|
||||
if (_loads.TryRemove(id, out var load)) load.Completion.TrySetCanceled();
|
||||
if (_loads.TryRemove(id, out var load)) load.Finish(null);
|
||||
Volatile.Read(ref _diag)?.TrySetCanceled();
|
||||
_outbound.Writer.TryComplete();
|
||||
// Resumes the callers just completed above, then ends; a later Post runs inline.
|
||||
_dispatcher.Stop();
|
||||
}
|
||||
|
||||
/// <summary>The compile timer of a started load fired: kill the worker, blaming this key alone (lmxopcua#790).</summary>
|
||||
private void OnLoadTimeout(LoadInFlight load)
|
||||
{
|
||||
if (!_loads.ContainsKey(load.Id)) return;
|
||||
load.Entry.TimedOut = true;
|
||||
_log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] has run for {Timeout} s since it started; killing the worker",
|
||||
load.Key, Shard, _loadTimeout.TotalSeconds);
|
||||
Kill(WorkerExitReason.KilledLoadTimeout, load.Key);
|
||||
}
|
||||
|
||||
private static string SafeExitCode(Process process)
|
||||
@@ -708,10 +818,52 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ }
|
||||
}
|
||||
|
||||
/// <summary>One load sent to the worker and not yet answered: its compile timer runs from <c>LoadStarted</c>.</summary>
|
||||
private sealed class LoadInFlight(ulong id, string key, LoadEntry entry, WorkerConnection owner)
|
||||
{
|
||||
private Timer? _timer;
|
||||
private int _finished;
|
||||
|
||||
public ulong Id { get; } = id;
|
||||
|
||||
public string Key { get; } = key;
|
||||
|
||||
public LoadEntry Entry { get; } = entry;
|
||||
|
||||
/// <summary>The worker has started compiling it: arm the compile timeout (once).</summary>
|
||||
public void Started(TimeSpan timeout)
|
||||
{
|
||||
if (Volatile.Read(ref _finished) != 0 || _timer is not null) return;
|
||||
var timer = new Timer(static s => { var l = (LoadInFlight)s!; l.Owner.OnLoadTimeout(l); }, this, Timeout.Infinite, Timeout.Infinite);
|
||||
if (Interlocked.CompareExchange(ref _timer, timer, null) is not null) { timer.Dispose(); return; }
|
||||
timer.Change(timeout, Timeout.InfiniteTimeSpan);
|
||||
if (Volatile.Read(ref _finished) != 0) timer.Dispose();
|
||||
}
|
||||
|
||||
private WorkerConnection Owner { get; } = owner;
|
||||
|
||||
/// <summary>The worker answered (<paramref name="result"/>) or ended (null).</summary>
|
||||
public void Finish(LoadResult? result)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _finished, 1) != 0) return;
|
||||
Volatile.Read(ref _timer)?.Dispose();
|
||||
if (result is null)
|
||||
{
|
||||
Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
|
||||
Entry.Complete(null);
|
||||
return;
|
||||
}
|
||||
var outcome = result.Outcomes.FirstOrDefault(o => o.Key == Key)
|
||||
?? new LoadOutcome { Key = Key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } };
|
||||
if (!outcome.Ok) Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
|
||||
Entry.Complete(outcome);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A key's compiled form in this worker.</summary>
|
||||
internal sealed class LoadEntry
|
||||
{
|
||||
private readonly TaskCompletionSource<LoadEntry?> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource<LoadEntry> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>Set once the runner has counted this load in its metrics.</summary>
|
||||
public int Counted;
|
||||
@@ -732,10 +884,17 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <summary>Path → slot in the registered order.</summary>
|
||||
public IReadOnlyDictionary<string, uint> Slots { get; }
|
||||
|
||||
/// <summary>The worker's verdict (null until it arrives).</summary>
|
||||
/// <summary>The worker's verdict; null until it arrives, and null for good when the worker ended first.</summary>
|
||||
public LoadOutcome? Outcome { get; private set; }
|
||||
|
||||
public Task<LoadEntry?> Ready => _ready.Task;
|
||||
/// <summary>
|
||||
/// Set when this connection killed the worker because THIS compile overran its timeout from its start
|
||||
/// (lmxopcua#790); the waiters then report a compile timeout instead of a worker restart.
|
||||
/// </summary>
|
||||
public volatile bool TimedOut;
|
||||
|
||||
/// <summary>Completes with this entry once the load has an outcome (or the worker ended).</summary>
|
||||
public Task<LoadEntry> Ready => _ready.Task;
|
||||
|
||||
public bool SameSlots(IReadOnlyList<string> inputPaths)
|
||||
{
|
||||
@@ -748,20 +907,42 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
public void Complete(LoadOutcome? outcome)
|
||||
{
|
||||
Outcome = outcome;
|
||||
_ready.TrySetResult(outcome is null ? null : this);
|
||||
_ready.TrySetResult(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One evaluation outstanding on a worker.</summary>
|
||||
/// <summary>
|
||||
/// One evaluation outstanding on a worker. Its deadlines share ONE timer (lmxopcua#789: the per-eval
|
||||
/// <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
|
||||
/// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
|
||||
/// </summary>
|
||||
internal sealed class PendingEval
|
||||
{
|
||||
private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
/// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
|
||||
internal static readonly EvalResult SoftTimedOut = new();
|
||||
|
||||
public PendingEval(string key, ulong evalId)
|
||||
private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource<EvalResult?> _outcome;
|
||||
private readonly ResultDispatcher? _dispatcher;
|
||||
private readonly Action? _onFinished;
|
||||
private Timer? _timer;
|
||||
private Action<PendingEval>? _onHardDeadline;
|
||||
private TimeSpan _grace;
|
||||
private int _phase;
|
||||
private int _done;
|
||||
|
||||
public PendingEval(string key, ulong evalId, Action? onFinished = null, ResultDispatcher? dispatcher = null)
|
||||
{
|
||||
Key = key;
|
||||
EvalId = evalId;
|
||||
_onFinished = onFinished;
|
||||
_dispatcher = dispatcher;
|
||||
// With a dispatcher the caller resumes ON the dispatcher's thread (synchronous continuation, no pool hop);
|
||||
// without one, on the thread pool as usual.
|
||||
_outcome = new TaskCompletionSource<EvalResult?>(dispatcher is null
|
||||
? TaskCreationOptions.RunContinuationsAsynchronously
|
||||
: TaskCreationOptions.None);
|
||||
}
|
||||
|
||||
/// <summary>The scoped key.</summary>
|
||||
@@ -770,8 +951,59 @@ internal sealed class PendingEval
|
||||
/// <summary>The eval id on the wire.</summary>
|
||||
public ulong EvalId { get; }
|
||||
|
||||
/// <summary>T + G, for the kill log line.</summary>
|
||||
public TimeSpan HardDeadline { get; private set; }
|
||||
|
||||
/// <summary>The worker's result, or null when the worker ended first.</summary>
|
||||
public Task<EvalResult?> Completion => _completion.Task;
|
||||
|
||||
public void Complete(EvalResult? result) => _completion.TrySetResult(result);
|
||||
/// <summary>
|
||||
/// What the caller waits on: the worker's result, null when the worker ended first, or
|
||||
/// <see cref="SoftTimedOut"/> at the soft deadline (the run itself is still outstanding then).
|
||||
/// </summary>
|
||||
public Task<EvalResult?> Outcome => _outcome.Task;
|
||||
|
||||
/// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
|
||||
internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
|
||||
{
|
||||
_grace = grace;
|
||||
_onHardDeadline = onHardDeadline;
|
||||
HardDeadline = soft + grace;
|
||||
// Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
|
||||
_timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
|
||||
_timer.Change(soft, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
|
||||
private void OnTimer()
|
||||
{
|
||||
if (Volatile.Read(ref _done) != 0) return;
|
||||
if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
|
||||
{
|
||||
_outcome.TrySetResult(SoftTimedOut);
|
||||
try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* answered meanwhile */ }
|
||||
return;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
|
||||
/// reader, then hand the caller's resumption to the dispatcher (or the pool).
|
||||
/// </summary>
|
||||
public void Complete(EvalResult? result)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _done, 1) != 0) return;
|
||||
_timer?.Dispose();
|
||||
try { _onFinished?.Invoke(); }
|
||||
finally
|
||||
{
|
||||
_completion.TrySetResult(result);
|
||||
if (_dispatcher is { } dispatcher) dispatcher.Post(this, result);
|
||||
else _outcome.TrySetResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Resumes the caller (on the dispatcher's thread). A soft timeout may already have done so.</summary>
|
||||
internal void SignalOutcome(EvalResult? result) => _outcome.TrySetResult(result);
|
||||
}
|
||||
|
||||
@@ -45,6 +45,12 @@ internal sealed class WorkerShard : IAsyncDisposable
|
||||
internal static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(30);
|
||||
internal static readonly TimeSpan HealthyResetAfter = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// While a shard keeps failing to start, at most one Warning per this interval (lmxopcua#788); the other retries
|
||||
/// log at Debug. The first failure of a down period is the one Error (with the exception).
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan DefaultStartFailureWarningInterval = TimeSpan.FromSeconds(60);
|
||||
|
||||
private static readonly Counter<long> StartsCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
|
||||
"otopcua.script.worker.starts", unit: "{process}", description: "ScriptWorker processes started (tag: shard).");
|
||||
|
||||
@@ -58,6 +64,7 @@ internal sealed class WorkerShard : IAsyncDisposable
|
||||
private readonly object _gate = new();
|
||||
private readonly Random _jitter = new();
|
||||
private readonly CancellationTokenSource _disposing = new();
|
||||
private readonly TimeSpan _startFailureWarningInterval;
|
||||
private WorkerConnection? _current;
|
||||
private Task<WorkerConnection?>? _starting;
|
||||
private TaskCompletionSource? _recovered;
|
||||
@@ -67,9 +74,15 @@ internal sealed class WorkerShard : IAsyncDisposable
|
||||
private int _consecutiveFailures;
|
||||
private bool _disposed;
|
||||
|
||||
// The current run of failed starts (lmxopcua#788): how many, since when, and when the last Warning went out.
|
||||
private int _startFailures;
|
||||
private long _startFailingSince;
|
||||
private long _lastStartFailureWarning;
|
||||
|
||||
public WorkerShard(string label, bool isolation, Func<Action<WorkerExit>, CancellationToken, Task<WorkerConnection>> launch,
|
||||
Action<WorkerShard, WorkerExit> onExit, ILogger log)
|
||||
Action<WorkerShard, WorkerExit> onExit, ILogger log, TimeSpan? startFailureWarningInterval = null)
|
||||
{
|
||||
_startFailureWarningInterval = startFailureWarningInterval ?? DefaultStartFailureWarningInterval;
|
||||
Label = label;
|
||||
IsIsolation = isolation;
|
||||
_launch = launch;
|
||||
@@ -298,7 +311,21 @@ internal sealed class WorkerShard : IAsyncDisposable
|
||||
{
|
||||
connection = await _launch(Attribute, CancellationToken.None).ConfigureAwait(false);
|
||||
StartsCounter.Add(1, new KeyValuePair<string, object?>("shard", Label));
|
||||
_log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
|
||||
int failedBefore;
|
||||
long failingSince;
|
||||
lock (_gate)
|
||||
{
|
||||
failedBefore = _startFailures;
|
||||
failingSince = _startFailingSince;
|
||||
_startFailures = 0;
|
||||
_startFailingSince = 0;
|
||||
_lastStartFailureWarning = 0;
|
||||
}
|
||||
if (failedBefore == 0)
|
||||
_log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
|
||||
else
|
||||
_log.Information("ScriptWorker[{Shard}] started, pid {Pid}, after {Attempts} failed start attempt(s) over {Outage}",
|
||||
Label, connection.Pid, failedBefore, Stopwatch.GetElapsedTime(failingSince));
|
||||
var started = connection;
|
||||
_ = started.Exited.ContinueWith(t => OnExited(started, t.Result), CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
|
||||
@@ -320,13 +347,31 @@ internal sealed class WorkerShard : IAsyncDisposable
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does", Label);
|
||||
int attempt;
|
||||
TimeSpan failingFor;
|
||||
bool warn;
|
||||
lock (_gate)
|
||||
{
|
||||
_starting = null;
|
||||
MarkDownLocked();
|
||||
ScheduleBackoffLocked();
|
||||
attempt = ++_startFailures;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
if (attempt == 1) _startFailingSince = now;
|
||||
failingFor = Stopwatch.GetElapsedTime(_startFailingSince, now);
|
||||
warn = attempt > 1 && (_lastStartFailureWarning == 0
|
||||
|| Stopwatch.GetElapsedTime(_lastStartFailureWarning, now) >= _startFailureWarningInterval);
|
||||
if (attempt == 1 || warn) _lastStartFailureWarning = now;
|
||||
}
|
||||
// lmxopcua#788: one Error with the stack per down period; the retries (every backoff, capped at 30 s, per
|
||||
// shard) are a rate-limited Warning without it, and the rest Debug. Recovery is one Information line above.
|
||||
if (attempt == 1)
|
||||
_log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does (retrying with backoff)", Label);
|
||||
else if (warn)
|
||||
_log.Warning("ScriptWorker[{Shard}] still failing to start: {Attempts} failed attempt(s) over {Outage}; last error: {Error}",
|
||||
Label, attempt, failingFor, ex.Message);
|
||||
else
|
||||
_log.Debug("ScriptWorker[{Shard}] start attempt {Attempts} failed: {Error}", Label, attempt, ex.Message);
|
||||
if (connection is not null) await connection.DisposeAsync().ConfigureAwait(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ message Envelope {
|
||||
DiagnosticsRequest diag_request = 21; // host -> worker (tests / ops)
|
||||
DiagnosticsReply diag_reply = 22; // worker -> host
|
||||
Cancel cancel = 23; // host -> worker, at the soft deadline
|
||||
LoadStarted load_started = 24; // worker -> host, when a compile thread picks a load up (lmxopcua#790)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +40,7 @@ message Hello {
|
||||
uint32 max_log_lines = 7; // outbound log buffer cap
|
||||
string pe_cache_dir = 8; // the node's compiled-PE cache (design §7); empty = no cache
|
||||
bytes pe_cache_key = 9; // its HMAC key; the worker loads only entries that verify under it
|
||||
uint32 compile_threads = 10; // dedicated compile threads; 0 = the worker's default (min(8, max(1, CPUs/2)))
|
||||
}
|
||||
|
||||
message Ready {
|
||||
@@ -81,6 +83,13 @@ enum PeCache {
|
||||
PE_CACHE_REJECTED = 3; // an entry failed verification, was deleted, and the source was compiled
|
||||
}
|
||||
|
||||
// A compile thread has taken the load off the worker's queue and is about to compile (or PE-load) it. The host's load
|
||||
// timeout runs from here, not from when the host sent the load: a load queued behind other compiles is not stuck and
|
||||
// must not be blamed for the queue (lmxopcua#790).
|
||||
message LoadStarted {
|
||||
uint64 load_id = 1;
|
||||
}
|
||||
|
||||
message LoadResult {
|
||||
uint64 load_id = 1;
|
||||
repeated LoadOutcome outcomes = 2;
|
||||
|
||||
@@ -62,30 +62,14 @@ public static class ScriptWorkerFrameCodec
|
||||
if (got < 4) throw new ScriptWorkerProtocolException("truncated frame header");
|
||||
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(header);
|
||||
if (size == 0) throw new ScriptWorkerProtocolException("zero-length frame");
|
||||
if (size < 0 || size > maxFrameBytes)
|
||||
throw new ScriptWorkerProtocolException($"frame length {size} is outside 1..{maxFrameBytes}");
|
||||
CheckLength(size, maxFrameBytes);
|
||||
|
||||
var body = ArrayPool<byte>.Shared.Rent(size);
|
||||
try
|
||||
{
|
||||
var read = await ReadFullyAsync(stream, body.AsMemory(0, size), ct).ConfigureAwait(false);
|
||||
if (read < size) throw new ScriptWorkerProtocolException($"truncated frame: {read} of {size} bytes");
|
||||
Envelope envelope;
|
||||
try
|
||||
{
|
||||
envelope = Envelope.Parser.ParseFrom(body, 0, size);
|
||||
}
|
||||
catch (InvalidProtocolBufferException ex)
|
||||
{
|
||||
throw new ScriptWorkerProtocolException($"unparseable frame: {ex.Message}", ex);
|
||||
}
|
||||
if (envelope.ProtocolVersion != ProtocolVersion)
|
||||
throw new ScriptWorkerProtocolException(
|
||||
$"protocol version {envelope.ProtocolVersion} is not the supported {ProtocolVersion}");
|
||||
if (envelope.BodyCase == Envelope.BodyOneofCase.None)
|
||||
throw new ScriptWorkerProtocolException("frame carries no body");
|
||||
return envelope;
|
||||
return ParseBody(body.AsSpan(0, size));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -93,6 +77,104 @@ public static class ScriptWorkerFrameCodec
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether this platform's pipe should be driven with blocking I/O (lmxopcua#789). On Unix a .NET pipe is a
|
||||
/// socket, and ONE asynchronous operation on it switches it to non-blocking for good, after which even a
|
||||
/// "blocking" read goes through the socket engine (an epoll-thread hand-off and a spinning wait per frame) —
|
||||
/// measured as most of the host's per-evaluation CPU. Kept blocking, a read is a plain kernel wait on a
|
||||
/// dedicated thread. Not on Windows, where an overlapped pipe handle is needed for full duplex and its
|
||||
/// asynchronous path is the efficient one.
|
||||
/// </summary>
|
||||
public static bool UseBlockingIo => !OperatingSystem.IsWindows();
|
||||
|
||||
/// <summary>Writes one frame synchronously (the blocking-I/O handshake). Not thread-safe.</summary>
|
||||
/// <param name="stream">The pipe.</param>
|
||||
/// <param name="envelope">The envelope.</param>
|
||||
/// <param name="maxFrameBytes">The cap.</param>
|
||||
public static void Write(Stream stream, Envelope envelope, int maxFrameBytes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
ArgumentNullException.ThrowIfNull(envelope);
|
||||
var size = envelope.CalculateSize();
|
||||
if (size == 0) throw new ScriptWorkerProtocolException("refusing to write an empty frame");
|
||||
if (size > maxFrameBytes)
|
||||
throw new ScriptWorkerProtocolException($"frame of {size} bytes exceeds the {maxFrameBytes}-byte cap");
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(size + 4);
|
||||
try
|
||||
{
|
||||
BinaryPrimitives.WriteInt32LittleEndian(buffer, size);
|
||||
envelope.WriteTo(new Span<byte>(buffer, 4, size));
|
||||
stream.Write(buffer, 0, size + 4);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads exactly one frame synchronously, never past it (the blocking-I/O handshake: what follows belongs to the
|
||||
/// connection's own reader).
|
||||
/// </summary>
|
||||
/// <param name="stream">The pipe.</param>
|
||||
/// <param name="maxFrameBytes">The cap.</param>
|
||||
/// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
|
||||
public static Envelope? Read(Stream stream, int maxFrameBytes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
Span<byte> header = stackalloc byte[4];
|
||||
var got = stream.ReadAtLeast(header, 4, throwOnEndOfStream: false);
|
||||
if (got == 0) return null;
|
||||
if (got < 4) throw new ScriptWorkerProtocolException("truncated frame header");
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(header);
|
||||
CheckLength(size, maxFrameBytes);
|
||||
var body = ArrayPool<byte>.Shared.Rent(size);
|
||||
try
|
||||
{
|
||||
var read = stream.ReadAtLeast(body.AsSpan(0, size), size, throwOnEndOfStream: false);
|
||||
if (read < size) throw new ScriptWorkerProtocolException($"truncated frame: {read} of {size} bytes");
|
||||
return ParseBody(body.AsSpan(0, size));
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(body);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Validates a frame's length prefix against the cap.</summary>
|
||||
/// <param name="size">The decoded length.</param>
|
||||
/// <param name="maxFrameBytes">The negotiated cap.</param>
|
||||
/// <exception cref="ScriptWorkerProtocolException">A zero, negative or oversize length.</exception>
|
||||
internal static void CheckLength(int size, int maxFrameBytes)
|
||||
{
|
||||
if (size == 0) throw new ScriptWorkerProtocolException("zero-length frame");
|
||||
if (size < 0 || size > maxFrameBytes)
|
||||
throw new ScriptWorkerProtocolException($"frame length {size} is outside 1..{maxFrameBytes}");
|
||||
}
|
||||
|
||||
/// <summary>Parses and validates one frame body (everything after the length prefix).</summary>
|
||||
/// <param name="body">The body bytes.</param>
|
||||
/// <returns>The envelope.</returns>
|
||||
/// <exception cref="ScriptWorkerProtocolException">Unparseable, the wrong protocol version, or no body.</exception>
|
||||
internal static Envelope ParseBody(ReadOnlySpan<byte> body)
|
||||
{
|
||||
Envelope envelope;
|
||||
try
|
||||
{
|
||||
envelope = Envelope.Parser.ParseFrom(body);
|
||||
}
|
||||
catch (InvalidProtocolBufferException ex)
|
||||
{
|
||||
throw new ScriptWorkerProtocolException($"unparseable frame: {ex.Message}", ex);
|
||||
}
|
||||
if (envelope.ProtocolVersion != ProtocolVersion)
|
||||
throw new ScriptWorkerProtocolException(
|
||||
$"protocol version {envelope.ProtocolVersion} is not the supported {ProtocolVersion}");
|
||||
if (envelope.BodyCase == Envelope.BodyOneofCase.None)
|
||||
throw new ScriptWorkerProtocolException("frame carries no body");
|
||||
return envelope;
|
||||
}
|
||||
|
||||
private static async ValueTask<int> ReadFullyAsync(Stream stream, Memory<byte> buffer, CancellationToken ct)
|
||||
{
|
||||
var total = 0;
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Concurrent;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// A buffered reader of <see cref="ScriptWorkerFrameCodec"/> frames (lmxopcua#789). One read from the pipe pulls in
|
||||
/// every frame the peer has written so far, and they are then parsed from the buffer without another read, so a
|
||||
/// batch of results costs one wake-up instead of two reads per frame. Same validation as
|
||||
/// <see cref="ScriptWorkerFrameCodec.ReadAsync"/>: zero-length, oversize, truncated, unparseable, wrong-version and
|
||||
/// bodiless frames are a <see cref="ScriptWorkerProtocolException"/>. Not thread-safe: one reader per stream.
|
||||
/// </summary>
|
||||
public sealed class ScriptWorkerFrameReader
|
||||
{
|
||||
private const int InitialBuffer = 64 * 1024;
|
||||
|
||||
private readonly Stream _stream;
|
||||
private byte[] _buffer = new byte[InitialBuffer];
|
||||
private int _start;
|
||||
private int _end;
|
||||
|
||||
/// <summary>Initializes the reader.</summary>
|
||||
/// <param name="stream">The pipe.</param>
|
||||
/// <param name="maxFrameBytes">The frame cap.</param>
|
||||
public ScriptWorkerFrameReader(Stream stream, int maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
_stream = stream;
|
||||
MaxFrameBytes = maxFrameBytes;
|
||||
}
|
||||
|
||||
/// <summary>The negotiated frame cap.</summary>
|
||||
public int MaxFrameBytes { get; set; }
|
||||
|
||||
/// <summary>Reads the next frame.</summary>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
|
||||
public async ValueTask<Envelope?> ReadAsync(CancellationToken ct = default)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (TryTake(out var envelope, out var need)) return envelope;
|
||||
Prepare(need);
|
||||
var n = await _stream.ReadAsync(_buffer.AsMemory(_end), ct).ConfigureAwait(false);
|
||||
if (n == 0) return EndOfStream();
|
||||
_end += n;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reads the next frame, blocking (for a dedicated reader thread).</summary>
|
||||
/// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
|
||||
public Envelope? Read()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (TryTake(out var envelope, out var need)) return envelope;
|
||||
Prepare(need);
|
||||
var n = _stream.Read(_buffer, _end, _buffer.Length - _end);
|
||||
if (n == 0) return EndOfStream();
|
||||
_end += n;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryTake(out Envelope? envelope, out int need)
|
||||
{
|
||||
envelope = null;
|
||||
var available = _end - _start;
|
||||
if (available < 4)
|
||||
{
|
||||
need = 4;
|
||||
return false;
|
||||
}
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(_start, 4));
|
||||
ScriptWorkerFrameCodec.CheckLength(size, MaxFrameBytes);
|
||||
need = size + 4;
|
||||
if (available < need) return false;
|
||||
envelope = ScriptWorkerFrameCodec.ParseBody(_buffer.AsSpan(_start + 4, size));
|
||||
_start += need;
|
||||
if (_start == _end) _start = _end = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Makes room for a frame of <paramref name="need"/> bytes (prefix included) starting at <c>_start</c>.</summary>
|
||||
private void Prepare(int need)
|
||||
{
|
||||
if (_start + need <= _buffer.Length && _end < _buffer.Length) return;
|
||||
var pending = _end - _start;
|
||||
if (need > _buffer.Length)
|
||||
{
|
||||
var bigger = new byte[Math.Max(need, _buffer.Length * 2)];
|
||||
Buffer.BlockCopy(_buffer, _start, bigger, 0, pending);
|
||||
_buffer = bigger;
|
||||
}
|
||||
else if (_start > 0)
|
||||
{
|
||||
Buffer.BlockCopy(_buffer, _start, _buffer, 0, pending);
|
||||
}
|
||||
_start = 0;
|
||||
_end = pending;
|
||||
}
|
||||
|
||||
private Envelope? EndOfStream()
|
||||
{
|
||||
var pending = _end - _start;
|
||||
if (pending == 0) return null;
|
||||
if (pending < 4) throw new ScriptWorkerProtocolException("truncated frame header");
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(_start, 4));
|
||||
throw new ScriptWorkerProtocolException($"truncated frame: {pending - 4} of {size} bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A combining frame writer (lmxopcua#789): the thread that queues a frame writes it itself when no write is in
|
||||
/// progress, so the common case needs no hand-off to a writer task (no thread-pool wake-up, no spin-wait), and
|
||||
/// frames queued while a write is in progress are written by that writer in the same pass. Within a pass consecutive
|
||||
/// <see cref="EvalBatch"/> items are merged into one batch and consecutive <see cref="EvalResults"/> items into one
|
||||
/// (design §4.5 opportunistic batching), every frame is serialised into one buffer, and the buffer goes out in one
|
||||
/// write. Frame order is preserved.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Asynchronous mode: a write that cannot complete at once (a full pipe) continues on the I/O completion, and
|
||||
/// frames queued meanwhile wait for it. Synchronous mode (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, so
|
||||
/// the socket under a Unix pipe never turns non-blocking): the draining thread blocks while the pipe is full. Only
|
||||
/// one thread drains at a time — every other producer just queues and returns — and a peer that stops reading is
|
||||
/// killed by its supervisor (the host's heartbeat check), which fails the write and releases that thread.</para>
|
||||
/// <para>A failed write completes the writer and reports the exception once through <c>onFault</c>.</para>
|
||||
/// </remarks>
|
||||
public sealed class ScriptWorkerFrameWriter
|
||||
{
|
||||
private readonly Stream _stream;
|
||||
private readonly Action<Exception> _onFault;
|
||||
private readonly ConcurrentQueue<Envelope> _queue = new();
|
||||
private readonly ArrayBufferWriter<byte> _pass = new(16 * 1024);
|
||||
private readonly bool _synchronous;
|
||||
private int _draining;
|
||||
private int _completed;
|
||||
private int _faulted;
|
||||
private long _seq;
|
||||
|
||||
/// <summary>Initializes the writer.</summary>
|
||||
/// <param name="stream">The pipe.</param>
|
||||
/// <param name="onFault">Called once when a write fails (or a frame is refused); the writer is then completed.</param>
|
||||
/// <param name="maxFrameBytes">The frame cap.</param>
|
||||
/// <param name="firstSeq">The last sequence number already used (the handshake's).</param>
|
||||
/// <param name="synchronous">Write with blocking calls (see the remarks).</param>
|
||||
public ScriptWorkerFrameWriter(Stream stream, Action<Exception> onFault,
|
||||
int maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes, long firstSeq = 0, bool synchronous = false)
|
||||
{
|
||||
_synchronous = synchronous;
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
ArgumentNullException.ThrowIfNull(onFault);
|
||||
_stream = stream;
|
||||
_onFault = onFault;
|
||||
MaxFrameBytes = maxFrameBytes;
|
||||
_seq = firstSeq;
|
||||
}
|
||||
|
||||
/// <summary>The negotiated frame cap.</summary>
|
||||
public int MaxFrameBytes { get; set; }
|
||||
|
||||
/// <summary>Whether <see cref="Complete"/> has been called or a write failed.</summary>
|
||||
public bool IsCompleted => Volatile.Read(ref _completed) != 0;
|
||||
|
||||
/// <summary>
|
||||
/// Queues a frame and, when no write is in progress, writes it on the calling thread. Returns false once the
|
||||
/// writer is completed; the frame is then dropped.
|
||||
/// </summary>
|
||||
/// <param name="envelope">The frame (its version and sequence number are set here).</param>
|
||||
/// <returns>Whether the frame was accepted.</returns>
|
||||
public bool TryWrite(Envelope envelope)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(envelope);
|
||||
// A full fence before the completion check, so a caller that registered state (a pending evaluation) before
|
||||
// calling here and a completer that completes and then sweeps that state cannot both miss each other.
|
||||
Interlocked.MemoryBarrier();
|
||||
if (Volatile.Read(ref _completed) != 0) return false;
|
||||
envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
|
||||
_queue.Enqueue(envelope);
|
||||
if (Interlocked.CompareExchange(ref _draining, 1, 0) == 0) _ = DrainAsync();
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Stops accepting frames. Frames already queued are still written by a pass in progress.</summary>
|
||||
public void Complete() => Interlocked.Exchange(ref _completed, 1);
|
||||
|
||||
private async Task DrainAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
FillPass();
|
||||
if (_pass.WrittenCount > 0)
|
||||
{
|
||||
if (_synchronous) _stream.Write(_pass.WrittenSpan);
|
||||
else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false);
|
||||
_pass.ResetWrittenCount();
|
||||
// Synchronous on purpose: a pipe has no write buffer to flush, and PipeStream does not override
|
||||
// FlushAsync, so the base Stream.FlushAsync would queue a thread-pool work item per pass.
|
||||
_stream.Flush();
|
||||
}
|
||||
|
||||
Volatile.Write(ref _draining, 0);
|
||||
// A frame queued after the last TryDequeue but before the flag dropped saw _draining == 1 and left it
|
||||
// to us: take the flag back and go round again, unless another thread already has.
|
||||
if (_queue.IsEmpty || Interlocked.CompareExchange(ref _draining, 1, 0) != 0) return;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Interlocked.Exchange(ref _completed, 1);
|
||||
_pass.ResetWrittenCount();
|
||||
while (_queue.TryDequeue(out _)) { }
|
||||
Volatile.Write(ref _draining, 0);
|
||||
if (Interlocked.Exchange(ref _faulted, 1) == 0) _onFault(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void FillPass()
|
||||
{
|
||||
EvalBatch? evals = null;
|
||||
EvalResults? results = null;
|
||||
while (_queue.TryDequeue(out var envelope))
|
||||
{
|
||||
switch (envelope.BodyCase)
|
||||
{
|
||||
case Envelope.BodyOneofCase.Eval:
|
||||
if (results is not null) { Append(new Envelope { Results = results }); results = null; }
|
||||
(evals ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
|
||||
break;
|
||||
case Envelope.BodyOneofCase.Results:
|
||||
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
|
||||
(results ??= new EvalResults()).Items.AddRange(envelope.Results.Items);
|
||||
break;
|
||||
default:
|
||||
if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
|
||||
if (results is not null) { Append(new Envelope { Results = results }); results = null; }
|
||||
Append(envelope);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (evals is not null) Append(new Envelope { Eval = evals });
|
||||
if (results is not null) Append(new Envelope { Results = results });
|
||||
}
|
||||
|
||||
private void Append(Envelope envelope)
|
||||
{
|
||||
envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
|
||||
envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
|
||||
var size = envelope.CalculateSize();
|
||||
if (size == 0) throw new ScriptWorkerProtocolException("refusing to write an empty frame");
|
||||
if (size > MaxFrameBytes)
|
||||
throw new ScriptWorkerProtocolException($"frame of {size} bytes exceeds the {MaxFrameBytes}-byte cap");
|
||||
var span = _pass.GetSpan(size + 4);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(span, size);
|
||||
envelope.WriteTo(span.Slice(4, size));
|
||||
_pass.Advance(size + 4);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user