feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results. The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged. - Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill. - Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider carries the runner to DriverHostActor. - Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial); design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms; batched p99 <= 53 ms at 1,000. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+159
-26
@@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
|
||||
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
|
||||
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
|
||||
int compileThreads = 0)
|
||||
int compileThreads = 0, int evalThreads = 0)
|
||||
{
|
||||
_compileThreads = compileThreads;
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
@@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
|
||||
if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
|
||||
|
||||
var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
|
||||
// Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
|
||||
// queue-behind-a-busy-thread case deterministic).
|
||||
if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
|
||||
_shards = new WorkerShard[_worker.Shards];
|
||||
for (var i = 0; i < _shards.Length; i++)
|
||||
{
|
||||
@@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
/// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
|
||||
internal Action<string>? SlotReleasingForTest { get; set; }
|
||||
|
||||
/// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
|
||||
internal Action<int>? BatchSubmittedForTest { get; set; }
|
||||
|
||||
/// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
|
||||
internal IReadOnlyCollection<string> SuspectKeys =>
|
||||
_keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
|
||||
@@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
|
||||
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
|
||||
// run by then.
|
||||
var heldGate = gate;
|
||||
var released = 0;
|
||||
void ReleaseSlot()
|
||||
{
|
||||
if (Interlocked.Exchange(ref released, 1) != 0) return;
|
||||
SlotReleasingForTest?.Invoke(key);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
heldGate?.Release();
|
||||
}
|
||||
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
|
||||
_worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
|
||||
_worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
|
||||
handedOff = true;
|
||||
return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return ScriptRunOutcome.Failed("evaluation cancelled");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!handedOff)
|
||||
{
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
gate?.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
||||
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
||||
// shard has no live worker (or for the isolation gate).
|
||||
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
|
||||
/// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
|
||||
/// <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
|
||||
/// bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
|
||||
/// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
|
||||
/// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
|
||||
/// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
|
||||
/// <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
|
||||
/// </remarks>
|
||||
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invocations);
|
||||
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
|
||||
Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
|
||||
for (var i = 0; i < invocations.Count; i++)
|
||||
{
|
||||
var invocation = invocations[i];
|
||||
ArgumentNullException.ThrowIfNull(invocation);
|
||||
switch (TryPrepareBatchItem(invocation, out var item, out var answered))
|
||||
{
|
||||
case BatchPreparation.Submit:
|
||||
groups ??= [];
|
||||
if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
|
||||
group.Add((i, item));
|
||||
break;
|
||||
case BatchPreparation.Answered:
|
||||
outcomes[i] = Task.FromResult(answered!);
|
||||
break;
|
||||
default:
|
||||
outcomes[i] = RunAsync(invocation, ct);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (groups is null) return outcomes;
|
||||
|
||||
foreach (var (connection, group) in groups)
|
||||
{
|
||||
var submissions = new BatchSubmission[group.Count];
|
||||
for (var j = 0; j < group.Count; j++)
|
||||
{
|
||||
var item = group[j].Item;
|
||||
submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
|
||||
item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
|
||||
}
|
||||
var submitted = Stopwatch.GetTimestamp();
|
||||
BatchSubmittedForTest?.Invoke(submissions.Length);
|
||||
var pendings = connection.SubmitBatch(submissions);
|
||||
for (var j = 0; j < group.Count; j++)
|
||||
{
|
||||
var (index, item) = group[j];
|
||||
outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
|
||||
/// where nothing has to be waited for. Takes the key's single-flight slot when it returns
|
||||
/// <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
|
||||
/// </summary>
|
||||
private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
|
||||
{
|
||||
item = null;
|
||||
answered = null;
|
||||
if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
|
||||
|
||||
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
|
||||
var state = Admit(key, invocation.Kind, invocation.Source);
|
||||
if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
|
||||
|
||||
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
|
||||
if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
|
||||
if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
|
||||
|
||||
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
|
||||
{
|
||||
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
|
||||
answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
return BatchPreparation.Answered;
|
||||
}
|
||||
state.ScriptLogger = invocation.ScriptLogger;
|
||||
NoteLoad(entry);
|
||||
item = new BatchItem(key, state, invocation, shard, connection, entry);
|
||||
return BatchPreparation.Submit;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
|
||||
/// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
|
||||
/// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
|
||||
/// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
|
||||
/// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
|
||||
/// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
|
||||
/// quarantine, suspect) has already run by then.
|
||||
/// </summary>
|
||||
private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
|
||||
{
|
||||
var released = 0;
|
||||
return () =>
|
||||
{
|
||||
if (Interlocked.Exchange(ref released, 1) != 0) return;
|
||||
SlotReleasingForTest?.Invoke(key);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
heldGate?.Release();
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
|
||||
private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
|
||||
WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = ct.CanBeCanceled
|
||||
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
|
||||
: await pending.Outcome.ConfigureAwait(false);
|
||||
@@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
{
|
||||
return ScriptRunOutcome.Failed("evaluation cancelled");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!handedOff)
|
||||
{
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
gate?.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
|
||||
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
|
||||
// shard has no live worker (or for the isolation gate).
|
||||
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
|
||||
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
|
||||
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
|
||||
|
||||
private enum BatchPreparation
|
||||
{
|
||||
/// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
|
||||
Full,
|
||||
|
||||
/// <summary>Answered without running (refused).</summary>
|
||||
Answered,
|
||||
|
||||
/// <summary>Submit it with the batch; its slot is taken.</summary>
|
||||
Submit,
|
||||
}
|
||||
|
||||
private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
|
||||
WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
|
||||
|
||||
private sealed class KeyState(string source, ScriptContextKind kind)
|
||||
{
|
||||
private byte[]? _sha;
|
||||
|
||||
@@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One evaluation of a <see cref="WorkerConnection.SubmitBatch"/>.</summary>
|
||||
/// <param name="Key">The scoped key.</param>
|
||||
/// <param name="Request">The request (its id is assigned on submit).</param>
|
||||
/// <param name="SoftTimeout">T.</param>
|
||||
/// <param name="HardGrace">G.</param>
|
||||
/// <param name="OnFinished">Runs once when the worker has answered or ended, before the caller resumes (the slot release).</param>
|
||||
internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished);
|
||||
|
||||
/// <summary>What the host needs to launch one worker.</summary>
|
||||
/// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
|
||||
/// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
|
||||
@@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
return pending;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Submits several evaluations as ONE <see cref="EvalBatch"/> frame (lmxopcua#780 batch submit, design §4.5). Each
|
||||
/// item gets its own deadlines and slot release exactly as through <see cref="Submit(string, EvalRequest, TimeSpan?, TimeSpan, Action?)"/>;
|
||||
/// an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts.
|
||||
/// </summary>
|
||||
/// <param name="items">The evaluations (their ids are assigned here).</param>
|
||||
/// <returns>One pending evaluation per item, in order.</returns>
|
||||
public PendingEval[] SubmitBatch(IReadOnlyList<BatchSubmission> items)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
var pendings = new PendingEval[items.Count];
|
||||
var batch = new EvalBatch();
|
||||
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
|
||||
var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher);
|
||||
pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline);
|
||||
_pending[item.Request.EvalId] = pending;
|
||||
batch.Items.Add(item.Request);
|
||||
pendings[i] = pending;
|
||||
}
|
||||
if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted))
|
||||
{
|
||||
foreach (var pending in pendings)
|
||||
if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null);
|
||||
}
|
||||
return pendings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The compiled form of <paramref name="key"/> in this worker when it is loaded at <paramref name="sourceHashHex"/>
|
||||
/// with <paramref name="inputPaths"/> and compiled cleanly — the batch fast path, which never waits for a load.
|
||||
/// </summary>
|
||||
/// <param name="key">The scoped key.</param>
|
||||
/// <param name="sourceHashHex">The source hash (hex).</param>
|
||||
/// <param name="inputPaths">The slot order.</param>
|
||||
/// <returns>The entry, or null when the key needs a (re)load or its load failed.</returns>
|
||||
public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList<string> inputPaths)
|
||||
=> _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths)
|
||||
&& existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut
|
||||
? existing
|
||||
: null;
|
||||
|
||||
private void OnHardDeadline(PendingEval pending)
|
||||
{
|
||||
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
|
||||
@@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
|
||||
// trusts"), except for one that was already completed by a kill race — ignored then.
|
||||
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
|
||||
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
|
||||
else CheckIssued(result.EvalId, "result");
|
||||
}
|
||||
// Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines
|
||||
// restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time.
|
||||
// The two notices are unordered with respect to each other (see PendingEval.Start).
|
||||
foreach (var id in frame.Results.StartedEvalIds)
|
||||
{
|
||||
if (_pending.TryGetValue(id, out var started)) started.Start();
|
||||
else CheckIssued(id, "start notice");
|
||||
}
|
||||
foreach (var id in frame.Results.DeferredEvalIds)
|
||||
{
|
||||
if (_pending.TryGetValue(id, out var deferred)) deferred.Defer();
|
||||
else CheckIssued(id, "deferral notice");
|
||||
}
|
||||
break;
|
||||
case Envelope.BodyOneofCase.LoadStarted:
|
||||
@@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>An id the host never issued is a protocol violation; one it issued and already completed is a benign race.</summary>
|
||||
private void CheckIssued(ulong evalId, string what)
|
||||
{
|
||||
if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId))
|
||||
throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}");
|
||||
}
|
||||
|
||||
private async Task PipeEndedAsync(WorkerExitReason? fault)
|
||||
{
|
||||
// The pipe is gone: the worker is dead or dying. Make sure, then report.
|
||||
@@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
|
||||
/// <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
|
||||
/// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind
|
||||
/// busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full
|
||||
/// T, then G, from the start — without a stale tick firing early.
|
||||
/// </remarks>
|
||||
internal sealed class PendingEval
|
||||
{
|
||||
/// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
|
||||
@@ -926,9 +1003,15 @@ internal sealed class PendingEval
|
||||
private readonly TaskCompletionSource<EvalResult?> _outcome;
|
||||
private readonly ResultDispatcher? _dispatcher;
|
||||
private readonly Action? _onFinished;
|
||||
private readonly object _clock = new();
|
||||
private Timer? _timer;
|
||||
private Action<PendingEval>? _onHardDeadline;
|
||||
private TimeSpan _soft;
|
||||
private TimeSpan _grace;
|
||||
private long _softDue;
|
||||
private long _hardDue;
|
||||
private bool _deferred;
|
||||
private bool _started;
|
||||
private int _phase;
|
||||
private int _done;
|
||||
|
||||
@@ -954,6 +1037,12 @@ internal sealed class PendingEval
|
||||
/// <summary>T + G, for the kill log line.</summary>
|
||||
public TimeSpan HardDeadline { get; private set; }
|
||||
|
||||
/// <summary>Whether the worker reported this run as queued and has not reported it started (tests / diagnostics).</summary>
|
||||
internal bool IsDeferred
|
||||
{
|
||||
get { lock (_clock) return _deferred; }
|
||||
}
|
||||
|
||||
/// <summary>The worker's result, or null when the worker ended first.</summary>
|
||||
public Task<EvalResult?> Completion => _completion.Task;
|
||||
|
||||
@@ -966,27 +1055,96 @@ internal sealed class PendingEval
|
||||
/// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
|
||||
internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
|
||||
{
|
||||
_soft = soft;
|
||||
_grace = grace;
|
||||
_onHardDeadline = onHardDeadline;
|
||||
HardDeadline = soft + grace;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
_softDue = now + Ticks(soft);
|
||||
_hardDue = _softDue + Ticks(grace);
|
||||
// Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
|
||||
_timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
|
||||
_timer.Change(soft, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running,
|
||||
/// so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire.
|
||||
/// </summary>
|
||||
internal void Defer()
|
||||
{
|
||||
lock (_clock)
|
||||
{
|
||||
if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return;
|
||||
_deferred = true;
|
||||
TryChange(Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline
|
||||
/// had already been reported). A run that was never deferred keeps the clock it has had since submit.
|
||||
/// </summary>
|
||||
internal void Start()
|
||||
{
|
||||
lock (_clock)
|
||||
{
|
||||
_started = true;
|
||||
if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return;
|
||||
_deferred = false;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
if (Volatile.Read(ref _phase) == 0)
|
||||
{
|
||||
_softDue = now + Ticks(_soft);
|
||||
_hardDue = _softDue + Ticks(_grace);
|
||||
TryChange(_soft);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets
|
||||
// the whole T + G from its start before it can be blamed for a kill.
|
||||
_hardDue = now + Ticks(_soft + _grace);
|
||||
TryChange(_soft + _grace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnTimer()
|
||||
{
|
||||
if (Volatile.Read(ref _done) != 0) return;
|
||||
if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
|
||||
bool soft = false, hard = false;
|
||||
lock (_clock)
|
||||
{
|
||||
_outcome.TrySetResult(SoftTimedOut);
|
||||
try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* answered meanwhile */ }
|
||||
return;
|
||||
if (_deferred || Volatile.Read(ref _done) != 0) return;
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
if (Volatile.Read(ref _phase) == 0)
|
||||
{
|
||||
// A tick armed before a deferral/start can land early: re-arm for what is left.
|
||||
if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; }
|
||||
Interlocked.Exchange(ref _phase, 1);
|
||||
soft = true;
|
||||
TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero);
|
||||
}
|
||||
else if (Volatile.Read(ref _phase) == 1)
|
||||
{
|
||||
if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; }
|
||||
Interlocked.Exchange(ref _phase, 2);
|
||||
hard = true;
|
||||
}
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
|
||||
// Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
|
||||
if (soft) _outcome.TrySetResult(SoftTimedOut);
|
||||
if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
|
||||
}
|
||||
|
||||
private void TryChange(TimeSpan due)
|
||||
{
|
||||
try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* answered meanwhile */ }
|
||||
}
|
||||
|
||||
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
|
||||
|
||||
/// <summary>
|
||||
/// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
|
||||
/// reader, then hand the caller's resumption to the dispatcher (or the pool).
|
||||
|
||||
Reference in New Issue
Block a user