Merge #843 #840 #841 load flakes, ScriptRunGuard slot-release race, DurationGood conformance, CI QuestDB pin
v2-ci / unit-tests (push) Failing after 1h12m19s
v2-ci / unit-tests (push) Failing after 1h12m19s
This commit is contained in:
@@ -105,6 +105,13 @@ public sealed class ScriptRunGuard
|
||||
_logger = logger ?? Log.Logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test seam (lmxopcua#843): invoked on the run's own thread just before its slot is released, with the key.
|
||||
/// A sleep here widens the release window, so a test can prove the release is ordered before the caller
|
||||
/// resumes rather than winning a race.
|
||||
/// </summary>
|
||||
internal Action<string>? SlotReleasingForTest { get; set; }
|
||||
|
||||
/// <summary>Raised (on the thread that detected it) when a script is quarantined.</summary>
|
||||
public event EventHandler<ScriptQuarantinedEventArgs>? Quarantined;
|
||||
|
||||
@@ -233,34 +240,46 @@ public sealed class ScriptRunGuard
|
||||
Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp());
|
||||
started.TrySetResult();
|
||||
var before = GC.GetAllocatedBytesForCurrentThread();
|
||||
var succeeded = false;
|
||||
try
|
||||
{
|
||||
// ScriptEvaluator.RunAsync executes the compiled delegate synchronously and returns a completed
|
||||
// task, so the after-sample is taken on the same thread, right after the script returned.
|
||||
var pending = evaluator.RunAsync(context, runToken);
|
||||
slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before;
|
||||
return await pending.ConfigureAwait(false);
|
||||
var value = await pending.ConfigureAwait(false);
|
||||
succeeded = true;
|
||||
return value;
|
||||
}
|
||||
catch
|
||||
{
|
||||
slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before;
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Release the slot when the run REALLY finishes — which for an abandoned run is long after
|
||||
// RunAsync has thrown ScriptTimeoutException — and BEFORE runTask completes (lmxopcua#843). This
|
||||
// used to be a ContinueWith on runTask: a task is observably complete before its continuations
|
||||
// run, so a caller that saw the completion (e.g. the virtual-tag evaluator's immediate retry after a
|
||||
// raced ObjectDisposedException, or simply the next trigger) could
|
||||
// start the key's NEXT run while this one still held the slot and be refused "previous
|
||||
// evaluation still running" — a Bad publish for a script that had finished. Seen under load:
|
||||
// 24 of 4,800 sequential same-key evaluations. A late-finishing orphan that breached the
|
||||
// allocation budget quarantines its key first, so no re-trigger slips in between.
|
||||
if (succeeded && BreachedBudget(slot.AllocatedBytes))
|
||||
Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation");
|
||||
SlotReleasingForTest?.Invoke(key);
|
||||
Volatile.Write(ref state.RunStartedTimestamp, 0);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
Interlocked.Decrement(ref _inFlight);
|
||||
}
|
||||
});
|
||||
|
||||
// Release the slot when the run REALLY finishes — which for an abandoned run is long after this
|
||||
// method has thrown ScriptTimeoutException. Registered before the wait so it fires regardless of
|
||||
// which way the wait below comes out. A late-finishing orphan that breached the allocation budget
|
||||
// still quarantines its key here.
|
||||
_ = runTask.ContinueWith(t =>
|
||||
{
|
||||
Volatile.Write(ref state.RunStartedTimestamp, 0);
|
||||
Interlocked.Exchange(ref state.InFlight, 0);
|
||||
Interlocked.Decrement(ref _inFlight);
|
||||
runCts.Dispose();
|
||||
if (t.IsFaulted) _ = t.Exception; // observed below by the awaiter when it is still listening
|
||||
if (t.IsCompletedSuccessfully && BreachedBudget(slot.AllocatedBytes))
|
||||
Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation");
|
||||
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
|
||||
// The budget clock starts when the pool starts the run, not when we queued it (P5): pool starvation
|
||||
@@ -313,7 +332,7 @@ public sealed class ScriptRunGuard
|
||||
&& !string.Equals(state.SourceIdentity, sourceIdentity, StringComparison.Ordinal))
|
||||
{
|
||||
// The script changed under this key: start from a clean slate. The old state's orphan (if any)
|
||||
// releases the GLOBAL in-flight count through its own continuation; nothing else references it.
|
||||
// releases the GLOBAL in-flight count when it finishes; nothing else references it.
|
||||
_keys.TryUpdate(key, new KeyState(sourceIdentity), state);
|
||||
continue;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user