Joseph Doherty
8582ce91bc
v2-ci / unit-tests (push) Has been cancelled
#789 (partial, further tuning deferred): combining inline frame writer, buffered frame reader, one timer per evaluation for T and T+G, slot release on the reader before the caller resumes, a non-spinning per-connection result dispatcher, and blocking pipe I/O on Unix (both ends) so the socket never enters the socket engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2: 228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % -> 123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache worker RSS, and times kill latency from the soft-timeout answer. #790: the worker sends LoadStarted; the compile timeout is armed then, so a load queued behind other compiles is never killed or struck. #788: one Error with stack per down period, rate-limited Warnings with attempt count, one Information on recovery. Design doc §8.2 notes + §12.2 record. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
702 lines
41 KiB
C#
702 lines
41 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using System.Diagnostics.Metrics;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using Serilog;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#780 P1 supervision gaps: the quarantine race, victim re-evaluation on a shard's return, the idle
|
|
/// isolation worker, the startup orphan sweep, culprit-finding for a compile that crashes its worker, and the
|
|
/// runner state / crash-loop circuit. Real worker processes throughout.
|
|
/// </summary>
|
|
[Collection(ScriptWorkerCollection.Name)]
|
|
[Trait("Category", "ScriptWorker")]
|
|
public sealed class SupervisionTests
|
|
{
|
|
private const string Healthy = "return 7;";
|
|
private const string Runaway = "while (true) { }";
|
|
private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
|
|
|
|
private static IReadOnlyDictionary<string, DataValueSnapshot> Ms(int ms) => Scripts.Input("ms", ms);
|
|
|
|
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
|
|
|
private static string VtKey(string key) => InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, key);
|
|
|
|
// ---------------------------------------------------------------- the quarantine race
|
|
|
|
/// <summary>
|
|
/// The ordering invariant behind the race fix, pinned deterministically at the connection: the exit attribution
|
|
/// callback runs BEFORE any outstanding evaluation or compile is completed and before <c>Exited</c> is signalled.
|
|
/// Mutation check: moving the attribution call after the completion loop in <c>WorkerConnection.Finish</c> turns
|
|
/// both assertions red.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Exit_attribution_runs_before_any_outstanding_evaluation_completes()
|
|
{
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions()),
|
|
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromSeconds(1),
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15));
|
|
PendingEval? pending = null;
|
|
WorkerConnection? connection = null;
|
|
bool? pendingCompletedAtAttribution = null, exitedCompletedAtAttribution = null;
|
|
IReadOnlyList<string>? inFlightAtAttribution = null;
|
|
connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, exit =>
|
|
{
|
|
pendingCompletedAtAttribution = pending?.Completion.IsCompleted;
|
|
exitedCompletedAtAttribution = connection?.Exited.IsCompleted;
|
|
inFlightAtAttribution = exit.InFlightKeys;
|
|
}, Ct);
|
|
await using var _ = connection;
|
|
|
|
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
|
System.Security.Cryptography.SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
entry!.Outcome!.Ok.ShouldBeTrue();
|
|
pending = connection.Submit("k", new EvalRequest { Key = "k", SoftTimeoutMs = 60_000, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks });
|
|
await Scripts.Eventually(() => connection.LastHeartbeat is { Inflight: 1 }, TimeSpan.FromSeconds(10), "the run in flight");
|
|
|
|
Process.GetProcessById(connection.Pid).Kill();
|
|
await connection.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
(await pending.Completion.WaitAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull("the worker went away");
|
|
|
|
inFlightAtAttribution.ShouldBe(["k"]);
|
|
pendingCompletedAtAttribution.ShouldBe(false, "attribution must precede the evaluation's completion");
|
|
exitedCompletedAtAttribution.ShouldBe(false, "attribution must precede Exited");
|
|
}
|
|
|
|
/// <summary>
|
|
/// The other half of the race fix, pinned without timing: the shard hands the connection an attribution callback
|
|
/// that reaches the runner's exit handler SYNCHRONOUSLY (the pre-fix shard called it from an Exited continuation,
|
|
/// after the evaluations had already been completed). Mutation check: launching with a no-op callback and
|
|
/// attributing from the Exited continuation instead leaves <c>attributed</c> false when the callback returns.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task The_shard_attributes_an_exit_synchronously_inside_the_connections_callback()
|
|
{
|
|
Action<WorkerExit>? captured = null;
|
|
var attributed = false;
|
|
await using var shard = new WorkerShard("t", isolation: false,
|
|
(onExiting, _) =>
|
|
{
|
|
captured = onExiting;
|
|
throw new InvalidOperationException("no process in this test");
|
|
},
|
|
(_, _) => attributed = true,
|
|
new LoggerConfiguration().CreateLogger());
|
|
(await shard.GetReadyAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull();
|
|
|
|
captured.ShouldNotBeNull();
|
|
captured(new WorkerExit(WorkerExitReason.Crash, null, 1, ["k"]));
|
|
attributed.ShouldBeTrue("the runner's verdict is recorded before the callback returns");
|
|
}
|
|
|
|
/// <summary>
|
|
/// A burst of cold compiles must not starve the worker's heartbeat. Found on windev by the §8 perf harness: 500
|
|
/// loads at once, each compiled synchronously on its own thread-pool work item, queued the heartbeat timer and
|
|
/// the pipe writer behind them; the host saw no frame for 3 s, killed the shard (KilledHeartbeat) and made all 500
|
|
/// keys suspects. Here a 100 ms heartbeat (kill after 300 ms of silence) and 400 distinct cold scripts at once:
|
|
/// every script answers and the shard's worker is the one that started. Mutation check: compile on the thread
|
|
/// pool again and the worker is killed for silence.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_burst_of_cold_compiles_does_not_starve_the_heartbeat()
|
|
{
|
|
var sink = new CapturingSink();
|
|
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromMilliseconds(50), loadTimeout: TimeSpan.FromSeconds(120));
|
|
(await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
|
|
const int count = 1500;
|
|
var outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(
|
|
Scripts.Vt("burst-" + i, $"return (int)ctx.GetTag(\"x\").Value * {i} + 1;", TimeSpan.FromSeconds(60),
|
|
Scripts.Input("x", 2)), Ct)));
|
|
|
|
var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)).Take(5));
|
|
for (var i = 0; i < count; i++) outcomes[i].Value.ShouldBe(2 * i + 1, $"{outcomes[i].Status} {outcomes[i].Reason}\n{log}");
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
|
|
worker.Exited.IsCompleted.ShouldBeFalse(log);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Single flight must not refuse a caller's NEXT run because of the run it already awaited: the slot is freed
|
|
/// before the awaiting caller resumes. The §8 perf harness found the slot freed by a continuation unordered
|
|
/// against the caller's await, so 32 of 200 back-to-back runs of one key came back "previous evaluation still
|
|
/// running" — which the VirtualTag and Calculation seams publish as Bad. Since lmxopcua#789 the release runs in
|
|
/// the connection, on the reader, before the caller's resumption is handed to the result dispatcher; the second
|
|
/// phase makes every release slow (the test hook sleeps inside it) so the ordering, not luck, is what keeps the
|
|
/// caller from being refused. Mutation check: release after handing off the resumption and the second phase goes
|
|
/// red.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_caller_that_awaited_its_run_is_never_refused_by_it()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options());
|
|
var refused = new List<string>();
|
|
for (var i = 0; i < 1_000; i++)
|
|
{
|
|
var outcome = await runner.RunAsync(
|
|
Scripts.Vt("seq", "return (int)ctx.GetTag(\"x\").Value + 7;", inputs: Scripts.Input("x", i)), Ct);
|
|
if (outcome.Status != ScriptRunStatus.Ok || !Equals(outcome.Value, i + 7)) refused.Add($"#{i}: {outcome.Reason}");
|
|
}
|
|
refused.ShouldBeEmpty();
|
|
|
|
runner.SlotReleasingForTest = _ => Thread.Sleep(5);
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
var outcome = await runner.RunAsync(
|
|
Scripts.Vt("seq", "return (int)ctx.GetTag(\"x\").Value + 7;", inputs: Scripts.Input("x", i)), Ct);
|
|
if (outcome.Status != ScriptRunStatus.Ok || !Equals(outcome.Value, i + 7)) refused.Add($"slow release #{i}: {outcome.Reason}");
|
|
}
|
|
refused.ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>
|
|
/// The race itself, end to end: with QuarantineAfterKills = 1, the moment the killed script's single-flight slot is
|
|
/// released the quarantine is already in place — so no trigger can re-run it in the restart gap, and the next
|
|
/// trigger is refused without any wait. Repeated, since the old ordering lost this race only some of the time.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_killed_script_is_quarantined_before_its_slot_is_released()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(250), quarantineAfterKills: 1));
|
|
var observed = new ConcurrentQueue<bool>();
|
|
runner.SlotReleasingForTest = key =>
|
|
{
|
|
if (key == VtKey("runaway")) observed.Enqueue(runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Contains("runaway"));
|
|
};
|
|
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
|
|
|
for (var round = 1; round <= 3; round++)
|
|
{
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
(await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(250)), Ct)).Reason!
|
|
.ShouldStartWith("script timed out after ");
|
|
await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
await Scripts.Eventually(() => observed.Count == round, TimeSpan.FromSeconds(5), "the slot release");
|
|
|
|
// No Eventually: the very next trigger after the kill is refused.
|
|
var next = await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(250)), Ct);
|
|
next.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
|
runner.Reset(ScriptContextKind.VirtualTag);
|
|
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
|
}
|
|
observed.ShouldAllBe(quarantined => quarantined, "the slot was released before the kill strike was recorded");
|
|
}
|
|
|
|
// ---------------------------------------------------------------- victims re-evaluated on return
|
|
|
|
/// <summary>
|
|
/// A victim's Unavailable outcome carries a Recovery task that completes when its shard is back — and the shard
|
|
/// comes back on its own (proactive restart), with no further trigger. Mutation check: without the shard's
|
|
/// recovery loop, Recovery never completes and no worker is running.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_victims_recovery_completes_when_its_shard_restarts_by_itself()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options());
|
|
(await runner.RunAsync(Scripts.Vt("victim", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
|
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
|
|
var inFlight = runner.RunAsync(Scripts.Vt("sibling", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
|
await Scripts.Eventually(() => worker.InFlightCount == 1, TimeSpan.FromSeconds(10), "the run in flight");
|
|
Process.GetProcessById(worker.Pid).Kill();
|
|
|
|
var victim = await inFlight.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
victim.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
|
_ = victim.Recovery.ShouldNotBeNull("a runner failure tells the seam when to re-evaluate");
|
|
victim.ToVirtualTagEvalResult().RetryWhen.ShouldBeSameAs(victim.Recovery);
|
|
|
|
// The crash made "sibling" a suspect, so its recovery is the isolation worker's; the crashed shard restarts by
|
|
// itself as well, so the scripts placed on it find it warm.
|
|
await victim.Recovery!.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
|
runner.Isolation.Current.ShouldNotBeNull("recovered without any new trigger");
|
|
await Scripts.Eventually(() => runner.Shards[0].Current, TimeSpan.FromSeconds(15), "the shard back");
|
|
runner.Shards[0].Current!.Pid.ShouldNotBe(worker.Pid);
|
|
(await runner.RunAsync(Scripts.Vt("sibling", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- the idle isolation worker
|
|
|
|
/// <summary>
|
|
/// The isolation worker stops after IsolationIdleTimeout with nothing to run — an Idle exit, not a failure — and
|
|
/// starts again for the next suspect. Mutation check: with the maintenance pass not stopping it, it stays up.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task An_idle_isolation_worker_is_stopped_and_restarts_on_demand()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options(isolationIdle: TimeSpan.FromSeconds(1)),
|
|
maintenanceInterval: TimeSpan.FromMilliseconds(200));
|
|
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
var run = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
|
await Scripts.Eventually(() => worker.InFlightCount == 1, TimeSpan.FromSeconds(10), "the run in flight");
|
|
Process.GetProcessById(worker.Pid).Kill();
|
|
(await run.WaitAsync(TimeSpan.FromSeconds(10), Ct)).Status.ShouldBe(ScriptRunStatus.Unavailable);
|
|
runner.SuspectKeys.ShouldBe([VtKey("spin")]);
|
|
|
|
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
|
var iso = runner.Isolation.Current.ShouldNotBeNull();
|
|
|
|
var exit = await iso.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
exit.Reason.ShouldBe(WorkerExitReason.Idle);
|
|
runner.Isolation.Current.ShouldBeNull();
|
|
runner.Isolation.ConsecutiveFailures.ShouldBe(0, "an idle stop is not a failure");
|
|
runner.Isolation.DownFor.ShouldBeNull("an idle stop does not make the isolation worker 'down'");
|
|
await Scripts.Eventually(() => runner.State == ScriptRunnerState.Ready, TimeSpan.FromSeconds(15), "the runner ready again");
|
|
|
|
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
|
runner.Isolation.Current.ShouldNotBeNull().Pid.ShouldNotBe(iso.Pid, "a fresh isolation worker for the next suspect run");
|
|
}
|
|
|
|
// ---------------------------------------------------------------- the orphan sweep
|
|
|
|
private static Process StartDecoyWorker()
|
|
{
|
|
// A real ScriptWorker that will sit waiting (up to 10 s) for a pipe that does not exist: the shape of a worker
|
|
// its host abandoned.
|
|
var psi = new ProcessStartInfo(WorkerConnection.ResolveMuxer())
|
|
{
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true,
|
|
RedirectStandardError = true,
|
|
RedirectStandardOutput = true,
|
|
};
|
|
psi.ArgumentList.Add("exec");
|
|
psi.ArgumentList.Add(OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions()));
|
|
psi.Environment[ScriptWorkerEnvironment.PipeName] = "otopcua-sw-d" + Guid.NewGuid().ToString("N")[..8];
|
|
psi.Environment[ScriptWorkerEnvironment.Nonce] = "decoy";
|
|
var p = Process.Start(psi)!;
|
|
Thread.Sleep(300);
|
|
if (p.HasExited) throw new InvalidOperationException($"the decoy worker exited {p.ExitCode}: {p.StandardError.ReadToEnd()}");
|
|
return p;
|
|
}
|
|
|
|
private static (int Pid, long StartTicks) DeadProcess()
|
|
{
|
|
var psi = new ProcessStartInfo(WorkerConnection.ResolveMuxer(), "--version")
|
|
{
|
|
UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true,
|
|
};
|
|
using var p = Process.Start(psi)!;
|
|
var start = p.StartTime.ToUniversalTime().Ticks;
|
|
var pid = p.Id;
|
|
p.WaitForExit();
|
|
return (pid, start);
|
|
}
|
|
|
|
private static string WriteMarker(string dir, Process worker, int hostPid, long hostStartTicks, long? workerStartTicks = null)
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var record = new WorkerLaunchMarker.MarkerRecord(worker.Id, workerStartTicks ?? worker.StartTime.ToUniversalTime().Ticks,
|
|
worker.ProcessName, hostPid, hostStartTicks, "decoy");
|
|
var path = Path.Combine(dir, worker.Id + ".worker");
|
|
File.WriteAllText(path, JsonSerializer.Serialize(record));
|
|
return path;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Design §3.3: a worker left behind by a DEAD host of this instance is killed at the next start — through the
|
|
/// runner's constructor, so the wiring is pinned too. Mutation check: without the Sweep call the decoy survives.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task The_startup_sweep_kills_a_worker_whose_host_is_gone()
|
|
{
|
|
using var dir = new Scripts.TempDir();
|
|
using var decoy = StartDecoyWorker();
|
|
var (deadPid, deadStart) = DeadProcess();
|
|
var marker = WriteMarker(dir.Path, decoy, deadPid, deadStart);
|
|
|
|
await using var runner = Scripts.Runner(Scripts.Options(), markerDirectory: dir.Path);
|
|
(await Task.Run(() => decoy.WaitForExit(5000), Ct)).ShouldBeTrue("the orphan was killed");
|
|
File.Exists(marker).ShouldBeFalse();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Never an unrelated process: a marker whose host is ALIVE (another instance, or a sibling runner in this
|
|
/// process) is left alone, and a marker whose pid now belongs to a different process (start time differs —
|
|
/// pid reuse) never kills it. Mutation checks: dropping either the host-alive check or the start-time check
|
|
/// kills the decoy.
|
|
/// </summary>
|
|
[Fact]
|
|
public void The_sweep_never_kills_a_live_hosts_worker_or_a_reused_pid()
|
|
{
|
|
using var dir = new Scripts.TempDir();
|
|
using var decoy = StartDecoyWorker();
|
|
using var self = Process.GetCurrentProcess();
|
|
try
|
|
{
|
|
var liveHostMarker = WriteMarker(dir.Path, decoy, self.Id, self.StartTime.ToUniversalTime().Ticks);
|
|
WorkerLaunchMarker.Sweep(dir.Path, new LoggerConfiguration().CreateLogger()).ShouldBeEmpty();
|
|
decoy.HasExited.ShouldBeFalse("a live host's worker is not an orphan");
|
|
File.Exists(liveHostMarker).ShouldBeTrue("the live host still owns its marker");
|
|
|
|
var (deadPid, deadStart) = DeadProcess();
|
|
var reused = WriteMarker(dir.Path, decoy, deadPid, deadStart,
|
|
workerStartTicks: decoy.StartTime.ToUniversalTime().AddHours(-1).Ticks);
|
|
WorkerLaunchMarker.Sweep(dir.Path, new LoggerConfiguration().CreateLogger()).ShouldBeEmpty();
|
|
decoy.HasExited.ShouldBeFalse("a pid whose start time does not match is some other process");
|
|
File.Exists(reused).ShouldBeFalse("a stale marker is cleaned up");
|
|
}
|
|
finally
|
|
{
|
|
try { decoy.Kill(); } catch { /* already gone */ }
|
|
}
|
|
}
|
|
|
|
/// <summary>A worker's marker exists while it runs and is removed when it stops cleanly.</summary>
|
|
[Fact]
|
|
public async Task A_running_worker_has_a_launch_marker_that_its_clean_stop_removes()
|
|
{
|
|
using var dir = new Scripts.TempDir();
|
|
var runner = Scripts.Runner(Scripts.Options(), markerDirectory: dir.Path);
|
|
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
|
var pid = runner.Shards[0].Current!.Pid;
|
|
var marker = Path.Combine(dir.Path, pid + ".worker");
|
|
File.Exists(marker).ShouldBeTrue();
|
|
JsonSerializer.Deserialize<WorkerLaunchMarker.MarkerRecord>(File.ReadAllText(marker))!.HostPid.ShouldBe(Environment.ProcessId);
|
|
await runner.DisposeAsync();
|
|
File.Exists(marker).ShouldBeFalse();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- a compile that crashes its worker
|
|
|
|
/// <summary>
|
|
/// Design §5.3.5 culprit-finding: a worker that dies while a script is COMPILING (no evaluation in flight) makes
|
|
/// that script a suspect; it then compiles alone in the isolation worker, where a second death is unambiguous and
|
|
/// quarantines it — so a poison source cannot crash-loop its shard forever. Mutation check: leaving the compiling
|
|
/// keys out of the attribution leaves the key unsuspected and unquarantined.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_script_whose_compile_keeps_crashing_its_worker_is_found_in_isolation_and_quarantined()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options(quarantineAfterKills: 1));
|
|
var body = new StringBuilder("long x = 0;\n");
|
|
for (var i = 0; i < 3000; i++) body.Append("x = x * 31 + ").Append(i).Append(";\n");
|
|
body.Append("return x;");
|
|
var poison = body.ToString();
|
|
|
|
async Task<ScriptRunOutcome> CrashDuringCompile(Func<WorkerConnection?> worker)
|
|
{
|
|
var run = runner.RunAsync(Scripts.Vt("poison", poison), Ct);
|
|
var live = await Scripts.Eventually(worker, TimeSpan.FromSeconds(15), "a live worker");
|
|
await Scripts.Eventually(() => live.LoadingCount == 1, TimeSpan.FromSeconds(15), "the compile in flight");
|
|
live.InFlightCount.ShouldBe(0, "no evaluation: only the compile is in flight");
|
|
Process.GetProcessById(live.Pid).Kill();
|
|
return await run.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
|
}
|
|
|
|
var first = await CrashDuringCompile(() => runner.Shards[0].Current);
|
|
first.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
|
runner.SuspectKeys.ShouldBe([VtKey("poison")], "a death during its compile makes it a suspect");
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("ambiguous on a shard: no strike");
|
|
|
|
await CrashDuringCompile(() => runner.Isolation.Current);
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["poison"]);
|
|
var refused = await runner.RunAsync(Scripts.Vt("poison", poison), Ct);
|
|
refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
|
refused.Reason.ShouldContain("compile crashed the isolation worker");
|
|
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- state, circuit, gauges
|
|
|
|
[Theory]
|
|
[InlineData(new double[] { }, null, ScriptRunnerState.Ready)]
|
|
[InlineData(new double[] { -1, -1 }, null, ScriptRunnerState.Ready)]
|
|
[InlineData(new double[] { -1, 5 }, null, ScriptRunnerState.Degraded)]
|
|
[InlineData(new double[] { -1, -1 }, 5.0, ScriptRunnerState.Degraded)]
|
|
[InlineData(new double[] { 200, 5 }, null, ScriptRunnerState.Degraded)]
|
|
[InlineData(new double[] { 200, 130 }, null, ScriptRunnerState.Unavailable)]
|
|
public void The_runner_state_rule(double[] shardsDownSeconds, double? isolationDownSeconds, ScriptRunnerState expected)
|
|
{
|
|
var shards = shardsDownSeconds.Select(s => s < 0 ? (TimeSpan?)null : TimeSpan.FromSeconds(s)).ToArray();
|
|
var iso = isolationDownSeconds is { } i ? TimeSpan.FromSeconds(i) : (TimeSpan?)null;
|
|
OutOfProcessScriptRunner.ComputeState(shards, iso, TimeSpan.FromMinutes(2)).ShouldBe(expected);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The crash-loop circuit: every shard down past CircuitOpenAfter makes the runner Unavailable (gauge value 2),
|
|
/// logs one Error, and answers evaluations at once instead of after UnavailableGrace. Mutation check: without the
|
|
/// zero grace under an open circuit, the evaluation waits the full 3 s.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Every_shard_down_opens_the_circuit_answers_at_once_and_reports_state_2()
|
|
{
|
|
var sink = new CapturingSink();
|
|
var missing = Path.Combine(Path.GetTempPath(), "no-such-dir-" + Guid.NewGuid().ToString("N"), OutOfProcessScriptRunner.WorkerAssemblyFileName);
|
|
await using var runner = Scripts.Runner(
|
|
Scripts.Options(workerPath: missing, unavailableGrace: TimeSpan.FromSeconds(3), circuitOpenAfter: TimeSpan.FromSeconds(1)),
|
|
sink.CreateLogger(), maintenanceInterval: TimeSpan.FromMilliseconds(100));
|
|
runner.State.ShouldBe(ScriptRunnerState.Ready, "nothing needed yet");
|
|
|
|
(await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct)).Status.ShouldBe(ScriptRunStatus.Unavailable);
|
|
runner.State.ShouldNotBe(ScriptRunnerState.Ready);
|
|
await Scripts.Eventually(() => runner.State == ScriptRunnerState.Unavailable, TimeSpan.FromSeconds(10), "the circuit open");
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
var fast = await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct);
|
|
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1), "an open circuit answers without waiting out the 3 s grace");
|
|
fast.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
|
fast.Reason.ShouldBe(ScriptRunOutcome.RunnerUnavailableReason);
|
|
|
|
await Scripts.Eventually(() => sink.Rendered.Any(m => m.Contains("circuit OPEN")), TimeSpan.FromSeconds(5), "the circuit Error");
|
|
await Task.Delay(500, Ct);
|
|
sink.Events.Count(e => e.Level == Serilog.Events.LogEventLevel.Error && e.RenderMessage().Contains("circuit OPEN"))
|
|
.ShouldBe(1, "logged once, not per maintenance pass");
|
|
|
|
var states = new List<int>();
|
|
using var listener = new MeterListener();
|
|
listener.InstrumentPublished = (instrument, l) =>
|
|
{
|
|
if (instrument.Name == "otopcua.script.runner.state") l.EnableMeasurementEvents(instrument);
|
|
};
|
|
listener.SetMeasurementEventCallback<int>((_, value, _, _) => states.Add(value));
|
|
listener.Start();
|
|
listener.RecordObservableInstruments();
|
|
states.ShouldContain(2);
|
|
runner.Status().State.ShouldBe(ScriptRunnerState.Unavailable);
|
|
}
|
|
|
|
/// <summary>The per-shard gauges observe the live worker (positive control: the shard is up and reports its pid).</summary>
|
|
[Fact]
|
|
public async Task The_shard_gauges_report_a_live_worker()
|
|
{
|
|
await using var runner = Scripts.Runner(Scripts.Options());
|
|
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
|
await Scripts.Eventually(() => runner.Shards[0].Current?.LastHeartbeat is not null, TimeSpan.FromSeconds(5), "a heartbeat");
|
|
|
|
var up = new List<(int Value, string? Shard)>();
|
|
var heap = new List<long>();
|
|
using var listener = new MeterListener();
|
|
listener.InstrumentPublished = (instrument, l) =>
|
|
{
|
|
if (instrument.Name is "otopcua.script.worker.up" or "otopcua.script.worker.heap_bytes") l.EnableMeasurementEvents(instrument);
|
|
};
|
|
listener.SetMeasurementEventCallback<int>((_, value, tags, _) =>
|
|
{
|
|
foreach (var tag in tags) if (tag.Key == "shard") up.Add((value, tag.Value as string));
|
|
});
|
|
listener.SetMeasurementEventCallback<long>((_, value, _, _) => heap.Add(value));
|
|
listener.Start();
|
|
listener.RecordObservableInstruments();
|
|
|
|
up.ShouldContain((1, "0"));
|
|
up.ShouldContain((0, "iso"), "the isolation worker is not running");
|
|
heap.ShouldContain(h => h > 0);
|
|
var status = runner.Status();
|
|
status.State.ShouldBe(ScriptRunnerState.Ready);
|
|
status.Shards.Single(s => s.Label == "0").Pid.ShouldBe(runner.Shards[0].Current!.Pid);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- lmxopcua#789: who resumes the caller
|
|
|
|
/// <summary>
|
|
/// lmxopcua#789: callers resume on the connection's result dispatcher, never on the pipe reader — so a consumer
|
|
/// that blocks after its await (the Calculation pump publishes to OPC UA subscribers there) cannot stop the reader
|
|
/// from taking heartbeats and get a healthy worker killed for silence. Here a 100 ms heartbeat (kill after 300 ms
|
|
/// of silence) and a continuation that blocks for 1.5 s. Mutation check: resume callers inline on the reader (a
|
|
/// synchronous-continuation TaskCompletionSource completed by the reader, no dispatcher) and the worker is killed
|
|
/// (KilledHeartbeat).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_caller_that_blocks_after_its_result_does_not_starve_the_reader()
|
|
{
|
|
var sink = new CapturingSink();
|
|
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromMilliseconds(100), loadTimeout: TimeSpan.FromSeconds(60));
|
|
(await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
|
|
string? resumedOn = null;
|
|
await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct).ContinueWith(t =>
|
|
{
|
|
// Runs wherever RunAsync's own continuation completed it: the thread that resumed the caller.
|
|
resumedOn = Thread.CurrentThread.Name;
|
|
t.Result.Value.ShouldBe(7);
|
|
Thread.Sleep(1_500);
|
|
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
|
|
|
// Behaviour first: the worker was not killed for silence while the caller blocked.
|
|
(await Task.WhenAny(worker.Exited, Task.Delay(1_000, Ct))).ShouldNotBe(worker.Exited,
|
|
string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal))));
|
|
var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)));
|
|
resumedOn.ShouldBe("ScriptWorker[0].Results", log);
|
|
(await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- lmxopcua#788: start-failure logging
|
|
|
|
/// <summary>
|
|
/// lmxopcua#788: a shard that keeps failing to start logs ONE Error with the exception for the down period; its
|
|
/// retries are a Warning without a stack, carrying the attempt count, rate-limited (the rest at Debug); and the
|
|
/// start that ends the outage is one Information line naming how many attempts failed. The docker-dev drill logged
|
|
/// 28 Error+stack entries in two minutes on one node before this. Mutation check: log every failure at Error with
|
|
/// the exception again and the single-Error assertion goes red (3 Errors).
|
|
/// </summary>
|
|
[Theory]
|
|
[InlineData(0, 2, 0)]
|
|
[InlineData(3_600_000, 0, 2)]
|
|
public async Task A_shard_that_cannot_start_logs_one_error_then_concise_rate_limited_retries_then_its_recovery(
|
|
int warnEveryMs, int expectedWarnings, int expectedDebugs)
|
|
{
|
|
var sink = new CapturingSink();
|
|
var log = sink.CreateLogger();
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
|
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(5), HeartbeatInterval: TimeSpan.FromSeconds(1),
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15));
|
|
var attempts = 0;
|
|
await using var shard = new WorkerShard("t", isolation: false,
|
|
(onExiting, ct) => Interlocked.Increment(ref attempts) <= 3
|
|
? throw new InvalidOperationException("the worker exited (code 1) before connecting to its pipe")
|
|
: WorkerConnection.StartAsync(spec, log, _ => { }, onExiting, ct),
|
|
(_, _) => { }, log, startFailureWarningInterval: TimeSpan.FromMilliseconds(warnEveryMs));
|
|
|
|
// The background recovery retries at the backoff (250 ms, 500 ms, 1 s, ±20 %) until the 4th attempt starts.
|
|
await shard.WhenRecoveredAsync(null).WaitAsync(TimeSpan.FromSeconds(30), Ct);
|
|
shard.Current.ShouldNotBeNull();
|
|
attempts.ShouldBe(4);
|
|
|
|
var events = sink.Events.Where(e => e.MessageTemplate.Text.StartsWith("ScriptWorker[{Shard}]", StringComparison.Ordinal)).ToList();
|
|
string Dump() => string.Join(Environment.NewLine, events.Select(e => $"{e.Level} {e.RenderMessage()} {(e.Exception is null ? "" : "+exception")}"));
|
|
static int Attempts(Serilog.Events.LogEvent e) => int.Parse(e.Properties["Attempts"].ToString(), System.Globalization.CultureInfo.InvariantCulture);
|
|
|
|
var errors = events.Where(e => e.Level == Serilog.Events.LogEventLevel.Error).ToList();
|
|
errors.Count.ShouldBe(1, Dump());
|
|
errors[0].Exception.ShouldNotBeNull("the first failure of the outage carries the stack");
|
|
|
|
var warnings = events.Where(e => e.Level == Serilog.Events.LogEventLevel.Warning).ToList();
|
|
warnings.Count.ShouldBe(expectedWarnings, Dump());
|
|
warnings.ShouldAllBe(e => e.Exception == null);
|
|
warnings.Select(Attempts).ShouldBe(expectedWarnings == 0 ? [] : [2, 3]);
|
|
events.Count(e => e.Level == Serilog.Events.LogEventLevel.Debug && e.Properties.ContainsKey("Attempts")).ShouldBe(expectedDebugs, Dump());
|
|
|
|
var recovered = events.Single(e => e.Level == Serilog.Events.LogEventLevel.Information && e.Properties.ContainsKey("Attempts"));
|
|
Attempts(recovered).ShouldBe(3);
|
|
recovered.RenderMessage().ShouldContain("after 3 failed start attempt(s)");
|
|
}
|
|
|
|
// ---------------------------------------------------------------- lmxopcua#790: compile timeout from compile start
|
|
|
|
/// <summary>A script of <paramref name="lines"/> statements: Roslyn needs time roughly proportional to it.</summary>
|
|
private static string Lines(int lines, int salt)
|
|
{
|
|
var body = new StringBuilder("var a = ").Append(salt).Append(";\n");
|
|
for (var i = 0; i < lines; i++) body.Append("a += (int)ctx.GetTag(\"x\").Value * ").Append(i % 97).Append(";\n");
|
|
return body.Append("return a;").ToString();
|
|
}
|
|
|
|
private static int LinesValue(int lines, int salt) => salt + Enumerable.Range(0, lines).Sum(i => i % 97);
|
|
|
|
/// <summary>
|
|
/// lmxopcua#790: the compile timeout runs from when a compile thread STARTS a load, so a queue of compiles that
|
|
/// takes several load timeouts to drain kills nothing and strikes nobody. One compile thread; the compiler is
|
|
/// warmed under a generous timeout (a cold Roslyn start alone takes seconds on the Windows VM), then the live
|
|
/// connection's timeout drops to four times one measured compile (at least 3 s; a 500-statement script takes
|
|
/// ~0.1 s on an M-series Mac and ~1.5 s on the Windows VM) and a burst of such scripts, sized from that and grown
|
|
/// until it does, queues for over three timeouts — asserted, so the test cannot pass vacuously
|
|
/// — while each compile stays far inside the timeout. Mutation check: time the load from when the host SENT it
|
|
/// (the old <c>WaitAsync(LoadTimeout)</c> around the load) and the worker is killed and the queued keys are
|
|
/// struck.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Compiles_queued_behind_others_longer_than_the_load_timeout_are_not_killed_or_struck()
|
|
{
|
|
const int lines = 500;
|
|
var sink = new CapturingSink();
|
|
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(120), compileThreads: 1);
|
|
(await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
|
|
ScriptInvocation Script(string prefix, int i)
|
|
=> Scripts.Vt($"{prefix}-{i}", Lines(lines, i), TimeSpan.FromSeconds(300), Scripts.Input("x", 1));
|
|
|
|
// Warm the compiler, then calibrate: the compile time of one such script on this box.
|
|
for (var i = 0; i < 3; i++) (await runner.RunAsync(Script("warm", i), Ct)).Value.ShouldBe(LinesValue(lines, i));
|
|
var probe = Stopwatch.StartNew();
|
|
for (var i = 0; i < 3; i++) (await runner.RunAsync(Script("probe", i), Ct)).Value.ShouldBe(LinesValue(lines, i));
|
|
var perCompile = probe.Elapsed / 3;
|
|
var loadTimeout = TimeSpan.FromTicks(Math.Max(TimeSpan.FromSeconds(3).Ticks, perCompile.Ticks * 4));
|
|
worker.LoadTimeout = loadTimeout;
|
|
var count = (int)Math.Clamp(Math.Ceiling(7 * loadTimeout / perCompile), 10, 300);
|
|
|
|
// A burst compiles faster per script than the sequential probe (the compiler keeps warming up), so grow the burst
|
|
// until it really outlasts three load timeouts; each burst must pass the same no-kill checks.
|
|
ScriptRunOutcome[] outcomes = [];
|
|
var elapsed = TimeSpan.Zero;
|
|
for (var round = 0; round < 4; round++)
|
|
{
|
|
runner.Reset(ScriptContextKind.VirtualTag); // unload the previous round's scripts (worker heap)
|
|
var sw = Stopwatch.StartNew();
|
|
var prefix = "queued" + round;
|
|
outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(Script(prefix, i), Ct)));
|
|
elapsed = sw.Elapsed;
|
|
for (var i = 0; i < count; i++) outcomes[i].Value.ShouldBe(LinesValue(lines, i), $"round {round}: {outcomes[i].Status} {outcomes[i].Reason}");
|
|
if (elapsed > loadTimeout * 3) break;
|
|
count = Math.Min(1_200, count * 2);
|
|
}
|
|
|
|
var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)).Take(5));
|
|
elapsed.ShouldBeGreaterThan(loadTimeout * 3,
|
|
$"precondition: {count} compiles (probe {perCompile.TotalMilliseconds:F0} ms each) must queue for several load timeouts, or this proves nothing");
|
|
worker.Exited.IsCompleted.ShouldBeFalse(log);
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
|
runner.SuspectKeys.ShouldBeEmpty();
|
|
sink.Rendered.ShouldNotContain(l => l.Contains("killing the worker", StringComparison.Ordinal));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The positive control for the test above: a compile that is still running <c>LoadTimeout</c> after it STARTED is
|
|
/// the script's fault — the worker is killed (<c>KilledLoadTimeout</c>), the caller gets the compile-timeout
|
|
/// failure, the key takes a kill strike (the second quarantines it), and the healthy key on the same shard is
|
|
/// not blamed. The healthy key comes from a primed PE cache, so a fresh worker's cold Roslyn start never counts
|
|
/// against the short timeout. Mutation check: never arm the timer on <c>LoadStarted</c> and the huge compile
|
|
/// simply finishes (status Ok, red).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_compile_still_running_a_load_timeout_after_it_started_is_killed_and_struck()
|
|
{
|
|
using var dir = new Scripts.TempDir();
|
|
var cache = Path.Combine(dir.Path, "pe-cache");
|
|
await using (var primer = Scripts.Runner(Scripts.Options(peCacheDir: cache)))
|
|
(await primer.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
|
|
|
|
var sink = new CapturingSink();
|
|
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(quarantineAfterKills: 2, peCacheDir: cache),
|
|
sink.CreateLogger(), null, heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(2), compileThreads: 1);
|
|
(await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
|
|
|
|
// 20,000 statements: about 4 s of Roslyn on an M-series Mac, longer elsewhere.
|
|
var huge = Scripts.Vt("huge", Lines(20_000, 0), TimeSpan.FromSeconds(30), Scripts.Input("x", 1));
|
|
for (var strike = 1; strike <= 2; strike++)
|
|
{
|
|
var before = runner.Shards[0].Current ?? await runner.Shards[0].GetReadyAsync(TimeSpan.FromSeconds(30), Ct);
|
|
var outcome = await runner.RunAsync(huge, Ct).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
outcome.Status.ShouldBe(ScriptRunStatus.Failed, $"strike {strike}");
|
|
outcome.Reason.ShouldNotBeNull().ShouldContain("compile did not finish");
|
|
var exit = await before.ShouldNotBeNull().Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
exit.Reason.ShouldBe(WorkerExitReason.KilledLoadTimeout);
|
|
exit.CulpritKey.ShouldBe(VtKey("huge"));
|
|
}
|
|
|
|
var third = await runner.RunAsync(huge, Ct);
|
|
third.Reason.ShouldNotBeNull().ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["huge"]);
|
|
(await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7, "the healthy key on the same shard is not blamed");
|
|
}
|
|
}
|