feat(scripting): PE cache, quarantine-race fix, recovery, readiness for the out-of-process runner (lmxopcua#780 P1)
v2-ci / unit-tests (push) Failing after 1h12m10s
v2-ci / unit-tests (push) Failing after 1h12m10s
Second P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md; still opt-in behind Scripting:Runner=OutOfProcess (InProcess unchanged). - Compiled-PE cache (design §7, §14 #8): Scripting:Worker:PeCacheDir holds <dir>/pe/<id>.pe entries authenticated with HMAC-SHA256 under a per-node random key (<dir>/pe-cache.key, created by the host, 0600/0700 on Unix), handed to the worker in Hello. Entry id = SHA-256(compile fingerprint + kind + source); the fingerprint covers the Roslyn version, the Core.Scripting build (analyzer, wrapper, stack guard), the sandbox reference set and the runtime. Edited, truncated, other-key or renamed entries are rejected, deleted and recompiled; the worker never loads unverified bytes. Pruned by recency at startup (30 d / 50 000). ScriptEvaluator splits into CompileToImage + FromImage. Measured on 40 scripts, restart to every first result: macOS 820 ms no cache / 840 ms empty / 85 ms warm; windev (Windows) 9.5 s / 9.4 s / 1.0 s. - Quarantine race: WorkerConnection runs exit attribution synchronously BEFORE completing the evaluations/loads it held (and before Exited), and the shard passes it straight to the runner — a kill strike is in place before the killed key's slot is released. - Supervision: a dead worker restarts by itself after its backoff; Unavailable outcomes carry ScriptRunOutcome.Recovery (-> VirtualTagEvalResult.RetryWhen) and the VT actor / Calculation driver re-evaluate once against current inputs when it completes. Isolation worker stops after Scripting:Worker:IsolationIdleTimeout (10 min). Keys compiling when a worker dies are suspects too (isolation finds a poison compile and quarantines it). Startup orphan sweep by launch marker: kills only a process whose marker's host is gone and whose pid + start time + image match. - Crash-loop circuit (Scripting:Worker:CircuitOpenAfter, 2 min): runner Unavailable, one Error, evaluations answered at once while shards keep retrying at the cap. - Observability (§10): otopcua.script.runner.state, worker.up/inflight/queue_depth/heap_bytes (shard), ipc.roundtrip + eval.duration (kind), compile{cache}, isolation.suspects, exits reason=idle. `script-runner` health check on the ready tier, at worst Degraded (§14 #12: degraded but still ready). Left on #780: self-contained worker publish, installer (account, pipe ACL, firewall), docker-dev live drill, scripted-alarm seam, the §4.5 per-seam batch submit, the §8 perf harness. Tests: SupervisionTests, PeCacheStoreTests/PeCacheWorkerTests (real workers), ScriptRunnerHealthCheckTests, VT + Calc recovery seam tests; 14 mutations each turned their test red. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,79 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 design §10, owner decision §14 #12: the script runner is a READY-tier component that is degraded but
|
||||
/// still ready when its circuit is open — a sick runner must never fail readiness (on a redundant pair that would move
|
||||
/// ServiceLevel to the peer over a script problem while OPC UA still serves driver data).
|
||||
/// </summary>
|
||||
public sealed class ScriptRunnerHealthCheckTests
|
||||
{
|
||||
private static ScriptWorkerShardStatus Shard(string label, bool up, double? downSeconds = null)
|
||||
=> new(label, label == "iso", up, downSeconds is { } s ? TimeSpan.FromSeconds(s) : null, up ? 1234 : null, up ? 0 : 3, 0, 0, 0);
|
||||
|
||||
private static ScriptRunnerStatus Status(ScriptRunnerState state, params ScriptWorkerShardStatus[] shards)
|
||||
=> new(state, shards, SuspectCount: 0, QuarantinedCount: 0, PeCacheEnabled: true);
|
||||
|
||||
[Fact]
|
||||
public void In_process_is_healthy()
|
||||
=> ScriptRunnerHealthDecision.Evaluate(null).Status.ShouldBe(HealthStatus.Healthy);
|
||||
|
||||
[Fact]
|
||||
public void Ready_is_healthy()
|
||||
=> ScriptRunnerHealthDecision.Evaluate(Status(ScriptRunnerState.Ready, Shard("0", true), Shard("1", true))).Status
|
||||
.ShouldBe(HealthStatus.Healthy);
|
||||
|
||||
[Fact]
|
||||
public void A_shard_down_is_degraded_and_names_it()
|
||||
{
|
||||
var (status, description, data) = ScriptRunnerHealthDecision.Evaluate(
|
||||
Status(ScriptRunnerState.Degraded, Shard("0", true), Shard("1", false, 4)));
|
||||
status.ShouldBe(HealthStatus.Degraded);
|
||||
description.ShouldContain("ScriptWorker 1 down");
|
||||
data["shard.1"].ShouldBe("down 4 s (3 failure(s))");
|
||||
data["state"].ShouldBe("Degraded");
|
||||
}
|
||||
|
||||
/// <summary>★ decision 12: the circuit-open runner is Degraded — never Unhealthy, so /health/ready stays 200.</summary>
|
||||
[Fact]
|
||||
public void An_open_circuit_is_degraded_never_unhealthy()
|
||||
{
|
||||
var (status, description, _) = ScriptRunnerHealthDecision.Evaluate(
|
||||
Status(ScriptRunnerState.Unavailable, Shard("0", false, 200), Shard("1", false, 200)));
|
||||
status.ShouldBe(HealthStatus.Degraded);
|
||||
description.ShouldContain("circuit open");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, false)]
|
||||
[InlineData(true, true)]
|
||||
public async Task It_is_registered_on_the_ready_tier_only_on_driver_nodes(bool hasDriver, bool registered)
|
||||
{
|
||||
var registrations = new ServiceCollection()
|
||||
.AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver)
|
||||
.BuildServiceProvider()
|
||||
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
||||
.Value.Registrations
|
||||
.Where(r => r.Name == ScriptRunnerHealthCheck.Name)
|
||||
.ToList();
|
||||
if (!registered)
|
||||
{
|
||||
registrations.ShouldBeEmpty();
|
||||
return;
|
||||
}
|
||||
var registration = registrations.ShouldHaveSingleItem();
|
||||
registration.Tags.ShouldBe([ZbHealthTags.Ready]);
|
||||
// A node without a runner selection (in-process, or a harness) resolves it as Healthy.
|
||||
var check = registration.Factory(new ServiceCollection().BuildServiceProvider()).ShouldBeOfType<ScriptRunnerHealthCheck>();
|
||||
(await check.CheckHealthAsync(new HealthCheckContext { Registration = registration }, CancellationToken.None))
|
||||
.Status.ShouldBe(HealthStatus.Healthy);
|
||||
}
|
||||
}
|
||||
+30
@@ -54,6 +54,36 @@ public sealed class VirtualTagActorRunnerUnavailableTests : RuntimeActorTestBase
|
||||
scriptFault.Value.ShouldBe(5);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Design §9: a victim of a worker restart is re-evaluated when the runner is back, with the latest inputs — not
|
||||
/// left Bad until an input happens to change (a static input never does). Mutation check: without the actor
|
||||
/// arming <see cref="VirtualTagEvalResult.RetryWhen"/>, no second result arrives.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_runner_unavailable_result_is_re_evaluated_once_when_the_runner_recovers()
|
||||
{
|
||||
var parent = CreateTestProbe();
|
||||
var evaluator = new QueuedEvaluator();
|
||||
var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Ok(5));
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Unavailable("script worker restarted") with { RetryWhen = recovered.Task });
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Ok(7));
|
||||
var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "x", evaluator: evaluator, dependencyRefs: ["a"]));
|
||||
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
|
||||
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Value.ShouldBe(5);
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
|
||||
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget).StatusCode.ShouldBe(OpcUaStatus.BadResourceUnavailable);
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // nothing re-runs before the runner is back
|
||||
|
||||
recovered.SetResult();
|
||||
var again = parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget);
|
||||
again.StatusCode.ShouldBe(OpcUaStatus.Good, "re-evaluated on recovery, with no new input");
|
||||
again.Value.ShouldBe(7);
|
||||
evaluator.Results.ShouldBeEmpty();
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // exactly one re-evaluation
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failure_without_a_status_keeps_BadInternalError()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 design §7 (owner decision §14 #8): the node-local HMAC'd compiled-PE cache. The store is tested
|
||||
/// directly (authentication, eviction, keying); the worker-backed cases prove a restart loads from it instead of
|
||||
/// recompiling, and that the worker never runs bytes that did not verify.
|
||||
/// </summary>
|
||||
public sealed class PeCacheStoreTests
|
||||
{
|
||||
private static readonly byte[] Fingerprint = SHA256.HashData("fp"u8);
|
||||
|
||||
private static (ScriptPeCacheStore Store, Scripts.TempDir Dir) NewStore()
|
||||
{
|
||||
var dir = new Scripts.TempDir();
|
||||
var key = ScriptPeCacheStore.LoadOrCreateKey(dir.Path);
|
||||
return (new ScriptPeCacheStore(dir.Path, key), dir);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_written_entry_reads_back_and_an_absent_one_misses()
|
||||
{
|
||||
var (store, dir) = NewStore();
|
||||
using var cleanup = dir;
|
||||
var id = ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 1;");
|
||||
store.TryRead(id, out _).ShouldBe(PeCacheLookup.Miss);
|
||||
store.TryWrite(id, [1, 2, 3, 4]).ShouldBeTrue();
|
||||
store.TryRead(id, out var pe).ShouldBe(PeCacheLookup.Hit);
|
||||
pe.ShouldBe(new byte[] { 1, 2, 3, 4 });
|
||||
}
|
||||
|
||||
/// <summary>Every way an entry can be made to lie is rejected AND deleted. Positive control: the untouched entry hits.</summary>
|
||||
[Theory]
|
||||
[InlineData("flip-pe")]
|
||||
[InlineData("flip-tag")]
|
||||
[InlineData("truncate")]
|
||||
[InlineData("magic")]
|
||||
[InlineData("other-key")]
|
||||
[InlineData("renamed")]
|
||||
public void A_tampered_or_mismatched_entry_is_rejected_and_evicted(string tamper)
|
||||
{
|
||||
var (store, dir) = NewStore();
|
||||
using var cleanup = dir;
|
||||
var id = ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 1;");
|
||||
var otherId = ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 2;");
|
||||
store.TryWrite(id, [10, 20, 30, 40, 50]).ShouldBeTrue();
|
||||
store.TryRead(id, out _).ShouldBe(PeCacheLookup.Hit, "positive control");
|
||||
var path = store.EntryPath(id);
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
var readId = id;
|
||||
switch (tamper)
|
||||
{
|
||||
case "flip-pe": bytes[^1] ^= 0xFF; File.WriteAllBytes(path, bytes); break;
|
||||
case "flip-tag": bytes[10] ^= 0x01; File.WriteAllBytes(path, bytes); break;
|
||||
case "truncate": File.WriteAllBytes(path, bytes[..(bytes.Length - 2)]); break;
|
||||
case "magic": bytes[0] = (byte)'X'; File.WriteAllBytes(path, bytes); break;
|
||||
case "other-key":
|
||||
new ScriptPeCacheStore(dir.Path, RandomNumberGenerator.GetBytes(32)).TryWrite(id, [10, 20, 30, 40, 50]).ShouldBeTrue();
|
||||
break;
|
||||
case "renamed":
|
||||
// A valid entry for one source copied under another source's id: the tag binds the id.
|
||||
File.Move(path, store.EntryPath(otherId));
|
||||
path = store.EntryPath(otherId);
|
||||
readId = otherId;
|
||||
break;
|
||||
}
|
||||
|
||||
store.TryRead(readId, out var pe).ShouldBe(PeCacheLookup.Rejected);
|
||||
pe.ShouldBeNull();
|
||||
File.Exists(path).ShouldBeFalse("a rejected entry is evicted");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_entry_id_changes_with_the_compiler_fingerprint_the_kind_and_the_source()
|
||||
{
|
||||
var baseline = ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 1;");
|
||||
ScriptPeCacheStore.EntryId(SHA256.HashData("fp2"u8), 1, "return 1;").ShouldNotBe(baseline, "a compiler/analyzer change misses");
|
||||
ScriptPeCacheStore.EntryId(Fingerprint, 2, "return 1;").ShouldNotBe(baseline);
|
||||
ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 1; ").ShouldNotBe(baseline);
|
||||
ScriptPeCacheStore.EntryId(Fingerprint, 1, "return 1;").ShouldBe(baseline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_compile_fingerprint_covers_the_context_and_result_types()
|
||||
{
|
||||
ScriptEvaluator<VirtualTagContext, object?>.CacheFingerprint.Length.ShouldBe(32);
|
||||
ScriptEvaluator<VirtualTagContext, object?>.CacheFingerprint
|
||||
.ShouldNotBe(ScriptEvaluator<ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.AlarmPredicateContext, bool>.CacheFingerprint);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_key_is_created_once_private_and_reused()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
var key = ScriptPeCacheStore.LoadOrCreateKey(dir.Path);
|
||||
key.Length.ShouldBe(ScriptPeCacheStore.KeyLength);
|
||||
ScriptPeCacheStore.LoadOrCreateKey(dir.Path).ShouldBe(key, "the second start reuses it");
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
File.GetUnixFileMode(Path.Combine(dir.Path, ScriptPeCacheStore.KeyFileName))
|
||||
.ShouldBe(UnixFileMode.UserRead | UnixFileMode.UserWrite, "readable only by the host account");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_malformed_key_is_refused_not_silently_replaced()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
File.WriteAllBytes(Path.Combine(dir.Path, ScriptPeCacheStore.KeyFileName), [1, 2, 3]);
|
||||
Should.Throw<IOException>(() => ScriptPeCacheStore.LoadOrCreateKey(dir.Path));
|
||||
File.ReadAllBytes(Path.Combine(dir.Path, ScriptPeCacheStore.KeyFileName)).Length.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Prune_drops_stale_entries_then_the_least_recently_used()
|
||||
{
|
||||
var (store, dir) = NewStore();
|
||||
using var cleanup = dir;
|
||||
var ids = Enumerable.Range(0, 5).Select(i => ScriptPeCacheStore.EntryId(Fingerprint, 1, "s" + i)).ToArray();
|
||||
for (var i = 0; i < ids.Length; i++)
|
||||
{
|
||||
store.TryWrite(ids[i], [1]).ShouldBeTrue();
|
||||
File.SetLastWriteTimeUtc(store.EntryPath(ids[i]), DateTime.UtcNow.AddDays(-i * 10));
|
||||
}
|
||||
store.Prune(maxAge: TimeSpan.FromDays(25), maxEntries: 2).ShouldBe(3);
|
||||
File.Exists(store.EntryPath(ids[0])).ShouldBeTrue();
|
||||
File.Exists(store.EntryPath(ids[1])).ShouldBeTrue();
|
||||
File.Exists(store.EntryPath(ids[2])).ShouldBeFalse("LRU beyond the cap");
|
||||
File.Exists(store.EntryPath(ids[3])).ShouldBeFalse("older than the max age");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The PE cache through real workers.</summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class PeCacheWorkerTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static string Source(int i) => $"var x = (int)ctx.GetTag(\"x\").Value; var acc = 0L; for (var j = 0; j < 10; j++) acc += x * {i} + j; return acc;";
|
||||
|
||||
private static long Expected(int i) => Enumerable.Range(0, 10).Sum(j => 3L * i + j);
|
||||
|
||||
/// <summary>Time from a fresh runner (fresh worker process) to the first result of every one of <paramref name="count"/> scripts.</summary>
|
||||
private static async Task<(TimeSpan Elapsed, (long Hits, long Misses, long Rejected) Counts)> ColdStart(string? cacheDir, int count)
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(peCacheDir: cacheDir));
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("vt-" + i, Source(i), inputs: Scripts.Input("x", 3)), Ct);
|
||||
outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason);
|
||||
outcome.Value.ShouldBe(Expected(i));
|
||||
}
|
||||
return (sw.Elapsed, runner.PeCacheCounts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The measurement the slice asks for: worker restart to first result for N scripts — with no cache (every
|
||||
/// start recompiles), with an empty cache (compiles and writes) and with a warm cache (loads only). The warm
|
||||
/// start must serve every script from the cache and beat the cold one. Mutation check: a worker that ignores the
|
||||
/// cache reports misses on the warm start.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_warm_restart_loads_every_script_from_the_cache_and_is_faster_than_a_cold_one()
|
||||
{
|
||||
const int count = 40;
|
||||
using var dir = new Scripts.TempDir();
|
||||
var cacheDir = Path.Combine(dir.Path, "pe-cache");
|
||||
|
||||
var uncached = await ColdStart(null, count);
|
||||
var cold = await ColdStart(cacheDir, count);
|
||||
var warm = await ColdStart(cacheDir, count);
|
||||
|
||||
TestContext.Current.TestOutputHelper?.WriteLine(
|
||||
$"{count} scripts, restart to all first results: no cache {uncached.Elapsed.TotalMilliseconds:F0} ms; " +
|
||||
$"empty cache {cold.Elapsed.TotalMilliseconds:F0} ms; warm cache {warm.Elapsed.TotalMilliseconds:F0} ms");
|
||||
|
||||
uncached.Counts.ShouldBe((0L, 0L, 0L), "no cache configured");
|
||||
cold.Counts.ShouldBe((0L, (long)count, 0L), "the first start compiles and writes every entry");
|
||||
warm.Counts.ShouldBe(((long)count, 0L, 0L), "the restart loads every script from the cache");
|
||||
warm.Elapsed.ShouldBeLessThan(cold.Elapsed, "loading a PE is cheaper than entering Roslyn");
|
||||
Directory.EnumerateFiles(Path.Combine(cacheDir, ScriptPeCacheStore.EntriesDirectoryName), "*.pe").Count().ShouldBe(count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The security property: a cache entry replaced with a DIFFERENT, validly framed PE (one that returns 666) under
|
||||
/// a forged tag is never run — the worker rejects it, deletes it, recompiles the real source and returns 1.
|
||||
/// Positive control: the forged image itself runs and returns 666 when loaded directly.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_forged_cache_entry_is_rejected_and_the_real_source_recompiled()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
var cacheDir = Path.Combine(dir.Path, "pe-cache");
|
||||
await using (var first = Scripts.Runner(Scripts.Options(peCacheDir: cacheDir)))
|
||||
(await first.RunAsync(Scripts.Vt("vt", "return 1;"), Ct)).Value.ShouldBe(1);
|
||||
var entry = Directory.EnumerateFiles(Path.Combine(cacheDir, ScriptPeCacheStore.EntriesDirectoryName), "*.pe").ShouldHaveSingleItem();
|
||||
|
||||
var evil = ScriptEvaluator<VirtualTagContext, object?>.CompileToImage("return 666;");
|
||||
using (var direct = ScriptEvaluator<VirtualTagContext, object?>.FromImage(evil))
|
||||
{
|
||||
(await direct.RunAsync(new VirtualTagContext(Scripts.NoInputs, static (_, _) => { }, Serilog.Core.Logger.None, () => DateTime.UtcNow), Ct))
|
||||
.ShouldBe(666, "positive control: the forged image is a working script");
|
||||
}
|
||||
var original = File.ReadAllBytes(entry);
|
||||
var forged = new byte[8 + 32 + evil.Length];
|
||||
original.AsSpan(0, 8).CopyTo(forged); // the real magic
|
||||
RandomNumberGenerator.GetBytes(32).CopyTo(forged, 8); // a tag the attacker cannot compute
|
||||
evil.CopyTo(forged, 40);
|
||||
File.WriteAllBytes(entry, forged);
|
||||
|
||||
await using var second = Scripts.Runner(Scripts.Options(peCacheDir: cacheDir));
|
||||
var outcome = await second.RunAsync(Scripts.Vt("vt", "return 1;"), Ct);
|
||||
outcome.Value.ShouldBe(1, "the forged bytes never ran");
|
||||
second.PeCacheCounts.Rejected.ShouldBe(1);
|
||||
var key = File.ReadAllBytes(Path.Combine(cacheDir, ScriptPeCacheStore.KeyFileName));
|
||||
new ScriptPeCacheStore(cacheDir, key).TryRead(Path.GetFileNameWithoutExtension(entry), out _)
|
||||
.ShouldBe(PeCacheLookup.Hit, "the entry was rewritten from the real compile");
|
||||
}
|
||||
|
||||
/// <summary>A compile error is never cached (the host's negative cache covers it) and does not poison the directory.</summary>
|
||||
[Fact]
|
||||
public async Task A_compile_error_writes_no_entry()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
var cacheDir = Path.Combine(dir.Path, "pe-cache");
|
||||
await using var runner = Scripts.Runner(Scripts.Options(peCacheDir: cacheDir));
|
||||
(await runner.RunAsync(Scripts.Vt("bad", "return nope;"), Ct)).Reason!.ShouldStartWith("compile error:");
|
||||
Directory.EnumerateFiles(Path.Combine(cacheDir, ScriptPeCacheStore.EntriesDirectoryName), "*.pe").ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>An unusable cache directory is a Warning and a cache-less runner, never a failure.</summary>
|
||||
[Fact]
|
||||
public async Task An_unusable_cache_directory_degrades_to_no_cache()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
var blocker = Path.Combine(dir.Path, "a-file");
|
||||
File.WriteAllText(blocker, "not a directory");
|
||||
var sink = new CapturingSink();
|
||||
await using var runner = Scripts.Runner(Scripts.Options(peCacheDir: Path.Combine(blocker, "pe-cache")), sink.CreateLogger());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", "return 5;"), Ct)).Value.ShouldBe(5);
|
||||
runner.Status().PeCacheEnabled.ShouldBeFalse();
|
||||
sink.Rendered.ShouldContain(m => m.Contains("is unusable"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 P1 supervision gaps: the quarantine race, victim re-evaluation on a shard's return, the idle
|
||||
/// isolation worker, the startup orphan sweep, culprit-finding for a compile that crashes its worker, and the
|
||||
/// runner state / crash-loop circuit. Real worker processes throughout.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class SupervisionTests
|
||||
{
|
||||
private const string Healthy = "return 7;";
|
||||
private const string Runaway = "while (true) { }";
|
||||
private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
|
||||
|
||||
private static IReadOnlyDictionary<string, DataValueSnapshot> Ms(int ms) => Scripts.Input("ms", ms);
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static string VtKey(string key) => InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, key);
|
||||
|
||||
// ---------------------------------------------------------------- the quarantine race
|
||||
|
||||
/// <summary>
|
||||
/// The ordering invariant behind the race fix, pinned deterministically at the connection: the exit attribution
|
||||
/// callback runs BEFORE any outstanding evaluation or compile is completed and before <c>Exited</c> is signalled.
|
||||
/// Mutation check: moving the attribution call after the completion loop in <c>WorkerConnection.Finish</c> turns
|
||||
/// both assertions red.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Exit_attribution_runs_before_any_outstanding_evaluation_completes()
|
||||
{
|
||||
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions()),
|
||||
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromSeconds(1),
|
||||
HandshakeTimeout: TimeSpan.FromSeconds(15));
|
||||
PendingEval? pending = null;
|
||||
WorkerConnection? connection = null;
|
||||
bool? pendingCompletedAtAttribution = null, exitedCompletedAtAttribution = null;
|
||||
IReadOnlyList<string>? inFlightAtAttribution = null;
|
||||
connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, exit =>
|
||||
{
|
||||
pendingCompletedAtAttribution = pending?.Completion.IsCompleted;
|
||||
exitedCompletedAtAttribution = connection?.Exited.IsCompleted;
|
||||
inFlightAtAttribution = exit.InFlightKeys;
|
||||
}, Ct);
|
||||
await using var _ = connection;
|
||||
|
||||
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
||||
System.Security.Cryptography.SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
||||
entry!.Outcome!.Ok.ShouldBeTrue();
|
||||
pending = connection.Submit("k", new EvalRequest { Key = "k", SoftTimeoutMs = 60_000, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks });
|
||||
await Scripts.Eventually(() => connection.LastHeartbeat is { Inflight: 1 }, TimeSpan.FromSeconds(10), "the run in flight");
|
||||
|
||||
Process.GetProcessById(connection.Pid).Kill();
|
||||
await connection.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
(await pending.Completion.WaitAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull("the worker went away");
|
||||
|
||||
inFlightAtAttribution.ShouldBe(["k"]);
|
||||
pendingCompletedAtAttribution.ShouldBe(false, "attribution must precede the evaluation's completion");
|
||||
exitedCompletedAtAttribution.ShouldBe(false, "attribution must precede Exited");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The other half of the race fix, pinned without timing: the shard hands the connection an attribution callback
|
||||
/// that reaches the runner's exit handler SYNCHRONOUSLY (the pre-fix shard called it from an Exited continuation,
|
||||
/// after the evaluations had already been completed). Mutation check: launching with a no-op callback and
|
||||
/// attributing from the Exited continuation instead leaves <c>attributed</c> false when the callback returns.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_shard_attributes_an_exit_synchronously_inside_the_connections_callback()
|
||||
{
|
||||
Action<WorkerExit>? captured = null;
|
||||
var attributed = false;
|
||||
await using var shard = new WorkerShard("t", isolation: false,
|
||||
(onExiting, _) =>
|
||||
{
|
||||
captured = onExiting;
|
||||
throw new InvalidOperationException("no process in this test");
|
||||
},
|
||||
(_, _) => attributed = true,
|
||||
new LoggerConfiguration().CreateLogger());
|
||||
(await shard.GetReadyAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull();
|
||||
|
||||
captured.ShouldNotBeNull();
|
||||
captured(new WorkerExit(WorkerExitReason.Crash, null, 1, ["k"]));
|
||||
attributed.ShouldBeTrue("the runner's verdict is recorded before the callback returns");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The race itself, end to end: with QuarantineAfterKills = 1, the moment the killed script's single-flight slot is
|
||||
/// released the quarantine is already in place — so no trigger can re-run it in the restart gap, and the next
|
||||
/// trigger is refused without any wait. Repeated, since the old ordering lost this race only some of the time.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_killed_script_is_quarantined_before_its_slot_is_released()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(250), quarantineAfterKills: 1));
|
||||
var observed = new ConcurrentQueue<bool>();
|
||||
runner.SlotReleasingForTest = key =>
|
||||
{
|
||||
if (key == VtKey("runaway")) observed.Enqueue(runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Contains("runaway"));
|
||||
};
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
|
||||
for (var round = 1; round <= 3; round++)
|
||||
{
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
(await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(250)), Ct)).Reason!
|
||||
.ShouldStartWith("script timed out after ");
|
||||
await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
await Scripts.Eventually(() => observed.Count == round, TimeSpan.FromSeconds(5), "the slot release");
|
||||
|
||||
// No Eventually: the very next trigger after the kill is refused.
|
||||
var next = await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(250)), Ct);
|
||||
next.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
runner.Reset(ScriptContextKind.VirtualTag);
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
}
|
||||
observed.ShouldAllBe(quarantined => quarantined, "the slot was released before the kill strike was recorded");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- victims re-evaluated on return
|
||||
|
||||
/// <summary>
|
||||
/// A victim's Unavailable outcome carries a Recovery task that completes when its shard is back — and the shard
|
||||
/// comes back on its own (proactive restart), with no further trigger. Mutation check: without the shard's
|
||||
/// recovery loop, Recovery never completes and no worker is running.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_victims_recovery_completes_when_its_shard_restarts_by_itself()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("victim", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var inFlight = runner.RunAsync(Scripts.Vt("sibling", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
||||
await Scripts.Eventually(() => worker.InFlightCount == 1, TimeSpan.FromSeconds(10), "the run in flight");
|
||||
Process.GetProcessById(worker.Pid).Kill();
|
||||
|
||||
var victim = await inFlight.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
victim.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
_ = victim.Recovery.ShouldNotBeNull("a runner failure tells the seam when to re-evaluate");
|
||||
victim.ToVirtualTagEvalResult().RetryWhen.ShouldBeSameAs(victim.Recovery);
|
||||
|
||||
// The crash made "sibling" a suspect, so its recovery is the isolation worker's; the crashed shard restarts by
|
||||
// itself as well, so the scripts placed on it find it warm.
|
||||
await victim.Recovery!.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
||||
runner.Isolation.Current.ShouldNotBeNull("recovered without any new trigger");
|
||||
await Scripts.Eventually(() => runner.Shards[0].Current, TimeSpan.FromSeconds(15), "the shard back");
|
||||
runner.Shards[0].Current!.Pid.ShouldNotBe(worker.Pid);
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the idle isolation worker
|
||||
|
||||
/// <summary>
|
||||
/// The isolation worker stops after IsolationIdleTimeout with nothing to run — an Idle exit, not a failure — and
|
||||
/// starts again for the next suspect. Mutation check: with the maintenance pass not stopping it, it stays up.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task An_idle_isolation_worker_is_stopped_and_restarts_on_demand()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(isolationIdle: TimeSpan.FromSeconds(1)),
|
||||
maintenanceInterval: TimeSpan.FromMilliseconds(200));
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
var run = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
||||
await Scripts.Eventually(() => worker.InFlightCount == 1, TimeSpan.FromSeconds(10), "the run in flight");
|
||||
Process.GetProcessById(worker.Pid).Kill();
|
||||
(await run.WaitAsync(TimeSpan.FromSeconds(10), Ct)).Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
runner.SuspectKeys.ShouldBe([VtKey("spin")]);
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
var iso = runner.Isolation.Current.ShouldNotBeNull();
|
||||
|
||||
var exit = await iso.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
exit.Reason.ShouldBe(WorkerExitReason.Idle);
|
||||
runner.Isolation.Current.ShouldBeNull();
|
||||
runner.Isolation.ConsecutiveFailures.ShouldBe(0, "an idle stop is not a failure");
|
||||
runner.Isolation.DownFor.ShouldBeNull("an idle stop does not make the isolation worker 'down'");
|
||||
await Scripts.Eventually(() => runner.State == ScriptRunnerState.Ready, TimeSpan.FromSeconds(15), "the runner ready again");
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
runner.Isolation.Current.ShouldNotBeNull().Pid.ShouldNotBe(iso.Pid, "a fresh isolation worker for the next suspect run");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the orphan sweep
|
||||
|
||||
private static Process StartDecoyWorker()
|
||||
{
|
||||
// A real ScriptWorker that will sit waiting (up to 10 s) for a pipe that does not exist: the shape of a worker
|
||||
// its host abandoned.
|
||||
var psi = new ProcessStartInfo(WorkerConnection.ResolveMuxer())
|
||||
{
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
};
|
||||
psi.ArgumentList.Add("exec");
|
||||
psi.ArgumentList.Add(OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions()));
|
||||
psi.Environment[ScriptWorkerEnvironment.PipeName] = "otopcua-sw-d" + Guid.NewGuid().ToString("N")[..8];
|
||||
psi.Environment[ScriptWorkerEnvironment.Nonce] = "decoy";
|
||||
var p = Process.Start(psi)!;
|
||||
Thread.Sleep(300);
|
||||
if (p.HasExited) throw new InvalidOperationException($"the decoy worker exited {p.ExitCode}: {p.StandardError.ReadToEnd()}");
|
||||
return p;
|
||||
}
|
||||
|
||||
private static (int Pid, long StartTicks) DeadProcess()
|
||||
{
|
||||
var psi = new ProcessStartInfo(WorkerConnection.ResolveMuxer(), "--version")
|
||||
{
|
||||
UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true,
|
||||
};
|
||||
using var p = Process.Start(psi)!;
|
||||
var start = p.StartTime.ToUniversalTime().Ticks;
|
||||
var pid = p.Id;
|
||||
p.WaitForExit();
|
||||
return (pid, start);
|
||||
}
|
||||
|
||||
private static string WriteMarker(string dir, Process worker, int hostPid, long hostStartTicks, long? workerStartTicks = null)
|
||||
{
|
||||
Directory.CreateDirectory(dir);
|
||||
var record = new WorkerLaunchMarker.MarkerRecord(worker.Id, workerStartTicks ?? worker.StartTime.ToUniversalTime().Ticks,
|
||||
worker.ProcessName, hostPid, hostStartTicks, "decoy");
|
||||
var path = Path.Combine(dir, worker.Id + ".worker");
|
||||
File.WriteAllText(path, JsonSerializer.Serialize(record));
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Design §3.3: a worker left behind by a DEAD host of this instance is killed at the next start — through the
|
||||
/// runner's constructor, so the wiring is pinned too. Mutation check: without the Sweep call the decoy survives.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_startup_sweep_kills_a_worker_whose_host_is_gone()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
using var decoy = StartDecoyWorker();
|
||||
var (deadPid, deadStart) = DeadProcess();
|
||||
var marker = WriteMarker(dir.Path, decoy, deadPid, deadStart);
|
||||
|
||||
await using var runner = Scripts.Runner(Scripts.Options(), markerDirectory: dir.Path);
|
||||
(await Task.Run(() => decoy.WaitForExit(5000), Ct)).ShouldBeTrue("the orphan was killed");
|
||||
File.Exists(marker).ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Never an unrelated process: a marker whose host is ALIVE (another instance, or a sibling runner in this
|
||||
/// process) is left alone, and a marker whose pid now belongs to a different process (start time differs —
|
||||
/// pid reuse) never kills it. Mutation checks: dropping either the host-alive check or the start-time check
|
||||
/// kills the decoy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void The_sweep_never_kills_a_live_hosts_worker_or_a_reused_pid()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
using var decoy = StartDecoyWorker();
|
||||
using var self = Process.GetCurrentProcess();
|
||||
try
|
||||
{
|
||||
var liveHostMarker = WriteMarker(dir.Path, decoy, self.Id, self.StartTime.ToUniversalTime().Ticks);
|
||||
WorkerLaunchMarker.Sweep(dir.Path, new LoggerConfiguration().CreateLogger()).ShouldBeEmpty();
|
||||
decoy.HasExited.ShouldBeFalse("a live host's worker is not an orphan");
|
||||
File.Exists(liveHostMarker).ShouldBeTrue("the live host still owns its marker");
|
||||
|
||||
var (deadPid, deadStart) = DeadProcess();
|
||||
var reused = WriteMarker(dir.Path, decoy, deadPid, deadStart,
|
||||
workerStartTicks: decoy.StartTime.ToUniversalTime().AddHours(-1).Ticks);
|
||||
WorkerLaunchMarker.Sweep(dir.Path, new LoggerConfiguration().CreateLogger()).ShouldBeEmpty();
|
||||
decoy.HasExited.ShouldBeFalse("a pid whose start time does not match is some other process");
|
||||
File.Exists(reused).ShouldBeFalse("a stale marker is cleaned up");
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { decoy.Kill(); } catch { /* already gone */ }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A worker's marker exists while it runs and is removed when it stops cleanly.</summary>
|
||||
[Fact]
|
||||
public async Task A_running_worker_has_a_launch_marker_that_its_clean_stop_removes()
|
||||
{
|
||||
using var dir = new Scripts.TempDir();
|
||||
var runner = Scripts.Runner(Scripts.Options(), markerDirectory: dir.Path);
|
||||
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var pid = runner.Shards[0].Current!.Pid;
|
||||
var marker = Path.Combine(dir.Path, pid + ".worker");
|
||||
File.Exists(marker).ShouldBeTrue();
|
||||
JsonSerializer.Deserialize<WorkerLaunchMarker.MarkerRecord>(File.ReadAllText(marker))!.HostPid.ShouldBe(Environment.ProcessId);
|
||||
await runner.DisposeAsync();
|
||||
File.Exists(marker).ShouldBeFalse();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- a compile that crashes its worker
|
||||
|
||||
/// <summary>
|
||||
/// Design §5.3.5 culprit-finding: a worker that dies while a script is COMPILING (no evaluation in flight) makes
|
||||
/// that script a suspect; it then compiles alone in the isolation worker, where a second death is unambiguous and
|
||||
/// quarantines it — so a poison source cannot crash-loop its shard forever. Mutation check: leaving the compiling
|
||||
/// keys out of the attribution leaves the key unsuspected and unquarantined.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_script_whose_compile_keeps_crashing_its_worker_is_found_in_isolation_and_quarantined()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(quarantineAfterKills: 1));
|
||||
var body = new StringBuilder("long x = 0;\n");
|
||||
for (var i = 0; i < 3000; i++) body.Append("x = x * 31 + ").Append(i).Append(";\n");
|
||||
body.Append("return x;");
|
||||
var poison = body.ToString();
|
||||
|
||||
async Task<ScriptRunOutcome> CrashDuringCompile(Func<WorkerConnection?> worker)
|
||||
{
|
||||
var run = runner.RunAsync(Scripts.Vt("poison", poison), Ct);
|
||||
var live = await Scripts.Eventually(worker, TimeSpan.FromSeconds(15), "a live worker");
|
||||
await Scripts.Eventually(() => live.LoadingCount == 1, TimeSpan.FromSeconds(15), "the compile in flight");
|
||||
live.InFlightCount.ShouldBe(0, "no evaluation: only the compile is in flight");
|
||||
Process.GetProcessById(live.Pid).Kill();
|
||||
return await run.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
||||
}
|
||||
|
||||
var first = await CrashDuringCompile(() => runner.Shards[0].Current);
|
||||
first.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
runner.SuspectKeys.ShouldBe([VtKey("poison")], "a death during its compile makes it a suspect");
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("ambiguous on a shard: no strike");
|
||||
|
||||
await CrashDuringCompile(() => runner.Isolation.Current);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["poison"]);
|
||||
var refused = await runner.RunAsync(Scripts.Vt("poison", poison), Ct);
|
||||
refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
refused.Reason.ShouldContain("compile crashed the isolation worker");
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- state, circuit, gauges
|
||||
|
||||
[Theory]
|
||||
[InlineData(new double[] { }, null, ScriptRunnerState.Ready)]
|
||||
[InlineData(new double[] { -1, -1 }, null, ScriptRunnerState.Ready)]
|
||||
[InlineData(new double[] { -1, 5 }, null, ScriptRunnerState.Degraded)]
|
||||
[InlineData(new double[] { -1, -1 }, 5.0, ScriptRunnerState.Degraded)]
|
||||
[InlineData(new double[] { 200, 5 }, null, ScriptRunnerState.Degraded)]
|
||||
[InlineData(new double[] { 200, 130 }, null, ScriptRunnerState.Unavailable)]
|
||||
public void The_runner_state_rule(double[] shardsDownSeconds, double? isolationDownSeconds, ScriptRunnerState expected)
|
||||
{
|
||||
var shards = shardsDownSeconds.Select(s => s < 0 ? (TimeSpan?)null : TimeSpan.FromSeconds(s)).ToArray();
|
||||
var iso = isolationDownSeconds is { } i ? TimeSpan.FromSeconds(i) : (TimeSpan?)null;
|
||||
OutOfProcessScriptRunner.ComputeState(shards, iso, TimeSpan.FromMinutes(2)).ShouldBe(expected);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The crash-loop circuit: every shard down past CircuitOpenAfter makes the runner Unavailable (gauge value 2),
|
||||
/// logs one Error, and answers evaluations at once instead of after UnavailableGrace. Mutation check: without the
|
||||
/// zero grace under an open circuit, the evaluation waits the full 3 s.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Every_shard_down_opens_the_circuit_answers_at_once_and_reports_state_2()
|
||||
{
|
||||
var sink = new CapturingSink();
|
||||
var missing = Path.Combine(Path.GetTempPath(), "no-such-dir-" + Guid.NewGuid().ToString("N"), OutOfProcessScriptRunner.WorkerAssemblyFileName);
|
||||
await using var runner = Scripts.Runner(
|
||||
Scripts.Options(workerPath: missing, unavailableGrace: TimeSpan.FromSeconds(3), circuitOpenAfter: TimeSpan.FromSeconds(1)),
|
||||
sink.CreateLogger(), maintenanceInterval: TimeSpan.FromMilliseconds(100));
|
||||
runner.State.ShouldBe(ScriptRunnerState.Ready, "nothing needed yet");
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct)).Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
runner.State.ShouldNotBe(ScriptRunnerState.Ready);
|
||||
await Scripts.Eventually(() => runner.State == ScriptRunnerState.Unavailable, TimeSpan.FromSeconds(10), "the circuit open");
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var fast = await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct);
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1), "an open circuit answers without waiting out the 3 s grace");
|
||||
fast.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
fast.Reason.ShouldBe(ScriptRunOutcome.RunnerUnavailableReason);
|
||||
|
||||
await Scripts.Eventually(() => sink.Rendered.Any(m => m.Contains("circuit OPEN")), TimeSpan.FromSeconds(5), "the circuit Error");
|
||||
await Task.Delay(500, Ct);
|
||||
sink.Events.Count(e => e.Level == Serilog.Events.LogEventLevel.Error && e.RenderMessage().Contains("circuit OPEN"))
|
||||
.ShouldBe(1, "logged once, not per maintenance pass");
|
||||
|
||||
var states = new List<int>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Name == "otopcua.script.runner.state") l.EnableMeasurementEvents(instrument);
|
||||
};
|
||||
listener.SetMeasurementEventCallback<int>((_, value, _, _) => states.Add(value));
|
||||
listener.Start();
|
||||
listener.RecordObservableInstruments();
|
||||
states.ShouldContain(2);
|
||||
runner.Status().State.ShouldBe(ScriptRunnerState.Unavailable);
|
||||
}
|
||||
|
||||
/// <summary>The per-shard gauges observe the live worker (positive control: the shard is up and reports its pid).</summary>
|
||||
[Fact]
|
||||
public async Task The_shard_gauges_report_a_live_worker()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
||||
await Scripts.Eventually(() => runner.Shards[0].Current?.LastHeartbeat is not null, TimeSpan.FromSeconds(5), "a heartbeat");
|
||||
|
||||
var up = new List<(int Value, string? Shard)>();
|
||||
var heap = new List<long>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Name is "otopcua.script.worker.up" or "otopcua.script.worker.heap_bytes") l.EnableMeasurementEvents(instrument);
|
||||
};
|
||||
listener.SetMeasurementEventCallback<int>((_, value, tags, _) =>
|
||||
{
|
||||
foreach (var tag in tags) if (tag.Key == "shard") up.Add((value, tag.Value as string));
|
||||
});
|
||||
listener.SetMeasurementEventCallback<long>((_, value, _, _) => heap.Add(value));
|
||||
listener.Start();
|
||||
listener.RecordObservableInstruments();
|
||||
|
||||
up.ShouldContain((1, "0"));
|
||||
up.ShouldContain((0, "iso"), "the isolation worker is not running");
|
||||
heap.ShouldContain(h => h > 0);
|
||||
var status = runner.Status();
|
||||
status.State.ShouldBe(ScriptRunnerState.Ready);
|
||||
status.Shards.Single(s => s.Label == "0").Pid.ShouldBe(runner.Shards[0].Current!.Pid);
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,8 @@ internal static class Scripts
|
||||
|
||||
/// <summary>Options for a single-shard runner with short budgets, so the containment tests run in seconds.</summary>
|
||||
public static ScriptingOptions Options(int shards = 1, TimeSpan? hardGrace = null, int quarantineAfterKills = 2,
|
||||
int probation = 20, TimeSpan? unavailableGrace = null, string? workerPath = null, long heapLimit = 256L * 1024 * 1024)
|
||||
int probation = 20, TimeSpan? unavailableGrace = null, string? workerPath = null, long heapLimit = 256L * 1024 * 1024,
|
||||
string? peCacheDir = null, TimeSpan? isolationIdle = null, TimeSpan? circuitOpenAfter = null)
|
||||
=> new()
|
||||
{
|
||||
Runner = ScriptRunnerMode.OutOfProcess,
|
||||
@@ -51,12 +52,34 @@ internal static class Scripts
|
||||
UnavailableGrace = unavailableGrace ?? TimeSpan.FromSeconds(30),
|
||||
ExecutablePath = workerPath,
|
||||
HeapLimitBytes = heapLimit,
|
||||
PeCacheDir = peCacheDir,
|
||||
IsolationIdleTimeout = isolationIdle ?? ScriptWorkerOptions.DefaultIsolationIdleTimeout,
|
||||
CircuitOpenAfter = circuitOpenAfter ?? ScriptWorkerOptions.DefaultCircuitOpenAfter,
|
||||
},
|
||||
};
|
||||
|
||||
public static OutOfProcessScriptRunner Runner(ScriptingOptions options, ILogger? log = null)
|
||||
/// <summary>A fresh, empty temp directory, deleted by <see cref="IDisposable.Dispose"/>.</summary>
|
||||
public sealed class TempDir : IDisposable
|
||||
{
|
||||
public TempDir()
|
||||
{
|
||||
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "otopcua-sw-test-" + Guid.NewGuid().ToString("N")[..12]);
|
||||
Directory.CreateDirectory(Path);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { Directory.Delete(Path, recursive: true); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
public static OutOfProcessScriptRunner Runner(ScriptingOptions options, ILogger? log = null,
|
||||
TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
|
||||
=> new(options, log ?? new LoggerConfiguration().CreateLogger(), null,
|
||||
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60));
|
||||
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60),
|
||||
maintenanceInterval: maintenanceInterval, markerDirectory: markerDirectory);
|
||||
|
||||
public static async Task<T> Eventually<T>(Func<T?> probe, TimeSpan timeout, string what) where T : class
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user