f50b29abae
v2-ci / unit-tests (push) Failing after 1h12m10s
Second P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md; still opt-in behind Scripting:Runner=OutOfProcess (InProcess unchanged). - Compiled-PE cache (design §7, §14 #8): Scripting:Worker:PeCacheDir holds <dir>/pe/<id>.pe entries authenticated with HMAC-SHA256 under a per-node random key (<dir>/pe-cache.key, created by the host, 0600/0700 on Unix), handed to the worker in Hello. Entry id = SHA-256(compile fingerprint + kind + source); the fingerprint covers the Roslyn version, the Core.Scripting build (analyzer, wrapper, stack guard), the sandbox reference set and the runtime. Edited, truncated, other-key or renamed entries are rejected, deleted and recompiled; the worker never loads unverified bytes. Pruned by recency at startup (30 d / 50 000). ScriptEvaluator splits into CompileToImage + FromImage. Measured on 40 scripts, restart to every first result: macOS 820 ms no cache / 840 ms empty / 85 ms warm; windev (Windows) 9.5 s / 9.4 s / 1.0 s. - Quarantine race: WorkerConnection runs exit attribution synchronously BEFORE completing the evaluations/loads it held (and before Exited), and the shard passes it straight to the runner — a kill strike is in place before the killed key's slot is released. - Supervision: a dead worker restarts by itself after its backoff; Unavailable outcomes carry ScriptRunOutcome.Recovery (-> VirtualTagEvalResult.RetryWhen) and the VT actor / Calculation driver re-evaluate once against current inputs when it completes. Isolation worker stops after Scripting:Worker:IsolationIdleTimeout (10 min). Keys compiling when a worker dies are suspects too (isolation finds a poison compile and quarantines it). Startup orphan sweep by launch marker: kills only a process whose marker's host is gone and whose pid + start time + image match. - Crash-loop circuit (Scripting:Worker:CircuitOpenAfter, 2 min): runner Unavailable, one Error, evaluations answered at once while shards keep retrying at the cap. - Observability (§10): otopcua.script.runner.state, worker.up/inflight/queue_depth/heap_bytes (shard), ipc.roundtrip + eval.duration (kind), compile{cache}, isolation.suspects, exits reason=idle. `script-runner` health check on the ready tier, at worst Degraded (§14 #12: degraded but still ready). Left on #780: self-contained worker publish, installer (account, pipe ACL, firewall), docker-dev live drill, scripted-alarm seam, the §4.5 per-seam batch submit, the §8 perf harness. Tests: SupervisionTests, PeCacheStoreTests/PeCacheWorkerTests (real workers), ScriptRunnerHealthCheckTests, VT + Calc recovery seam tests; 14 mutations each turned their test red. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
106 lines
4.5 KiB
C#
106 lines
4.5 KiB
C#
using System.Collections.Concurrent;
|
|
using Serilog;
|
|
using Serilog.Core;
|
|
using Serilog.Events;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
|
|
|
/// <summary>Collects Serilog events in memory.</summary>
|
|
internal sealed class CapturingSink : ILogEventSink
|
|
{
|
|
public ConcurrentQueue<LogEvent> Events { get; } = new();
|
|
|
|
public void Emit(LogEvent logEvent) => Events.Enqueue(logEvent);
|
|
|
|
public ILogger CreateLogger() => new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(this).CreateLogger();
|
|
|
|
public IEnumerable<string> Rendered => Events.Select(e => e.RenderMessage());
|
|
}
|
|
|
|
internal static class Scripts
|
|
{
|
|
public static readonly IReadOnlyDictionary<string, DataValueSnapshot> NoInputs =
|
|
new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
|
|
|
public static IReadOnlyDictionary<string, DataValueSnapshot> Input(string path, object? value, uint status = 0)
|
|
=> new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
|
|
{
|
|
[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow),
|
|
};
|
|
|
|
public static ScriptInvocation Vt(string key, string source, TimeSpan? timeout = null,
|
|
IReadOnlyDictionary<string, DataValueSnapshot>? inputs = null, ILogger? logger = null)
|
|
=> new(key, ScriptContextKind.VirtualTag, source, inputs ?? NoInputs, timeout ?? TimeSpan.FromSeconds(5), logger);
|
|
|
|
/// <summary>Options for a single-shard runner with short budgets, so the containment tests run in seconds.</summary>
|
|
public static ScriptingOptions Options(int shards = 1, TimeSpan? hardGrace = null, int quarantineAfterKills = 2,
|
|
int probation = 20, TimeSpan? unavailableGrace = null, string? workerPath = null, long heapLimit = 256L * 1024 * 1024,
|
|
string? peCacheDir = null, TimeSpan? isolationIdle = null, TimeSpan? circuitOpenAfter = null)
|
|
=> new()
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
QuarantineAfterConsecutiveTimeouts = 100,
|
|
Worker = new ScriptWorkerOptions
|
|
{
|
|
Shards = shards,
|
|
HardGrace = hardGrace ?? TimeSpan.FromMilliseconds(500),
|
|
QuarantineAfterKills = quarantineAfterKills,
|
|
IsolationProbationRuns = probation,
|
|
UnavailableGrace = unavailableGrace ?? TimeSpan.FromSeconds(30),
|
|
ExecutablePath = workerPath,
|
|
HeapLimitBytes = heapLimit,
|
|
PeCacheDir = peCacheDir,
|
|
IsolationIdleTimeout = isolationIdle ?? ScriptWorkerOptions.DefaultIsolationIdleTimeout,
|
|
CircuitOpenAfter = circuitOpenAfter ?? ScriptWorkerOptions.DefaultCircuitOpenAfter,
|
|
},
|
|
};
|
|
|
|
/// <summary>A fresh, empty temp directory, deleted by <see cref="IDisposable.Dispose"/>.</summary>
|
|
public sealed class TempDir : IDisposable
|
|
{
|
|
public TempDir()
|
|
{
|
|
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "otopcua-sw-test-" + Guid.NewGuid().ToString("N")[..12]);
|
|
Directory.CreateDirectory(Path);
|
|
}
|
|
|
|
public string Path { get; }
|
|
|
|
public void Dispose()
|
|
{
|
|
try { Directory.Delete(Path, recursive: true); } catch { /* best effort */ }
|
|
}
|
|
}
|
|
|
|
public static OutOfProcessScriptRunner Runner(ScriptingOptions options, ILogger? log = null,
|
|
TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
|
|
=> new(options, log ?? new LoggerConfiguration().CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60),
|
|
maintenanceInterval: maintenanceInterval, markerDirectory: markerDirectory);
|
|
|
|
public static async Task<T> Eventually<T>(Func<T?> probe, TimeSpan timeout, string what) where T : class
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (probe() is { } value) return value;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException($"timed out waiting for {what}");
|
|
}
|
|
|
|
public static async Task Eventually(Func<bool> probe, TimeSpan timeout, string what)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (probe()) return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException($"timed out waiting for {what}");
|
|
}
|
|
}
|