1ead8f458a
v2-ci / unit-tests (push) Waiting to run
- A self-contained Host publish (-r win-x64 --self-contained, the Windows installs) now publishes the ScriptWorker self-contained for the same RID into script-worker/. The launcher starts its apphost directly when runtimeconfig.json lists includedFrameworks. A framework-dependent worker (docker-dev) keeps running through the dotnet muxer. - docker-dev/docker-compose.script-runner.yml: OutOfProcess for the central pair's VT + Calc seams. Live drill recorded in design §12.1 (runaway kill + quarantine, kill -9 recovery, open circuit with /health/ready still 200, metrics). - ScriptRunnerPerfTests (Category=Perf, opt-in via OTOPCUA_SCRIPT_PERF=1): InProcess vs OutOfProcess at K=1/2/4. The accepted 5k evals/s/node gate passes on macOS and windev. The host-CPU, RSS and Windows latency budget misses are tracked as #789. - Fix: the single-flight slot is released before the awaiting caller resumes. Before, a caller that started its next run right after the previous one returned could be refused with "previous evaluation still running", which the seams publish as Bad. - Fix: the worker compiles on dedicated threads, not the thread pool. A burst of cold loads starved the heartbeat, and the host killed the shard (found on windev). Load-timeout queueing is #790. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
116 lines
5.1 KiB
C#
116 lines
5.1 KiB
C#
using System.Collections.Concurrent;
|
|
using Serilog;
|
|
using Serilog.Core;
|
|
using Serilog.Events;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
|
|
|
/// <summary>Collects Serilog events in memory.</summary>
|
|
internal sealed class CapturingSink : ILogEventSink
|
|
{
|
|
public ConcurrentQueue<LogEvent> Events { get; } = new();
|
|
|
|
public void Emit(LogEvent logEvent) => Events.Enqueue(logEvent);
|
|
|
|
public ILogger CreateLogger() => new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(this).CreateLogger();
|
|
|
|
public IEnumerable<string> Rendered => Events.Select(e => e.RenderMessage());
|
|
}
|
|
|
|
internal static class Scripts
|
|
{
|
|
public static readonly IReadOnlyDictionary<string, DataValueSnapshot> NoInputs =
|
|
new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
|
|
|
public static IReadOnlyDictionary<string, DataValueSnapshot> Input(string path, object? value, uint status = 0)
|
|
=> new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
|
|
{
|
|
[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow),
|
|
};
|
|
|
|
public static ScriptInvocation Vt(string key, string source, TimeSpan? timeout = null,
|
|
IReadOnlyDictionary<string, DataValueSnapshot>? inputs = null, ILogger? logger = null)
|
|
=> new(key, ScriptContextKind.VirtualTag, source, inputs ?? NoInputs, timeout ?? TimeSpan.FromSeconds(5), logger);
|
|
|
|
/// <summary>
|
|
/// Name of the environment variable that points the suite at a different worker entry than the one built beside
|
|
/// the tests — e.g. the apphost of a self-contained <c>script-worker/</c> from a <c>-r win-x64</c> Host publish.
|
|
/// </summary>
|
|
public const string WorkerUnderTestVariable = "OTOPCUA_SCRIPTWORKER_TEST_PATH";
|
|
|
|
/// <summary>The worker entry named by <see cref="WorkerUnderTestVariable"/>, or null for the built one.</summary>
|
|
public static string? WorkerUnderTest
|
|
=> Environment.GetEnvironmentVariable(WorkerUnderTestVariable) is { Length: > 0 } path ? path : null;
|
|
|
|
/// <summary>Options for a single-shard runner with short budgets, so the containment tests run in seconds.</summary>
|
|
public static ScriptingOptions Options(int shards = 1, TimeSpan? hardGrace = null, int quarantineAfterKills = 2,
|
|
int probation = 20, TimeSpan? unavailableGrace = null, string? workerPath = null, long heapLimit = 256L * 1024 * 1024,
|
|
string? peCacheDir = null, TimeSpan? isolationIdle = null, TimeSpan? circuitOpenAfter = null)
|
|
=> new()
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
QuarantineAfterConsecutiveTimeouts = 100,
|
|
Worker = new ScriptWorkerOptions
|
|
{
|
|
Shards = shards,
|
|
HardGrace = hardGrace ?? TimeSpan.FromMilliseconds(500),
|
|
QuarantineAfterKills = quarantineAfterKills,
|
|
IsolationProbationRuns = probation,
|
|
UnavailableGrace = unavailableGrace ?? TimeSpan.FromSeconds(30),
|
|
ExecutablePath = workerPath ?? WorkerUnderTest,
|
|
HeapLimitBytes = heapLimit,
|
|
PeCacheDir = peCacheDir,
|
|
IsolationIdleTimeout = isolationIdle ?? ScriptWorkerOptions.DefaultIsolationIdleTimeout,
|
|
CircuitOpenAfter = circuitOpenAfter ?? ScriptWorkerOptions.DefaultCircuitOpenAfter,
|
|
},
|
|
};
|
|
|
|
/// <summary>A fresh, empty temp directory, deleted by <see cref="IDisposable.Dispose"/>.</summary>
|
|
public sealed class TempDir : IDisposable
|
|
{
|
|
public TempDir()
|
|
{
|
|
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "otopcua-sw-test-" + Guid.NewGuid().ToString("N")[..12]);
|
|
Directory.CreateDirectory(Path);
|
|
}
|
|
|
|
public string Path { get; }
|
|
|
|
public void Dispose()
|
|
{
|
|
try { Directory.Delete(Path, recursive: true); } catch { /* best effort */ }
|
|
}
|
|
}
|
|
|
|
public static OutOfProcessScriptRunner Runner(ScriptingOptions options, ILogger? log = null,
|
|
TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
|
|
=> new(options, log ?? new LoggerConfiguration().CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60),
|
|
maintenanceInterval: maintenanceInterval, markerDirectory: markerDirectory);
|
|
|
|
public static async Task<T> Eventually<T>(Func<T?> probe, TimeSpan timeout, string what) where T : class
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (probe() is { } value) return value;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException($"timed out waiting for {what}");
|
|
}
|
|
|
|
public static async Task Eventually(Func<bool> probe, TimeSpan timeout, string what)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (probe()) return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException($"timed out waiting for {what}");
|
|
}
|
|
}
|