feat(scripting): self-contained script worker publish, docker-dev drill, §8 perf harness (lmxopcua#780 P1)

- A self-contained Host publish (-r win-x64 --self-contained, the Windows installs) now publishes the
  ScriptWorker self-contained for the same RID into script-worker/. The launcher starts its apphost
  directly when runtimeconfig.json lists includedFrameworks. A framework-dependent worker (docker-dev)
  keeps running through the dotnet muxer.
- docker-dev/docker-compose.script-runner.yml: OutOfProcess for the central pair's VT + Calc seams.
  Live drill recorded in design §12.1 (runaway kill + quarantine, kill -9 recovery, open circuit
  with /health/ready still 200, metrics).
- ScriptRunnerPerfTests (Category=Perf, opt-in via OTOPCUA_SCRIPT_PERF=1): InProcess vs OutOfProcess
  at K=1/2/4. The accepted 5k evals/s/node gate passes on macOS and windev. The host-CPU, RSS and
  Windows latency budget misses are tracked as #789.
- Fix: the single-flight slot is released before the awaiting caller resumes. Before, a caller that
  started its next run right after the previous one returned could be refused with "previous
  evaluation still running", which the seams publish as Bad.
- Fix: the worker compiles on dedicated threads, not the thread pool. A burst of cold loads starved
  the heartbeat, and the host killed the shard (found on windev). Load-timeout queueing is #790.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs

@@ -278,11 +278,61 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     /// <summary>The default worker path, or the configured one.</summary>
     /// <param name="worker">The worker options.</param>
     /// <returns>The absolute worker path.</returns>
+    /// <remarks>
+    ///     With no <see cref="ScriptWorkerOptions.ExecutablePath"/>, the worker is looked up in <c>script-worker/</c>
+    ///     under the host's base directory by <see cref="ResolveWorkerEntry"/>: a self-contained worker (the Windows
+    ///     installs, which have no .NET runtime) is started through its apphost, a framework-dependent one (docker-dev)
+    ///     through the dotnet muxer.
+    /// </remarks>
     public static string ResolveWorkerPath(ScriptWorkerOptions worker)
         => string.IsNullOrWhiteSpace(worker.ExecutablePath)
-            ? Path.Combine(AppContext.BaseDirectory, WorkerFolderName, WorkerAssemblyFileName)
+            ? ResolveWorkerEntry(Path.Combine(AppContext.BaseDirectory, WorkerFolderName))
             : Path.GetFullPath(worker.ExecutablePath);
 
+    /// <summary>The worker's apphost file name on this OS.</summary>
+    public static string WorkerAppHostFileName
+        => OperatingSystem.IsWindows() ? "ZB.MOM.WW.OtOpcUa.ScriptWorker.exe" : "ZB.MOM.WW.OtOpcUa.ScriptWorker";
+
+    /// <summary>
+    ///     The entry a worker folder should be launched through: its apphost when the folder holds a
+    ///     <b>self-contained</b> worker, otherwise the entry assembly (run through the dotnet muxer).
+    /// </summary>
+    /// <param name="folder">The worker folder.</param>
+    /// <returns>The absolute path of the apphost or of the entry assembly (which may not exist).</returns>
+    /// <remarks>
+    ///     A framework-dependent build also emits an apphost, but that apphost needs an installed runtime the
+    ///     self-contained Windows hosts do not have, and the muxer path is what docker-dev and the tests have always
+    ///     used, so the apphost is chosen only for a self-contained worker. Self-contained is read from the worker's
+    ///     <c>runtimeconfig.json</c>: the SDK writes <c>includedFrameworks</c> for a self-contained app and
+    ///     <c>framework</c>/<c>frameworks</c> for a framework-dependent one.
+    /// </remarks>
+    public static string ResolveWorkerEntry(string folder)
+    {
+        var full = Path.GetFullPath(folder);
+        var appHost = Path.Combine(full, WorkerAppHostFileName);
+        return File.Exists(appHost) && IsSelfContainedWorker(full) ? appHost : Path.Combine(full, WorkerAssemblyFileName);
+    }
+
+    /// <summary>Whether the worker in <paramref name="folder"/> was published self-contained.</summary>
+    /// <param name="folder">The worker folder.</param>
+    /// <returns>True when its <c>runtimeconfig.json</c> lists <c>includedFrameworks</c>.</returns>
+    public static bool IsSelfContainedWorker(string folder)
+    {
+        var config = Path.Combine(folder, Path.GetFileNameWithoutExtension(WorkerAssemblyFileName) + ".runtimeconfig.json");
+        if (!File.Exists(config)) return false;
+        try
+        {
+            using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(config),
+                new System.Text.Json.JsonDocumentOptions { CommentHandling = System.Text.Json.JsonCommentHandling.Skip, AllowTrailingCommas = true });
+            return doc.RootElement.TryGetProperty("runtimeOptions", out var options)
+                   && options.TryGetProperty("includedFrameworks", out _);
+        }
+        catch (Exception ex) when (ex is System.Text.Json.JsonException or IOException or UnauthorizedAccessException)
+        {
+            return false;
+        }
+    }
+
     /// <inheritdoc />
     public async Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default)
     {
@@ -369,17 +419,26 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
             // death, attribution (strike, quarantine, suspect) has already run when this continuation fires.
             handedOff = true;
             var heldGate = gate;
-            _ = pending.Completion.ContinueWith(_ =>
+            var released = 0;
+            void ReleaseSlot()
             {
+                if (Interlocked.Exchange(ref released, 1) != 0) return;
                 SlotReleasingForTest?.Invoke(key);
                 Interlocked.Exchange(ref state.InFlight, 0);
                 heldGate?.Release();
-            }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+            }
+            _ = pending.Completion.ContinueWith(_ => ReleaseSlot(),
+                CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
 
             EvalResult? result;
             try
             {
                 result = await pending.Completion.WaitAsync(invocation.Timeout, ct).ConfigureAwait(false);
+                // The worker has answered (or died, with attribution already run): free the slot HERE, before this
+                // caller resumes. The continuation above is not ordered against this await, so relying on it alone
+                // let a caller that awaited one run and then started the next see "previous evaluation still running"
+                // (found by the §8 perf harness: 32 of 200 sequential same-key runs refused). Idempotent.
+                ReleaseSlot();
             }
             catch (TimeoutException)
             {

src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs

@@ -38,6 +38,17 @@ internal sealed class WorkerHost
     private readonly ConcurrentDictionary<string, LoadedScript> _scripts = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<long, RunningEval> _running = new();
     private readonly BlockingCollection<EvalRequest> _queue = new(new ConcurrentQueue<EvalRequest>());
+
+    /// <summary>
+    ///     Loads waiting for a compile thread. Compiles are long, synchronous and CPU-bound, so they run on
+    ///     <see cref="CompileThreads"/> dedicated threads, never on the thread pool: a burst of cold loads (a node start
+    ///     or a deploy with an empty PE cache) used to put one blocking pool item per load in front of the heartbeat
+    ///     timer and the pipe writer, and the host killed the worker for silence (found on windev, lmxopcua#780 §8).
+    /// </summary>
+    private readonly BlockingCollection<LoadScripts> _loads = new(new ConcurrentQueue<LoadScripts>());
+
+    /// <summary>Dedicated compile threads: half the processors, at least 1, at most 8.</summary>
+    internal static int CompileThreads => Math.Clamp(Environment.ProcessorCount / 2, 1, 8);
     private readonly ForwardingLogSink _logSink;
     private readonly ILogger _scriptLogger;
     private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
@@ -116,6 +127,10 @@ internal sealed class WorkerHost
         {
             new Thread(EvalThreadLoop) { IsBackground = true, Name = $"ScriptWorker.Eval{i}" }.Start();
         }
+        for (var i = 0; i < CompileThreads; i++)
+        {
+            new Thread(CompileThreadLoop) { IsBackground = true, Name = $"ScriptWorker.Compile{i}" }.Start();
+        }
         new Thread(WatchdogLoop) { IsBackground = true, Name = "ScriptWorker.Watchdog" }.Start();
         _ = Task.Run(HeartbeatLoopAsync);
         _ = Task.Run(LogFlushLoopAsync);
@@ -146,9 +161,9 @@ internal sealed class WorkerHost
             {
                 case Envelope.BodyOneofCase.Load:
                 {
-                    // Compile off the reader, so evaluations for already-loaded keys keep flowing during a load.
-                    var load = frame.Load;
-                    _ = Task.Run(() => Load(load));
+                    // Compile off the reader, so evaluations for already-loaded keys keep flowing during a load, and
+                    // off the thread pool, so the heartbeat and the writer keep flowing during a burst of them.
+                    _loads.Add(frame.Load);
                     break;
                 }
                 case Envelope.BodyOneofCase.Unload:
@@ -179,6 +194,11 @@ internal sealed class WorkerHost
         }
     }
 
+    private void CompileThreadLoop()
+    {
+        foreach (var load in _loads.GetConsumingEnumerable()) Load(load);
+    }
+
     private void Load(LoadScripts load)
     {
         var result = new LoadResult { LoadId = load.LoadId };

src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/build/ScriptWorker.Deploy.targets

@@ -5,9 +5,13 @@
     the worker test project. The worker is its own app (own deps.json / runtimeconfig.json), so it lives in its own folder
     rather than beside the host's assemblies.
 
-    Publish note: the copy is the worker's framework-dependent BUILD output, run through the dotnet muxer. That suits the
-    docker-dev image (aspnet:10.0 runtime). A self-contained host publish (the Windows deploys) needs a self-contained
-    worker publish too; that is part of the installer work tracked on lmxopcua#780.
+    Publish: a framework-dependent host publish (docker-dev, aspnet:10.0 runtime) copies the worker's
+    framework-dependent BUILD output, which the launcher runs through the dotnet muxer. A self-contained host publish
+    (a self-contained `-r win-x64` publish: the Windows installs, which have no .NET runtime) instead PUBLISHES the worker
+    self-contained for the same RuntimeIdentifier into $(PublishDir)script-worker/, and the launcher starts its
+    apphost directly (OutOfProcessScriptRunner.ResolveWorkerEntry reads runtimeconfig.json `includedFrameworks`).
+    The worker then carries its own copy of the runtime (~70 MB); it is not trimmed or single-file, because Roslyn
+    compiles against the runtime's reference set on disk.
   -->
   <PropertyGroup>
     <_OtScriptWorkerProject>$(MSBuildThisFileDirectory)../ZB.MOM.WW.OtOpcUa.ScriptWorker.csproj</_OtScriptWorkerProject>
@@ -18,7 +22,12 @@
   </ItemGroup>
 
   <Target Name="_OtResolveScriptWorkerOutput">
-    <MSBuild Projects="$(_OtScriptWorkerProject)" Targets="GetTargetPath" Properties="Configuration=$(Configuration)">
+    <PropertyGroup>
+      <_OtScriptWorkerResolveProps>Configuration=$(Configuration)</_OtScriptWorkerResolveProps>
+      <!-- A RID-specific host build builds the referenced worker for that RID too (bin/<cfg>/<tfm>/<rid>/). -->
+      <_OtScriptWorkerResolveProps Condition="'$(RuntimeIdentifier)' != ''">$(_OtScriptWorkerResolveProps);RuntimeIdentifier=$(RuntimeIdentifier)</_OtScriptWorkerResolveProps>
+    </PropertyGroup>
+    <MSBuild Projects="$(_OtScriptWorkerProject)" Targets="GetTargetPath" Properties="$(_OtScriptWorkerResolveProps)">
       <Output TaskParameter="TargetOutputs" ItemName="_OtScriptWorkerTargetPath" />
     </MSBuild>
     <PropertyGroup>
@@ -35,9 +44,23 @@
           SkipUnchangedFiles="true" />
   </Target>
 
-  <Target Name="_OtCopyScriptWorkerAfterPublish" AfterTargets="Publish" DependsOnTargets="_OtResolveScriptWorkerOutput">
+  <Target Name="_OtCopyScriptWorkerAfterPublish" AfterTargets="Publish" DependsOnTargets="_OtResolveScriptWorkerOutput"
+          Condition="'$(SelfContained)' != 'true'">
     <Copy SourceFiles="@(_OtScriptWorkerFiles)"
           DestinationFiles="@(_OtScriptWorkerFiles->'$(PublishDir)script-worker/%(RecursiveDir)%(Filename)%(Extension)')"
           SkipUnchangedFiles="true" />
   </Target>
+
+  <!-- Self-contained host publish: publish the worker self-contained for the same RID. PublishDir is made absolute so
+       the nested publish does not resolve it against the worker's project directory. -->
+  <Target Name="_OtPublishSelfContainedScriptWorker" AfterTargets="Publish"
+          Condition="'$(SelfContained)' == 'true'">
+    <Error Condition="'$(RuntimeIdentifier)' == ''"
+           Text="lmxopcua#780: a self-contained publish needs a RuntimeIdentifier to publish the script worker for." />
+    <PropertyGroup>
+      <_OtScriptWorkerPublishDir>$([System.IO.Path]::GetFullPath('$(PublishDir)script-worker/'))</_OtScriptWorkerPublishDir>
+    </PropertyGroup>
+    <MSBuild Projects="$(_OtScriptWorkerProject)" Targets="Publish"
+             Properties="Configuration=$(Configuration);RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=true;PublishSingleFile=false;PublishTrimmed=false;PublishDir=$(_OtScriptWorkerPublishDir)" />
+  </Target>
 </Project>