merge(r2): r2-plan03

src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Adapters/MxGatewayDataConnection.cs

@@ -105,14 +105,16 @@ public class MxGatewayDataConnection : IDataConnection, IBrowsableDataConnection
 
         // Background event loop: route each value change to the matching subscription callback.
         _eventLoopCts = new CancellationTokenSource();
-        _ = Task.Run(() => RunEventLoopAsync(_eventLoopCts.Token));
+        var loopToken = _eventLoopCts.Token;   // capture NOW: the field may be replaced by a
+        var loopClient = _client;              // subsequent reconnect before Task.Run executes (N1)
+        _ = Task.Run(() => RunEventLoopAsync(loopClient, loopToken));
     }
 
-    private async Task RunEventLoopAsync(CancellationToken ct)
+    private async Task RunEventLoopAsync(IMxGatewayClient client, CancellationToken ct)
     {
         try
         {
-            await _client!.RunEventLoopAsync(update =>
+            await client.RunEventLoopAsync(update =>
             {
                 if (_tagToSub.TryGetValue(update.TagPath, out var subId) && _subs.TryGetValue(subId, out var s))
                     s.Callback(update.TagPath, new TagValue(update.Value, update.Quality, update.Timestamp));
@@ -122,6 +124,18 @@ public class MxGatewayDataConnection : IDataConnection, IBrowsableDataConnection
         {
             // Normal shutdown (DisconnectAsync / DisposeAsync cancelled the loop).
         }
+        catch (Exception ex) when (ct.IsCancellationRequested)
+        {
+            // Teardown fault of a superseded loop (N1): Cancel() is not synchronous with
+            // the loop's exit — a cancelled/disposed gRPC stream commonly surfaces as
+            // RpcException(StatusCode.Cancelled) (Grpc.Net default without
+            // ThrowOperationCanceledOnCancellation) or ObjectDisposedException from the
+            // concurrent DisposeAsync, milliseconds AFTER ConnectAsync has already reset
+            // _disconnectFired. Any fault on a cancelled token is normal shutdown;
+            // raising Disconnected here would flap the NEW healthy session.
+            _logger.LogDebug(ex,
+                "MxGateway event loop faulted after cancellation — superseded loop teardown; not signalling disconnect");
+        }
         catch (Exception ex)
         {
             _logger.LogWarning(ex, "MxGateway event stream faulted; signalling disconnect");

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs

@@ -186,6 +186,10 @@ public class AlarmActor : ReceiveActor
 
         // Handle the off-dispatcher trigger-expression evaluation result (P1).
         Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
+
+        // Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
+        // instead of stranding the alarm trigger with an unhandled Status.Failure.
+        Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
     }
 
     /// <inheritdoc />
@@ -551,7 +555,8 @@ public class AlarmActor : ReceiveActor
             }
         }, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
             ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
-            success: r => new ExpressionEvalResult(r));
+            success: r => new ExpressionEvalResult(r),
+            failure: ex => new ExpressionEvalFailed(ex));
     }
 
     /// <summary>
@@ -576,6 +581,22 @@ public class AlarmActor : ReceiveActor
         if (_evalPending) { _evalPending = false; StartExpressionEvaluation(); }
     }
 
+    /// <summary>
+    /// Applies a faulted off-dispatcher evaluation task (N2): logs the fault, then
+    /// delegates to the false result (per the class contract — a throwing expression
+    /// is treated as false so the state machine still runs, an Active alarm clears)
+    /// which clears in-flight and drains any pending evaluation. Without this the
+    /// alarm trigger would be permanently parked with _evalInFlight stuck true.
+    /// </summary>
+    private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
+    {
+        _healthCollector?.IncrementAlarmError();
+        _logger.LogError(msg.Cause,
+            "Alarm {Alarm} trigger-expression evaluation task faulted on {Instance}; treated as false",
+            _alarmName, _instanceName);
+        HandleExpressionEvalResult(new ExpressionEvalResult(false));
+    }
+
     /// <summary>
     /// HiLo level evaluator: returns the most-severe matching band for the
     /// given value. Severity order checked from highest to lowest so that a
@@ -795,6 +816,15 @@ public class AlarmActor : ReceiveActor
     /// actor thread.
     /// </summary>
     private sealed record ExpressionEvalResult(bool Result);
+
+    /// <summary>
+    /// Piped back to self when the off-dispatcher evaluation TASK itself faults
+    /// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
+    /// during shutdown) — the inner body's catch never sees that. Without this
+    /// mapping the actor receives an unhandled Status.Failure and _evalInFlight
+    /// stays true forever, permanently parking the expression trigger (N2).
+    /// </summary>
+    internal sealed record ExpressionEvalFailed(Exception Cause);
 }
 
 internal enum RateOfChangeDirection { Either, Rising, Falling }

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -518,7 +518,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         // delete/disable commands, breaking that ordering. A deploy is an
         // infrequent admin command, so briefly holding the singleton for a pure
         // Roslyn compile is acceptable; the central deployer already Asks and
-        // waits for the DeploymentStatusResponse.
+        // waits for the DeploymentStatusResponse. Redeploys and multi-instance
+        // deploys of unchanged scripts hit the process-wide compile cache
+        // (SiteScriptCompileCache), so the synchronous gate recompiles only
+        // genuinely new code and the Instance Actor's PreStart reuses the gate's
+        // compile (N4).
         var errors = _deployCompileValidator.Validate(command.FlattenedConfigurationJson);
         if (errors.Count > 0)
         {

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs

@@ -159,6 +159,10 @@ public class ScriptActor : ReceiveActor, IWithTimers
 
         // Handle the off-dispatcher trigger-expression evaluation result (P1).
         Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
+
+        // Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
+        // instead of stranding the trigger with an unhandled Status.Failure.
+        Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
     }
 
     /// <inheritdoc />
@@ -310,7 +314,8 @@ public class ScriptActor : ReceiveActor, IWithTimers
             }
         }, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
             ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
-            success: r => new ExpressionEvalResult(r));
+            success: r => new ExpressionEvalResult(r),
+            failure: ex => new ExpressionEvalFailed(ex));
     }
 
     /// <summary>
@@ -333,6 +338,18 @@ public class ScriptActor : ReceiveActor, IWithTimers
         if (_evalPending) { _evalPending = false; StartExpressionEvaluation(); }
     }
 
+    /// <summary>
+    /// Applies a faulted off-dispatcher evaluation task (N2): logs the fault via the
+    /// existing error path (which also increments the health counter), then reuses the
+    /// false-result path — clearing in-flight, applying the false edge, and draining any
+    /// pending evaluation — so a transient scheduler fault cannot permanently park the trigger.
+    /// </summary>
+    private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
+    {
+        LogExpressionError(msg.Cause);
+        HandleExpressionEvalResult(new ExpressionEvalResult(false));
+    }
+
     /// <summary>
     /// Applies a WhileTrue trigger's condition-state transition: on the
     /// false→true edge, fire once and start the re-fire timer; on the
@@ -621,6 +638,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
     /// the actor thread.
     /// </summary>
     private sealed record ExpressionEvalResult(bool Result);
+
+    /// <summary>
+    /// Piped back to self when the off-dispatcher evaluation TASK itself faults
+    /// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
+    /// during shutdown) — the inner body's catch never sees that. Without this
+    /// mapping the actor receives an unhandled Status.Failure and _evalInFlight
+    /// stays true forever, permanently parking the expression trigger (N2).
+    /// </summary>
+    internal sealed record ExpressionEvalFailed(Exception Cause);
 }
 
 // ── Trigger config types ──

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Deployment/DeployCompileValidator.cs

@@ -14,8 +14,11 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
 /// deployment; no partial state is applied).
 ///
 /// Because it holds no actor state and only calls the (stateless)
-/// <see cref="ScriptCompilationService"/>, it is safe to run off the singleton
-/// mailbox on a <c>Task.Run</c> thread — the deploy path never blocks on Roslyn.
+/// <see cref="ScriptCompilationService"/>, it is pure and thread-safe; the Deployment
+/// Manager nevertheless invokes it <em>synchronously on the singleton's actor thread</em>
+/// as a pure prefix step, preserving mailbox-FIFO deploy ordering (see
+/// <see cref="Actors.DeploymentManagerActor.HandleDeploy"/>). Compiles of unchanged
+/// script bodies are deduped by the process-wide compile cache.
 /// </summary>
 public sealed class DeployCompileValidator
 {

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptCompilationService.cs

@@ -13,6 +13,13 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
 /// <see cref="ScriptTrustValidator"/>; this service keeps the real
 /// execution-path compile of the script against <see cref="ScriptGlobals"/> /
 /// <see cref="TriggerExpressionGlobals"/>.
+///
+/// Compile results — success AND failure — are memoised process-wide in
+/// <see cref="SiteScriptCompileCache"/> (keyed on code hash + globals type), so an
+/// unchanged script body compiles once per process lifetime and the deploy gate,
+/// the Instance Actor's PreStart recompile, and SharedScriptLibrary all reuse one
+/// compiled <c>Script&lt;object?&gt;</c>. Error text is name-free by construction, so a
+/// cached failure renders correctly for every caller.
 /// </summary>
 public class ScriptCompilationService
 {
@@ -92,10 +99,33 @@ public class ScriptCompilationService
         => CompileCore(name, expression, typeof(TriggerExpressionGlobals));
 
     /// <summary>
-    /// Shared compilation path: validates the trust model, builds the script
-    /// against the given globals type, and returns the compiled result.
+    /// Shared compilation funnel for every site-side compile (the deploy gate, the
+    /// Instance Actor's PreStart recompile, and SharedScriptLibrary). Results — success
+    /// AND failure — are memoised process-wide in <see cref="SiteScriptCompileCache"/>,
+    /// keyed on code hash + globals type: an unchanged script body compiles once per
+    /// process lifetime and the compiled <c>Script&lt;object?&gt;</c> is shared across all
+    /// call sites (N4). Cached error text is name-free by construction (Roslyn diagnostics /
+    /// the compile-exception message never embed the caller's script name), so a cached
+    /// failure renders correctly for every caller.
     /// </summary>
     private ScriptCompilationResult CompileCore(string name, string code, Type globalsType)
+    {
+        var result = SiteScriptCompileCache.GetOrAdd(code, globalsType,
+            () => CompileUncached(name, code, globalsType));
+        if (!result.IsSuccess)
+            _logger.LogDebug(
+                "Script {Script}: returning cached compile failure ({ErrorCount} error(s))",
+                name, result.Errors.Count); // miss-path logs name the FIRST caller; keep this caller visible
+        return result;
+    }
+
+    /// <summary>
+    /// The uncached compile: validates the trust model, builds the script against the
+    /// given globals type, and returns the compiled result. The verdict is deterministic
+    /// on the code (plus the compile-time-static trust policy and globals surface), so it
+    /// is safe to cache alongside the compiled script.
+    /// </summary>
+    private ScriptCompilationResult CompileUncached(string name, string code, Type globalsType)
     {
         // Validate trust model
         var violations = ValidateTrustModel(code);

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs

@@ -0,0 +1,84 @@
+using System.Collections.Concurrent;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+/// <summary>
+/// Process-wide cache of site-side script <see cref="ScriptCompilationResult"/>s, keyed on
+/// the globals type plus a SHA-256 hash of the script code. Unlike the TemplateEngine's
+/// <c>ScriptCompileVerdictCache</c> (which caches central-validation <em>verdicts</em>), this
+/// caches the full compiled result — the immutable Roslyn <c>Script&lt;object?&gt;</c> itself —
+/// so the synchronous deploy gate and the <c>InstanceActor.PreStart</c> recompile share one
+/// Roslyn compile per script body per process lifetime (N4).
+///
+/// <para>Soundness rests on three invariants:</para>
+/// <list type="number">
+///   <item>A compile result is a pure function of (code, globals type, compile-time-static
+///   trust policy) — none of which vary at runtime for a given key.</item>
+///   <item>The stored error text is name-free (Roslyn <c>Diagnostic.GetMessage()</c> /
+///   <c>"Compilation exception: …"</c>); callers prefix their own script-name context, so a
+///   shared cached failure renders correctly for every caller. <em>Failures are cached too</em> —
+///   a bad script body fails identically for every caller.</item>
+///   <item>Roslyn <c>Script&lt;T&gt;</c> is immutable and thread-safe (<c>RunAsync</c> creates
+///   per-run state), so one compiled instance may be shared across actors and threads.</item>
+/// </list>
+///
+/// <para>
+/// Bounded at <see cref="MaxEntries"/> entries — deliberately smaller than the verdict cache's
+/// 4096 because entries pin compiled assemblies, not just verdict strings. On overflow the cache
+/// is cleared wholesale (results are recomputable, so a coarse reset avoids eviction bookkeeping).
+/// <see cref="Hits"/>/<see cref="Count"/>/<see cref="Clear"/> are exposed for tests and diagnostics.
+/// </para>
+/// </summary>
+internal static class SiteScriptCompileCache
+{
+    /// <summary>Upper bound on cached entries; the cache is cleared wholesale on overflow.</summary>
+    internal const int MaxEntries = 1024;
+
+    private static readonly ConcurrentDictionary<string, ScriptCompilationResult> Cache = new();
+    private static long _hits;
+
+    /// <summary>Number of cache hits observed since the last <see cref="Clear"/>.</summary>
+    public static long Hits => Interlocked.Read(ref _hits);
+
+    /// <summary>Current number of cached compile results.</summary>
+    public static int Count => Cache.Count;
+
+    /// <summary>
+    /// Returns the cached compile result for <paramref name="code"/> against
+    /// <paramref name="globalsType"/>, or computes it via <paramref name="factory"/> and caches
+    /// it on a miss. A hit increments <see cref="Hits"/>. Both success and failure results are
+    /// cached — the error text is name-free by construction.
+    /// </summary>
+    /// <param name="code">The script source code to look up (hashed to form the cache key).</param>
+    /// <param name="globalsType">The Roslyn globals surface the script compiles against; part of the key so identical source under different globals stays distinct.</param>
+    /// <param name="factory">Computes the compile result on a cache miss.</param>
+    /// <returns>The cached or newly computed compile result.</returns>
+    public static ScriptCompilationResult GetOrAdd(string code, Type globalsType, Func<ScriptCompilationResult> factory)
+    {
+        var key = globalsType.FullName + ":" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
+
+        if (Cache.TryGetValue(key, out var result))
+        {
+            Interlocked.Increment(ref _hits);
+            return result;
+        }
+
+        result = factory();
+
+        // Coarse bound: on overflow drop everything rather than track evictions.
+        if (Cache.Count >= MaxEntries)
+            Cache.Clear();
+
+        Cache[key] = result;
+        return result;
+    }
+
+    /// <summary>Clears all cached compile results and resets the hit counter.</summary>
+    public static void Clear()
+    {
+        Cache.Clear();
+        Interlocked.Exchange(ref _hits, 0);
+    }
+}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs

@@ -56,5 +56,14 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
         builder.RequireThat(options.ConfigFetchRetryCount >= 0,
             $"ScadaBridge:SiteRuntime:ConfigFetchRetryCount must be >= 0 " +
             $"(was {options.ConfigFetchRetryCount}).");
+
+        builder.RequireThat(options.TagSubscribeRetryIntervalMs > 0,
+            $"ScadaBridge:SiteRuntime:TagSubscribeRetryIntervalMs must be greater than 0 " +
+            $"(was {options.TagSubscribeRetryIntervalMs}); a zero interval hot-loops the tag-subscribe " +
+            "retry and a negative one throws inside the Instance Actor's retry scheduling.");
+
+        builder.RequireThat(options.StuckScriptGraceMs >= 0,
+            $"ScadaBridge:SiteRuntime:StuckScriptGraceMs must be >= 0 " +
+            $"(was {options.StuckScriptGraceMs}); a negative grace throws inside the stuck-script watchdog's delay.");
     }
 }