Merge local master 7bcb6331 into remed/lane-FOCAS

Refs #740
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs

@@ -26,7 +26,11 @@ public sealed class Deployment
     public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow;
 
     /// <summary>Gets or sets the serialized artifact blob containing the configuration.</summary>
-    public byte[] ArtifactBlob { get; init; } = Array.Empty<byte>();
+    /// <remarks>
+    ///     Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob
+    ///     is "no artifact" to every reader (#485), so the audit row survives with its bytes retired.
+    /// </remarks>
+    public byte[] ArtifactBlob { get; set; } = Array.Empty<byte>();
 
     /// <summary>Gets or sets the row version for optimistic concurrency control.</summary>
     public byte[] RowVersion { get; set; } = Array.Empty<byte>();

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs

@@ -0,0 +1,36 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
+{
+    /// <summary>
+    ///     Enables <c>ALLOW_SNAPSHOT_ISOLATION</c> on the config database so the deploy path can read every
+    ///     table inside one <c>SNAPSHOT</c> transaction (<c>DraftSnapshotFactory.FromConfigDbAsync</c>):
+    ///     the validator and the artifact composer then see one consistent state instead of two reads
+    ///     an AdminUI save can land between. No schema change — the model snapshot is untouched.
+    /// </summary>
+    /// <remarks>
+    ///     <c>ALTER DATABASE</c> cannot run inside the user transaction the migrator wraps each migration in,
+    ///     hence <c>suppressTransaction: true</c>. Turning the option ON does not need exclusive access (unlike
+    ///     <c>READ_COMMITTED_SNAPSHOT</c>); it waits for in-flight transactions and then takes effect. Down
+    ///     leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF
+    ///     would fail any deploy running at that instant.
+    /// </remarks>
+    public partial class EnableSnapshotIsolation : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(
+                "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;",
+                suppressTransaction: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            // Deliberately a no-op; see the class remarks.
+        }
+    }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs

@@ -66,6 +66,23 @@ public sealed class DraftSnapshot
     /// silently dropped when the permission trie is built, so the grant has to be caught here instead.</summary>
     public IReadOnlyList<NodeAcl> NodeAcls { get; init; } = [];
 
+    /// <summary>
+    ///     Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster
+    ///     topology (<see cref="DraftValidator.ValidateClusterTopology"/>) and composes the artifact from
+    ///     the SAME read as every other table — a second read between validate and compose is exactly how
+    ///     an edit ships unvalidated.
+    /// </summary>
+    public IReadOnlyList<ServerCluster> ServerClusters { get; init; } = [];
+
+    /// <summary>Cluster-node rows; see <see cref="ServerClusters"/> for why they ride the snapshot.</summary>
+    public IReadOnlyList<ClusterNode> ClusterNodes { get; init; } = [];
+
+    /// <summary>
+    ///     Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact
+    ///     composer ships the keys a driver-only node authenticates against from the one consistent read.
+    /// </summary>
+    public IReadOnlyList<ApiKey> ApiKeys { get; init; } = [];
+
     /// <summary>Prior Equipment rows (any generation, same cluster) for stability checks.</summary>
     public IReadOnlyList<Equipment> PriorEquipment { get; init; } = [];
 

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs

@@ -1,3 +1,4 @@
+using System.Data;
 using Microsoft.EntityFrameworkCore;
 
 namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation;
@@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation;
 ///         / <see cref="DraftSnapshot.Site"/> are left null so the path-length rule uses its
 ///         conservative upper bound.
 ///     </para>
+///     <para>
+///         <b>One consistent read.</b> On a relational provider the tables are read inside a
+///         <see cref="IsolationLevel.Snapshot"/> transaction, so a <c>/uns</c> or <c>/raw</c> save that
+///         lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path
+///         validates AND composes its artifact from this one snapshot, which is what makes "the artifact the
+///         validator never saw" unrepresentable. SQL Server requires <c>ALLOW_SNAPSHOT_ISOLATION ON</c>
+///         (migration <c>EnableSnapshotIsolation</c>); without it the first query fails with error 3952 and
+///         the deploy is rejected rather than silently read unisolated. A non-relational provider (the
+///         InMemory test provider) and a context already inside a transaction read without one.
+///     </para>
 /// </remarks>
 public static class DraftSnapshotFactory
 {
@@ -29,6 +40,29 @@ public static class DraftSnapshotFactory
     /// <param name="ct">Cancellation token.</param>
     /// <returns>A snapshot populated from the live DB, ready for <see cref="DraftValidator.Validate"/>.</returns>
     public static async Task<DraftSnapshot> FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(db);
+
+        var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null;
+        var tx = ownsTransaction
+            ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false)
+            : null;
+        await using (tx)
+        {
+            var snapshot = await ReadAsync(db, ct).ConfigureAwait(false);
+            // Read-only: committing releases the snapshot's version-store reservation; nothing was written.
+            if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false);
+            return snapshot;
+        }
+    }
+
+    // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment
+    // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the
+    // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server —
+    // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1;
+    // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would
+    // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted.
+    private static async Task<DraftSnapshot> ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct)
         => new DraftSnapshot
         {
             // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the
@@ -36,23 +70,26 @@ public static class DraftSnapshotFactory
             // EquipmentUuid checks run in separate validator passes — no rule here reads these fields.
             GenerationId = 0,            // generation model dropped; placeholder (no rule reads it)
             ClusterId = string.Empty,    // global snapshot; rules compare entity ClusterId fields, not this
+            ServerClusters     = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
+            ClusterNodes       = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
             // v3: Namespace entity retired — no namespace list to materialize. The Raw tree
             // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length,
             // effective-leaf uniqueness).
-            RawFolders         = await db.RawFolders.AsNoTracking().ToListAsync(ct),
-            DriverInstances    = await db.DriverInstances.AsNoTracking().ToListAsync(ct),
-            Devices            = await db.Devices.AsNoTracking().ToListAsync(ct),
-            TagGroups          = await db.TagGroups.AsNoTracking().ToListAsync(ct),
-            UnsAreas           = await db.UnsAreas.AsNoTracking().ToListAsync(ct),
-            UnsLines           = await db.UnsLines.AsNoTracking().ToListAsync(ct),
-            Equipment          = await db.Equipment.AsNoTracking().ToListAsync(ct),
-            Tags               = await db.Tags.AsNoTracking().ToListAsync(ct),
-            UnsTagReferences   = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct),
-            VirtualTags        = await db.VirtualTags.AsNoTracking().ToListAsync(ct),
-            ScriptedAlarms     = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct),
-            Scripts            = await db.Scripts.AsNoTracking().ToListAsync(ct),
-            PollGroups         = await db.PollGroups.AsNoTracking().ToListAsync(ct),
-            NodeAcls           = await db.NodeAcls.AsNoTracking().ToListAsync(ct),
+            RawFolders         = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
+            DriverInstances    = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
+            Devices            = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
+            TagGroups          = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
+            UnsAreas           = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
+            UnsLines           = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
+            Equipment          = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
+            Tags               = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
+            UnsTagReferences   = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
+            VirtualTags        = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
+            ScriptedAlarms     = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
+            Scripts            = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
+            PollGroups         = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
+            NodeAcls           = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
+            ApiKeys            = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
             PriorEquipment     = [],   // intentional: no prior-generation table to diff against at this level
             ActiveReservations = await db.ExternalIdReservations
                 .AsNoTracking()

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs

@@ -452,6 +452,58 @@ public static class DraftValidator
                 "Calculation tags form a dependency cycle (members: " + string.Join(", ", cycle) +
                 "); a calc→calc cycle is a live oscillation loop and must be broken before deploy",
                 cycle.Count > 0 ? cycle[0] : null));
+
+        // Rule 3 — effective-trigger gate (R07:U5). A calc tag evaluates on a declared dependency
+        // changing (change-trigger: the literal ctx.GetTag paths) and/or on a timer. A change-triggered
+        // tag whose script has NO literal reads (`return ctx.Now.Hour;`, a dynamic-path read) and no timer
+        // never evaluates at all: it sits BadWaitingForInitialData forever while the driver reports
+        // Healthy, and nothing warned. Same for a tag that is neither change- nor timer-triggered.
+        foreach (var (t, _, scriptId) in calcTags)
+        {
+            var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig);
+            if (timerIntervalMs is > 0) continue;
+            var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null;
+            // An unresolved scriptId is already CalculationScriptMissing / CalculationScriptNotFound; with no
+            // source there are no reads to count, so a change-triggered tag would get a second, misleading
+            // "no literal ctx.GetTag read" error. (changeTriggered=false + no timer is source-independent.)
+            if (changeTriggered && source is null) continue;
+            var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count;
+            if (changeTriggered && depCount > 0) continue;
+            errors.Add(new("CalculationNoEffectiveTrigger",
+                changeTriggered
+                    ? $"Calculation tag '{t.TagId}' is change-triggered but its script has no literal ctx.GetTag read, " +
+                      "and it has no timerIntervalMs — nothing would ever evaluate it. Add a dependency read or a timer."
+                    : $"Calculation tag '{t.TagId}' has changeTriggered=false and no timerIntervalMs — nothing would ever evaluate it. " +
+                      "Enable the change trigger or add a timer.",
+                t.TagId));
+        }
+    }
+
+    /// <summary>Read the trigger fields from a calc tag's <c>TagConfig</c>: <c>changeTriggered</c>
+    /// (default <c>true</c>, the driver's own default) and <c>timerIntervalMs</c> (null when absent or not a
+    /// number). Never throws — malformed JSON yields the defaults.</summary>
+    private static (bool ChangeTriggered, int? TimerIntervalMs) ParseTriggers(string? tagConfig)
+    {
+        if (string.IsNullOrWhiteSpace(tagConfig)) return (true, null);
+        try
+        {
+            using var doc = System.Text.Json.JsonDocument.Parse(tagConfig);
+            var root = doc.RootElement;
+            if (root.ValueKind != System.Text.Json.JsonValueKind.Object) return (true, null);
+            var changeTriggered = true;
+            if (root.TryGetProperty("changeTriggered", out var ct)
+                && ct.ValueKind is System.Text.Json.JsonValueKind.False or System.Text.Json.JsonValueKind.True)
+                changeTriggered = ct.GetBoolean();
+            int? timer = null;
+            if (root.TryGetProperty("timerIntervalMs", out var tm)
+                && tm.ValueKind == System.Text.Json.JsonValueKind.Number && tm.TryGetInt32(out var ms))
+                timer = ms;
+            return (changeTriggered, timer);
+        }
+        catch (System.Text.Json.JsonException)
+        {
+            return (true, null);
+        }
     }
 
     /// <summary>Extract the <c>scriptId</c> string from a calc tag's schemaless <c>TagConfig</c> JSON. Never

src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs

@@ -92,6 +92,12 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
     private bool? _lastGateDecision;
 
     private long _queuedRowCount;
+    // Cached dead-letter depth (R05:7). GetStatus used to run a synchronous COUNT(*) table scan per
+    // call — one per /healthz probe and per AdminUI poll on the shared LocalDb file. Kept in step by
+    // every path that flips dead_lettered (drain outcomes, corrupt rows, RetryDeadLettered, retention
+    // purge) and resynced from storage on the same cadence as _queuedRowCount, so a row the pair peer
+    // replicates in is visible after the next resync rather than never.
+    private long _deadLetterRowCount;
     // Probe counter — incremented every time we actually issue a real COUNT(*) for
     // capacity enforcement. Public for test instrumentation only.
     private long _capacityProbeCount;
@@ -145,9 +151,16 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         _clock = clock ?? (() => DateTime.UtcNow);
         _drainGate = drainGate ?? (static () => true);
 
-        _queuedRowCount = ProbeQueuedRowCount();
+        _queuedRowCount = ProbeRowCount(deadLettered: false);
+        _deadLetterRowCount = ProbeRowCount(deadLettered: true);
     }
 
+    /// <summary>Test-only: re-sync both cached counters (queued + dead-lettered) from storage — the same
+    /// resync the enqueue cadence drives every <see cref="ResyncEnqueueInterval"/> enqueues.</summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A task that completes once both counters match storage.</returns>
+    public Task DebugResyncCountersAsync(CancellationToken ct) => ResyncRowCountsAsync(ct);
+
     /// <summary>
     ///     Start the background drain worker. Not started automatically so tests can
     ///     drive <see cref="DrainOnceAsync"/> deterministically.
@@ -260,7 +273,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         // has a second way to fall behind that no local code path can observe.
         if (enqueuesSinceResync >= ResyncEnqueueInterval)
         {
-            await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false);
+            await ResyncRowCountsAsync(ct).ConfigureAwait(false);
             cached = Interlocked.Read(ref _queuedRowCount);
             Interlocked.Exchange(ref _enqueuesSinceResync, 0);
         }
@@ -273,22 +286,26 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         await EnforceCapacityAsync(ct).ConfigureAwait(false);
     }
 
-    /// <summary>Synchronously query <c>COUNT(*)</c> of non-dead-lettered rows. Used at startup.</summary>
-    private long ProbeQueuedRowCount()
+    /// <summary>Synchronously query <c>COUNT(*)</c> of queued or dead-lettered rows. Used at startup.</summary>
+    private long ProbeRowCount(bool deadLettered)
     {
         Interlocked.Increment(ref _capacityProbeCount);
         using var conn = _db.CreateConnection();
         using var cmd = conn.CreateCommand();
-        cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0";
+        cmd.CommandText = deadLettered
+            ? "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"
+            : "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0";
         return (long)(cmd.ExecuteScalar() ?? 0L);
     }
 
-    /// <summary>Re-sync the in-memory counter from storage (async path).</summary>
-    private async Task ResyncQueuedRowCountAsync(CancellationToken ct)
+    /// <summary>Re-sync both in-memory counters from storage (async path).</summary>
+    private async Task ResyncRowCountsAsync(CancellationToken ct)
     {
         Interlocked.Increment(ref _capacityProbeCount);
         var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false);
         Interlocked.Exchange(ref _queuedRowCount, live);
+        var dead = await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false);
+        Interlocked.Exchange(ref _deadLetterRowCount, dead);
     }
 
     private async Task<long> CountAsync(string predicate, CancellationToken ct)
@@ -356,6 +373,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
                     await corruptTx.CommitAsync(ct).ConfigureAwait(false);
                 }
                 Interlocked.Add(ref _queuedRowCount, -corruptIds.Count);
+                Interlocked.Add(ref _deadLetterRowCount, corruptIds.Count);
                 _logger.Warning(
                     "Dead-lettered {Count} historian queue row(s) with un-deserializable payload",
                     corruptIds.Count);
@@ -405,6 +423,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
             }
 
             int rowsLeavingQueue = 0;
+            int rowsDeadLettered = 0;
             await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false))
             {
                 for (var i = 0; i < outcomes.Count; i++)
@@ -423,6 +442,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
                         case HistorianWriteOutcome.PermanentFail:
                             await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false);
                             rowsLeavingQueue++;
+                            rowsDeadLettered++;
                             break;
                         case HistorianWriteOutcome.RetryPlease:
                             // finding 002: cap retries so a perpetually-RetryPlease (poison)
@@ -434,6 +454,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
                             {
                                 await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false);
                                 rowsLeavingQueue++;
+                                rowsDeadLettered++;
                             }
                             else
                             {
@@ -449,6 +470,8 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
             // cap (finding 002) — keep the counter aligned.
             if (rowsLeavingQueue > 0)
                 Interlocked.Add(ref _queuedRowCount, -rowsLeavingQueue);
+            if (rowsDeadLettered > 0)
+                Interlocked.Add(ref _deadLetterRowCount, rowsDeadLettered);
 
             var acks = outcomes.Count(o => o == HistorianWriteOutcome.Ack);
             lock (_statusLock)
@@ -523,13 +546,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         var queued = Interlocked.Read(ref _queuedRowCount);
         if (queued < 0) queued = 0;
 
-        long deadlettered;
-        using (var conn = _db.CreateConnection())
-        using (var cmd = conn.CreateCommand())
-        {
-            cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1";
-            deadlettered = (long)(cmd.ExecuteScalar() ?? 0L);
-        }
+        // Cached, never a live scan (R05:7) — see the field's remarks for the resync cadence.
+        var deadlettered = Interlocked.Read(ref _deadLetterRowCount);
+        if (deadlettered < 0) deadlettered = 0;
 
         DateTime? lastDrain, lastSuccess;
         string? lastError;
@@ -568,7 +587,11 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         var revived = cmd.ExecuteNonQuery();
         // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory
         // counter aligned.
-        if (revived > 0) Interlocked.Add(ref _queuedRowCount, revived);
+        if (revived > 0)
+        {
+            Interlocked.Add(ref _queuedRowCount, revived);
+            Interlocked.Add(ref _deadLetterRowCount, -revived);
+        }
         return revived;
     }
 
@@ -646,8 +669,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
         Interlocked.Increment(ref _capacityProbeCount);
         var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false);
 
-        // Resync the in-memory counter while we have a fresh number.
+        // Resync the in-memory counters while we have a fresh number.
         Interlocked.Exchange(ref _queuedRowCount, count);
+        Interlocked.Exchange(ref _deadLetterRowCount, await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false));
         if (count < _capacity) return;
 
         var toEvict = count - _capacity + 1;
@@ -684,7 +708,10 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
             ct).ConfigureAwait(false);
 
         if (purged > 0)
+        {
+            Interlocked.Add(ref _deadLetterRowCount, -purged);
             _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged);
+        }
     }
 
     private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1);

src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs

@@ -73,6 +73,14 @@ public sealed class ScriptedAlarmEngine : IDisposable
     private readonly Func<DateTime> _clock;
     private readonly TimeSpan _scriptTimeout;
 
+    /// <summary>
+    ///     Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a
+    ///     predicate run while the alarm's previous (timed-out, abandoned) run is still executing and
+    ///     quarantines after three consecutive timeouts (a refusal is not a strike unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — one orphan
+    ///     per alarm instead of one per upstream push. Reset on every <see cref="LoadAsync"/> (a redeploy).
+    /// </summary>
+    private readonly ScriptRunGuard _guard;
+
     // ConcurrentDictionary, not a plain Dictionary: every mutation happens under
     // _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds,
     // RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request
@@ -162,6 +170,33 @@ public sealed class ScriptedAlarmEngine : IDisposable
         => _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null;
     private readonly ConcurrentDictionary<string, DataValueSnapshot> _valueCache
         = new(StringComparer.Ordinal);
+
+    /// <summary>Test hook — the number of paths currently held in the value cache.</summary>
+    internal int ValueCacheCountForTest => _valueCache.Count;
+
+    /// <summary>Test hook — whether <paramref name="path"/> is currently held in the value cache.</summary>
+    internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path);
+
+    /// <summary>
+    ///     How long <see cref="Dispose"/> waits for in-flight background work (a re-evaluation or shelving
+    ///     check mid-<c>SaveAsync</c>) before cancelling it through <see cref="_shutdownCts"/> and
+    ///     returning. 5 s, like <c>PollGroupEngine</c>'s drain; a hung store must not hang host shutdown.
+    /// </summary>
+    internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5);
+
+    /// <summary>Cancelled by <see cref="Dispose"/> once the drain bound expires — every background task
+    /// (<c>DrainDirtyAsync</c>, <c>ShelvingCheckAsync</c>) runs on its token, so a store call that honours
+    /// cancellation returns instead of outliving the engine.</summary>
+    private readonly CancellationTokenSource _shutdownCts = new();
+
+    /// <summary><see cref="_shutdownCts"/>'s token, captured once: background work reads this, never the CTS,
+    /// so <see cref="Dispose"/> can dispose the CTS without racing a late reader.</summary>
+    private readonly CancellationToken _shutdownToken;
+
+    private volatile bool _shutdownCtsDisposed;
+
+    /// <summary>Test hook — whether <see cref="Dispose"/> disposed <see cref="_shutdownCts"/>.</summary>
+    internal bool ShutdownCtsDisposedForTest => _shutdownCtsDisposed;
     private readonly Dictionary<string, HashSet<string>> _alarmsReferencing
         = new(StringComparer.Ordinal); // tag path -> alarm ids
 
@@ -204,13 +239,15 @@ public sealed class ScriptedAlarmEngine : IDisposable
     /// <param name="engineLogger">The logger for engine-level diagnostics.</param>
     /// <param name="clock">Optional function providing the current UTC time; defaults to DateTime.UtcNow.</param>
     /// <param name="scriptTimeout">Optional timeout for script execution; defaults to the evaluator's default timeout.</param>
+    /// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
     public ScriptedAlarmEngine(
         ITagUpstreamSource upstream,
         IAlarmStateStore store,
         ScriptLoggerFactory loggerFactory,
         ILogger engineLogger,
         Func<DateTime>? clock = null,
-        TimeSpan? scriptTimeout = null)
+        TimeSpan? scriptTimeout = null,
+        ScriptingOptions? scripting = null)
     {
         _upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
         _store = store ?? throw new ArgumentNullException(nameof(store));
@@ -218,8 +255,21 @@ public sealed class ScriptedAlarmEngine : IDisposable
         _engineLogger = engineLogger ?? throw new ArgumentNullException(nameof(engineLogger));
         _clock = clock ?? (() => DateTime.UtcNow);
         _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator<AlarmPredicateContext, bool>.DefaultTimeout;
+        _guard = new ScriptRunGuard(scripting, _engineLogger);
+        _shutdownToken = _shutdownCts.Token;
     }
 
+    /// <summary>The run guard every predicate evaluation goes through (diagnostics / tests).</summary>
+    public ScriptRunGuard RunGuard => _guard;
+
+    /// <summary>
+    ///     OPC UA <c>BadInternalError</c> — the quality a condition is annotated with while its predicate
+    ///     cannot be evaluated (timed out, refused because the previous run is still executing, or
+    ///     quarantined). The condition's state is frozen; the quality says why it cannot be trusted. The
+    ///     next completed evaluation restores the input-derived quality through the normal bucket tracking.
+    /// </summary>
+    private const uint StatusBadInternalError = 0x80020000u;
+
     /// <summary>Raised for every emission the Part9StateMachine produces that the engine should publish.</summary>
     public event EventHandler<ScriptedAlarmEvent>? OnEvent;
 
@@ -259,6 +309,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
             UnsubscribeFromUpstream();
             _alarms.Clear();
             _alarmsReferencing.Clear();
+            // The value cache is re-seeded below for exactly the paths this generation references. It
+            // used to survive reloads untouched, so every path a retired alarm had ever referenced stayed
+            // cached forever — a slow leak over many deploys with a churning tag set.
+            _valueCache.Clear();
             // Drop the prior generation's per-alarm scratch buffers — definitions may
             // have changed (different Inputs, different Logger), so any reuse would be
             // unsafe.
@@ -268,6 +322,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
             // recompile this one. Skipping this is what made the earlier fix a
             // no-op in production.
             _compileCache.Clear();
+            // A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts
+            // its own through the guard's source identity; this covers "redeployed to retry").
+            _guard.ResetAll();
 
             var compileFailures = new List<string>();
             foreach (var def in definitions)
@@ -284,15 +341,17 @@ public sealed class ScriptedAlarmEngine : IDisposable
 
                     // Route through CompiledScriptCache so the emitted assembly's
                     // collectible ALC participates in publish-replace cleanup.
+                    // The compiled script is run through _guard (timeout + quarantine) at evaluation time —
+                    // the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not
+                    // see that the run it abandoned was still executing.
                     var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource);
-                    var timed = new TimedScriptEvaluator<AlarmPredicateContext, bool>(evaluator, _scriptTimeout);
                     var logger = _loggerFactory.Create(def.AlarmId);
 
                     var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate);
                     var allInputs = new HashSet<string>(extraction.Reads, StringComparer.Ordinal);
                     foreach (var t in templateTokens) allInputs.Add(t);
 
-                    _alarms[def.AlarmId] = new AlarmState(def, timed, extraction.Reads, templateTokens, logger,
+                    _alarms[def.AlarmId] = new AlarmState(def, evaluator, extraction.Reads, templateTokens, logger,
                         AlarmConditionState.Fresh(def.AlarmId, _clock()));
 
                     foreach (var path in allInputs)
@@ -546,8 +605,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
     /// <param name="value">The new data value snapshot.</param>
     internal void OnUpstreamChange(string path, DataValueSnapshot value)
     {
+        if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
         _valueCache[path] = value;
-        if (_disposed) return; // don't queue new work against a disposing engine
         if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
 
         bool startDrain;
@@ -562,7 +621,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
 
         // Started outside the lock: DrainDirtyAsync runs synchronously up to its
         // first real await, and nothing it does should happen under _dirtyLock.
-        if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None));
+        if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownToken));
     }
 
     /// <summary>
@@ -705,6 +764,11 @@ public sealed class ScriptedAlarmEngine : IDisposable
             }
             finally { _evalGate.Release(); }
         }
+        catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested)
+        {
+            // Dispose cut the drain short after its bound — expected, not a failure.
+            return;
+        }
         catch (Exception ex)
         {
             _engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed");
@@ -744,7 +808,6 @@ public sealed class ScriptedAlarmEngine : IDisposable
         // short-circuit so an outright-Bad input is still observed. A bucket change with no state
         // transition is delivered out of band as a QualityChanged emission (see below).
         var worstStatus = WorstInputStatus(scratch.ReadCache);
-        var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus);
 
         // Cold-start guard — skip the predicate when any referenced upstream tag has no
         // cached value yet (the upstream subscription hasn't delivered its first push).
@@ -757,7 +820,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
             // The condition is frozen (can't trust its state), but its source quality just changed
             // bucket — annotate it out of band so a comms-lost / Bad-input scripted condition reports
             // Bad, mirroring the native OT path.
-            if (qualityBucketChanged)
+            if (TrackQualityBucket(state.Definition.AlarmId, worstStatus))
                 pendingEmissions.Add(BuildQualityEmission(state, seed, worstStatus));
             return seed;
         }
@@ -767,22 +830,41 @@ public sealed class ScriptedAlarmEngine : IDisposable
         bool predicateTrue;
         try
         {
-            predicateTrue = await state.Evaluator.RunAsync(context, ct).ConfigureAwait(false);
+            predicateTrue = await _guard.RunAsync(
+                    state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context,
+                    _scriptTimeout, ct)
+                .ConfigureAwait(false);
         }
-        catch (OperationCanceledException)
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
         {
             throw;
         }
         catch (ScriptTimeoutException tex)
         {
-            state.Logger.Warning("Alarm predicate timed out after {Timeout} — state unchanged", tex.Timeout);
-            return seed;
+            return PredicateDidNotComplete(state, seed, scratch,
+                $"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions);
+        }
+        catch (ScriptRunRefusedException ex)
+        {
+            return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
+        }
+        catch (ScriptQuarantinedException ex)
+        {
+            return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
         }
         catch (Exception ex)
         {
+            scratch.LastNonCompletion = null;
             state.Logger.Error(ex, "Alarm predicate threw — state unchanged");
             return seed;
         }
+        scratch.LastNonCompletion = null;
+
+        // Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last
+        // bucket this alarm was annotated with; tracking the input bucket before the attempt flipped it to
+        // Good on every push, so a refused / quarantined predicate (which annotates Bad) re-emitted
+        // QualityChanged on every push instead of once per real bucket change.
+        var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus);
 
         var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc);
         var transition = result.Emission != EmissionKind.None
@@ -802,6 +884,28 @@ public sealed class ScriptedAlarmEngine : IDisposable
         return result.State;
     }
 
+    /// <summary>
+    ///     The predicate did not return (timed out, refused because the previous run is still executing, or
+    ///     quarantined): hold the condition at <paramref name="seed"/>, log the reason <b>once per change of
+    ///     reason</b> (a quarantined alarm is re-attempted on every push and would otherwise log every time),
+    ///     and annotate the condition <see cref="StatusBadInternalError"/> out of band — the frozen state
+    ///     cannot be trusted and the quality is how the operator sees that. The next completed evaluation
+    ///     restores the input-derived quality through the normal bucket tracking.
+    /// </summary>
+    private AlarmConditionState PredicateDidNotComplete(
+        AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason,
+        List<ScriptedAlarmEvent> pendingEmissions)
+    {
+        if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal))
+        {
+            scratch.LastNonCompletion = reason;
+            state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason);
+        }
+        if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError))
+            pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError));
+        return seed;
+    }
+
     /// <summary>
     ///     Refill <paramref name="cache"/> in place from <c>_valueCache</c>, falling
     ///     back to a synchronous <c>ITagUpstreamSource.ReadTag</c> for paths whose
@@ -986,7 +1090,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
     {
         if (_disposed) return;
         var ids = _alarms.Keys.ToArray();
-        TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None));
+        TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownToken));
     }
 
     /// <summary>
@@ -1125,6 +1229,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
             }
             finally { _evalGate.Release(); }
         }
+        catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested)
+        {
+            return; // Dispose cut the check short after its bound
+        }
         catch (Exception ex)
         {
             _engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed");
@@ -1165,11 +1273,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
         // flight can outlive the engine and write to a (possibly disposed) store
         // after Dispose() has returned. The tasks re-check _disposed after
         // acquiring the gate and bail out, but the await still has to complete.
+        // BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store
+        // (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that
+        // calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a
+        // store that honours cancellation returns promptly), log, and give them one short grace period.
         Task[] toAwait;
         lock (_inFlightLock) { toAwait = [.. _inFlight]; }
         if (toAwait.Length > 0)
         {
-            try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); }
+            var all = Task.WhenAll(toAwait);
+            try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); }
+            catch (TimeoutException)
+            {
+                _engineLogger.Warning(
+                    "ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)",
+                    toAwait.Length, DisposeDrainTimeout);
+                _shutdownCts.Cancel();
+                try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); }
+                catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ }
+            }
             catch (Exception ex)
             {
                 // Background task failures already logged inside ReevaluateAsync /
@@ -1192,6 +1314,16 @@ public sealed class ScriptedAlarmEngine : IDisposable
         _alarmsReferencing.Clear();
         _scratchByAlarmId.Clear();
         _lastQualityBucketByAlarmId.Clear();
+
+        // Release the shutdown CTS once nothing can still be observing it through a registration: every
+        // drained task has completed. A task still stuck past the grace keeps its registration, so the CTS is
+        // left to the GC in that case (logged above).
+        if (toAwait.All(t => t.IsCompleted))
+        {
+            _shutdownCts.Dispose();
+            _shutdownCtsDisposed = true;
+        }
+
         // Dispose every compiled-predicate ALC so the engine's shutdown actually
         // releases the emitted assemblies. The drain above ensures no evaluator is
         // mid-call; CompiledScriptCache.Dispose internally guards against use-after-
@@ -1201,7 +1333,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
 
     private sealed record AlarmState(
         ScriptedAlarmDefinition Definition,
-        TimedScriptEvaluator<AlarmPredicateContext, bool> Evaluator,
+        ScriptEvaluator<AlarmPredicateContext, bool> Evaluator,
         IReadOnlySet<string> Inputs,
         IReadOnlyList<string> TemplateTokens,
         ILogger Logger,
@@ -1227,6 +1359,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
         /// <summary>Gets the predicate evaluation context.</summary>
         public AlarmPredicateContext Context { get; }
 
+        /// <summary>The reason the last evaluation did not complete (timeout / refused / quarantined), or null
+        /// after a completed run — the once-per-change log gate for <c>PredicateDidNotComplete</c>.</summary>
+        public string? LastNonCompletion { get; set; }
+
         /// <summary>
         /// Initializes a new AlarmScratch with the specified inputs, logger, and clock.
         /// </summary>

src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using Serilog;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 
 namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
@@ -23,19 +24,27 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
 ///         adapter doesn't maintain per-subscription state beyond the filter set — it
 ///         checks each emission against every live subscription.
 ///     </para>
+///     <para>
+///         A throwing <see cref="OnAlarmEvent"/> subscriber is logged at Warning (naming the alarm) and
+///         does not crash the adapter — the same posture as the engine's own <c>FireEvent</c>. It used to
+///         be swallowed silently: a lost alarm event with no trace.
+///     </para>
 /// </remarks>
 public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable
 {
     private readonly ScriptedAlarmEngine _engine;
+    private readonly ILogger _logger;
     private readonly ConcurrentDictionary<string, Subscription> _subscriptions
         = new(StringComparer.Ordinal);
     private bool _disposed;
 
     /// <summary>Initializes a new instance of the <see cref="ScriptedAlarmSource"/> class.</summary>
     /// <param name="engine">The scripted alarm engine to expose.</param>
-    public ScriptedAlarmSource(ScriptedAlarmEngine engine)
+    /// <param name="logger">Where a throwing subscriber is reported; null ⇒ Serilog's global logger.</param>
+    public ScriptedAlarmSource(ScriptedAlarmEngine engine, ILogger? logger = null)
     {
         _engine = engine ?? throw new ArgumentNullException(nameof(engine));
+        _logger = logger ?? Log.Logger;
         _engine.OnEvent += OnEngineEvent;
     }
 
@@ -98,7 +107,13 @@ public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable
                 Severity: evt.Severity,
                 SourceTimestampUtc: evt.TimestampUtc);
             try { OnAlarmEvent?.Invoke(this, payload); }
-            catch { /* subscriber exceptions don't crash the adapter */ }
+            catch (Exception ex)
+            {
+                // Subscriber exceptions don't crash the adapter — but they are not silent either: a lost
+                // alarm event must leave a trace the operator can find.
+                _logger.Warning(ex, "ScriptedAlarmSource OnAlarmEvent subscriber threw for {AlarmId} ({Emission})",
+                    evt.AlarmId, evt.Emission);
+            }
         }
     }
 

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs

@@ -17,13 +17,15 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
 /// </summary>
 /// <remarks>
 ///     <para>
-///         Deny-list is the authoritative Phase 7 plan set:
+///         Deny-list is the authoritative Phase 7 plan set, hardened since:
 ///         <c>System.IO</c>, <c>System.Net</c>, <c>System.Diagnostics</c>,
-///         <c>System.Reflection</c>, <c>System.Threading.Thread</c>,
-///         <c>System.Threading.Tasks</c> (scripts are synchronous predicates — no
-///         legitimate need to start background tasks; a <c>Task.Run</c> fan-out outlives
-///         the evaluation timeout entirely), <c>System.Runtime.InteropServices</c>,
-///         <c>Microsoft.Win32</c>.
+///         <c>System.Reflection</c>, <c>System.Threading.Tasks</c> and the whole of
+///         <c>System.Threading</c> (scripts are synchronous predicates — no legitimate need
+///         to start background work, and no blocking primitive may park the evaluation's pool
+///         thread past its budget; <c>CancellationToken</c> / <c>Interlocked</c> /
+///         <c>Volatile</c> are allow-listed back in), <c>System.Runtime.InteropServices</c>,
+///         <c>System.Runtime.Loader</c>, <c>Microsoft.Win32</c>; plus the member-granular
+///         <see cref="ForbiddenMembers"/> (<c>System.Type</c>'s reflection entry points).
 ///     </para>
 ///     <para>
 ///         Deny-list prefix match. <c>System.Net</c> catches <c>System.Net.Http</c>,
@@ -63,13 +65,19 @@ public static class ForbiddenTypeAnalyzer
         "System.Net",
         "System.Diagnostics",           // catches Process, ProcessStartInfo, EventLog, Trace/Debug file sinks
         "System.Reflection",
-        // System.Threading.Thread is NOT in this list: Thread's containing namespace is
-        // "System.Threading" (not "System.Threading.Thread"), so a prefix check on
-        // "System.Threading.Thread" never matches. Thread is denied type-granularly via
-        // ForbiddenFullTypeNames instead so the check actually fires.
         "System.Threading.Tasks",       // Task.Run / Parallel — scripts are synchronous predicates
                                         // and have no legitimate need to start background work;
                                         // a Task fan-out outlives the evaluation timeout entirely.
+                                        // Listed BEFORE the parent prefix so its rejection names it.
+        "System.Threading",             // Thread / ThreadPool / Timer (background work that outlives
+                                        // the timeout) AND every blocking primitive — SemaphoreSlim,
+                                        // Monitor, SpinWait, ManualResetEvent(Slim), Mutex,
+                                        // EventWaitHandle — which park the script's pool thread past
+                                        // its budget (the same orphan ScriptRunGuard exists to bound)
+                                        // and, for the named kernel objects, touch system-global
+                                        // state. The few sync-free primitives a script may
+                                        // legitimately name are allow-listed in
+                                        // AllowedTypesInForbiddenNamespaces.
         "System.Runtime.InteropServices",
         "System.Runtime.Loader",        // AssemblyLoadContext + AssemblyDependencyResolver —
                                         // arbitrary DLL load into the host process.
@@ -79,11 +87,55 @@ public static class ForbiddenTypeAnalyzer
         "Microsoft.Win32",              // registry
     ];
 
+    /// <summary>
+    ///     Types that stay reachable although their namespace is in
+    ///     <see cref="ForbiddenNamespacePrefixes"/>. Matched by exact fully-qualified name against the
+    ///     resolved type symbol (generic constructions unwrapped). The whole of <c>System.Threading</c> is
+    ///     denied; these three are the sync-free primitives with no blocking surface — a
+    ///     <see cref="CancellationToken"/> is read-only state, <see cref="Interlocked"/> and
+    ///     <see cref="Volatile"/> are single-instruction memory operations.
+    /// </summary>
+    public static readonly IReadOnlyList<string> AllowedTypesInForbiddenNamespaces =
+    [
+        "System.Threading.CancellationToken",
+        "System.Threading.Interlocked",
+        "System.Threading.Volatile",
+    ];
+
+    /// <summary>
+    ///     Members scripts are NOT allowed to call on otherwise-allowed types, as
+    ///     <c>(declaring type, member name)</c>. <c>System.Type</c> must stay reachable
+    ///     (<c>typeof</c>, <c>Name</c>, <c>IsValueType</c> are ordinary metadata reads) but its reflection
+    ///     entry points obtain a type or member by NAME — <c>Type.GetType("System.IO.File")</c> reaches a
+    ///     forbidden type without ever naming it syntactically, and <c>GetMethod*</c> / <c>GetProperty</c>
+    ///     / <c>InvokeMember</c> / … hand back a <c>System.Reflection</c> handle (which pass 2 catches when
+    ///     declared, but not when consumed inline). Instance <c>object.GetType()</c> is declared on
+    ///     <c>System.Object</c> and is not matched.
+    /// </summary>
+    public static readonly IReadOnlyList<(string Type, string Member)> ForbiddenMembers =
+    [
+        ("System.Type", "GetType"),
+        ("System.Type", "GetMethod"),
+        ("System.Type", "GetMethods"),
+        ("System.Type", "InvokeMember"),
+        ("System.Type", "GetProperty"),
+        ("System.Type", "GetProperties"),
+        ("System.Type", "GetField"),
+        ("System.Type", "GetFields"),
+        ("System.Type", "GetMember"),
+        ("System.Type", "GetMembers"),
+        ("System.Type", "GetConstructor"),
+        ("System.Type", "GetConstructors"),
+        ("System.Type", "GetEvent"),
+        ("System.Type", "GetEvents"),
+        ("System.Type", "GetDefaultMembers"),
+    ];
+
     /// <summary>
     ///     Fully-qualified type names scripts are NOT allowed to reference, regardless of
     ///     namespace. Used when a dangerous type shares its namespace with legitimate types
-    ///     (e.g. <c>System.Threading</c> hosts both <c>CancellationToken</c> and
-    ///     <c>Thread</c>) so a namespace-prefix rule would over-block. Matched by exact
+    ///     (e.g. <c>System.Runtime.CompilerServices</c> hosts both <c>MethodImplOptions</c> and
+    ///     <c>RuntimeHelpers</c>) so a namespace-prefix rule would over-block. Matched by exact
     ///     fully-qualified name against the resolved <em>type</em> symbol — every member of
     ///     the type (including read-only ones) is therefore rejected.
     /// </summary>
@@ -99,19 +151,22 @@ public static class ForbiddenTypeAnalyzer
     ///         <item><c>System.Activator</c> — <c>CreateInstance</c> is a
     ///         reflection-equivalent escape that constructs a forbidden type by name
     ///         without ever naming it syntactically.</item>
-    ///         <item><c>System.Threading.Thread</c> — raw thread creation bypasses the
-    ///         per-evaluation timeout; denied type-granularly because its containing
-    ///         namespace is <c>System.Threading</c> (shared with allowed types like
-    ///         <c>CancellationToken</c>), so a namespace-prefix rule cannot reach it
-    ///         without blocking unrelated types.</item>
-    ///         <item><c>System.Threading.ThreadPool</c> / <c>System.Threading.Timer</c> —
-    ///         background-work vectors that outlive the per-evaluation timeout; same
-    ///         threat as <c>Task.Run</c>.</item>
     ///         <item><c>System.Runtime.CompilerServices.Unsafe</c> — exposes raw type-
     ///         reinterpretation (<c>As&lt;TFrom,TTo&gt;</c>, <c>As&lt;T&gt;(object)</c>)
     ///         that bypasses the CLR type system without requiring an <c>unsafe</c> context at
     ///         the call site; enables type-confusion and managed heap corruption.</item>
+    ///         <item><c>System.Runtime.CompilerServices.RuntimeHelpers</c> —
+    ///         <c>GetUninitializedObject</c> constructs an instance of any type (a forbidden
+    ///         one included) with no constructor and no syntactic type name beyond a
+    ///         <c>Type</c> value; <c>PrepareMethod</c> / <c>RunClassConstructor</c> reach
+    ///         runtime internals.</item>
     ///     </list>
+    ///     <para>
+    ///         <c>System.Threading.Thread</c> / <c>ThreadPool</c> / <c>Timer</c> used to be listed
+    ///         here; since the whole <c>System.Threading</c> namespace is denied (with a type-granular
+    ///         allow-list, see <see cref="AllowedTypesInForbiddenNamespaces"/>) the prefix rule reaches
+    ///         them first.
+    ///     </para>
     /// </remarks>
     public static readonly IReadOnlyList<string> ForbiddenFullTypeNames =
     [
@@ -119,26 +174,6 @@ public static class ForbiddenTypeAnalyzer
         "System.AppDomain",
         "System.GC",
         "System.Activator",
-        // System.Threading.Thread lives in the System.Threading namespace (shared with
-        // CancellationToken, SemaphoreSlim, etc.), so a namespace-prefix deny-list cannot
-        // target it without blocking those legitimate types. Denied type-granularly here.
-        "System.Threading.Thread",
-        // Broadening the references list to the BCL trusted-platform-assemblies set
-        // re-exposed two background-work vectors the original deny-list missed. Both live
-        // in System.Threading (shared with allowed sync primitives like CancellationToken /
-        // SemaphoreSlim), so they must be denied type-granularly:
-        //
-        //   System.Threading.ThreadPool — QueueUserWorkItem / UnsafeQueueUserWorkItem
-        //                                 re-introduce the background-fanout threat
-        //                                 closed against System.Threading.Tasks.
-        //   System.Threading.Timer      — Timer(callback, …) schedules unbounded work
-        //                                 that outlives the per-evaluation timeout.
-        //
-        // System.Runtime.Loader.AssemblyLoadContext is also covered, but at the namespace-
-        // prefix level above (System.Runtime.Loader) so future BCL additions to that
-        // namespace are denied by default.
-        "System.Threading.ThreadPool",
-        "System.Threading.Timer",
         // System.Runtime.CompilerServices.Unsafe exposes raw type-reinterpretation
         // (Unsafe.As<TFrom, TTo>, Unsafe.As<T>(object)) that bypasses the CLR type system
         // without requiring an 'unsafe' compilation context at the call site. In a script,
@@ -148,6 +183,11 @@ public static class ForbiddenTypeAnalyzer
         // compile-time-only types (CallerMemberNameAttribute, MethodImplAttribute,
         // MethodImplOptions) that scripts may legitimately reference.
         "System.Runtime.CompilerServices.Unsafe",
+        // RuntimeHelpers.GetUninitializedObject(Type) builds an instance of ANY type — a
+        // forbidden one obtained through Type.GetType (itself denied via ForbiddenMembers)
+        // included — without a constructor or a syntactic type name. Same namespace caveat
+        // as Unsafe above, so type-granular.
+        "System.Runtime.CompilerServices.RuntimeHelpers",
     ];
 
     /// <summary>
@@ -277,28 +317,61 @@ public static class ForbiddenTypeAnalyzer
         if (rejections.Any(r => r.Span == span && r.TypeName == typeName))
             return;
 
-        var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty;
-        foreach (var forbidden in ForbiddenNamespacePrefixes)
-        {
-            if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal))
-            {
-                rejections.Add(new ForbiddenTypeRejection(
-                    Span: span,
-                    TypeName: typeName,
-                    Namespace: ns,
-                    Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " +
-                             $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules."));
-                return;
-            }
-        }
-
-        // Type-granular deny-list — dangerous types that live in the allow-listed
-        // System namespace and so cannot be caught by ForbiddenNamespacePrefixes.
-        // Matched on the full type name; OriginalDefinition unwraps any generic
-        // construction before naming.
+        // Full name of the (unwrapped) type — shared by the allow-list, the type deny-list and the
+        // member deny-list below. OriginalDefinition unwraps any generic construction before naming.
         var fullTypeName = typeSymbol.OriginalDefinition.ToDisplayString(
             SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle(
                 SymbolDisplayGlobalNamespaceStyle.Omitted));
+
+        var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty;
+
+        // Member deny-list first: the containing type (System.Type) is allowed, so neither list
+        // below would ever look at the member name.
+        if (symbol is IMethodSymbol method)
+        {
+            var declaring = method.ContainingType.OriginalDefinition.ToDisplayString(
+                SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle(
+                    SymbolDisplayGlobalNamespaceStyle.Omitted));
+            foreach (var (type, member) in ForbiddenMembers)
+            {
+                if (declaring == type && method.Name == member)
+                {
+                    rejections.Add(new ForbiddenTypeRejection(
+                        Span: span,
+                        TypeName: $"{type}.{member}",
+                        Namespace: ns,
+                        Message: $"Member '{type}.{member}' is on the sandbox forbidden-member deny-list: " +
+                                 "it obtains a type or member by name, which bypasses the type allow-list " +
+                                 "(see ForbiddenTypeAnalyzer.ForbiddenMembers for the authoritative list)."));
+                    return;
+                }
+            }
+        }
+
+        // A type on the allow-list stays reachable even though its namespace is denied; it is still
+        // subject to the type-granular deny-list below (nothing is on both today).
+        var allowListed = AllowedTypesInForbiddenNamespaces.Contains(fullTypeName, StringComparer.Ordinal);
+
+        if (!allowListed)
+        {
+            foreach (var forbidden in ForbiddenNamespacePrefixes)
+            {
+                if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal))
+                {
+                    rejections.Add(new ForbiddenTypeRejection(
+                        Span: span,
+                        TypeName: typeName,
+                        Namespace: ns,
+                        Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " +
+                                 $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules."));
+                    return;
+                }
+            }
+        }
+
+        // Type-granular deny-list — dangerous types that live in an allow-listed namespace
+        // (System, System.Runtime.CompilerServices) and so cannot be caught by
+        // ForbiddenNamespacePrefixes.
         foreach (var forbiddenType in ForbiddenFullTypeNames)
         {
             if (fullTypeName == forbiddenType)

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs

@@ -0,0 +1,15 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+/// <summary>
+///     Implemented by a script seam that runs its scripts through a <see cref="ScriptRunGuard"/>, so the
+///     host that drives its deploy boundary can lift every quarantine on a redeploy (the guard's own
+///     source-change detection covers an edited script; this covers "the operator redeployed to retry").
+/// </summary>
+public interface IScriptQuarantineOwner
+{
+    /// <summary>Lift every quarantine and forget every consecutive-timeout count (<see cref="ScriptRunGuard.ResetAll"/>).</summary>
+    void ResetQuarantines();
+
+    /// <summary>The script keys currently quarantined.</summary>
+    IReadOnlyCollection<string> QuarantinedScripts { get; }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs

@@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
 ///         Original exception (if any) is attached so the main log's diagnostics keep
 ///         the full stack trace.
 ///     </para>
+///     <para>
+///         <b>Rate-limited per script.</b> A predicate that throws on every push would otherwise mirror
+///         one Warning per push into the main log; a <see cref="ScriptLogRateLimiter"/> keyed by script
+///         name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The
+///         scripts-*.log file sink is unaffected and keeps the full stream.
+///     </para>
 /// </remarks>
 public sealed class ScriptLogCompanionSink : ILogEventSink
 {
     private readonly ILogger _mainLogger;
     private readonly LogEventLevel _minMirrorLevel;
+    private readonly ScriptLogRateLimiter _limiter;
 
     /// <summary>Initializes a new instance of the ScriptLogCompanionSink.</summary>
     /// <param name="mainLogger">The companion logger to mirror events to.</param>
     /// <param name="minMirrorLevel">The minimum log level to mirror (default: Error).</param>
-    public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error)
+    /// <param name="limiter">Per-script rate limiter; null ⇒ the default 20 per minute.</param>
+    public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error,
+        ScriptLogRateLimiter? limiter = null)
     {
         _mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger));
         _minMirrorLevel = minMirrorLevel;
+        _limiter = limiter ?? new ScriptLogRateLimiter();
     }
 
-    /// <summary>Emits a log event to the companion logger if it meets the minimum level.</summary>
+    /// <summary>Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget.</summary>
     /// <param name="logEvent">The log event to emit.</param>
     public void Emit(LogEvent logEvent)
     {
@@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink
             scriptName = s;
         }
 
+        var admission = _limiter.Admit(scriptName);
+        if (admission.SuppressedInPreviousWindow > 0)
+        {
+            _mainLogger.Warning(
+                "[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log",
+                scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds);
+        }
+        if (!admission.Admitted) return;
+
         var rendered = logEvent.RenderMessage();
         if (logEvent.Exception is not null)
         {

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs

@@ -0,0 +1,126 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+/// <summary>
+///     Per-source fixed-window rate limit for the script-log fan-out: the first
+///     <see cref="DefaultLimitPerWindow"/> entries a source emits in a <see cref="DefaultWindow"/> are
+///     admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first
+///     admission after the window rolls — so the caller can emit one summary line in the dropped entries'
+///     place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot
+///     shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the
+///     cluster bus / main log.
+/// </summary>
+/// <remarks>
+///     Used by <see cref="ScriptLogTopicSink"/> (key = script + virtual-tag + alarm identity),
+///     <see cref="ScriptLogCompanionSink"/> (key = script name) and the virtual-tag actor's per-failure
+///     log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources
+///     are swept every <see cref="SweepEvery"/> admissions so a churning tag set cannot grow the table
+///     without bound.
+/// </remarks>
+public sealed class ScriptLogRateLimiter
+{
+    /// <summary>Entries admitted per source per window by default: 20.</summary>
+    public const int DefaultLimitPerWindow = 20;
+
+    /// <summary>The default window: one minute.</summary>
+    public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1);
+
+    private const int SweepEvery = 256;
+
+    private readonly Dictionary<string, Source> _sources = new(StringComparer.Ordinal);
+    private readonly int _limit;
+    private readonly TimeSpan _window;
+    private readonly Func<DateTime> _clock;
+    private int _admitsSinceSweep;
+
+    /// <summary>Initializes a limiter.</summary>
+    /// <param name="limitPerWindow">Entries admitted per source per window; values below 1 are treated as 1.</param>
+    /// <param name="window">The fixed window; null ⇒ <see cref="DefaultWindow"/>.</param>
+    /// <param name="clock">UTC clock; null ⇒ <see cref="DateTime.UtcNow"/>.</param>
+    public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func<DateTime>? clock = null)
+    {
+        _limit = Math.Max(1, limitPerWindow);
+        _window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow;
+        _clock = clock ?? (static () => DateTime.UtcNow);
+    }
+
+    /// <summary>Entries admitted per source per window, as configured (after clamping to at least 1).</summary>
+    public int LimitPerWindow => _limit;
+
+    /// <summary>The window length.</summary>
+    public TimeSpan Window => _window;
+
+    /// <summary>
+    ///     Ask whether <paramref name="sourceKey"/>'s next entry may be emitted. When the window has rolled
+    ///     since the last call, <see cref="ScriptLogAdmission.SuppressedInPreviousWindow"/> carries the
+    ///     number of entries dropped in the window that just closed (zero otherwise) — the caller emits one
+    ///     summary for them BEFORE the admitted entry.
+    /// </summary>
+    /// <param name="sourceKey">Identity of the emitting script instance.</param>
+    /// <returns>The admission decision.</returns>
+    public ScriptLogAdmission Admit(string sourceKey)
+    {
+        ArgumentNullException.ThrowIfNull(sourceKey);
+        var now = _clock();
+        lock (_sources)
+        {
+            if (++_admitsSinceSweep >= SweepEvery)
+            {
+                _admitsSinceSweep = 0;
+                Sweep(now);
+            }
+
+            if (!_sources.TryGetValue(sourceKey, out var source))
+            {
+                source = new Source { WindowStart = now };
+                _sources[sourceKey] = source;
+            }
+
+            long summary = 0;
+            if (now - source.WindowStart >= _window)
+            {
+                summary = source.Suppressed;
+                source.WindowStart = now;
+                source.Admitted = 0;
+                source.Suppressed = 0;
+            }
+
+            source.LastSeen = now;
+            if (source.Admitted < _limit)
+            {
+                source.Admitted++;
+                return new ScriptLogAdmission(true, summary);
+            }
+
+            source.Suppressed++;
+            return new ScriptLogAdmission(false, summary);
+        }
+    }
+
+    private void Sweep(DateTime now)
+    {
+        var idle = now - (_window + _window);
+        List<string>? dead = null;
+        foreach (var (key, source) in _sources)
+        {
+            // A source that still has a pending summary is kept: its next entry must report the count.
+            if (source.LastSeen < idle && source.Suppressed == 0)
+                (dead ??= []).Add(key);
+        }
+        if (dead is null) return;
+        foreach (var key in dead) _sources.Remove(key);
+    }
+
+    private sealed class Source
+    {
+        public DateTime WindowStart;
+        public DateTime LastSeen;
+        public int Admitted;
+        public long Suppressed;
+    }
+}
+
+/// <summary>Result of <see cref="ScriptLogRateLimiter.Admit"/>.</summary>
+/// <param name="Admitted">Whether the entry may be emitted.</param>
+/// <param name="SuppressedInPreviousWindow">Entries dropped in the window that just closed — non-zero at
+/// most once per window roll; the caller emits one summary line for them.</param>
+public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow);

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs

@@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
 ///         property so pipelines that pre-date this sink continue to work correctly. If
 ///         neither property is present <c>ScriptId</c> defaults to <c>"unknown"</c>.
 ///     </para>
+///     <para>
+///         <b>Rate-limited per source.</b> A script that logs at Information on every evaluation of a 1 Hz
+///         input put one rendered string per second per script onto the cluster bus; 200 such scripts shed
+///         everyone's entries out of the bounded telemetry channels. A <see cref="ScriptLogRateLimiter"/>
+///         keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per
+///         source and publishes one summary entry naming the dropped count when the window rolls.
+///     </para>
 /// </remarks>
 public sealed class ScriptLogTopicSink : ILogEventSink
 {
     private readonly IScriptLogPublisher _publisher;
     private readonly LogEventLevel _min;
+    private readonly ScriptLogRateLimiter _limiter;
 
     /// <summary>Initializes a new instance of the <see cref="ScriptLogTopicSink"/> class.</summary>
     /// <param name="publisher">The publisher that routes entries to the cluster bus. Must not be <c>null</c>.</param>
     /// <param name="min">Minimum log level to forward. Events below this level are discarded (default: <see cref="LogEventLevel.Information"/>).</param>
+    /// <param name="limiter">Per-source rate limiter; null ⇒ the default 20 per minute.</param>
     /// <exception cref="ArgumentNullException">Thrown when <paramref name="publisher"/> is <c>null</c>.</exception>
-    public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information)
+    public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information,
+        ScriptLogRateLimiter? limiter = null)
     {
         _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
         _min = min;
+        _limiter = limiter ?? new ScriptLogRateLimiter();
     }
 
     /// <summary>
     ///     Converts the <paramref name="logEvent"/> to a <see cref="ScriptLogEntry"/> and
-    ///     publishes it, unless the event is below the configured minimum level.
+    ///     publishes it, unless the event is below the configured minimum level or the source is over its
+    ///     per-minute budget.
     /// </summary>
     /// <param name="logEvent">The Serilog event to process. Silently ignored when <c>null</c>.</param>
     public void Emit(LogEvent logEvent)
@@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink
                 ? s
                 : null;
 
+        var scriptId = P(ScriptLoggerFactory.ScriptIdProperty)
+                       ?? P(ScriptLoggerFactory.ScriptNameProperty)
+                       ?? "unknown";
+        var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty);
+        var alarmId = P(ScriptLoggerFactory.AlarmIdProperty);
+        var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty);
+
+        var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId));
+        if (admission.SuppressedInPreviousWindow > 0)
+        {
+            _publisher.Publish(new ScriptLogEntry(
+                ScriptId: scriptId,
+                Level: nameof(LogEventLevel.Warning),
+                Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " +
+                         $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " +
+                         $"(limit {_limiter.LimitPerWindow}/window); see scripts-*.log for the full stream",
+                TimestampUtc: logEvent.Timestamp.UtcDateTime,
+                VirtualTagId: virtualTagId,
+                AlarmId: alarmId,
+                EquipmentId: equipmentId));
+        }
+        if (!admission.Admitted) return;
+
         _publisher.Publish(new ScriptLogEntry(
-            ScriptId: P(ScriptLoggerFactory.ScriptIdProperty)
-                      ?? P(ScriptLoggerFactory.ScriptNameProperty)
-                      ?? "unknown",
+            ScriptId: scriptId,
             Level: logEvent.Level.ToString(),
             Message: logEvent.RenderMessage(),
             TimestampUtc: logEvent.Timestamp.UtcDateTime,
-            VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty),
-            AlarmId: P(ScriptLoggerFactory.AlarmIdProperty),
-            EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty)));
+            VirtualTagId: virtualTagId,
+            AlarmId: alarmId,
+            EquipmentId: equipmentId));
     }
 }

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs

@@ -0,0 +1,420 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using Serilog;
+using ZB.MOM.WW.OtOpcUa.Commons.Observability;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+/// <summary>
+///     Per-script run quarantine shared by every script seam on the node. Wraps each evaluation of a
+///     <see cref="ScriptEvaluator{TContext, TResult}"/> with the wall-clock timeout
+///     <see cref="TimedScriptEvaluator{TContext, TResult}"/> used to provide, and adds what that wrapper
+///     could not: knowledge of whether the script it abandoned is <em>still running</em>.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>The leak this closes.</b> A timed-out script is abandoned, not stopped — a CPU-bound
+///         <c>while(true)</c> keeps its thread-pool thread forever. Each seam re-evaluated on every trigger,
+///         so a runaway script on a 1 Hz input leaked one pool thread per second until the pool starved and
+///         unrelated work (Akka dispatchers, the OPC UA publish thread, gRPC) stalled. The guard keys every
+///         run by script instance and:
+///     </para>
+///     <list type="number">
+///         <item><b>Refuses to start a run while that script's previous run is still executing</b> —
+///         <see cref="ScriptRunRefusedException"/> (<see cref="PreviousRunStillRunningReason"/>), counted on
+///         <see cref="RefusedWhileRunningCount"/>. One orphan per script, never one per trigger.</item>
+///         <item><b>Quarantines after <see cref="ScriptingOptions.QuarantineAfterConsecutiveTimeouts"/>
+///         consecutive timeouts</b> (only a timeout is a strike; a refusal is a Bad publish plus its counter
+///         and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a
+///         timeout plus two refusals and must not quarantine; a run that returns — a value or a throw —
+///         resets the streak), <b>or on a refusal behind a stuck orphan</b> — one that has been executing for
+///         longer than <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> × the timeout (20 s at the
+///         2 s default), which is how a never-returning <c>while(true)</c> (one timeout, then only refusals)
+///         gets quarantined: no further dispatch, every attempt throws
+///         <see cref="ScriptQuarantinedException"/>, one Error log, one <c>otopcua.script.quarantined</c>
+///         increment, one <see cref="Quarantined"/> event. Lifted by a source change (the operator fixed the
+///         script — a new source identity under the same key starts clean and the old orphan no longer blocks
+///         it), by <see cref="ResetAll"/> on redeploy, or by <see cref="Reset"/>.</item>
+///         <item><b>Quarantines on an allocation-budget breach</b>: the run's own-thread allocation is sampled
+///         with <see cref="GC.GetAllocatedBytesForCurrentThread"/> around the script delegate and compared
+///         with <see cref="ScriptingOptions.MaxAllocBytesPerRun"/>. This is observation after the fact — a
+///         single run that allocates enough to OOM the process is not stopped; that needs an out-of-process
+///         runner.</item>
+///     </list>
+///     <para>
+///         <b>The timeout clock starts when the script starts, not when it is queued.</b> The run is pushed to
+///         the pool (a CPU-bound script would otherwise execute inline and return an already-completed task
+///         that no timeout can fire on), and the wait only begins once the pool has actually picked the run
+///         up — so thread-pool starvation is charged to nobody's script. A caller-supplied cancellation wins
+///         over the timeout (<see cref="OperationCanceledException"/>, never
+///         <see cref="ScriptTimeoutException"/>) so shutdown paths are not misread as timeouts.
+///     </para>
+///     <para>
+///         <b>Cancellation reaches the evaluator, not (yet) the script.</b> On timeout the token handed to
+///         <see cref="ScriptEvaluator{TContext, TResult}.RunAsync"/> is cancelled; a script can only observe
+///         it once <c>ScriptContext</c> exposes it (follow-up — the context type lives in the Abstractions
+///         project).
+///     </para>
+/// </remarks>
+public sealed class ScriptRunGuard
+{
+    /// <summary>Failure reason a seam publishes when a run was refused because the previous one is still executing.</summary>
+    public const string PreviousRunStillRunningReason = "previous evaluation still running";
+
+    /// <summary>Prefix of every quarantine reason / <see cref="ScriptQuarantinedException"/> message.</summary>
+    public const string QuarantinedReasonPrefix = "script quarantined: ";
+
+    /// <summary>
+    ///     Counter <c>otopcua.script.quarantined</c> (Prometheus: <c>otopcua_script_quarantined</c>): one
+    ///     increment per script quarantined, tagged <c>reason</c> = <c>timeouts</c> | <c>stuck</c> | <c>allocation</c>.
+    /// </summary>
+    public static readonly Counter<long> QuarantinedCounter =
+        OtOpcUaTelemetry.Meter.CreateCounter<long>("otopcua.script.quarantined", unit: "{script}",
+            description: "Scripts quarantined by the run guard (reason: timeouts | stuck | allocation).");
+
+    /// <summary>
+    ///     Counter <c>otopcua.script.run_refused</c>: runs refused because the script's previous run was
+    ///     still executing.
+    /// </summary>
+    public static readonly Counter<long> RefusedCounter =
+        OtOpcUaTelemetry.Meter.CreateCounter<long>("otopcua.script.run_refused", unit: "{run}",
+            description: "Script runs refused because the previous run of the same script was still executing.");
+
+    private readonly ConcurrentDictionary<string, KeyState> _keys = new(StringComparer.Ordinal);
+    private readonly long _maxAllocBytesPerRun;
+    private readonly int _quarantineThreshold;
+    private readonly double _stuckFactor;
+    private readonly ILogger _logger;
+    private int _inFlight;
+    private long _started;
+    private long _refused;
+    private long _quarantined;
+
+    /// <summary>Initializes a new guard.</summary>
+    /// <param name="options">Budget + threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
+    /// <param name="logger">Where the quarantine Error line goes; null ⇒ Serilog's global logger.</param>
+    public ScriptRunGuard(ScriptingOptions? options = null, ILogger? logger = null)
+    {
+        options ??= new ScriptingOptions();
+        _maxAllocBytesPerRun = options.MaxAllocBytesPerRun;
+        _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts);
+        _stuckFactor = options.StuckOrphanQuarantineFactor;
+        _logger = logger ?? Log.Logger;
+    }
+
+    /// <summary>Raised (on the thread that detected it) when a script is quarantined.</summary>
+    public event EventHandler<ScriptQuarantinedEventArgs>? Quarantined;
+
+    /// <summary>Runs currently executing on a pool thread — abandoned (timed-out) orphans included.</summary>
+    public int InFlightCount => Volatile.Read(ref _inFlight);
+
+    /// <summary>Runs actually dispatched to the pool, lifetime.</summary>
+    public long StartedRunCount => Interlocked.Read(ref _started);
+
+    /// <summary>Runs refused because the same script's previous run was still executing, lifetime.</summary>
+    public long RefusedWhileRunningCount => Interlocked.Read(ref _refused);
+
+    /// <summary>Scripts quarantined, lifetime (a reset and re-quarantine counts twice).</summary>
+    public long QuarantinedCount => Interlocked.Read(ref _quarantined);
+
+    /// <summary>The keys currently quarantined.</summary>
+    public IReadOnlyCollection<string> QuarantinedKeys =>
+        _keys.Where(kv => kv.Value.QuarantineReason is not null).Select(kv => kv.Key).ToArray();
+
+    /// <summary>Whether <paramref name="key"/> is quarantined, and why.</summary>
+    /// <param name="key">The script instance key.</param>
+    /// <param name="reason">The quarantine reason (prefixed <see cref="QuarantinedReasonPrefix"/>), or null.</param>
+    /// <returns><c>true</c> when quarantined.</returns>
+    public bool IsQuarantined(string key, out string? reason)
+    {
+        reason = _keys.TryGetValue(key, out var state) ? state.QuarantineReason : null;
+        return reason is not null;
+    }
+
+    /// <summary>
+    ///     Forget everything about <paramref name="key"/>: clears its quarantine and its consecutive count, and
+    ///     abandons its in-flight slot (an orphan still running keeps counting on <see cref="InFlightCount"/>
+    ///     until it finishes, but no longer blocks the key).
+    /// </summary>
+    /// <param name="key">The script instance key.</param>
+    public void Reset(string key) => _keys.TryRemove(key, out _);
+
+    /// <summary>
+    ///     <see cref="Reset"/> for every key — the redeploy / compile-cache-clear boundary. Orphans keep
+    ///     counting on <see cref="InFlightCount"/> until they finish.
+    /// </summary>
+    public void ResetAll() => _keys.Clear();
+
+    /// <summary>
+    ///     Run <paramref name="evaluator"/> against <paramref name="context"/> under the guard.
+    /// </summary>
+    /// <typeparam name="TContext">The script context type.</typeparam>
+    /// <typeparam name="TResult">The script result type.</typeparam>
+    /// <param name="key">The script instance key (virtual-tag id, calc-tag RawPath, alarm id).</param>
+    /// <param name="sourceIdentity">
+    ///     The script source (or its hash). A change under the same key clears the key's quarantine and
+    ///     abandons its orphan slot — the operator fixed the script. Null ⇒ not tracked.
+    /// </param>
+    /// <param name="evaluator">The compiled script.</param>
+    /// <param name="context">The script's <c>ctx</c>.</param>
+    /// <param name="timeout">Wall-clock budget, measured from the moment the pool starts the run.</param>
+    /// <param name="ct">Caller cancellation; wins over the timeout.</param>
+    /// <returns>The script's result.</returns>
+    /// <exception cref="ScriptQuarantinedException">The key is (or just became) quarantined.</exception>
+    /// <exception cref="ScriptRunRefusedException">The key's previous run is still executing.</exception>
+    /// <exception cref="ScriptTimeoutException">The run exceeded <paramref name="timeout"/>.</exception>
+    /// <exception cref="OperationCanceledException"><paramref name="ct"/> was cancelled.</exception>
+    public async Task<TResult> RunAsync<TContext, TResult>(
+        string key,
+        string? sourceIdentity,
+        ScriptEvaluator<TContext, TResult> evaluator,
+        TContext context,
+        TimeSpan timeout,
+        CancellationToken ct = default)
+        where TContext : ScriptContext
+    {
+        ArgumentNullException.ThrowIfNull(key);
+        ArgumentNullException.ThrowIfNull(evaluator);
+        ArgumentNullException.ThrowIfNull(context);
+        if (timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive.");
+
+        var state = Admit(key, sourceIdentity);
+
+        // Slot acquisition is the whole mechanism: a second trigger landing while the previous run (orphan or
+        // not) still holds the slot is refused here, before anything is dispatched to the pool.
+        if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
+        {
+            // A refusal is a Bad publish plus its counter — NOT a strike. Only a timeout (and an allocation
+            // breach) counts toward quarantine: on a 1 Hz trigger with a 2 s budget a single 4 s overrun is
+            // one timeout followed by two refusals, and counting those would quarantine a script that merely
+            // ran long once. The refusal also leaves the timeout streak untouched.
+            Interlocked.Increment(ref _refused);
+            RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
+            // …unless the orphan is STUCK: a never-returning script times out once and is then only ever
+            // refused, so the timeout streak alone would never quarantine it. Past the stuck threshold the
+            // refusal itself quarantines (same path: Error log, metric, Bad, lifted on redeploy/source change).
+            if (_stuckFactor > 0 && Volatile.Read(ref state.RunStartedTimestamp) is var startedAt and not 0)
+            {
+                var running = Stopwatch.GetElapsedTime(startedAt);
+                var threshold = timeout * _stuckFactor;
+                if (running > threshold)
+                {
+                    Quarantine(key, state,
+                        $"its previous evaluation has been running for {running.TotalSeconds:F1} s, over " +
+                        $"{_stuckFactor:0.##}x its {timeout.TotalMilliseconds:F0} ms budget " +
+                        "(Scripting:StuckOrphanQuarantineFactor) — it is stuck; the script is not evaluated again " +
+                        "until its source changes or it is redeployed",
+                        "stuck");
+                    throw new ScriptQuarantinedException(key, state.QuarantineReason!);
+                }
+            }
+            throw new ScriptRunRefusedException(key);
+        }
+
+        Interlocked.Increment(ref _inFlight);
+        Interlocked.Increment(ref _started);
+
+        var slot = new RunSlot();
+        var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var runCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        var runToken = runCts.Token;
+
+        // Task.Run deliberately takes NO token: a pre-cancelled token would skip the lambda, `started` would
+        // never complete, and the slot would never be released. The evaluator observes the token instead.
+        var runTask = Task.Run(async () =>
+        {
+            Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp());
+            started.TrySetResult();
+            var before = GC.GetAllocatedBytesForCurrentThread();
+            try
+            {
+                // ScriptEvaluator.RunAsync executes the compiled delegate synchronously and returns a completed
+                // task, so the after-sample is taken on the same thread, right after the script returned.
+                var pending = evaluator.RunAsync(context, runToken);
+                slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before;
+                return await pending.ConfigureAwait(false);
+            }
+            catch
+            {
+                slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before;
+                throw;
+            }
+        });
+
+        // Release the slot when the run REALLY finishes — which for an abandoned run is long after this
+        // method has thrown ScriptTimeoutException. Registered before the wait so it fires regardless of
+        // which way the wait below comes out. A late-finishing orphan that breached the allocation budget
+        // still quarantines its key here.
+        _ = runTask.ContinueWith(t =>
+        {
+            Volatile.Write(ref state.RunStartedTimestamp, 0);
+            Interlocked.Exchange(ref state.InFlight, 0);
+            Interlocked.Decrement(ref _inFlight);
+            runCts.Dispose();
+            if (t.IsFaulted) _ = t.Exception; // observed below by the awaiter when it is still listening
+            if (t.IsCompletedSuccessfully && BreachedBudget(slot.AllocatedBytes))
+                Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation");
+        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+
+        // The budget clock starts when the pool starts the run, not when we queued it (P5): pool starvation
+        // is a host problem and must not be reported as "script timed out".
+        await started.Task.WaitAsync(ct).ConfigureAwait(false);
+
+        TResult result;
+        try
+        {
+            result = await runTask.WaitAsync(timeout, ct).ConfigureAwait(false);
+        }
+        catch (TimeoutException)
+        {
+            // The caller's cancel wins even when it lands a few µs after the timeout fired.
+            if (ct.IsCancellationRequested) throw new OperationCanceledException(ct);
+            runCts.Cancel();
+            if (NoteTimeout(key, state, timeout))
+                throw new ScriptQuarantinedException(key, state.QuarantineReason!);
+            throw new ScriptTimeoutException(timeout);
+        }
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception)
+        {
+            // The script returned control (by throwing) — that is a completion for quarantine purposes.
+            Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0);
+            throw;
+        }
+
+        Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0);
+        if (BreachedBudget(slot.AllocatedBytes))
+        {
+            // The continuation above has already (or will have) quarantined the key; make the reason visible
+            // to THIS caller too, so the breaching run itself publishes Bad rather than a suspect value.
+            Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation");
+            throw new ScriptQuarantinedException(key, state.QuarantineReason!);
+        }
+        return result;
+    }
+
+    /// <summary>Resolve the key's state, rolling it over when the source identity changed, and refuse a quarantined key.</summary>
+    private KeyState Admit(string key, string? sourceIdentity)
+    {
+        while (true)
+        {
+            var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), sourceIdentity);
+            if (sourceIdentity is not null && state.SourceIdentity is not null
+                && !string.Equals(state.SourceIdentity, sourceIdentity, StringComparison.Ordinal))
+            {
+                // The script changed under this key: start from a clean slate. The old state's orphan (if any)
+                // releases the GLOBAL in-flight count through its own continuation; nothing else references it.
+                _keys.TryUpdate(key, new KeyState(sourceIdentity), state);
+                continue;
+            }
+            if (sourceIdentity is not null && state.SourceIdentity is null)
+            {
+                state.SourceIdentity = sourceIdentity;
+            }
+            if (state.QuarantineReason is { } reason)
+                throw new ScriptQuarantinedException(key, reason);
+            return state;
+        }
+    }
+
+    /// <summary>Record a timeout strike; returns true when it tipped the key into quarantine. A completed run
+    /// (value or throw) resets the streak; a refusal neither counts nor resets.</summary>
+    private bool NoteTimeout(string key, KeyState state, TimeSpan timeout)
+    {
+        var count = Interlocked.Increment(ref state.ConsecutiveTimeouts);
+        if (count < _quarantineThreshold) return false;
+        Quarantine(key, state,
+            $"{count} consecutive evaluation(s) timed out (budget {timeout.TotalMilliseconds:F0} ms); " +
+            "the script is not evaluated again until its source changes or it is redeployed",
+            "timeouts");
+        return true;
+    }
+
+    private bool BreachedBudget(long allocatedBytes) =>
+        _maxAllocBytesPerRun > 0 && allocatedBytes > _maxAllocBytesPerRun;
+
+    private string AllocationReason(long allocatedBytes) =>
+        $"one evaluation allocated {allocatedBytes / (1024.0 * 1024.0):F1} MB, over the " +
+        $"{_maxAllocBytesPerRun / (1024.0 * 1024.0):F0} MB budget (Scripting:MaxAllocBytesPerRun); " +
+        "the script is not evaluated again until its source changes or it is redeployed";
+
+    private void Quarantine(string key, KeyState state, string why, string metricReason)
+    {
+        var reason = QuarantinedReasonPrefix + why;
+        // First writer wins: a late orphan and the awaiter can both reach here for the same run.
+        if (Interlocked.CompareExchange(ref state.QuarantineReason, reason, null) is not null) return;
+
+        Interlocked.Increment(ref _quarantined);
+        QuarantinedCounter.Add(1,
+            new KeyValuePair<string, object?>("script", key),
+            new KeyValuePair<string, object?>("reason", metricReason));
+        _logger.Error("Script {ScriptKey} quarantined: {Reason}", key, why);
+        try { Quarantined?.Invoke(this, new ScriptQuarantinedEventArgs(key, reason)); }
+        catch (Exception ex)
+        {
+            _logger.Warning(ex, "ScriptRunGuard Quarantined subscriber threw for {ScriptKey}", key);
+        }
+    }
+
+    private sealed class KeyState(string? sourceIdentity)
+    {
+        public string? SourceIdentity = sourceIdentity;
+        public int InFlight;
+        public int ConsecutiveTimeouts;
+        /// <summary><see cref="Stopwatch.GetTimestamp"/> when the pool started the in-flight run; 0 when idle.</summary>
+        public long RunStartedTimestamp;
+        public string? QuarantineReason;
+    }
+
+    private sealed class RunSlot
+    {
+        public long AllocatedBytes;
+    }
+}
+
+/// <summary>Carried by <see cref="ScriptRunGuard.Quarantined"/>.</summary>
+/// <param name="Key">The quarantined script instance key.</param>
+/// <param name="Reason">Why, prefixed <see cref="ScriptRunGuard.QuarantinedReasonPrefix"/>.</param>
+public sealed record ScriptQuarantinedEventArgs(string Key, string Reason);
+
+/// <summary>
+///     Thrown by <see cref="ScriptRunGuard.RunAsync{TContext, TResult}"/> when the script's previous run is
+///     still executing (an abandoned, timed-out run that never returned). Nothing was dispatched; the seam
+///     publishes Bad with <see cref="ScriptRunGuard.PreviousRunStillRunningReason"/>.
+/// </summary>
+public sealed class ScriptRunRefusedException : Exception
+{
+    /// <summary>The script instance key.</summary>
+    public string Key { get; }
+
+    /// <summary>Initializes the exception for <paramref name="key"/>.</summary>
+    /// <param name="key">The script instance key.</param>
+    public ScriptRunRefusedException(string key) : base(ScriptRunGuard.PreviousRunStillRunningReason)
+    {
+        Key = key;
+    }
+}
+
+/// <summary>
+///     Thrown by <see cref="ScriptRunGuard.RunAsync{TContext, TResult}"/> for a quarantined script: repeated
+///     timeouts, a stuck orphan or an allocation-budget breach. Nothing is dispatched while quarantined; the seam
+///     publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or
+///     redeploying.
+/// </summary>
+public sealed class ScriptQuarantinedException : Exception
+{
+    /// <summary>The script instance key.</summary>
+    public string Key { get; }
+
+    /// <summary>Initializes the exception with the quarantine reason (already prefixed).</summary>
+    /// <param name="key">The script instance key.</param>
+    /// <param name="reason">The quarantine reason, prefixed <see cref="ScriptRunGuard.QuarantinedReasonPrefix"/>.</param>
+    public ScriptQuarantinedException(string key, string reason) : base(reason)
+    {
+        Key = key;
+    }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs

@@ -0,0 +1,56 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+/// <summary>
+///     Node-wide script runtime-safety knobs (the <c>Scripting</c> appsettings section), consumed by
+///     <see cref="ScriptRunGuard"/>. One instance is shared by every script seam on the node — the
+///     virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting
+///     bounds every authored script the same way.
+/// </summary>
+/// <remarks>
+///     <para>
+///         The in-process sandbox cannot pre-empt a script: a <c>while(true)</c> is abandoned on its pool
+///         thread at the timeout, and an allocation is observed only after the run returns. These knobs bound
+///         the <em>damage</em> — one orphan per script, no further runs until the source changes — not the
+///         first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner.
+///     </para>
+/// </remarks>
+public sealed class ScriptingOptions
+{
+    /// <summary>The appsettings section this binds from.</summary>
+    public const string SectionName = "Scripting";
+
+    /// <summary>Default <see cref="MaxAllocBytesPerRun"/>: 64 MB.</summary>
+    public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024;
+
+    /// <summary>Default <see cref="QuarantineAfterConsecutiveTimeouts"/>: 3.</summary>
+    public const int DefaultQuarantineAfterConsecutiveTimeouts = 3;
+
+    /// <summary>Default <see cref="StuckOrphanQuarantineFactor"/>: 10.</summary>
+    public const double DefaultStuckOrphanQuarantineFactor = 10;
+
+    /// <summary>
+    ///     Managed bytes one script run may allocate on its own thread (sampled with
+    ///     <see cref="GC.GetAllocatedBytesForCurrentThread"/> around the run) before the script is
+    ///     quarantined. Zero or negative disables the budget. Default 64 MB.
+    /// </summary>
+    public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun;
+
+    /// <summary>
+    ///     Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every
+    ///     evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit
+    ///     reset. Only a timeout is a strike — a run refused because the previous one is still executing is
+    ///     reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or
+    ///     throw) resets it. Default 3; values below 1 are treated as 1.
+    /// </summary>
+    public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts;
+
+    /// <summary>
+    ///     A refusal quarantines the script immediately when the abandoned run it was refused behind has been
+    ///     executing for longer than this many times the script's timeout (20 s at the 2 s default). This is
+    ///     what quarantines a permanently stuck script (<c>while(true)</c>): it times out once and is then
+    ///     only ever refused, so the timeout streak alone would never reach
+    ///     <see cref="QuarantineAfterConsecutiveTimeouts"/>. A young orphan (a short overrun) is still just a
+    ///     refusal. Zero or negative disables the rule. Default 10.
+    /// </summary>
+    public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor;
+}

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs

@@ -19,10 +19,12 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
 ///         task continues running on a thread-pool thread until the Roslyn runtime
 ///         returns. In the CPU-bound-infinite-loop case that's effectively "leaked" —
 ///         the thread is tied up until the runtime decides to return, which it may
-///         never do. Phase 7 plan Stream A.4 accepts this as a known trade-off; tighter
-///         CPU budgeting would require an out-of-process script runner, which is a v3
-///         concern. In practice, the timeout + structured warning log surfaces the
-///         offending script so the operator can fix it; the orphan thread is rare.
+///         never do. This wrapper has no memory of that orphan, so re-running the same
+///         script through it on every trigger leaks one more thread per trigger — which
+///         is why the production seams (virtual tags, Calculation, scripted alarms) run
+///         through <see cref="ScriptRunGuard"/> instead: same timeout, plus a per-script
+///         "previous run still executing" refusal and a quarantine. This class remains
+///         for callers that evaluate once (tests, the authoring preview).
 ///     </para>
 ///     <para>
 ///         Caller-supplied <see cref="CancellationToken"/> is honored — if the caller

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs

@@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus
     // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync,
     // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse.
-    private readonly object _evalGate = new();
+    // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async —
+    // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's
+    // budget clock starts only once the pool has picked the script up, so pool starvation is never
+    // reported as a script timeout).
+    private readonly SemaphoreSlim _evalGate = new(1, 1);
+
+    // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight
+    // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false).
+    private readonly CancellationTokenSource _evalCts = new();
 
     // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single
     // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an
@@ -103,7 +111,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     private CalculationTimerScheduler? _timers;
     private volatile bool _running;
     private DateTime? _lastComputeUtc;
-    private bool _disposed;
+    private volatile bool _disposed;
 
     /// <summary>Occurs when a calc tag's computed value changes (or transitions Good↔Bad).</summary>
     public event EventHandler<DataChangeEventArgs>? OnDataChange;
@@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         // _logger (ILogger<CalculationDriver>) implements the non-generic ILogger, so it is passed straight
         // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution
         // throws) land on the SAME logger the driver itself was constructed with, not a separate one.
-        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout);
+        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting);
 
         RebuildTagTable(_options);
     }
@@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                     {
                         RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
                         RunTimeout = _options.RunTimeout,
+                        Scripting = _options.Scripting,
                     };
                 }
             }
@@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                     _driverInstanceId);
             }
         }
-        return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout };
+        return new CalculationDriverOptions
+        {
+            RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting,
+        };
     }
 
     /// <summary>Re-bind the tag table from <paramref name="driverConfigJson"/> — shared by
@@ -326,6 +338,11 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         StopTimers();
         // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout.
         lock (_lock) _pending.Clear();
+        // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so
+        // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole
+        // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An
+        // in-flight run that races this is already handled by the evaluator's retry-once path.
+        _evaluator.ClearCompiledScripts();
         return Task.CompletedTask;
     }
 
@@ -351,11 +368,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     /// <inheritdoc />
     public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc)
     {
-        if (!_running) return;
         lock (_lock)
         {
+            // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction,
+            // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync.
+            // Dropping them left every calc tag waiting for each input to change again; keeping them means
+            // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated.
             _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc);
             _arrivedDeps.Add(rawPath);
+            if (!_running) return;
             if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return;
 
             // QUEUE each change-triggered dependent — never evaluate here. This
@@ -491,13 +512,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     {
         if (_pumpScheduled) return;
         _pumpScheduled = true;
-        _ = Task.Run(RunEvaluationPump);
+        _ = Task.Run(RunEvaluationPumpAsync);
     }
 
-    /// <summary>Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most
-    /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the
-    /// change path evaluated inline under the state lock.</summary>
-    private void RunEvaluationPump()
+    /// <summary>Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a
+    /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated
+    /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread
+    /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured
+    /// false timeouts under pool starvation).</summary>
+    private async Task RunEvaluationPumpAsync()
     {
         while (true)
         {
@@ -515,39 +538,69 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
 
             // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still
             // evaluate one at a time while no host thread can block behind a script.
-            lock (_evalGate)
+            if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
+            try
             {
-                try
-                {
-                    RunEvaluation(next);
-                }
-                catch (Exception ex)
-                {
-                    // The evaluator never throws for script faults, so this is a defect path only — it
-                    // must not kill the pump and strand every queued evaluation behind it.
-                    _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
-                        _driverInstanceId, next.Definition.RawPath);
-                }
+                await RunEvaluationAsync(next).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                // The evaluator never throws for script faults, so this is a defect path only — it
+                // must not kill the pump and strand every queued evaluation behind it.
+                _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
+                    _driverInstanceId, next.Definition.RawPath);
+            }
+            finally
+            {
+                _evalGate.Release();
             }
         }
     }
 
-    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's
-    /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work).</summary>
-    private void EvaluateTagFromTimer(string rawPath)
+    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
+    /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
+    /// completes).</summary>
+    private async Task EvaluateTagFromTimerAsync(string rawPath)
     {
         if (!_running) return;
         PendingEvaluation? prepared;
         lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow);
         if (prepared is null) return;
-        lock (_evalGate) RunEvaluation(prepared);
+        if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
+        try
+        {
+            await RunEvaluationAsync(prepared).ConfigureAwait(false);
+        }
+        finally
+        {
+            _evalGate.Release();
+        }
+    }
+
+    /// <summary>Acquire <see cref="_evalGate"/>; false once <see cref="Dispose"/> has disposed it (a pump
+    /// iteration or timer tick that raced the teardown simply stops).</summary>
+    private async Task<bool> TryEnterEvalGateAsync()
+    {
+        if (_disposed) return false;
+        try
+        {
+            await _evalGate.WaitAsync().ConfigureAwait(false);
+        }
+        catch (ObjectDisposedException)
+        {
+            return false;
+        }
+        if (!_disposed) return true;
+        // Dispose landed while we waited (it gave up the gate without disposing it): back out.
+        _evalGate.Release();
+        return false;
     }
 
     /// <summary>Run one prepared evaluation and publish the outcome. MUST be called under
     /// <see cref="_evalGate"/> and NOT under <see cref="_lock"/>: the script run is the whole point of the
     /// split, and the script log + <see cref="OnDataChange"/> fan-out both invoke subscriber code.</summary>
     /// <param name="pending">The prepared evaluation.</param>
-    private void RunEvaluation(PendingEvaluation pending)
+    private async Task RunEvaluationAsync(PendingEvaluation pending)
     {
         var def = pending.Definition;
 
@@ -572,9 +625,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
 
         // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are
         // then the input's real ones rather than a fabricated Good stamped at evaluation time.
-        var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies);
+        var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
+            .ConfigureAwait(false);
 
-        string? failureReason = null;
+        string? failureToLog = null;
         (object? Value, uint StatusCode)? toPublish;
         lock (_lock)
         {
@@ -584,16 +638,24 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
 
             if (!result.Success)
             {
-                failureReason = result.Reason ?? "evaluator failure";
+                var reason = result.Reason ?? "evaluator failure";
                 toPublish = MarkEvaluationFailed(st, pending.TimestampUtc);
+                // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes
+                // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which
+                // flooded the script log at the input's cadence. The Bad publish was already once per
+                // transition; the log now matches it.
+                if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal))
+                    failureToLog = reason;
+                st.LastFailureReason = reason;
             }
             else
             {
+                st.LastFailureReason = null;
                 toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc);
             }
         }
 
-        if (failureReason is not null) PublishScriptLog(def, failureReason);
+        if (failureToLog is not null) PublishScriptLog(def, failureToLog);
         if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc);
     }
 
@@ -612,7 +674,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     private (object? Value, uint StatusCode)? MarkEvaluationSucceeded(
         TagState st, object? value, uint statusCode, DateTime timestampUtc)
     {
-        if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value))
+        if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value))
             return null;
 
         st.HasValue = true;
@@ -624,6 +686,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         return (value, statusCode);
     }
 
+    /// <summary>Value equality for the dedup: structural for arrays (R07:H8 — <c>object.Equals</c> is reference
+    /// equality for a <c>double[]</c>, so an array-returning timer script republished every tick through
+    /// the dual-namespace fan-out and the historian), ordinary <see cref="object.Equals(object?, object?)"/>
+    /// otherwise.</summary>
+    private static bool ValuesEqual(object? a, object? b)
+    {
+        if (a is Array x && b is Array y)
+            return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y);
+        return Equals(a, b);
+    }
+
     /// <summary>Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the
     /// worst input's status — carrying the last known value — <b>once per change of that status</b>: a
     /// repeat of the same Bad is silent, while <c>BadNoCommunication</c> arriving over an earlier
@@ -688,7 +761,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     {
         var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray();
         if (timed.Length == 0) return;
-        _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger);
+        _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger);
         _timers.Start(timed);
     }
 
@@ -701,6 +774,12 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     /// <summary>Test/diagnostic accessor: number of timer ticks skipped for in-flight groups.</summary>
     internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0;
 
+    /// <summary>Test/diagnostic accessor: sources currently remembered as failing to compile.</summary>
+    internal int CompileFailureCountForTest => _evaluator.CompileFailureCount;
+
+    /// <summary>Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache.</summary>
+    internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount;
+
     /// <summary>Test/diagnostic accessor: <see langword="true"/> when no change-triggered evaluation is
     /// queued or in flight. A test that used to assert straight after <see cref="OnDependencyValue"/> —
     /// which evaluated inline — waits on this instead, so the assertion still runs against a settled
@@ -723,9 +802,34 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         _running = false;
         StopTimers();
         lock (_lock) _pending.Clear();
+        // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop
+        // the compiled scripts.
+        try { _evalCts.Cancel(); } catch (ObjectDisposedException) { }
         _evaluator.Dispose();
+
+        // Then take the gate: once Dispose holds it no evaluation is mid-run (the token — read only under
+        // the gate — has been observed), so the gate and the CTS can be disposed. A late pump iteration or
+        // timer tick sees _disposed / an ObjectDisposedException in TryEnterEvalGateAsync and stops. Bounded:
+        // a run that ignores the cancel cannot hold Dispose; in that case both are left to the GC.
+        if (_evalGate.Wait(DisposeGateTimeout))
+        {
+            _evalCts.Dispose();
+            _evalGate.Dispose();
+            _evalResourcesDisposed = true;
+        }
+        else
+        {
+            _logger.LogWarning("Calculation {Driver}: an evaluation did not release the gate within {Timeout} of Dispose; leaving it to the GC",
+                _driverInstanceId, DisposeGateTimeout);
+        }
     }
 
+    private static readonly TimeSpan DisposeGateTimeout = TimeSpan.FromSeconds(2);
+    private volatile bool _evalResourcesDisposed;
+
+    /// <summary>Test hook — whether <see cref="Dispose"/> disposed the evaluation gate and its CTS.</summary>
+    internal bool EvalResourcesDisposedForTest => _evalResourcesDisposed;
+
     /// <summary>Per-tag published state (guarded by <see cref="_lock"/>).</summary>
     private sealed class TagState
     {
@@ -741,6 +845,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         /// now carries the failing INPUT's sub-code rather than a fixed <c>BadInternalError</c>.</summary>
         public uint LastPublishedStatus;
         public DateTime LastTimestampUtc;
+
+        /// <summary>The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the
+        /// gate that keeps the script-log entry to the transition and to a CHANGE of reason.</summary>
+        public string? LastFailureReason;
     }
 
     private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle;

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs

@@ -71,11 +71,22 @@ public static class CalculationDriverFactoryExtensions
             ?? throw new InvalidOperationException(
                 $"Calculation driver config for '{driverInstanceId}' deserialised to null");
 
+        // Run-guard knobs are optional and default to the node-wide ScriptingOptions defaults; only an
+        // explicitly authored key overrides one (so a config carrying neither binds Scripting = null).
+        ScriptingOptions? scripting = null;
+        if (dto.MaxAllocBytesPerRun is not null || dto.QuarantineAfterConsecutiveTimeouts is not null)
+        {
+            scripting = new ScriptingOptions();
+            if (dto.MaxAllocBytesPerRun is { } alloc) scripting.MaxAllocBytesPerRun = alloc;
+            if (dto.QuarantineAfterConsecutiveTimeouts is { } n) scripting.QuarantineAfterConsecutiveTimeouts = n;
+        }
+
         var options = new CalculationDriverOptions
         {
             RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
             RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0
                 ? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2),
+            Scripting = scripting,
         };
 
         return new CalculationDriver(
@@ -92,5 +103,11 @@ public static class CalculationDriverFactoryExtensions
 
         /// <summary>Optional per-script evaluation wall-clock budget override, in milliseconds (default 2000).</summary>
         public int? RunTimeoutMs { get; init; }
+
+        /// <summary>Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables).</summary>
+        public long? MaxAllocBytesPerRun { get; init; }
+
+        /// <summary>Optional run-guard quarantine threshold: consecutive timeouts (refusals do not count) before a tag's script stops being evaluated (default 3).</summary>
+        public int? QuarantineAfterConsecutiveTimeouts { get; init; }
     }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs

@@ -1,20 +1,21 @@
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
 
-/// <summary>
-///     Bound options for a <see cref="CalculationDriver"/> instance. The Calculation pseudo-driver has
-///     no backend/endpoint config — its only input is the authored raw calc tags the deploy artifact
-///     injects as the <see cref="RawTags"/> list (each carrying the tag's <c>scriptId</c> + resolved
-///     <c>scriptSource</c> + trigger config inside its <see cref="RawTagEntry.TagConfig"/> blob).
-/// </summary>
+/// <summary>Bound options for one <see cref="CalculationDriver"/> instance.</summary>
 public sealed class CalculationDriverOptions
 {
-    /// <summary>The driver's authored raw calc tags (RawPath + TagConfig blob + WriteIdempotent), from
-    /// the deploy artifact. Each is mapped to a <see cref="CalculationTagDefinition"/> at construction.</summary>
+    /// <summary>The authored raw calc tags (RawPath + TagConfig JSON) this driver computes.</summary>
     public IReadOnlyList<RawTagEntry> RawTags { get; init; } = Array.Empty<RawTagEntry>();
 
-    /// <summary>Per-script wall-clock evaluation budget handed to the <see cref="CalculationEvaluator"/>.
-    /// Defaults to 2 seconds (parity with the VirtualTag evaluator).</summary>
+    /// <summary>Wall-clock budget per script run (driver-config <c>runTimeoutMs</c>); default 2 s.</summary>
     public TimeSpan RunTimeout { get; init; } = TimeSpan.FromSeconds(2);
+
+    /// <summary>
+    ///     Run-guard budget + quarantine threshold for this driver's scripts (driver-config
+    ///     <c>maxAllocBytesPerRun</c> / <c>quarantineAfterConsecutiveTimeouts</c>). Null ⇒ the
+    ///     <see cref="ScriptingOptions"/> defaults (64 MB, 3).
+    /// </summary>
+    public ScriptingOptions? Scripting { get; init; }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using Microsoft.Extensions.Logging;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
@@ -37,8 +38,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
 ///     apply-boundary clear, which is what stops collectible <c>AssemblyLoadContext</c>s accreting across
 ///     script edits.
 /// </para>
+/// <para>
+///     <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
+///     calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous
+///     (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
+///     timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — both surface as a <see cref="VirtualTagEvalResult.Failure"/>
+///     the driver maps to Bad. The driver's evaluation pump awaits <see cref="EvaluateAsync"/> rather than
+///     blocking a pool thread on the sync <see cref="Evaluate(string, string, IReadOnlyDictionary{string, DataValueSnapshot})"/>,
+///     and the guard starts the budget clock only once the pool has picked the script up, so pool starvation
+///     is never reported as a script timeout.
+/// </para>
 /// </summary>
-public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
+public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
 {
     // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose()
     // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the
@@ -47,6 +58,14 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
     private readonly ILogger _logger;
     private readonly SerilogLogger _scriptRoot;
     private readonly TimeSpan _runTimeout;
+    private readonly ScriptRunGuard _guard;
+
+    // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE,
+    // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a
+    // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the
+    // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the
+    // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source.
+    private readonly ConcurrentDictionary<string, string> _compileFailures = new(StringComparer.Ordinal);
     private bool _disposed;
 
     /// <summary>Initializes a new <see cref="CalculationEvaluator"/>.</summary>
@@ -56,16 +75,22 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
     /// implements this — instead of needing a second, separately-obtained logger instance.</param>
     /// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
     /// <param name="runTimeout">Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator).</param>
+    /// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
     public CalculationEvaluator(
         ILogger logger,
         ScriptRootLogger scriptRoot,
-        TimeSpan? runTimeout = null)
+        TimeSpan? runTimeout = null,
+        ScriptingOptions? scripting = null)
     {
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
         _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
+        _guard = new ScriptRunGuard(scripting, _scriptRoot);
     }
 
+    /// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
+    public ScriptRunGuard RunGuard => _guard;
+
     /// <summary>
     ///     Evaluate <paramref name="scriptSource"/> for the calc tag <paramref name="calcTagId"/> against
     ///     the snapshot in <paramref name="dependencies"/> (RawPath → last value). Never throws — script
@@ -112,8 +137,25 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
     /// <param name="scriptSource">The C# script source producing the tag's value.</param>
     /// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
     /// <returns>The evaluation outcome.</returns>
+    /// <remarks>Blocks the calling thread for the run. The driver's pump uses <see cref="EvaluateAsync"/>;
+    /// this overload remains for callers with no async context — chiefly tests.</remarks>
     public VirtualTagEvalResult Evaluate(
         string calcTagId, string scriptSource, IReadOnlyDictionary<string, DataValueSnapshot> dependencies)
+        => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult();
+
+    /// <summary>
+    ///     The async form of <see cref="Evaluate(string, string, IReadOnlyDictionary{string, DataValueSnapshot})"/>:
+    ///     the compiled script runs on the pool under the <see cref="ScriptRunGuard"/> and the caller awaits
+    ///     it instead of blocking a second pool thread for the duration. Never throws for script faults.
+    /// </summary>
+    /// <param name="calcTagId">Identity of the calc tag (bound to the script log).</param>
+    /// <param name="scriptSource">The C# script source producing the tag's value.</param>
+    /// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
+    /// <param name="ct">Caller cancellation (driver shutdown); wins over the run timeout.</param>
+    /// <returns>The evaluation outcome.</returns>
+    public async Task<VirtualTagEvalResult> EvaluateAsync(
+        string calcTagId, string scriptSource, IReadOnlyDictionary<string, DataValueSnapshot> dependencies,
+        CancellationToken ct = default)
     {
         if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed");
         if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression");
@@ -146,6 +188,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
                     calcTagId, path),
             logger: scriptLog);
 
+        // A source that already failed to compile fails the same way without touching Roslyn (R07:P4).
+        var sourceKey = HashSource(scriptSource);
+        if (_compileFailures.TryGetValue(sourceKey, out var knownFailure))
+            return VirtualTagEvalResult.Failure(knownFailure);
+
         // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent
         // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's
         // disposed guard) re-fetches and retries exactly once. The loop runs at most twice.
@@ -158,13 +205,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
             }
             catch (CompilationErrorException ex)
             {
-                _logger.LogWarning(ex, "Calculation {Id}: Roslyn compile failed", calcTagId);
-                return VirtualTagEvalResult.Failure($"compile error: {ex.Message}");
+                return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed");
             }
             catch (ScriptSandboxViolationException ex)
             {
-                _logger.LogWarning(ex, "Calculation {Id}: sandbox violation", calcTagId);
-                return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}");
+                return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation");
             }
             catch (ObjectDisposedException)
             {
@@ -180,11 +225,12 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
 
             try
             {
-                // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
-                // interrupt a CPU-bound/infinite-loop Roslyn script, so the timed wrapper is what bounds a
-                // runaway script to the wall-clock budget instead of wedging the driver's dispatch.
-                var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
-                var raw = timed.RunAsync(context).GetAwaiter().GetResult();
+                // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw
+                // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock
+                // budget is what bounds a runaway script — and the guard is what keeps the next trigger from
+                // stacking another orphan behind it (one per tag, then quarantine).
+                var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct)
+                    .ConfigureAwait(false);
                 return VirtualTagEvalResult.Ok(raw);
             }
             catch (ScriptTimeoutException)
@@ -192,6 +238,18 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
                 return VirtualTagEvalResult.Failure(
                     $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
             }
+            catch (ScriptRunRefusedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
+            catch (ScriptQuarantinedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)");
+            }
             catch (ObjectDisposedException) when (!_disposed && attempt == 0)
             {
                 // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the
@@ -211,8 +269,39 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
         }
     }
 
+    /// <summary>Record a compile failure for <paramref name="sourceKey"/> and log it ONCE — the next trigger
+    /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning.</summary>
+    private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what)
+    {
+        if (_compileFailures.TryAdd(sourceKey, reason))
+            _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what);
+        return VirtualTagEvalResult.Failure(reason);
+    }
+
+    private static string HashSource(string source)
+        => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source)));
+
+    /// <summary>Test/diagnostic: the number of sources currently remembered as failing to compile.</summary>
+    public int CompileFailureCount => _compileFailures.Count;
+
+    /// <summary>Test/diagnostic: the number of compiled scripts currently cached.</summary>
+    public int CompiledScriptCount => _cache.Count;
+
     /// <inheritdoc />
-    public void ClearCompiledScripts() => _cache.Clear();
+    public void ClearCompiledScripts()
+    {
+        _cache.Clear();
+        _compileFailures.Clear();
+        // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine
+        // (an edited source already lifts its own through the guard's source identity).
+        _guard.ResetAll();
+    }
+
+    /// <inheritdoc />
+    public void ResetQuarantines() => _guard.ResetAll();
+
+    /// <inheritdoc />
+    public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
 
     /// <summary>Disposes the evaluator, draining + disposing every cached compile's collectible ALC.</summary>
     public void Dispose()
@@ -220,5 +309,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
         if (_disposed) return;
         _disposed = true;
         _cache.Dispose();
+        _compileFailures.Clear();
     }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs

@@ -7,11 +7,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
 ///     <see cref="System.Threading.Timer"/> per distinct interval group (so the wire count stays low
 ///     regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for
 ///     the same group is still running. Each tick invokes the driver's own evaluate
-///     callback per tag in the group.
+///     callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script
+///     run, not just across the synchronous part of the callback, so a slow script still bounds its group
+///     to one outstanding tick.
 /// </summary>
 internal sealed class CalculationTimerScheduler : IDisposable
 {
-    private readonly Action<string> _evaluate;
+    private readonly Func<string, Task> _evaluate;
     private readonly ILogger _logger;
     private readonly List<Timer> _timers = [];
     private readonly List<TickGroup> _groups = [];
@@ -20,9 +22,9 @@ internal sealed class CalculationTimerScheduler : IDisposable
     private bool _disposed;
 
     /// <summary>Initializes a new instance of the <see cref="CalculationTimerScheduler"/> class.</summary>
-    /// <param name="evaluate">Callback invoked with a calc tag's RawPath when its timer group ticks.</param>
+    /// <param name="evaluate">Callback invoked with a calc tag's RawPath when its timer group ticks; awaited.</param>
     /// <param name="logger">Logger for diagnostics.</param>
-    public CalculationTimerScheduler(Action<string> evaluate, ILogger logger)
+    public CalculationTimerScheduler(Func<string, Task> evaluate, ILogger logger)
     {
         _evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -69,12 +71,20 @@ internal sealed class CalculationTimerScheduler : IDisposable
             return;
         }
 
+        // The Timer callback is void; the tick itself is async so the awaited script runs do not pin this
+        // pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the
+        // Timer (which would be an unobserved fault on a timer thread).
+        _ = TickAsync(ctx);
+    }
+
+    private async Task TickAsync(TickGroup ctx)
+    {
         try
         {
             foreach (var path in ctx.Paths)
             {
                 if (_cts.IsCancellationRequested) return;
-                try { _evaluate(path); }
+                try { await _evaluate(path).ConfigureAwait(false); }
                 catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); }
             }
         }

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs

@@ -28,6 +28,12 @@ public sealed class AdminOperationsActor : ReceiveActor
     private readonly IReadOnlyDictionary<string, IDriverProbe> _probesByType;
     private readonly TagConfigValidationMode _tagConfigValidationMode;
 
+    /// <summary>
+    ///     The coordinator's apply deadline. A <c>Dispatching</c> row with no expected-ack rows that is older
+    ///     than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7).
+    /// </summary>
+    private readonly TimeSpan _applyDeadline;
+
     /// <summary>
     ///     Where central→node commands are handed off (per-cluster mesh), or
     ///     <see langword="null"/> to publish on the mediator directly.
@@ -36,6 +42,9 @@ public sealed class AdminOperationsActor : ReceiveActor
 
     private readonly ILoggingAdapter _log = Context.GetLogger();
 
+    /// <summary>The StartDeployment whose compose is running off-actor, or null. See <see cref="HandleStartDeployment"/>.</summary>
+    private StartDeployment? _composeInFlight;
+
     /// <summary>Creates actor props for the admin operations actor.</summary>
     /// <param name="dbFactory">Factory for creating config database contexts.</param>
     /// <param name="coordinator">Reference to the deployment coordinator actor.</param>
@@ -45,15 +54,17 @@ public sealed class AdminOperationsActor : ReceiveActor
     ///     Central comm actor commands are handed to, or <see langword="null"/> to publish on the
     ///     DistributedPubSub mediator directly (the behaviour before the per-cluster mesh).
     /// </param>
+    /// <param name="applyDeadline">The coordinator's apply deadline; defaults to <see cref="DeployTimings.ApplyDeadline"/> (the coordinator's default source).</param>
     /// <returns>Props configured to create an AdminOperationsActor.</returns>
     public static Props Props(
         IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
         IActorRef coordinator,
         IEnumerable<IDriverProbe> probes,
         TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
-        IActorRef? meshRouter = null) =>
+        IActorRef? meshRouter = null,
+        TimeSpan? applyDeadline = null) =>
         Akka.Actor.Props.Create(() =>
-            new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter));
+            new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline));
 
     /// <summary>Initializes a new instance of the AdminOperationsActor.</summary>
     /// <param name="dbFactory">Factory for creating config database contexts.</param>
@@ -61,27 +72,48 @@ public sealed class AdminOperationsActor : ReceiveActor
     /// <param name="probes">Driver probes registered in DI; keyed by DriverType (case-insensitive).</param>
     /// <param name="tagConfigValidationMode">Deploy-gate TagConfig strictness mode (Warn default | Error opt-in).</param>
     /// <param name="meshRouter">Central comm actor, or <see langword="null"/> for the mediator.</param>
+    /// <param name="applyDeadline">The coordinator's apply deadline; defaults to <see cref="DeployTimings.ApplyDeadline"/> (the coordinator's default source).</param>
     public AdminOperationsActor(
         IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
         IActorRef coordinator,
         IEnumerable<IDriverProbe> probes,
         TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
-        IActorRef? meshRouter = null)
+        IActorRef? meshRouter = null,
+        TimeSpan? applyDeadline = null)
     {
+        // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS
+        // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree.
+        _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline;
         _dbFactory = dbFactory;
         _coordinator = coordinator;
         _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase);
         _tagConfigValidationMode = tagConfigValidationMode;
         _meshRouter = meshRouter;
 
-        ReceiveAsync<StartDeployment>(HandleStartDeploymentAsync);
-        ReceiveAsync<TestDriverConnect>(HandleTestDriverConnectAsync);
+        // Synchronous receives: the compose and the probe run off-actor and come back as messages, so
+        // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout.
+        Receive<StartDeployment>(HandleStartDeployment);
+        Receive<DeployComposed>(HandleDeployComposed);
+        Receive<DeployComposeFailed>(HandleDeployComposeFailed);
+        Receive<TestDriverConnect>(HandleTestDriverConnect);
         ReceiveAsync<RestartDriver>(HandleRestartDriverAsync);
         ReceiveAsync<ReconnectDriver>(HandleReconnectDriverAsync);
         Receive<AcknowledgeAlarmCommand>(HandleAcknowledgeAlarm);
         Receive<ShelveAlarmCommand>(HandleShelveAlarm);
     }
 
+    /// <summary>
+    ///     The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's
+    ///     correlation id, never <c>ex.Message</c> — a SqlException or UriFormatException message carries
+    ///     connection-string fragments and host names, and the reply reaches any ConfigEditor browser session.
+    ///     The detail is in the central log under the same correlation id.
+    /// </summary>
+    /// <param name="operation">The admin operation that failed.</param>
+    /// <param name="correlationId">The request's correlation token, as logged alongside the exception.</param>
+    /// <returns>The operator-facing message.</returns>
+    internal static string ServerFault(string operation, object correlationId) =>
+        $"{operation} failed on the server; see the central node log for correlation {correlationId}.";
+
     /// <summary>
     ///     Hands a central→node command to the mesh transport router, or publishes it directly when
     ///     no router is wired.
@@ -129,8 +161,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId);
-            replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId));
+            _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId);
+            replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
@@ -176,215 +208,364 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId);
-            replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId));
+            _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId);
+            replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
-    private async Task HandleStartDeploymentAsync(StartDeployment msg)
+    /// <summary>
+    ///     <c>StartDeployment</c>. The only serialised part is the actor-local <see cref="_composeInFlight"/>
+    ///     flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the
+    ///     thread pool and come back as a <see cref="DeployComposed"/> / <see cref="DeployComposeFailed"/>
+    ///     message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the
+    ///     in-flight query off the actor thread too) and alarm acks keep flowing while it runs.
+    /// </summary>
+    /// <remarks>
+    ///     <see cref="_composeInFlight"/> is what keeps the check correct once it is off-actor: it is set
+    ///     BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a
+    ///     second request is refused in memory for the whole check-plus-compose and two artifacts can never
+    ///     race for one coordinator.
+    /// </remarks>
+    private void HandleStartDeployment(StartDeployment msg)
     {
         var replyTo = Sender;
+        if (_composeInFlight is { } inFlightRequest)
+        {
+            replyTo.Tell(new StartDeploymentResult(
+                StartDeploymentOutcome.AnotherDeploymentInFlight,
+                DeploymentId: null,
+                RevisionHash: null,
+                Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.",
+                msg.CorrelationId));
+            return;
+        }
+
+        _composeInFlight = msg;
+        // Captured here: Context/Sender are not available on the thread-pool thread.
+        var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown";
+        var self = Self;
+        Task.Run(async () =>
+            {
+                var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false);
+                return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false);
+            })
+            .PipeTo(self,
+                success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch),
+                failure: ex => new DeployComposeFailed(msg, replyTo, ex));
+    }
+
+    /// <summary>
+    ///     The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight —
+    ///     keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first.
+    /// </summary>
+    /// <param name="msg">The request.</param>
+    /// <returns>The refusal, or null to proceed.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         <b>Lost dispatch (fix round 1 / I7).</b> A <c>Dispatching</c> row with ZERO
+    ///         <c>NodeDeploymentState</c> rows was never taken by the coordinator (its <c>HandleDispatch</c>
+    ///         seeds the expected set and flips the status in one <c>SaveChanges</c>). The coordinator's PreStart
+    ///         recovers such a row — but only one that already existed when it started. A row committed AFTER
+    ///         PreStart whose <c>Tell</c> never arrived (this actor died between the compose's commit and
+    ///         <see cref="HandleDeployComposed"/>) has no owner at all, and refused every later deploy forever.
+    ///         Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is
+    ///         closed <c>TimedOut</c> with a reason and the new deploy proceeds. A late <c>DispatchDeployment</c>
+    ///         for it is ignored by the coordinator's terminal-state guard.
+    ///     </para>
+    ///     <para>
+    ///         One indexed projection (<c>IX_Deployment_Status</c>) plus, only for a Dispatching row past the
+    ///         deadline, a count and a tracked update.
+    ///     </para>
+    /// </remarks>
+    private async Task<ComposeOutcome?> CheckInFlightAsync(StartDeployment msg)
+    {
+        await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false);
+        var inflight = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
+            .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc })
+            .ToListAsync()
+            .ConfigureAwait(false);
+
+        foreach (var row in inflight)
+        {
+            if (row.Status == DeploymentStatus.Dispatching
+                && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline
+                && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false)
+                && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false))
+            {
+                continue;
+            }
+
+            return new ComposeOutcome(
+                new StartDeploymentResult(
+                    StartDeploymentOutcome.AnotherDeploymentInFlight,
+                    DeploymentId: new DeploymentId(row.DeploymentId),
+                    RevisionHash: null,
+                    Message: $"Deployment {row.DeploymentId:N} is still in flight.",
+                    msg.CorrelationId),
+                Dispatch: null);
+        }
+        return null;
+    }
+
+    /// <summary>Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion).</summary>
+    /// <param name="db">The context.</param>
+    /// <param name="deploymentId">The row.</param>
+    /// <returns><see langword="true"/> when the row is now closed.</returns>
+    private async Task<bool> CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId)
+    {
+        var stamp = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching)
+            .Select(d => new { d.RowVersion })
+            .FirstOrDefaultAsync()
+            .ConfigureAwait(false);
+        if (stamp is null) return true;   // no longer Dispatching — not in flight as far as this check goes
+
+        // Stub update: the row's ArtifactBlob is never read just to change its status.
+        var stub = new Deployment
+        {
+            DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty,
+            RowVersion = stamp.RowVersion,
+            Status = DeploymentStatus.TimedOut,
+            FailureReason =
+                "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " +
+                "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.",
+        };
+        var entry = db.Deployments.Attach(stub);
+        entry.Property(d => d.Status).IsModified = true;
+        entry.Property(d => d.FailureReason).IsModified = true;
         try
         {
-            await using var db = await _dbFactory.CreateDbContextAsync();
+            await db.SaveChangesAsync().ConfigureAwait(false);
+        }
+        catch (DbUpdateConcurrencyException)
+        {
+            // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight.
+            return false;
+        }
+        _log.Warning(
+            "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " +
+            "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline);
+        return true;
+    }
 
-            // Refuse if any deployment is already in flight — keeps the coordinator's state
-            // unambiguous. The UI is expected to wait for the in-flight one to seal or fail.
-            var inflight = await db.Deployments
-                .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
-                .Select(d => d.DeploymentId)
-                .FirstOrDefaultAsync();
-            if (inflight != Guid.Empty)
-            {
-                replyTo.Tell(new StartDeploymentResult(
-                    StartDeploymentOutcome.AnotherDeploymentInFlight,
-                    DeploymentId: new DeploymentId(inflight),
-                    RevisionHash: null,
-                    Message: $"Deployment {inflight:N} is still in flight.",
-                    msg.CorrelationId));
-                return;
-            }
+    private void HandleDeployComposed(DeployComposed msg)
+    {
+        _composeInFlight = null;
+        // The row is committed (the compose task completed), so the coordinator can be told now; it reads
+        // the row back by id when it seeds the expected-ack set.
+        if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch);
+        msg.ReplyTo.Tell(msg.Result);
+    }
 
-            // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject
-            // on ANY validation error. The config is now canonical — the company overlay loader emits
-            // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId
-            // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions,
-            // …) gates the deploy. A green build in this repo does not prove the config is valid; this
-            // is the last guard before a bad address space (or a non-derived EquipmentId) ships.
-            // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can
-            // (rarely) be a false positive if an external-id reservation release has not yet been
-            // committed/visible when the snapshot is read — operators seeing a spurious one should
-            // check ExternalIdReservation state before re-submitting.
-            var draft = await DraftSnapshotFactory.FromConfigDbAsync(db);
-            var errors = DraftValidator.Validate(draft).ToList();
+    private void HandleDeployComposeFailed(DeployComposeFailed msg)
+    {
+        _composeInFlight = null;
+        _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId);
+        msg.ReplyTo.Tell(new StartDeploymentResult(
+            StartDeploymentOutcome.Rejected,
+            DeploymentId: null,
+            RevisionHash: null,
+            Message: ServerFault("StartDeployment", msg.Request.CorrelationId),
+            msg.Request.CorrelationId));
+    }
 
-            // Cluster-topology guard. The SQL
-            // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode)
-            // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an
-            // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at
-            // Hot/Warm and the constraint stays green, which would boot the runtime into an
-            // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on
-            // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must
-            // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the
-            // same reject summary alongside the snapshot rules so a deploy failing either check is
-            // rejected with both sets of messages. ClusterId-ordered for a deterministic summary.
-            var clusters = await db.ServerClusters
-                .AsNoTracking()
-                .OrderBy(c => c.ClusterId)
-                .ToListAsync();
-            var nodesByCluster = (await db.ClusterNodes
-                    .AsNoTracking()
-                    .ToListAsync())
-                .GroupBy(n => n.ClusterId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal);
-            foreach (var cluster in clusters)
-            {
-                var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns)
-                    ? (IReadOnlyList<ClusterNode>)ns
-                    : [];
-                errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes));
-            }
+    /// <summary>Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted.</summary>
+    private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch);
 
-            // Deploy-time TagConfig strictness inspection over equipment tags. The
-            // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would
-            // REJECT — a present-but-invalid enum value, an absent or malformed address field, a
-            // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads
-            // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum;
-            // it now rejects the tag, so these warnings describe rejections.) In Warn mode
-            // (default) these are non-blocking (logged + appended to the accepted result Message); in Error
-            // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed.
-            // Running servers are untouched either way — the gate only sees re-deploys.
-            // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId /
-            // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type.
-            var driverTypeById = draft.DriverInstances
-                .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal);
-            var driverByDevice = draft.Devices
-                .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal);
-            var deviceConfigById = draft.Devices
-                .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal);
-            var tagConfigWarnings = new List<(string TagId, string Text)>();
-            foreach (var t in draft.Tags)
-            {
-                if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue;
-                if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue;
-                foreach (var w in EquipmentTagConfigInspector.Inspect(
-                    driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId)))
-                    tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}"));
-            }
-            if (tagConfigWarnings.Count > 0)
-            {
-                if (_tagConfigValidationMode == TagConfigValidationMode.Error)
-                    errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId)));
-                else
-                    foreach (var x in tagConfigWarnings)
-                        _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text);
-            }
+    private sealed record DeployComposed(
+        StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch);
 
-            if (errors.Count > 0)
-            {
-                var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}"));
-                _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}",
-                    errors.Count, summary);
-                replyTo.Tell(new StartDeploymentResult(
+    private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error);
+
+    /// <summary>
+    ///     The deploy compose, off the actor thread. Touches no actor state: everything it needs is in
+    ///     <paramref name="msg"/>, <paramref name="sourceNode"/> and the readonly fields captured at
+    ///     construction (<see cref="_dbFactory"/>, <see cref="_tagConfigValidationMode"/>, <see cref="_log"/> —
+    ///     Akka's logging adapter is thread-safe).
+    /// </summary>
+    private async Task<ComposeOutcome> ComposeDeploymentAsync(StartDeployment msg, string sourceNode)
+    {
+        await using var db = await _dbFactory.CreateDbContextAsync();
+
+        // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject
+        // on ANY validation error. The config is now canonical — the company overlay loader emits
+        // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId
+        // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions,
+        // …) gates the deploy. A green build in this repo does not prove the config is valid; this
+        // is the last guard before a bad address space (or a non-derived EquipmentId) ships.
+        // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can
+        // (rarely) be a false positive if an external-id reservation release has not yet been
+        // committed/visible when the snapshot is read — operators seeing a spurious one should
+        // check ExternalIdReservation state before re-submitting.
+        var draft = await DraftSnapshotFactory.FromConfigDbAsync(db);
+        var errors = DraftValidator.Validate(draft).ToList();
+
+        // Cluster-topology guard. The SQL
+        // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode)
+        // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an
+        // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at
+        // Hot/Warm and the constraint stays green, which would boot the runtime into an
+        // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of
+        // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows.
+        // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing
+        // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary.
+        // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT
+        // transaction), not a second query that could observe a later edit.
+        var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList();
+        var nodesByCluster = draft.ClusterNodes
+            .GroupBy(n => n.ClusterId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal);
+        foreach (var cluster in clusters)
+        {
+            var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns)
+                ? (IReadOnlyList<ClusterNode>)ns
+                : [];
+            errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes));
+        }
+
+        // Deploy-time TagConfig strictness inspection over equipment tags. The
+        // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would
+        // REJECT — a present-but-invalid enum value, an absent or malformed address field, a
+        // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads
+        // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum;
+        // it now rejects the tag, so these warnings describe rejections.) In Warn mode
+        // (default) these are non-blocking (logged + appended to the accepted result Message); in Error
+        // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed.
+        // Running servers are untouched either way — the gate only sees re-deploys.
+        // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId /
+        // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type.
+        var driverTypeById = draft.DriverInstances
+            .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal);
+        var driverByDevice = draft.Devices
+            .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal);
+        var deviceConfigById = draft.Devices
+            .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal);
+        var tagConfigWarnings = new List<(string TagId, string Text)>();
+        foreach (var t in draft.Tags)
+        {
+            if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue;
+            if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue;
+            foreach (var w in EquipmentTagConfigInspector.Inspect(
+                driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId)))
+                tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}"));
+        }
+        if (tagConfigWarnings.Count > 0)
+        {
+            if (_tagConfigValidationMode == TagConfigValidationMode.Error)
+                errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId)));
+            else
+                foreach (var x in tagConfigWarnings)
+                    _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text);
+        }
+
+        if (errors.Count > 0)
+        {
+            var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}"));
+            _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}",
+                errors.Count, summary);
+            return new ComposeOutcome(
+                new StartDeploymentResult(
                     StartDeploymentOutcome.Rejected,
                     DeploymentId: null,
                     RevisionHash: null,
                     Message: summary,
-                    msg.CorrelationId));
-                return;
+                    msg.CorrelationId),
+                Dispatch: null);
+        }
+
+        // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough)
+        // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc
+        // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct
+        // sources are counted (the compile cache keys on source, so duplicates collapse to one unit).
+        // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard
+        // reject; this advisory rides in the Accepted Message so the UI can show it.
+        // advisory is declared outside the inner try so the Accepted reply below can still read it even
+        // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's
+        // failure branch is reserved for genuine seal/save failures.
+        string? advisory = null;
+        try
+        {
+            const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07)
+            // Projected from the snapshot already in hand — Script.SourceCode used to be read three times.
+            var compiled = draft.Scripts
+                .Select(s => s.SourceCode)
+                .Where(src => !PassthroughScript.TryMatch(src, out _))
+                .Distinct(StringComparer.Ordinal)
+                .Count();
+            if (compiled > 0)
+            {
+                var estMiB = compiled * PerScriptMiB;
+                _log.Warning(
+                    "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it",
+                    compiled, estMiB);
+                advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)";
             }
+        }
+        catch (Exception ex)
+        {
+            // Guardrail is advisory-only — a failed estimate must never block a valid deploy.
+            _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped");
+            advisory = null;
+        }
 
-            // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough)
-            // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc
-            // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct
-            // sources are counted (the compile cache keys on source, so duplicates collapse to one unit).
-            // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard
-            // reject; this advisory rides in the Accepted Message so the UI can show it.
-            // advisory is declared outside the inner try so the Accepted reply below can still read it even
-            // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a
-            // valid deploy — the outer catch is reserved for genuine seal/save failures.
-            string? advisory = null;
-            try
-            {
-                const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07)
-                var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync();
-                var compiled = scriptSources
-                    .Where(src => !PassthroughScript.TryMatch(src, out _))
-                    .Distinct(StringComparer.Ordinal)
-                    .Count();
-                if (compiled > 0)
-                {
-                    var estMiB = compiled * PerScriptMiB;
-                    _log.Warning(
-                        "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it",
-                        compiled, estMiB);
-                    advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)";
-                }
-            }
-            catch (Exception ex)
-            {
-                // Guardrail is advisory-only — a failed estimate must never block a valid deploy.
-                _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped");
-                advisory = null;
-            }
+        // Composed from the validated snapshot, never from a second read: an edit that lands between
+        // validation and here is simply not in this artifact (R03:3.2).
+        var artifact = ConfigComposer.Compose(draft);
+        var deploymentId = DeploymentId.NewId();
+        var revHash = RevisionHash.Parse(artifact.RevisionHash);
 
-            var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db);
-            var deploymentId = DeploymentId.NewId();
-            var revHash = RevisionHash.Parse(artifact.RevisionHash);
+        db.Deployments.Add(new Deployment
+        {
+            DeploymentId = deploymentId.Value,
+            RevisionHash = artifact.RevisionHash,
+            Status = DeploymentStatus.Dispatching,
+            CreatedBy = msg.CreatedBy,
+            ArtifactBlob = artifact.Blob,
+        });
 
-            db.Deployments.Add(new Deployment
-            {
-                DeploymentId = deploymentId.Value,
-                RevisionHash = artifact.RevisionHash,
-                Status = DeploymentStatus.Dispatching,
-                CreatedBy = msg.CreatedBy,
-                ArtifactBlob = artifact.Blob,
-            });
+        // Marker ConfigEdit row so the audit timeline shows the deployment snapshot.
+        db.ConfigEdits.Add(new ConfigEdit
+        {
+            EntityType = "Deployment",
+            EntityId = deploymentId.Value,
+            FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}",
+            EditedBy = msg.CreatedBy,
+            SourceNode = sourceNode,
+        });
 
-            // Marker ConfigEdit row so the audit timeline shows the deployment snapshot.
-            db.ConfigEdits.Add(new ConfigEdit
-            {
-                EntityType = "Deployment",
-                EntityId = deploymentId.Value,
-                FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}",
-                EditedBy = msg.CreatedBy,
-                SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown",
-            });
+        await db.SaveChangesAsync();
 
-            await db.SaveChangesAsync();
+        // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so
+        // the operator sees the non-blocking strictness findings on a successful deploy.
+        var messageParts = new List<string>();
+        if (advisory is not null) messageParts.Add(advisory);
+        messageParts.AddRange(tagConfigWarnings.Select(x => x.Text));
+        var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null;
 
-            _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId));
-
-            // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so
-            // the operator sees the non-blocking strictness findings on a successful deploy.
-            var messageParts = new List<string>();
-            if (advisory is not null) messageParts.Add(advisory);
-            messageParts.AddRange(tagConfigWarnings.Select(x => x.Text));
-            var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null;
-
-            replyTo.Tell(new StartDeploymentResult(
+        return new ComposeOutcome(
+            new StartDeploymentResult(
                 StartDeploymentOutcome.Accepted,
                 deploymentId,
                 revHash,
                 Message: acceptedMessage,
-                msg.CorrelationId));
-        }
-        catch (Exception ex)
-        {
-            _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy);
-            replyTo.Tell(new StartDeploymentResult(
-                StartDeploymentOutcome.Rejected,
-                DeploymentId: null,
-                RevisionHash: null,
-                Message: ex.Message,
-                msg.CorrelationId));
-        }
+                msg.CorrelationId),
+            new DispatchDeployment(deploymentId, revHash, msg.CorrelationId));
     }
 
-    private async Task HandleTestDriverConnectAsync(TestDriverConnect msg)
+    /// <summary>
+    ///     Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe
+    ///     against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every
+    ///     path inside the task yields a <see cref="TestDriverConnectResult"/>, so the requester never
+    ///     sees a <c>Status.Failure</c>.
+    /// </summary>
+    private void HandleTestDriverConnect(TestDriverConnect msg)
     {
         var replyTo = Sender;
         if (!_probesByType.TryGetValue(msg.DriverType, out var probe))
@@ -397,6 +578,11 @@ public sealed class AdminOperationsActor : ReceiveActor
             return;
         }
 
+        Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo);
+    }
+
+    private async Task<TestDriverConnectResult> ProbeAsync(IDriverProbe probe, TestDriverConnect msg)
+    {
         var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60);
         var timeout = TimeSpan.FromSeconds(clampedSec);
         using var cts = new CancellationTokenSource(timeout);
@@ -406,28 +592,28 @@ public sealed class AdminOperationsActor : ReceiveActor
         {
             var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token);
             sw.Stop();
-            replyTo.Tell(new TestDriverConnectResult(
+            return new TestDriverConnectResult(
                 result.Ok,
                 result.Message,
                 result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
         catch (OperationCanceledException)
         {
-            replyTo.Tell(new TestDriverConnectResult(
+            return new TestDriverConnectResult(
                 false,
                 $"Probe timed out after {clampedSec}s.",
                 null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType);
-            replyTo.Tell(new TestDriverConnectResult(
+            _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId);
+            return new TestDriverConnectResult(
                 false,
-                ex.Message,
+                ServerFault("Test Connect", msg.CorrelationId),
                 null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
     }
 
@@ -457,8 +643,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId);
-            replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId));
+            _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId);
+            replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
@@ -487,8 +673,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId);
-            replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId));
+            _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId);
+            replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId));
         }
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs

@@ -1,17 +1,17 @@
 using System.IO.Compression;
 using System.Security.Cryptography;
 using System.Text.Json;
-using Microsoft.EntityFrameworkCore;
 using ZB.MOM.WW.OtOpcUa.Configuration;
-using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
 
 namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
 
 /// <summary>
 /// Pure snapshot composer: reads the current live-edit state from <see cref="OtOpcUaConfigDbContext"/>
 /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash.
-/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots
-/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the
+/// Determinism comes from reading every collection ordered by its natural key on the server
+/// (<see cref="DraftSnapshotFactory"/>), so two snapshots over the same DB state always produce the same hash
+/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the
 /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see <see cref="Pack"/>.
 /// </summary>
 public static class ConfigComposer
@@ -25,45 +25,71 @@ public static class ConfigComposer
     };
 
     /// <summary>Reads the current configuration and returns a deterministic snapshot blob with revision hash.</summary>
+    /// <remarks>
+    ///     One consistent read (<see cref="DraftSnapshotFactory.FromConfigDbAsync"/>, a SNAPSHOT transaction on
+    ///     SQL Server) handed to <see cref="Compose"/>. The deploy path does not call this: it validates and
+    ///     composes from the one <see cref="DraftSnapshot"/> it already holds, so nothing can land between the
+    ///     validator's read and the composer's. This overload serves the drift probe and tests.
+    /// </remarks>
     /// <param name="db">The configuration database context.</param>
     /// <param name="ct">The cancellation token for the operation.</param>
     /// <returns>A deterministic <see cref="ConfigArtifact"/> containing the serialised snapshot bytes and its revision hash.</returns>
     public static async Task<ConfigArtifact> SnapshotAndFlattenAsync(
         OtOpcUaConfigDbContext db, CancellationToken ct = default)
+        => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false));
+
+    /// <summary>
+    ///     Serialises an already-read <see cref="DraftSnapshot"/> into the deterministic artifact. Pure over
+    ///     its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now.
+    /// </summary>
+    /// <remarks>
+    ///     Serialises the collections IN THE ORDER GIVEN. Determinism comes from
+    ///     <see cref="DraftSnapshotFactory.FromConfigDbAsync"/>, which reads every table ordered by its natural key
+    ///     on the server; a snapshot built any other way must be ordered the same way or its hash will differ.
+    /// </remarks>
+    /// <param name="snapshot">The rows to compose — the SAME rows the validator was run over.</param>
+    /// <returns>The compressed artifact blob and its revision hash.</returns>
+    public static ConfigArtifact Compose(DraftSnapshot snapshot)
     {
+        ArgumentNullException.ThrowIfNull(snapshot);
+
         // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig)
         // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS
         // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/
         // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit).
         // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256.
-        var snapshot = new
+        // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from
+        // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash,
+        // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys
+        // differently from the SQL collation and silently re-hashed every unchanged configuration.
+        var ordered = new
         {
-            Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
-            Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
-            RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
-            DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
-            Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
-            TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
-            Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
-            Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
-            UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
-            PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
-            UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
-            UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
-            NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
+            Clusters = snapshot.ServerClusters,
+            Nodes = snapshot.ClusterNodes,
+            RawFolders = snapshot.RawFolders,
+            DriverInstances = snapshot.DriverInstances,
+            Devices = snapshot.Devices,
+            TagGroups = snapshot.TagGroups,
+            Equipment = snapshot.Equipment,
+            Tags = snapshot.Tags,
+            UnsTagReferences = snapshot.UnsTagReferences,
+            PollGroups = snapshot.PollGroups,
+            UnsAreas = snapshot.UnsAreas,
+            UnsLines = snapshot.UnsLines,
+            NodeAcls = snapshot.NodeAcls,
             // the data-plane credentials travel with the config, because a driver-only node
             // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every
             // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the
             // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob
             // still cannot be turned into a working credential. This is also why revoking a key reaches a
             // driver-only node only on the next deployment; see the ApiKey entity's remarks.
-            ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
-            Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
-            VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
-            ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
+            ApiKeys = snapshot.ApiKeys,
+            Scripts = snapshot.Scripts,
+            VirtualTags = snapshot.VirtualTags,
+            ScriptedAlarms = snapshot.ScriptedAlarms,
         };
 
-        return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions));
+        return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions));
     }
 
     /// <summary>

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs

@@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
 /// Dedup is two-layer: in-buffer (the <see cref="Dictionary{TKey, TValue}"/> below collapses
 /// duplicate EventIds before flush) and at the database via the filtered unique index
 /// <c>UX_ConfigAuditLog_EventId</c> (cross-restart safety — a retry of an already-flushed
-/// batch hits the constraint and we drop the duplicate insert without losing the rest of
-/// the batch).
+/// batch hits the constraint; <see cref="SaveBatch"/> drops exactly the already-persisted EventIds
+/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch).
 ///
 /// Implements the shared <see cref="IAuditWriter"/> seam: <see cref="WriteAsync"/> is a
 /// best-effort, never-throwing entry point that simply <c>Tell</c>s this actor and returns
@@ -92,23 +92,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
 
         try
         {
-            using var db = _dbFactory.CreateDbContext();
-            foreach (var evt in snapshot)
-            {
-                db.ConfigAuditLogs.Add(new ConfigAuditLog
-                {
-                    Timestamp = evt.OccurredAtUtc.UtcDateTime,
-                    Principal = evt.Actor,
-                    EventType = $"{evt.Category}:{evt.Action}",
-                    NodeId = evt.SourceNode,
-                    DetailsJson = evt.DetailsJson,
-                    EventId = evt.EventId,
-                    CorrelationId = evt.CorrelationId,
-                    Outcome = evt.Outcome.ToString(),
-                });
-            }
-            db.SaveChanges();
-            _log.Debug("AuditWriter flushed {Count} events", snapshot.Count);
+            var dropped = SaveBatch(snapshot);
+            _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped);
         }
         catch (Exception ex)
         {
@@ -116,6 +101,102 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
         }
     }
 
+    private static ConfigAuditLog ToRow(AuditEvent evt) => new()
+    {
+        Timestamp = evt.OccurredAtUtc.UtcDateTime,
+        Principal = evt.Actor,
+        EventType = $"{evt.Category}:{evt.Action}",
+        NodeId = evt.SourceNode,
+        DetailsJson = evt.DetailsJson,
+        EventId = evt.EventId,
+        CorrelationId = evt.CorrelationId,
+        Outcome = evt.Outcome.ToString(),
+    };
+
+    /// <summary>
+    ///     Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4).
+    /// </summary>
+    /// <param name="batch">The flush.</param>
+    /// <returns>How many rows were dropped.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         <b>Why not detach <c>ex.Entries</c>.</b> SQL Server inserts the batch as one statement batch, so
+    ///         the <see cref="DbUpdateException"/> raised for ONE duplicate <c>EventId</c> names EVERY entry in
+    ///         the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for
+    ///         (pinned against a real SQL Server by <c>On_sql_server_a_duplicate_event_id_costs_only_that_row</c>).
+    ///     </para>
+    ///     <para>
+    ///         So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index):
+    ///         (1) ask the database which of the batch's EventIds it already holds and drop exactly those —
+    ///         the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if
+    ///         that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the
+    ///         rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state
+    ///         from a failed save leaks into the next one. A failure that names no entry (connection loss,
+    ///         deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves.
+    ///     </para>
+    /// </remarks>
+    private int SaveBatch(IReadOnlyList<AuditEvent> batch)
+    {
+        if (TrySave(batch, out var first)) return 0;
+        _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count);
+
+        HashSet<Guid> existing;
+        using (var db = _dbFactory.CreateDbContext())
+        {
+            var ids = batch.Select(e => (Guid?)e.EventId).ToList();
+            existing = db.ConfigAuditLogs
+                .AsNoTracking()
+                .Where(r => ids.Contains(r.EventId))
+                .Select(r => r.EventId!.Value)
+                .ToHashSet();
+        }
+        foreach (var id in existing)
+            _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id);
+
+        var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList();
+        var dropped = batch.Count - remaining.Count;
+        if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped;
+
+        // Still rejected with no duplicate left to blame: isolate the offending row(s).
+        foreach (var evt in remaining)
+        {
+            if (TrySave([evt], out var rowError)) continue;
+            dropped++;
+            if (IsUniqueViolation(rowError!))
+                _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId);
+            else
+                _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId);
+        }
+        return dropped;
+    }
+
+    /// <summary>Saves <paramref name="events"/> on a fresh context; a rejection that names entries is reported, anything else throws.</summary>
+    /// <param name="events">The rows to save.</param>
+    /// <param name="rejection">The row-level rejection, when this returns false.</param>
+    /// <returns><see langword="true"/> when saved.</returns>
+    private bool TrySave(IReadOnlyList<AuditEvent> events, out DbUpdateException? rejection)
+    {
+        using var db = _dbFactory.CreateDbContext();
+        foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt));
+        try
+        {
+            db.SaveChanges();
+            rejection = null;
+            return true;
+        }
+        catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex))
+        {
+            rejection = ex;
+            return false;
+        }
+    }
+
+    /// <summary>SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection.</summary>
+    /// <param name="ex">The update failure.</param>
+    /// <returns><see langword="true"/> for a duplicate-key rejection.</returns>
+    private static bool IsUniqueViolation(DbUpdateException ex) =>
+        ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 };
+
     /// <inheritdoc />
     protected override void PreRestart(Exception reason, object message)
     {

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs

@@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
             LoadContactsFromDb();
         });
         Receive<ContactsLoaded>(HandleContactsLoaded);
+        Receive<PskRefreshed>(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount));
+        Receive<PskRefreshFailed>(msg => _log.Warning(msg.Error,
+            "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh"));
         Receive<CommandDispatched>(HandleCommandDispatched);
 
         // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler
@@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
 
         _targets = msg.Targets;
 
+        // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port
+        // moved). Only the resolved dialer is asked; nothing is resolved just to prune it.
+        if (_dialer is { } dialer)
+        {
+            try
+            {
+                dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal));
+            }
+            catch (Exception ex)
+            {
+                _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh");
+            }
+        }
+
         if (_targets.Count == 0)
         {
             _log.Warning(
@@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
         }
 
         pskProvider.SetKnownClusters(msg.ClusterIds);
-        pskProvider.InvalidateAll();
+
+        // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically,
+        // instead of dropping the cache and making the first ack per cluster re-read the store on a
+        // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake)
+        // still drops its cache here, synchronously, exactly as before.
+        CancellationToken ct;
+        try
+        {
+            ct = _lifecycle.Token;
+        }
+        catch (ObjectDisposedException)
+        {
+            return; // Stopping.
+        }
+        var self = Self;
+        Task refresh;
+        try
+        {
+            refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct);
+        }
+        catch (Exception ex)
+        {
+            _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh");
+            refresh = Task.CompletedTask;
+        }
+        if (!refresh.IsCompletedSuccessfully)
+        {
+            refresh.PipeTo(self,
+                success: () => new PskRefreshed(msg.ClusterIds.Count),
+                failure: ex => new PskRefreshFailed(ex));
+        }
 
         if (!_gatePublished)
         {
@@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
     /// <summary>Internal tick asking for a contact-point refresh.</summary>
     public sealed record RefreshContacts;
 
+    /// <summary>The asynchronous PSK refresh (R03:3.4) completed.</summary>
+    /// <param name="ClusterCount">How many clusters were refreshed.</param>
+    public sealed record PskRefreshed(int ClusterCount);
+
+    /// <summary>The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service.</summary>
+    /// <param name="Error">The failure.</param>
+    public sealed record PskRefreshFailed(Exception Error);
+
     /// <summary>Result of a contact-point load, piped back from the DB read.</summary>
     /// <param name="Targets">
     ///     The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs

@@ -106,6 +106,27 @@ public interface IMeshPskProvider
 
     /// <summary>Drops every cached key.</summary>
     void InvalidateAll();
+
+    /// <summary>
+    ///     Re-reads the keys for <paramref name="clusterIds"/> from the store OFF the request path and
+    ///     swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep
+    ///     serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose
+    ///     secret the store reports ABSENT is revoked (no key from then on). Replaces
+    ///     "invalidate on every refresh", which made the first inbound ack per cluster per
+    ///     <c>ContactRefreshSeconds</c> block a gRPC thread on a synchronous, 3 s-bounded store read.
+    /// </summary>
+    /// <param name="clusterIds">The clusters central currently knows (the same set given to <see cref="SetKnownClusters"/>).</param>
+    /// <param name="ct">Cancels the refresh.</param>
+    /// <returns>Completes when the swap has happened.</returns>
+    /// <remarks>
+    ///     The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup
+    ///     re-read — so a fake or a non-caching provider needs nothing more.
+    /// </remarks>
+    Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
+    {
+        InvalidateAll();
+        return Task.CompletedTask;
+    }
 }
 
 /// <inheritdoc />
@@ -144,6 +165,16 @@ public sealed class MeshPskProvider : IMeshPskProvider
     private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
         new(StringComparer.OrdinalIgnoreCase);
 
+    /// <summary>
+    ///     Keys pre-resolved by <see cref="RefreshAsync"/>, consulted after the config map and before the
+    ///     lazy <see cref="_cache"/>. Swapped wholesale (volatile reference to an immutable dictionary),
+    ///     never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next
+    ///     one. A cluster missing from it — never refreshed, or whose read failed — falls through to the
+    ///     lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key".
+    /// </summary>
+    private volatile IReadOnlyDictionary<string, string> _refreshed =
+        new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
     /// <summary>
     ///     The known-cluster gate's membership set, or <see langword="null"/> when the gate is
     ///     inactive (never published, or published empty — see <see cref="SetKnownClusters"/>).
@@ -210,6 +241,13 @@ public sealed class MeshPskProvider : IMeshPskProvider
             return true;
         }
 
+        // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all.
+        if (_refreshed.TryGetValue(clusterId, out var refreshed))
+        {
+            psk = refreshed;
+            return true;
+        }
+
         var entry = _cache.GetOrAdd(
             clusterId,
             static (id, self) => new Lazy<string?>(
@@ -288,7 +326,124 @@ public sealed class MeshPskProvider : IMeshPskProvider
     }
 
     /// <inheritdoc />
-    public void InvalidateAll() => _cache.Clear();
+    public void InvalidateAll()
+    {
+        _cache.Clear();
+        _refreshed = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+    }
+
+    /// <summary>
+    ///     Ceiling on concurrent store reads during one <see cref="RefreshAsync"/>. A fleet of N clusters used to
+    ///     fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most
+    ///     this many at once now.
+    /// </summary>
+    public const int MaxConcurrentRefreshReads = 8;
+
+    /// <inheritdoc />
+    public async Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
+    {
+        ArgumentNullException.ThrowIfNull(clusterIds);
+
+        var previous = _refreshed;
+        var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
+        var results = new ConcurrentDictionary<string, StoreRead>(StringComparer.OrdinalIgnoreCase);
+
+        // Bounded fan-out; each read bounded like the synchronous path.
+        await Parallel.ForEachAsync(
+                ids,
+                new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct },
+                async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false))
+            .ConfigureAwait(false);
+
+        // Three outcomes per cluster (fix round 1 / I8):
+        //  - Found     → the new key;
+        //  - NotFound  → the store answered that the secret does not exist: a REVOCATION — the cluster gets no
+        //                key (and its lazily cached one is evicted below), so a deleted secret stops working
+        //                within one refresh instead of living on for the life of the process;
+        //  - Failed    → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key —
+        //                a transient fault must not open a window in which the cluster has no key at all.
+        var next = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var answered = new List<string>();
+        foreach (var id in ids)
+        {
+            var read = results[id];
+            switch (read.Outcome)
+            {
+                case StoreReadOutcome.Found:
+                    next[id] = read.Value!;
+                    answered.Add(id);
+                    break;
+                case StoreReadOutcome.NotFound:
+                    answered.Add(id);
+                    break;
+                default:
+                    if (previous.TryGetValue(id, out var kept)) next[id] = kept;
+                    break;
+            }
+        }
+
+        // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new
+        // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too).
+        // Entries for clusters whose read failed are left alone.
+        _refreshed = next;
+        foreach (var id in answered) _cache.TryRemove(id, out _);
+    }
+
+    private enum StoreReadOutcome { Found, NotFound, Failed }
+
+    private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value);
+
+    /// <summary>
+    ///     The asynchronous twin of <see cref="ResolveFromStore"/>: same name derivation, same bound — awaited
+    ///     instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor —
+    ///     but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read".
+    /// </summary>
+    /// <param name="clusterId">The application <c>Cluster</c> id.</param>
+    /// <param name="ct">The caller's token, linked under the store-read bound.</param>
+    /// <returns>The outcome and, when found, the key.</returns>
+    private async Task<StoreRead> ResolveFromStoreAsync(string clusterId, CancellationToken ct)
+    {
+        SecretName name;
+        try
+        {
+            name = new SecretName(SecretNamePrefix + clusterId);
+        }
+        catch (ArgumentException ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
+                + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or "
+                + "rename the cluster to [a-z0-9._/-].",
+                clusterId);
+            return new StoreRead(StoreReadOutcome.Failed, null);
+        }
+
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        cts.CancelAfter(StoreReadTimeout);
+        try
+        {
+            var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false);
+            if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value);
+
+            _logger.LogWarning(
+                "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is "
+                + "REVOKED — commands to that cluster and acks from it are refused until it is restored.",
+                name.Value,
+                clusterId);
+            return new StoreRead(StoreReadOutcome.NotFound, null);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the "
+                + "previously resolved key (if any) stays in service until the next refresh.",
+                clusterId,
+                name.Value);
+            return new StoreRead(StoreReadOutcome.Failed, null);
+        }
+    }
 
     /// <summary>
     ///     Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs

@@ -34,6 +34,15 @@ public interface IMeshCommandDialer
     ///     <see cref="StatusCode.Unauthenticated"/> rather than dialled unauthenticated.
     /// </exception>
     Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct);
+
+    /// <summary>
+    ///     Releases any per-endpoint resources for endpoints NOT in <paramref name="nodeEndpoints"/>
+    ///     (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a
+    ///     renamed host or moved port does not leave the old channel and its idle HTTP/2 connection
+    ///     alive for the life of the process. Default no-op for dialers that cache nothing.
+    /// </summary>
+    /// <param name="nodeEndpoints">The endpoints still in the dispatch set.</param>
+    void RetainOnly(IReadOnlyCollection<string> nodeEndpoints) { }
 }
 
 /// <inheritdoc />
@@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable
         await call.ResponseAsync.ConfigureAwait(false);
     }
 
+    /// <inheritdoc />
+    public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
+    {
+        ArgumentNullException.ThrowIfNull(nodeEndpoints);
+        if (_disposed) return;
+
+        var keep = nodeEndpoints as IReadOnlySet<string> ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal);
+        foreach (var endpoint in _channels.Keys)
+        {
+            if (keep.Contains(endpoint)) continue;
+            // A dial that already took this channel out of the cache before the removal fails once, as
+            // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever
+            // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant
+            // of a contact refresh, the same accepted race Dispose documents.
+            if (_channels.TryRemove(endpoint, out var channel))
+            {
+                channel.Dispose();
+                _log.LogInformation(
+                    "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set",
+                    endpoint);
+            }
+        }
+    }
+
     /// <summary>Marks the dialer disposed and disposes every cached gRPC channel.</summary>
     public void Dispose()
     {

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs

@@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self));
 
         using var db = _dbFactory.CreateDbContext();
+        // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never
+        // reads — and this query runs on every singleton start and hand-over.
         var inflight = db.Deployments
             .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
             .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc })
             .FirstOrDefault();
         if (inflight is null) return;
 
@@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             .AsNoTracking()
             .ToList();
 
+        // Resume the deadline timer using the remaining time. The deadline runs from when the
+        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
+        // proxy — we don't track a separate "dispatched at" column).
+        var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
+        var remaining = _applyDeadline - elapsed;
+
+        if (nodeStates.Count == 0)
+        {
+            // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by
+            // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed:
+            // HandleDispatch threw before its SaveChanges (this actor restarted and the message was
+            // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the
+            // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to
+            // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight
+            // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply
+            // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that
+            // keeps throwing ends up on the other branch once the deadline passes); past it the row is
+            // closed as TimedOut with a reason that names the condition.
+            if (remaining > TimeSpan.Zero)
+            {
+                _log.Warning(
+                    "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
+                    "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
+                    "re-dispatching with {Remaining} of the apply deadline left",
+                    _current, remaining);
+                Self.Tell(new DispatchDeployment(
+                    _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
+            }
+            else
+            {
+                Self.Tell(new DeadlineElapsed(_current.Value));
+            }
+            return;
+        }
+
         _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer);
         foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying))
         {
@@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
         }
 
-        // Resume the deadline timer using the remaining time. The deadline runs from when the
-        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
-        // proxy — we don't track a separate "dispatched at" column).
-        var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
-        var remaining = _applyDeadline - elapsed;
         if (remaining <= TimeSpan.Zero)
         {
             Self.Tell(new DeadlineElapsed(_current.Value));
@@ -201,6 +234,33 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
     private void HandleDispatch(DispatchDeployment msg)
     {
+        // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already
+        // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message
+        // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it.
+        //
+        // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1):
+        // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit
+        // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's
+        // RowVersion, any change since the guard read makes the flip a concurrency conflict instead.
+        byte[]? guardRowVersion;
+        using (var check = _dbFactory.CreateDbContext())
+        {
+            var guard = check.Deployments
+                .AsNoTracking()
+                .Where(d => d.DeploymentId == msg.DeploymentId.Value)
+                .Select(d => new { d.Status, d.RowVersion })
+                .FirstOrDefault();
+            if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut)
+            {
+                _log.Warning(
+                    "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)",
+                    msg.DeploymentId, guard.Status);
+                if (_current == msg.DeploymentId) ResetForNext();
+                return;
+            }
+            guardRowVersion = guard?.RowVersion;
+        }
+
         _current = msg.DeploymentId;
         _acks.Clear();
         _degraded.Clear();
@@ -208,22 +268,51 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack.
         using (var db = _dbFactory.CreateDbContext())
         {
-            // Discovery reuses this context rather than opening a second one — it reads the same
-            // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
-            // context keeps the expected set and the seeded rows consistent by construction.
-            _expectedAcks = DiscoverDriverNodes(db);
-
-            foreach (var node in _expectedAcks)
+            // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows
+            // already seeded for this deployment ARE the expected set — re-discovering would both duplicate
+            // the primary key and let a node toggled since the first attempt change who is expected.
+            var seeded = db.NodeDeploymentStates
+                .AsNoTracking()
+                .Where(x => x.DeploymentId == msg.DeploymentId.Value)
+                .Select(x => x.NodeId)
+                .ToList();
+            if (seeded.Count > 0)
             {
-                db.NodeDeploymentStates.Add(new NodeDeploymentState
-                {
-                    NodeId = node.Value,
-                    DeploymentId = msg.DeploymentId.Value,
-                    Status = NodeDeploymentStatus.Applying,
-                });
+                _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer);
+            }
+            else
+            {
+                // Discovery reuses this context rather than opening a second one — it reads the same
+                // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
+                // context keeps the expected set and the seeded rows consistent by construction.
+                _expectedAcks = DiscoverDriverNodes(db);
+
+                foreach (var node in _expectedAcks)
+                {
+                    db.NodeDeploymentStates.Add(new NodeDeploymentState
+                    {
+                        NodeId = node.Value,
+                        DeploymentId = msg.DeploymentId.Value,
+                        Status = NodeDeploymentStatus.Applying,
+                    });
+                }
+            }
+            UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion);
+            try
+            {
+                db.SaveChanges();
+            }
+            catch (DbUpdateConcurrencyException)
+            {
+                // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The
+                // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is
+                // published and the coordinator is free for the next deployment.
+                _log.Warning(
+                    "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " +
+                    "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId);
+                ResetForNext();
+                return;
             }
-            UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks);
-            db.SaveChanges();
         }
 
         // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing
@@ -323,11 +412,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on
             // each NodeDeploymentState row) and the seal logs at Warning.
             var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList();
-            var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value);
-            if (d is not null)
-                d.FailureReason = Truncate(
-                    $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
-                    "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.");
+            SetFailureReason(db, Truncate(
+                $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
+                "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."));
             db.SaveChanges();
             _log.Warning(
                 "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}",
@@ -360,7 +447,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
                 msg.DeploymentId, _current);
             return;
         }
-        if (_acks.Count == _expectedAcks.Count)
+        // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the
+        // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped.
+        if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count)
         {
             // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere.
             return;
@@ -368,6 +457,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
         using var db = _dbFactory.CreateDbContext();
         UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut);
+        if (_expectedAcks.Count == 0)
+        {
+            SetFailureReason(db, Truncate(
+                "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " +
+                "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."));
+            db.SaveChanges();
+            _log.Warning(
+                "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " +
+                "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy",
+                _current.Value, _applyDeadline);
+            ResetForNext();
+            return;
+        }
         db.SaveChanges();
         // Name the silent nodes, not just the counts. Since the expected set became DB-derived this
         // is the failure an operator meets after deploying while a configured node is switched off,
@@ -394,13 +496,63 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
     public sealed record DeadlineElapsed(DeploymentId DeploymentId);
 
+    /// <summary>
+    ///     Stages a status change WITHOUT materialising the row: the row carries the whole <c>ArtifactBlob</c>
+    ///     (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub
+    ///     carrying only the key and the row's current RowVersion is attached and the changed columns marked
+    ///     modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged
+    ///     stub stays tracked so <see cref="SetFailureReason"/> can add to the same UPDATE.
+    /// </summary>
     private static void UpdateDeploymentStatus(
-        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false)
+        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null)
     {
-        var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value);
-        if (d is null) return;
-        d.Status = status;
-        if (sealNow) d.SealedAtUtc = DateTime.UtcNow;
+        var entry = TrackStub(db, id, rowVersion);
+        if (entry is null) return;
+        entry.Entity.Status = status;
+        entry.Property(x => x.Status).IsModified = true;
+        if (sealNow)
+        {
+            entry.Entity.SealedAtUtc = DateTime.UtcNow;
+            entry.Property(x => x.SealedAtUtc).IsModified = true;
+        }
+    }
+
+    /// <summary>Stages the current deployment's failure reason onto the stub <see cref="UpdateDeploymentStatus"/> staged.</summary>
+    private void SetFailureReason(OtOpcUaConfigDbContext db, string reason)
+    {
+        if (_current is null) return;
+        var entry = TrackStub(db, _current.Value);
+        if (entry is null) return;
+        entry.Entity.FailureReason = reason;
+        entry.Property(x => x.FailureReason).IsModified = true;
+    }
+
+    /// <summary>The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone.</summary>
+    /// <param name="db">The context.</param>
+    /// <param name="id">The deployment.</param>
+    /// <param name="rowVersion">
+    ///     The RowVersion the update must be conditioned on, when the caller already read one (the dispatch
+    ///     guard); <see langword="null"/> reads the current one.
+    /// </param>
+    private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry<Deployment>? TrackStub(
+        OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null)
+    {
+        var tracked = db.ChangeTracker.Entries<Deployment>().FirstOrDefault(e => e.Entity.DeploymentId == id.Value);
+        if (tracked is not null) return tracked;
+        if (rowVersion is null)
+        {
+            var stamp = db.Deployments
+                .AsNoTracking()
+                .Where(x => x.DeploymentId == id.Value)
+                .Select(x => new { x.RowVersion })
+                .FirstOrDefault();
+            if (stamp is null) return null;
+            rowVersion = stamp.RowVersion;
+        }
+        return db.Deployments.Attach(new Deployment
+        {
+            DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion,
+        });
     }
 
     /// <summary>

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs

@@ -0,0 +1,190 @@
+using Akka.Actor;
+using Akka.Event;
+using Microsoft.EntityFrameworkCore;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
+
+/// <summary>
+///     Admin-role cluster singleton that periodically empties the <c>ArtifactBlob</c> of old, terminal
+///     deployments (R03:1.2). See <see cref="DeploymentRetentionOptions"/> for the policy and
+///     <see cref="SweepAsync"/> for the rule, which is pure over a context so it is unit-testable.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Empties, never deletes, and never nulls.</b> The column is <c>NOT NULL varbinary(max)</c>, and
+///         every reader (<c>DriverHostActor.LoadArtifactBlob</c>, <c>OpcUaPublishActor</c>,
+///         <c>DeploymentArtifactService</c>) already treats a zero-length blob as "no artifact" (#485: empty
+///         bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An
+///         empty blob therefore needs no schema change and no reader change, where a NULL would need both.
+///     </para>
+///     <para>
+///         <b>Never touches an in-flight row</b> (Dispatching / AwaitingApplyAcks): a node may still be
+///         fetching it. The newest <see cref="DeploymentRetentionOptions.KeepSealed"/> terminal rows keep
+///         their blob whatever their age, so the "latest sealed" every boot path reads is always intact.
+///     </para>
+/// </remarks>
+public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers
+{
+    private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
+    private readonly DeploymentRetentionOptions _options;
+    private readonly ILoggingAdapter _log = Context.GetLogger();
+    private const string SweepTimerKey = "retention-sweep";
+
+    /// <summary>Gets or sets the timer scheduler.</summary>
+    public ITimerScheduler Timers { get; set; } = null!;
+
+    /// <summary>Creates the actor props.</summary>
+    /// <param name="dbFactory">The config DB factory.</param>
+    /// <param name="options">The retention policy.</param>
+    /// <returns>The props.</returns>
+    public static Props Props(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentRetentionOptions options) =>
+        Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options));
+
+    /// <summary>Initializes a new instance of the <see cref="DeploymentRetentionActor"/> class.</summary>
+    /// <param name="dbFactory">The config DB factory.</param>
+    /// <param name="options">The retention policy.</param>
+    public DeploymentRetentionActor(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentRetentionOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(dbFactory);
+        ArgumentNullException.ThrowIfNull(options);
+        _dbFactory = dbFactory;
+        _options = options;
+
+        Receive<Sweep>(_ => RunSweep());
+        Receive<SweepCompleted>(msg =>
+        {
+            if (msg.Pruned > 0)
+                _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned);
+            else
+                _log.Debug("Deployment retention sweep found nothing to prune");
+            if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg);
+        });
+        Receive<SweepFailed>(msg =>
+        {
+            _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries");
+            if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg);
+        });
+    }
+
+    /// <inheritdoc />
+    protected override void PreStart()
+    {
+        if (!_options.Enabled)
+        {
+            _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever");
+            return;
+        }
+        var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes));
+        var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds));
+        Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval);
+    }
+
+    private void RunSweep()
+    {
+        var replyTo = Sender.IsNobody() ? null : Sender;
+        var self = Self;
+        var now = DateTime.UtcNow;
+        Task.Run(async () =>
+            {
+                await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false);
+                return await SweepAsync(db, _options, now).ConfigureAwait(false);
+            })
+            .PipeTo(self,
+                success: pruned => new SweepCompleted(pruned, replyTo),
+                failure: ex => new SweepFailed(ex, replyTo));
+    }
+
+    /// <summary>
+    ///     The retention rule. Empties the blob of every terminal deployment that is (a) not one of the
+    ///     newest <see cref="DeploymentRetentionOptions.KeepSealed"/> SEALED rows (at least one — the newest
+    ///     Sealed blob is never pruned), (b) not referenced by a
+    ///     <c>NodeDeploymentState</c> row started or applied within
+    ///     <see cref="DeploymentRetentionOptions.ReferencedDays"/>, and (c) still carrying bytes — at most
+    ///     <see cref="DeploymentRetentionOptions.MaxRowsPerSweep"/> per call.
+    /// </summary>
+    /// <param name="db">The config DB.</param>
+    /// <param name="options">The policy.</param>
+    /// <param name="utcNow">The sweep's clock.</param>
+    /// <param name="ct">Cancellation.</param>
+    /// <returns>How many rows were emptied.</returns>
+    public static async Task<int> SweepAsync(
+        OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(db);
+        ArgumentNullException.ThrowIfNull(options);
+
+        var keep = Math.Max(1, options.KeepSealed);
+        var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays));
+        var batch = Math.Max(1, options.MaxRowsPerSweep);
+
+        // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep.
+        var terminal = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.Status == DeploymentStatus.Sealed
+                        || d.Status == DeploymentStatus.PartiallyFailed
+                        || d.Status == DeploymentStatus.TimedOut)
+            .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 })
+            .ToListAsync(ct)
+            .ConfigureAwait(false);
+
+        // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst
+        // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last
+        // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all
+        // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate.
+        var keptSealed = terminal
+            .Where(d => d.Status == DeploymentStatus.Sealed)
+            .Take(keep)
+            .Select(d => d.DeploymentId)
+            .ToHashSet();
+        var aged = terminal
+            .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId))
+            .Select(d => d.DeploymentId)
+            .ToList();
+        if (aged.Count == 0) return 0;
+
+        var referenced = (await db.NodeDeploymentStates
+                .AsNoTracking()
+                .Where(s => aged.Contains(s.DeploymentId)
+                            && (s.StartedAtUtc >= referenceFloor
+                                || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor)))
+                .Select(s => s.DeploymentId)
+                .Distinct()
+                .ToListAsync(ct)
+                .ConfigureAwait(false))
+            .ToHashSet();
+
+        var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList();
+        if (targets.Count == 0) return 0;
+
+        // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on
+        // both providers, and the blob read is the one-time cost of retiring that row.
+        var rows = await db.Deployments
+            .Where(d => targets.Contains(d.DeploymentId))
+            .ToListAsync(ct)
+            .ConfigureAwait(false);
+        foreach (var row in rows) row.ArtifactBlob = [];
+        await db.SaveChangesAsync(ct).ConfigureAwait(false);
+        return rows.Count;
+    }
+
+    /// <summary>Asks for a sweep now; the sender (if any) receives the <see cref="SweepCompleted"/> / <see cref="SweepFailed"/>.</summary>
+    public sealed class Sweep
+    {
+        /// <summary>The singleton instance.</summary>
+        public static readonly Sweep Instance = new();
+        private Sweep() { }
+    }
+
+    /// <summary>A sweep finished.</summary>
+    /// <param name="Pruned">How many rows were emptied.</param>
+    /// <param name="ReplyTo">Who asked, when the sweep was requested rather than timer-driven.</param>
+    public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo);
+
+    /// <summary>A sweep threw.</summary>
+    /// <param name="Error">The failure.</param>
+    /// <param name="ReplyTo">Who asked, when the sweep was requested rather than timer-driven.</param>
+    public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo);
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs

@@ -0,0 +1,55 @@
+using Microsoft.Extensions.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
+
+/// <summary>
+///     <c>Deployments:Retention</c> — how long a sealed deployment keeps its <c>ArtifactBlob</c> (R03:1.2).
+///     Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine
+///     edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a
+///     row: the audit record (who deployed what revision, when, and how every node acked) survives.
+/// </summary>
+public sealed class DeploymentRetentionOptions
+{
+    /// <summary>The configuration section name.</summary>
+    public const string SectionName = "Deployments:Retention";
+
+    /// <summary>Master switch; <see langword="false"/> keeps every blob forever (the previous behaviour).</summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    ///     How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1).
+    ///     Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push
+    ///     the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so
+    ///     this must stay well above 1; 50 also keeps a comfortable re-deploy history.
+    /// </summary>
+    public int KeepSealed { get; set; } = 50;
+
+    /// <summary>
+    ///     A deployment any <c>NodeDeploymentState</c> row started or applied within this many days keeps its
+    ///     blob even when older than the <see cref="KeepSealed"/> window — a node that just applied it may
+    ///     still replay it on restart.
+    /// </summary>
+    public int ReferencedDays { get; set; } = 30;
+
+    /// <summary>How often the sweep runs.</summary>
+    public int IntervalMinutes { get; set; } = 60;
+
+    /// <summary>Delay before the first sweep after the singleton starts, so a boot is not a blob scan.</summary>
+    public int InitialDelaySeconds { get; set; } = 60;
+
+    /// <summary>
+    ///     Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its
+    ///     RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps.
+    /// </summary>
+    public int MaxRowsPerSweep { get; set; } = 200;
+
+    /// <summary>Binds the section from configuration, with defaults for anything absent.</summary>
+    /// <param name="configuration">The app configuration; null yields the defaults.</param>
+    /// <returns>The bound options.</returns>
+    public static DeploymentRetentionOptions Bind(IConfiguration? configuration)
+    {
+        var options = new DeploymentRetentionOptions();
+        configuration?.GetSection(SectionName).Bind(options);
+        return options;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs

@@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
@@ -15,6 +16,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
 using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 
 namespace ZB.MOM.WW.OtOpcUa.ControlPlane;
@@ -30,6 +32,7 @@ public static class ServiceCollectionExtensions
     public const string AuditWriterSingletonName = "audit-writer";
     public const string RedundancyStateSingletonName = "redundancy-state";
     public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler";
+    public const string DeploymentRetentionSingletonName = "deployment-retention";
 
     /// <summary>
     /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with
@@ -109,7 +112,7 @@ public static class ServiceCollectionExtensions
                 // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks
                 // inside a props factory and races the spawn it is waiting for.
                 var meshRouter = registry.Get<CentralCommunicationActorKey>();
-                return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter);
+                return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter);
             },
             singletonOptions);
 
@@ -126,7 +129,10 @@ public static class ServiceCollectionExtensions
                     resolver.GetService<IConfiguration>()?["Deployment:TagConfigValidationMode"],
                     ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn;
                 var meshRouter = registry.Get<CentralCommunicationActorKey>();
-                return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter);
+                // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it
+                // is past the coordinator's deadline, so the two must never drift.
+                return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter,
+                    applyDeadline: DeployTimings.ApplyDeadline);
             },
             singletonOptions);
 
@@ -181,6 +187,21 @@ public static class ServiceCollectionExtensions
             singletonOptions,
             createProxyToo: false);
 
+        // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the
+        // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same
+        // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb);
+        // proxy-less because nothing addresses it in production — a test sends Sweep directly.
+        builder.WithSingleton<DeploymentRetentionKey>(
+            DeploymentRetentionSingletonName,
+            (system, registry, resolver) =>
+            {
+                var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
+                var options = DeploymentRetentionOptions.Bind(resolver.GetService<IConfiguration>());
+                return DeploymentRetentionActor.Props(dbFactory, options);
+            },
+            singletonOptions,
+            createProxyToo: false);
+
         return builder;
     }
 
@@ -256,4 +277,5 @@ public sealed class AdminOperationsActorKey { }
 public sealed class AuditWriterActorKey { }
 public sealed class RedundancyStateActorKey { }
 public sealed class ClusterNodeAddressReconcilerKey { }
+public sealed class DeploymentRetentionKey { }
 public sealed class CentralCommunicationActorKey { }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs

@@ -0,0 +1,41 @@
+using ZB.MOM.WW.Configuration;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+/// <summary>
+///     Fail-fast startup validator for <see cref="ApiKeyOptions"/> (R03:2.3). The pepper used to be
+///     checked only when the first key was verified — <c>ApiKeySecret.Verify</c> throws on an empty one —
+///     so a site node deployed without <c>Security__ApiKeys__Pepper</c> booted green, passed health,
+///     applied deployments and denied every machine client with the same opaque "Invalid API key" until
+///     someone read the log; a central node with a wrong pepper minted keys that could verify nowhere.
+///     Every other security section already fails host start on a bad value; this one now does too.
+/// </summary>
+/// <remarks>
+///     Registered for <c>hasDriver || hasAdmin</c>: a driver node needs the pepper to VERIFY keys, an admin
+///     node to MINT them. The minimum length is a floor against a placeholder (<c>"x"</c>) passing as
+///     configured, not a strength claim — the pepper is a server-side secret, not a password.
+/// </remarks>
+public sealed class ApiKeyOptionsValidator : OptionsValidatorBase<ApiKeyOptions>
+{
+    /// <summary>The shortest pepper accepted (characters, after trimming).</summary>
+    public const int MinPepperLength = 16;
+
+    /// <inheritdoc />
+    protected override void Validate(ValidationBuilder builder, ApiKeyOptions options)
+    {
+        var pepper = options.Pepper?.Trim() ?? string.Empty;
+        builder.RequireThat(pepper.Length > 0,
+            $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " +
+            "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " +
+            "every API key (driver) or mint keys that can never verify (admin).");
+        builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength,
+            $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters.");
+        builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username),
+            $"{ApiKeyOptions.SectionName}:Username must not be blank.");
+        builder.RequireThat(options.StoreTimeoutMs >= 0,
+            $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline).");
+        builder.RequireThat(options.StoreCacheSeconds >= 0,
+            $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache).");
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs

@@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration
                         sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
                 sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
                 sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
-                sp.GetRequiredService<TimeProvider>()));
+                sp.GetRequiredService<TimeProvider>(),
+                // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as
+                // ApiKeyOptions, Host-owned knobs.
+                ApiKeyStoreGuardOptions.Bind(sp.GetService<IConfiguration>())));
             services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
 
             // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.

src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs

@@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines;
 ///     into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no
 ///     longer open this window at all.
 /// </para>
+/// <para>
+///     <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
+///     virtual-tag id with the expression as the source identity: a run is refused while the tag's previous
+///     (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
+///     timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as <see cref="VirtualTagEvalResult.Failure"/>
+///     so <c>VirtualTagActor</c> publishes Bad exactly as it does for a timeout. A quarantine lifts when the
+///     expression changes, on <see cref="ClearCompiledScripts"/> / <see cref="ResetQuarantines"/> (the deploy
+///     boundary), never by itself.
+/// </para>
 /// </summary>
-public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable
+public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
 {
     // Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and
     // Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary
@@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     private readonly ILogger<RoslynVirtualTagEvaluator> _logger;
     private readonly SerilogLogger _scriptRoot;
     private readonly TimeSpan _runTimeout;
+    private readonly ScriptRunGuard _guard;
     private bool _disposed;
 
     /// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
     /// <param name="logger">Logger for recording compilation and execution errors (host diagnostics).</param>
     /// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
     /// <param name="runTimeout">Maximum execution time for each script; defaults to 2 seconds if not specified.</param>
+    /// <param name="scripting">Run-guard budget + quarantine threshold (the <c>Scripting</c> section); null ⇒ defaults.</param>
     public RoslynVirtualTagEvaluator(
         ILogger<RoslynVirtualTagEvaluator> logger,
         ScriptRootLogger scriptRoot,
-        TimeSpan? runTimeout = null)
+        TimeSpan? runTimeout = null,
+        ScriptingOptions? scripting = null)
     {
         _logger = logger;
         _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
         _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
+        _guard = new ScriptRunGuard(scripting, _scriptRoot);
     }
 
+    /// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
+    public ScriptRunGuard RunGuard => _guard;
+
     // Value-only overload. Every dependency reaches the script stamped Good with an
     // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
     // overload instead; this one remains for callers that genuinely hold only
@@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
 
             try
             {
-                // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
-                // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
-                // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
-                // the wall-clock budget fires regardless of whether the inner task completes (the orphaned
-                // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
-                var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
-                var raw = timed.RunAsync(context).GetAwaiter().GetResult();
+                // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw
+                // CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll
+                // it), so the wall-clock budget is what returns control — and the guard is what stops the
+                // abandoned run from being joined by another on the next trigger (one orphan per tag, then
+                // quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed.
+                var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout)
+                    .GetAwaiter().GetResult();
                 return VirtualTagEvalResult.Ok(raw);
             }
             catch (ScriptTimeoutException)
@@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
                 return VirtualTagEvalResult.Failure(
                     $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
             }
+            catch (ScriptRunRefusedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
+            catch (ScriptQuarantinedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
             catch (ObjectDisposedException) when (!_disposed && attempt == 0)
             {
                 // S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our
@@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     }
 
     /// <inheritdoc />
-    public void ClearCompiledScripts() => _cache.Clear();
+    public void ClearCompiledScripts()
+    {
+        _cache.Clear();
+        // The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine
+        // (an edited expression already lifts its own through the guard's source identity).
+        _guard.ResetAll();
+    }
+
+    /// <inheritdoc />
+    public void ResetQuarantines() => _guard.ResetAll();
+
+    /// <inheritdoc />
+    public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
 
     /// <summary>Disposes the evaluator and cleans up all cached script evaluators.</summary>
     public void Dispose()

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs

@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
 
 namespace ZB.MOM.WW.OtOpcUa.Host.Health;
 
@@ -28,11 +29,40 @@ public static class HealthEndpoints
     /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
     /// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
     /// </param>
+    /// <param name="hasDriver">
+    /// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
+    /// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready tier, and the active tier on a
+    /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading
+    /// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
+    /// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
+    /// </param>
     /// <returns>The same service collection, for chaining.</returns>
-    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
+    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
     {
         var checks = services.AddHealthChecks();
 
+        // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve.
+        // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6):
+        //  - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may
+        //    route to it;
+        //  - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState
+        //    (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would
+        //    take the operator's only console out of rotation precisely when they need it to diagnose the
+        //    failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands
+        //    the peer takes Primary and the active tier follows it on its own.
+        // Factory-registered so the state singleton is resolved per probe.
+        if (hasDriver)
+        {
+            var tags = hasAdmin
+                ? new[] { ZbHealthTags.Ready }
+                : new[] { ZbHealthTags.Ready, ZbHealthTags.Active };
+            checks.Add(new HealthCheckRegistration(
+                OpcUaEndpointHealthCheck.Name,
+                sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
+                failureStatus: null,
+                tags: tags));
+        }
+
         // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
         // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
         // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs

@@ -0,0 +1,47 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Health;
+
+/// <summary>
+///     Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
+///     listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
+///     one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
+///     connection-refused.
+/// </summary>
+/// <remarks>
+///     Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
+///     admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused
+///     admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier
+///     is what Traefik routes the AdminUI on.
+/// </remarks>
+public sealed class OpcUaEndpointHealthCheck : IHealthCheck
+{
+    /// <summary>The registration name, as it appears in the health JSON.</summary>
+    public const string Name = "opcua-endpoint";
+
+    private readonly IOpcUaEndpointState _state;
+
+    /// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
+    /// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
+    public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
+    {
+        ArgumentNullException.ThrowIfNull(state);
+        _state = state;
+    }
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
+        var result = _state.Status switch
+        {
+            OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
+            OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
+            OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
+            _ => HealthCheckResult.Unhealthy(
+                "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
+        };
+        return Task.FromResult(result);
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs

@@ -0,0 +1,56 @@
+using Microsoft.Extensions.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>
+///     The pre-authentication guards <see cref="BoundedApiKeyStore"/> puts in front of the ConfigDb key
+///     lookup (R03:3.3). Bound from the same <c>Security:ApiKeys</c> section as <c>ApiKeyOptions</c> but
+///     owned by the Host, where the store lives.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Why.</b> The negative cache was keyed by key id, so a flood of DISTINCT well-formed
+///         <c>otua_&lt;random&gt;_x</c> credentials missed it every time and cost one indexed SQL query per
+///         attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close
+///         that: a <b>known-id set</b> (every <c>ApiKeyId</c> the ConfigDb holds, refreshed every
+///         <c>StoreCacheSeconds</c>) rejects an unknown id in memory, and a <b>miss budget</b> (token
+///         bucket) bounds how many lookups that reach the store and MISS are allowed per second when the
+///         set is unavailable.
+///     </para>
+///     <para>
+///         The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback
+///         (the <c>IOpcUaUserAuthenticator</c> seam) does not carry the client's address. Per-endpoint
+///         accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane.
+///     </para>
+/// </remarks>
+public sealed class ApiKeyStoreGuardOptions
+{
+    /// <summary>The configuration section the knobs live in (shared with <c>ApiKeyOptions</c>).</summary>
+    public const string SectionName = "Security:ApiKeys";
+
+    /// <summary>
+    ///     Whether the known-id set is consulted. <see langword="false"/> restores a store read per unknown
+    ///     id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact
+    ///     store is never wrapped).
+    /// </summary>
+    public bool KnownIdSet { get; set; } = true;
+
+    /// <summary>
+    ///     How many store MISSES (lookups that reach the inner store and find no key) may burst before
+    ///     further lookups for unknown ids are refused without a store read. <c>0</c> disables the budget.
+    /// </summary>
+    public int MissBurst { get; set; } = 100;
+
+    /// <summary>How many miss tokens are restored per second, up to <see cref="MissBurst"/>.</summary>
+    public double MissRefillPerSecond { get; set; } = 20;
+
+    /// <summary>Binds the section, with defaults for anything absent.</summary>
+    /// <param name="configuration">The app configuration; null yields the defaults.</param>
+    /// <returns>The bound options.</returns>
+    public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration)
+    {
+        var options = new ApiKeyStoreGuardOptions();
+        configuration?.GetSection(SectionName).Bind(options);
+        return options;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs

@@ -56,6 +56,33 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
     private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
         new(StringComparer.Ordinal);
+    private readonly ApiKeyStoreGuardOptions _guard;
+
+    // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds
+    // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first
+    // successful load, or when the inner store cannot list ids — then the per-id path runs as before.
+    // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile
+    // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read
+    // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed.
+    private volatile KnownIdSnapshot? _knownIds;
+    private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1);
+
+    // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is
+    // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked
+    // one store's capability into every other consumer of the same options).
+    private volatile bool _listingUnsupported;
+
+    /// <summary>The known-id set and the instant it was loaded, published as one unit.</summary>
+    /// <param name="Ids">The ids the inner store holds.</param>
+    /// <param name="StampedUtc">When they were read.</param>
+    private sealed record KnownIdSnapshot(IReadOnlySet<string> Ids, DateTimeOffset StampedUtc);
+
+    // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards
+    // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the
+    // configured rate instead of one SQL round trip per attempt under the SessionManager lock.
+    private readonly object _bucketGate = new();
+    private double _missTokens;
+    private DateTimeOffset _missRefilledUtc;
 
     /// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
     /// <param name="inner">The store being bounded.</param>
@@ -66,11 +93,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
     ///     <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
     ///     detail.
     /// </param>
+    /// <param name="guard">The pre-authentication guards (R03:3.3); defaults when omitted.</param>
     public BoundedApiKeyStore(
         IApiKeyStore inner,
         IOptions<ApiKeyOptions> options,
         ILogger<BoundedApiKeyStore> log,
-        TimeProvider time)
+        TimeProvider time,
+        ApiKeyStoreGuardOptions? guard = null)
     {
         ArgumentNullException.ThrowIfNull(inner);
         ArgumentNullException.ThrowIfNull(options);
@@ -81,8 +110,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
         _options = options;
         _log = log;
         _time = time;
+        _guard = guard ?? new ApiKeyStoreGuardOptions();
+        _missTokens = Math.Max(0, _guard.MissBurst);
+        _missRefilledUtc = time.GetUtcNow();
     }
 
+    /// <summary>The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards.</summary>
+    public IReadOnlySet<string>? KnownKeyIds => _knownIds?.Ids;
+
     /// <summary>The store being bounded.</summary>
     /// <remarks>
     ///     Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
@@ -115,12 +150,103 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
             return cached;
         }
 
+        // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked.
+        var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false);
+        if (known is not null && !known.Contains(apiKeyId))
+        {
+            OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "unknown_id"));
+            return null;
+        }
+
+        // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known
+        // to exist (a known id is a real key being verified, never a guess).
+        if (known is null && !TryTakeMissToken())
+        {
+            OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "miss_budget"));
+            throw new InvalidOperationException(
+                "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / "
+                + "MissRefillPerSecond); refusing this lookup without a store read.");
+        }
+
         var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
             .ConfigureAwait(false);
+        if (fresh is not null && known is null) RefundMissToken();
         if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
         return fresh;
     }
 
+    /// <summary>
+    ///     The known-id set, refreshed when older than <paramref name="ttl"/>. Single-flight: concurrent
+    ///     callers during a refresh use the previous set (or, before the first load, the per-id path). A
+    ///     refresh that fails or exceeds the deadline logs and keeps the previous set.
+    /// </summary>
+    /// <param name="ttl">The refresh interval (the cache TTL); non-positive disables the set.</param>
+    /// <param name="ct">The caller's token.</param>
+    /// <returns>The set, or null.</returns>
+    private async Task<IReadOnlySet<string>?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct)
+    {
+        if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null;
+
+        var now = _time.GetUtcNow();
+        var current = _knownIds;
+        if (current is not null && now - current.StampedUtc < ttl) return current.Ids;
+
+        if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids;
+        try
+        {
+            // Re-check under the gate: another caller may have refreshed while this one waited.
+            if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids;
+            var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false);
+            if (ids is null)
+            {
+                // Unsupported by the inner store: stop asking — for this decorator only.
+                _listingUnsupported = true;
+                return null;
+            }
+            _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow());
+            return ids;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
+        {
+            _log.LogWarning(ex,
+                "ApiKeyStore: refreshing the known key-id set failed; {Mode}",
+                current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force");
+            return current?.Ids;
+        }
+        finally
+        {
+            _knownIdsRefresh.Release();
+        }
+    }
+
+    /// <summary>Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled.</summary>
+    /// <returns><see langword="false"/> when the budget is exhausted.</returns>
+    private bool TryTakeMissToken()
+    {
+        var burst = _guard.MissBurst;
+        if (burst <= 0) return true;
+        lock (_bucketGate)
+        {
+            var now = _time.GetUtcNow();
+            var elapsed = (now - _missRefilledUtc).TotalSeconds;
+            if (elapsed > 0)
+            {
+                _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond));
+                _missRefilledUtc = now;
+            }
+            if (_missTokens < 1) return false;
+            _missTokens -= 1;
+            return true;
+        }
+    }
+
+    /// <summary>A lookup that reached the store and HIT was not a miss: its token goes back.</summary>
+    private void RefundMissToken()
+    {
+        if (_guard.MissBurst <= 0) return;
+        lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1);
+    }
+
     // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.
     /// <inheritdoc />
     public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs

@@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore
         return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false);
     }
 
+    /// <inheritdoc />
+    public async Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
+    {
+        await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false);
+        // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index
+        // scan — so an unknown id can be refused in memory instead of costing a query per attempt.
+        var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false);
+        return ids.ToHashSet(StringComparer.Ordinal);
+    }
+
     /// <summary>Loads a principal's grants, confined to this node's cluster.</summary>
     /// <remarks>
     ///     <b>The full grant set, not just the <c>Cluster</c>-scoped rows</b>. The old

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs

@@ -35,6 +35,16 @@ public interface IApiKeyStore
     /// <param name="ct">The cancellation token.</param>
     /// <returns>The matching ACL rows; empty when none.</returns>
     Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct);
+
+    /// <summary>
+    ///     Every <c>ApiKeyId</c> the store holds (enabled or not), for the pre-authentication known-id set
+    ///     (R03:3.3); <see langword="null"/> when the store cannot cheaply list them, in which case callers
+    ///     fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry
+    ///     and the secret are still decided by <see cref="FindAsync"/>.
+    /// </summary>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The id set, or null when unsupported.</returns>
+    Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult<IReadOnlySet<string>?>(null);
 }
 
 /// <summary>An API key resolved for authentication, with the ACL grants naming its principal.</summary>

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs

@@ -0,0 +1,73 @@
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
+public enum OpcUaEndpointStatus
+{
+    /// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
+    NotStarted,
+
+    /// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
+    Started,
+
+    /// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
+    Failed,
+
+    /// <summary>The hosted service stopped the SDK; the port is released.</summary>
+    Stopped,
+}
+
+/// <summary>
+///     The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
+///     start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
+///     elected driver Primary and acked deploys as Applied through the Null sink — while clients got
+///     connection-refused from the node the fleet called healthy.
+/// </summary>
+/// <remarks>
+///     Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
+///     ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
+///     <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
+///     node whose SDK failed is visible (503) but still Primary-eligible.
+/// </remarks>
+public interface IOpcUaEndpointState
+{
+    /// <summary>The endpoint's current lifecycle status.</summary>
+    OpcUaEndpointStatus Status { get; }
+
+    /// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
+    string? FailureReason { get; }
+}
+
+/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
+public sealed class OpcUaEndpointState : IOpcUaEndpointState
+{
+    private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
+    private volatile string? _failureReason;
+
+    /// <inheritdoc />
+    public OpcUaEndpointStatus Status => _status;
+
+    /// <inheritdoc />
+    public string? FailureReason => _failureReason;
+
+    /// <summary>Records a successful SDK start.</summary>
+    public void MarkStarted()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Started;
+    }
+
+    /// <summary>Records a failed SDK start.</summary>
+    /// <param name="reason">The failure, as the health probe should report it.</param>
+    public void MarkFailed(string reason)
+    {
+        _failureReason = reason;
+        _status = OpcUaEndpointStatus.Failed;
+    }
+
+    /// <summary>Records that the hosted service stopped the SDK.</summary>
+    public void MarkStopped()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Stopped;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.Authorization;
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
     private readonly HistorianUpdaterHolder _historianUpdater;
     private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
+    private readonly OpcUaEndpointState _endpointState;
+    private readonly bool _rethrowStartFailure;
 
     private OpcUaApplicationHost? _appHost;
     private OtOpcUaSdkServer? _server;
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
     /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
     /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
+    /// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
+    /// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
+    /// so a harness that never probes health is unchanged.</param>
+    /// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
+    /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
+    /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
+    /// the health probe instead. Null (harnesses) behaves as fused.</param>
     public OtOpcUaServerHostedService(
         IOptions<OpcUaApplicationHostOptions> options,
         DeferredAddressSpaceSink deferredSink,
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         ILoggerFactory loggerFactory,
         IDriverPrimaryRoleSource? primaryRoleSource = null,
         HistorianUpdaterHolder? historianUpdater = null,
-        IHistoryUpdateRoleView? historyUpdateRoleView = null)
+        IHistoryUpdateRoleView? historyUpdateRoleView = null,
+        OpcUaEndpointState? endpointState = null,
+        EffectiveRoles? roles = null)
     {
         ArgumentNullException.ThrowIfNull(aclOptions);
         ArgumentNullException.ThrowIfNull(time);
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         _primaryRoleSource = primaryRoleSource;
         _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
         _historyUpdateRoleView = historyUpdateRoleView;
+        _endpointState = endpointState ?? new OpcUaEndpointState();
+        _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
         _logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
     }
 
@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         }
         catch (Exception ex)
         {
+            // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
+            // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
+            // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
+            // the OpcUa:* config without a process bounce — but it is visibly not ready.
+            _endpointState.MarkFailed(ex.Message);
+            await ReleaseFailedStartAsync().ConfigureAwait(false);
+            if (_rethrowStartFailure)
+            {
+                _logger.LogCritical(ex,
+                    "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
+                    "node justifies staying up; failing host start so the supervisor restarts the process");
+                throw;
+            }
             _logger.LogError(ex,
-                "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
-            // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
-            // Operators see the failure via the logs + can correct config without a process bounce
-            // of the whole binary.
+                "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
+                "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
             return;
         }
 
@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogWarning(
                 "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
+            _endpointState.MarkFailed("SDK reported started but the node manager is null");
             return;
         }
 
@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogError(ex,
                 "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
+            _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
             try
             {
-                await StopAsync(cancellationToken).ConfigureAwait(false);
+                DetachSinks();
             }
             catch (Exception cleanupError)
             {
                 _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
             }
-            var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
-                async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
-                () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
-            if (released)
-            {
-                _appHost = null;
-                _server = null;
-            }
+            await ReleaseEndpointAsync().ConfigureAwait(false);
             return;
         }
 
@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // attached, so the first tick cannot land on a half-wired server.
         StartRevalidationTimer();
 
+        _endpointState.MarkStarted();
         _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
     }
 
+    /// <summary>
+    ///     Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
+    ///     listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
+    ///     transport is up); disposing it is the only way to be sure. The state stays Failed.
+    /// </summary>
+    private async Task ReleaseFailedStartAsync()
+    {
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is null) return;
+        try
+        {
+            await appHost.DisposeAsync().ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
+        }
+    }
+
+    /// <summary>
+    ///     Stops the SDK and disposes the application host, releasing endpoint ownership only when both
+    ///     succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
+    ///     fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
+    /// </summary>
+    private async Task ReleaseEndpointAsync()
+    {
+        var server = _server;
+        var appHost = _appHost;
+        if (server is null && appHost is null) return;
+        var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
+            async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
+            () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
+            _logger).ConfigureAwait(false);
+        if (released)
+        {
+            _appHost = null;
+            _server = null;
+        }
+    }
+
     /// <summary>
     ///     The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
     ///     (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     }
 
     /// <summary>
-    /// Stops the OPC UA server asynchronously.
+    /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
     /// </summary>
+    /// <remarks>
+    ///     R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
+    ///     the other hosted services' StopAsync), serving the last-written values with an emptied authority:
+    ///     reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
+    ///     up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
+    ///     stays idempotent with this: ownership is nulled once the stop succeeded.
+    /// </remarks>
     /// <param name="cancellationToken">Cancellation token.</param>
     /// <returns>A task that represents the asynchronous operation.</returns>
-    public Task StopAsync(CancellationToken cancellationToken)
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        DetachSinks();
+        await ReleaseEndpointAsync().ConfigureAwait(false);
+        if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
+    }
+
+    /// <summary>
+    ///     Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
+    ///     or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
+    ///     until <see cref="ReleaseEndpointAsync"/>.
+    /// </summary>
+    private void DetachSinks()
     {
         // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
         // authority and gate have already been cleared is the shape this whole teardown block exists to
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // request rather than let the seams fall back to their role checks — a stopping node should shed
         // enforcement in the same order it acquired it.
         _server?.SetDataPlaneAuthorizer(null);
-        return Task.CompletedTask;
     }
 
     /// <summary>
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     public async ValueTask DisposeAsync()
     {
         // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
-        // not leave a tick armed against a server that is about to go away.
+        // not leave a tick armed against a server that is about to go away. A StopAsync that released the
+        // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
+        // them set, and this is the retry.
         _revalidationTimer?.Dispose();
         _revalidationTimer = null;
 
-        if (_appHost is not null) await _appHost.DisposeAsync();
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is not null) await appHost.DisposeAsync();
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua");
 // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it
 // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string.
 // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected.
-// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it
-// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a
-// startup error.
-builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
+// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this
+// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3):
+// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is
+// present. A node with neither role (dev-only) has no key path and keeps the plain bind.
+if (hasDriver || hasAdmin)
+    builder.Services.AddValidatedOptions<ApiKeyOptions, ApiKeyOptionsValidator>(
+        builder.Configuration, ApiKeyOptions.SectionName);
+else
+    builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
 // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config
 // change rather than a redeploy, and what it swaps rather than adds.
 builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));
@@ -452,6 +457,11 @@ if (hasDriver)
     // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
     // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
     builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
+    // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
+    // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
+    // interface, so the writer and the probe cannot disagree.
+    builder.Services.AddSingleton<OpcUaEndpointState>();
+    builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
     builder.Services.AddHostedService<OtOpcUaServerHostedService>();
 }
 
@@ -554,8 +564,9 @@ if (hasAdmin)
 builder.Services.AddOtOpcUaSecrets(builder.Configuration);
 
 // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
-// probing it would throw when the readiness endpoint resolves the factory.
-builder.Services.AddOtOpcUaHealth(hasAdmin);
+// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
+// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
+builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
 builder.Services.AddOtOpcUaObservability(builder.Configuration);
 
 // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs

@@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
     // publish.
     private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData;
 
-    // Intentionally never pruned: cold-start semantics depend on retained last values so
-    // ReadTag can answer synchronously for any path that has ever been pushed.
+    // Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned
+    // ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every
+    // (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops
+    // costing a boxed value + two DateTimes for the process lifetime (R02:L3).
     private readonly ConcurrentDictionary<string, DataValueSnapshot> _cache
         = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, ImmutableList<Action<string, DataValueSnapshot>>> _observers
@@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
         }
     }
 
+    /// <summary>
+    ///     Drop every cached snapshot whose path is not in <paramref name="paths"/>. Observers are
+    ///     untouched. Called by the host after each successful engine (re)load with the union of the loaded
+    ///     alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm
+    ///     references any more stop being retained.
+    /// </summary>
+    /// <param name="paths">The paths to keep.</param>
+    /// <returns>The number of cached snapshots dropped.</returns>
+    public int RetainOnly(IReadOnlySet<string> paths)
+    {
+        ArgumentNullException.ThrowIfNull(paths);
+        var dropped = 0;
+        foreach (var key in _cache.Keys)
+        {
+            if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++;
+        }
+        return dropped;
+    }
+
+    /// <summary>Test/diagnostic: the number of paths currently retained.</summary>
+    public int RetainedPathCount => _cache.Count;
+
     /// <inheritdoc/>
     public DataValueSnapshot ReadTag(string path)
     {

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs

@@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
 ///     cluster <c>alerts</c> DistributedPubSub topic.
 ///
 ///     <para>
-///         The host is the engine's lifecycle owner: the caller (the driver-role startup) builds
-///         the <see cref="DependencyMuxTagUpstreamSource"/>, constructs a <see cref="ScriptedAlarmEngine"/>
-///         <b>around that same upstream</b>, and passes both here. The host disposes the engine in
-///         <see cref="PostStop"/>.
+///         The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the
+///         <b>factory</b> <see cref="Props(IActorRef, IActorRef?, Func{ValueTuple{DependencyMuxTagUpstreamSource, ScriptedAlarmEngine}}, NodeId?, Func{int}?, ITelemetryLocalHub?, TimeSpan?)"/>:
+///         every incarnation builds its own <see cref="DependencyMuxTagUpstreamSource"/> + a
+///         <see cref="ScriptedAlarmEngine"/> around it, and disposes them in <see cref="PostStop"/>.
+///         Akka's default restart runs <c>PostStop</c> then re-creates the actor from the SAME Props, so the
+///         legacy instance-taking Props — a fixed engine captured in the Props — restarted around a
+///         <em>disposed</em> engine and every later <c>LoadAsync</c> threw: the node's scripted alarms
+///         silently died. The legacy overload is kept source-compatible but is restart-safe only in the
+///         weaker sense that a restart now DETACHES rather than disposes its engine (see
+///         <see cref="PreRestart"/>); production should spawn through the factory.
 ///     </para>
 ///
 ///     <para>
@@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
     /// Warning on every clean shutdown.</summary>
     private sealed record AlarmsLoadCanceled;
 
+    /// <summary>Default bound on one inbound alarm command's engine op (<see cref="OnAlarmCommand"/>): 10 s.</summary>
+    public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10);
+
     private readonly IActorRef _publishActor;
     private readonly IActorRef? _mux;
     private readonly DependencyMuxTagUpstreamSource _upstream;
@@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
     private readonly ILoggingAdapter _log = Context.GetLogger();
     private readonly CancellationTokenSource _cts = new();
     private readonly EventHandler<ScriptedAlarmEvent> _onEngineEvent;
+    private readonly TimeSpan _commandTimeout;
+
+    /// <summary>True when THIS incarnation built its engine (factory Props) and therefore disposes it on
+    /// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the
+    /// incarnation, so a restart only detaches from it and a final stop disposes it.</summary>
+    private readonly bool _ownsEngine;
+
+    /// <summary>Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers can tell a restart
+    /// (the next incarnation reuses a legacy engine) from a genuine stop.</summary>
+    private bool _restarting;
 
     /// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
     /// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
@@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
     /// <param name="telemetryHub">Optional node-local live-telemetry hub (Phase 5) each cluster-wide
     /// <c>alerts</c> transition is also emitted into; null (tests) skips the emit.</param>
     /// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
+    /// <remarks>Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the
+    /// actor around these SAME instances, so prefer the factory overload in production.</remarks>
     public static Props Props(
         IActorRef publishActor,
         IActorRef? mux,
@@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         ITelemetryLocalHub? telemetryHub = null) =>
         Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));
 
+    /// <summary>
+    ///     Restart-safe factory Props (R02:H1): each incarnation of the actor calls
+    ///     <paramref name="engineFactory"/> in its constructor and owns what it returns — the engine must
+    ///     be constructed <b>around</b> the returned upstream — so a supervision restart gets a fresh,
+    ///     undisposed engine and the next <see cref="ApplyScriptedAlarms"/> loads cleanly.
+    /// </summary>
+    /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
+    /// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
+    /// <param name="engineFactory">Builds a fresh (upstream, engine) pair; called once per incarnation.</param>
+    /// <param name="localNode">The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit.</param>
+    /// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
+    /// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
+    /// <param name="commandTimeout">Bound on one inbound alarm command's engine op; null ⇒ <see cref="DefaultCommandTimeout"/>.</param>
+    /// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
+    public static Props Props(
+        IActorRef publishActor,
+        IActorRef? mux,
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null,
+        ITelemetryLocalHub? telemetryHub = null,
+        TimeSpan? commandTimeout = null) =>
+        Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout));
+
+    /// <summary>Factory-built incarnation: owns (and on every stop disposes) the engine it builds.</summary>
+    /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
+    /// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
+    /// <param name="engineFactory">Builds this incarnation's (upstream, engine) pair.</param>
+    /// <param name="localNode">The local cluster node id for the Primary alerts-emit gate.</param>
+    /// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
+    /// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
+    /// <param name="commandTimeout">Bound on one inbound alarm command's engine op.</param>
+    public ScriptedAlarmHostActor(
+        IActorRef publishActor,
+        IActorRef? mux,
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null,
+        ITelemetryLocalHub? telemetryHub = null,
+        TimeSpan? commandTimeout = null)
+        : this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout)
+    {
+    }
+
+    private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build(
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory)
+    {
+        ArgumentNullException.ThrowIfNull(engineFactory);
+        var pair = engineFactory();
+        ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream");
+        ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine");
+        return pair;
+    }
+
     /// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
     /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
     /// <param name="mux">Optional dependency multiplexer the host registers dependency interest with.</param>
@@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         NodeId? localNode = null,
         Func<int>? driverMemberCountProvider = null,
         ITelemetryLocalHub? telemetryHub = null)
+        : this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null)
+    {
+    }
+
+    private ScriptedAlarmHostActor(
+        IActorRef publishActor,
+        IActorRef? mux,
+        (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair,
+        NodeId? localNode,
+        Func<int>? driverMemberCountProvider,
+        ITelemetryLocalHub? telemetryHub,
+        bool ownsEngine,
+        TimeSpan? commandTimeout)
     {
         ArgumentNullException.ThrowIfNull(publishActor);
-        ArgumentNullException.ThrowIfNull(upstream);
-        ArgumentNullException.ThrowIfNull(engine);
+        ArgumentNullException.ThrowIfNull(pair.Upstream);
+        ArgumentNullException.ThrowIfNull(pair.Engine);
         _publishActor = publishActor;
         _mux = mux;
-        _upstream = upstream;
-        _engine = engine;
+        _upstream = pair.Upstream;
+        _engine = pair.Engine;
+        _ownsEngine = ownsEngine;
+        _commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout;
         _localNode = localNode;
         _driverMemberCountProvider = driverMemberCountProvider;
         _telemetryHub = telemetryHub;
@@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
         // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
         // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
-        _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self));
-        _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count);
+        // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
+        // supervision restart builds one around an EMPTY upstream) converges immediately instead of
+        // waiting for every input to change again.
+        _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
+        // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
+        // any more will never be pushed again, so its last value is dead weight for the process lifetime.
+        var dropped = _upstream.RetainOnly(new HashSet<string>(msg.DepRefs, StringComparer.Ordinal));
+        _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))",
+            msg.DepRefs.Count, dropped);
 
         ReassertConditionNodes();
     }
@@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
             return;
         }
 
+        // Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a
+        // ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung
+        // file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime
+        // token so a stop still cancels promptly.
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
+        cts.CancelAfter(_commandTimeout);
+        var ct = cts.Token;
         try
         {
             switch (cmd.Operation)
             {
                 case "Acknowledge":
-                    await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                     break;
                 case "Confirm":
-                    await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                     break;
                 case "OneShotShelve":
-                    await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct);
                     break;
                 case "TimedShelve":
                     // A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA
@@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                             cmd.AlarmId);
                         return;
                     }
-                    await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None);
+                    await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct);
                     break;
                 case "Unshelve":
-                    await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct);
                     break;
                 case "Enable":
-                    await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct);
                     break;
                 case "Disable":
-                    await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct);
                     break;
                 case "AddComment":
                     if (string.IsNullOrWhiteSpace(cmd.Comment))
@@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                         _log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId);
                         return;
                     }
-                    await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                     break;
                 default:
                     _log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}",
@@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                     break;
             }
         }
+        catch (OperationCanceledException) when (!_cts.IsCancellationRequested)
+        {
+            _log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled",
+                cmd.Operation, cmd.AlarmId, _commandTimeout);
+        }
         catch (Exception ex)
         {
             // A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style.
@@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         base.PreStart();
     }
 
+    /// <inheritdoc />
+    /// <remarks>Akka's default <c>PreRestart</c> stops the children and calls <see cref="PostStop"/>. The flag
+    /// lets that <see cref="PostStop"/> keep a legacy (Props-captured) engine alive for the next incarnation
+    /// instead of disposing the instance it is about to be re-created around.</remarks>
+    protected override void PreRestart(Exception reason, object message)
+    {
+        _restarting = true;
+        base.PreRestart(reason, message);
+    }
+
     /// <inheritdoc />
     protected override void PostStop()
     {
         // Unregister mux interest first so no further DependencyValueChanged arrives while we tear down.
         _mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self));
         // Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't
-        // Tell a stopping actor), then dispose the engine (which drains its background work + clears).
+        // Tell a stopping actor), then dispose the engine (which drains its background work + clears) —
+        // unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will
+        // re-attach to (R02:H1).
         _cts.Cancel();
         _engine.OnEvent -= _onEngineEvent;
-        _engine.Dispose();
+        if (_ownsEngine || !_restarting) _engine.Dispose();
         _cts.Dispose();
         base.PostStop();
     }

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs

@@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor
         var refs = _consumer.DependencyRefs.ToList();
         // A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on
         // re-register), so re-registering after an apply cleanly adopts an added/removed ref.
-        _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self));
+        // ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays
+        // the last value of every ref it retained so the calc tags compute from current inputs at once
+        // instead of sitting BadWaitingForInitialData until each input happens to change.
+        _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true));
         _log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count);
     }
 

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs

@@ -26,32 +26,134 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 ///     <see cref="VirtualTagActor.DependencyValueChanged"/> receive as well, for direct senders and tests.
 ///     The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and
 ///     would over-emit if it spanned the cluster.
+///
+///     <para>
+///         <b>Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2).</b> The mux
+///         retains the last <c>(value, status, timestamp)</c> of a ref <b>only while at least one subscriber
+///         registered with <see cref="RegisterInterest.ReplayRetained"/> names it</b> (the virtual-tag
+///         actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives
+///         one batch of the retained values for its refs, in the shape a driver burst arrives in. The case
+///         this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new
+///         driver instance + a new mux adapter, and before this every calc tag sat
+///         <c>BadWaitingForInitialData</c> until each of its inputs happened to change again. So when a
+///         ref's last replay-opting subscriber leaves, its value is kept for <see cref="RetentionGrace"/>
+///         (default <see cref="DefaultRetentionGrace"/>; a returning replay registration inside it cancels
+///         the release), then dropped together with the ref's entry.
+///     </para>
+///     <para>
+///         A subscriber that does NOT opt in retains nothing: the continuous-historization recorder
+///         registers every historized ref on the node (a replay there would record a stale sample per tag
+///         per deploy), and retaining on its behalf made the map grow with the historized address space and
+///         never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name,
+///         plus those released within the last grace window. A ref with no subscriber and nothing retained
+///         has no entry at all.
+///     </para>
+///     <para>
+///         <b>Source removal.</b> The mux records which driver instance last published each retained ref;
+///         <see cref="SourceRemoved"/> evicts every value that driver produced, so a removed driver's last
+///         Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained
+///         value with Bad. ⚠️ As of fix round 1 no production actor sends <see cref="SourceRemoved"/> yet —
+///         <c>DriverHostActor</c> (outside the scripting lane) must Tell it when it stops / removes a driver
+///         child. Until then a removed driver's last value can still be replayed to a replay-opting
+///         subscriber that re-registers for that ref while another replay-opting subscriber holds it, or
+///         within the grace window.
+///     </para>
 /// </summary>
-public sealed class DependencyMuxActor : ReceiveActor
+public sealed class DependencyMuxActor : ReceiveActor, IWithTimers
 {
     public const string ActorName = "dependency-mux";
 
+    /// <summary>How long a ref's retained value outlives its last replay-opting subscriber.</summary>
+    public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2);
+
     /// <summary>Register a subscriber's interest in a set of tag refs. Idempotent on re-register —
     /// the second call replaces the prior interest set for that subscriber.</summary>
-    public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber);
+    /// <param name="TagRefs">The refs the subscriber wants.</param>
+    /// <param name="Subscriber">The subscriber.</param>
+    /// <param name="ReplayRetained">When true, the mux retains the last value of every ref in
+    /// <paramref name="TagRefs"/> while this subscriber is registered, and immediately sends the subscriber one
+    /// <see cref="VirtualTagActor.DependencyValueBatch"/> holding the last retained value of every ref it has
+    /// one for (nothing when it has none).</param>
+    public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber, bool ReplayRetained = false);
 
     /// <summary>Unregister every interest held by <see cref="Subscriber"/>. Sent on PostStop by
     /// the subscriber, or by Terminated handling when the mux watches.</summary>
     public sealed record UnregisterInterest(IActorRef Subscriber);
 
-    private readonly Dictionary<string, HashSet<IActorRef>> _byRef = new(StringComparer.Ordinal);
-    private readonly Dictionary<IActorRef, HashSet<string>> _bySubscriber = new();
+    /// <summary>A driver instance was removed / stopped: evict every retained value it produced so it is
+    /// never replayed. Subscriptions are untouched.</summary>
+    /// <param name="DriverInstanceId">The removed driver instance.</param>
+    public sealed record SourceRemoved(string DriverInstanceId);
+
+    private sealed class SweepReleased
+    {
+        public static readonly SweepReleased Instance = new();
+    }
+
+    private const string SweepTimerKey = "sweep-released";
+
+    /// <summary>Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?".</summary>
+    private sealed class RefEntry
+    {
+        public readonly HashSet<IActorRef> Subscribers = new();
+
+        /// <summary>Registered subscribers of this ref that opted into replay.</summary>
+        public int Replayers;
+
+        /// <summary>When the last replay-opting subscriber left (retention in grace), or null.</summary>
+        public DateTime? ReleasedAtUtc;
+
+        public VirtualTagActor.DependencyValueChanged? Last;
+        public string? LastSource;
+
+        public bool Retains => Replayers > 0 || ReleasedAtUtc is not null;
+        public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null;
+
+        public void Forget()
+        {
+            Last = null;
+            LastSource = null;
+            ReleasedAtUtc = null;
+        }
+    }
+
+    private sealed record Registration(HashSet<string> Refs, bool Replay);
+
+    private readonly Dictionary<string, RefEntry> _byRef = new(StringComparer.Ordinal);
+    private readonly Dictionary<IActorRef, Registration> _bySubscriber = new();
     private readonly ILoggingAdapter _log = Context.GetLogger();
 
+    /// <summary>How long a released ref's value is kept.</summary>
+    public TimeSpan RetentionGrace { get; }
+
+    /// <inheritdoc />
+    public ITimerScheduler Timers { get; set; } = null!;
+
+    /// <summary>Test hook — refs that currently hold a retained value.</summary>
+    internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null);
+
+    /// <summary>Test hook — refs the mux holds any state for (subscribers or a retained value).</summary>
+    internal int TrackedRefCount => _byRef.Count;
+
     /// <summary>Creates props for the DependencyMuxActor.</summary>
+    /// <param name="retentionGrace">How long a ref's value outlives its last replay-opting subscriber;
+    /// null ⇒ <see cref="DefaultRetentionGrace"/>.</param>
     /// <returns>The props for creating this actor.</returns>
-    public static Props Props() => Akka.Actor.Props.Create<DependencyMuxActor>();
+    public static Props Props(TimeSpan? retentionGrace = null)
+    {
+        var grace = retentionGrace ?? DefaultRetentionGrace;
+        return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace));
+    }
 
     /// <summary>Initializes a new instance of the <see cref="DependencyMuxActor"/> class.</summary>
-    public DependencyMuxActor()
+    /// <param name="retentionGrace">See <see cref="Props"/>; non-positive ⇒ <see cref="DefaultRetentionGrace"/>.</param>
+    public DependencyMuxActor(TimeSpan retentionGrace)
     {
+        RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace;
         Receive<RegisterInterest>(OnRegister);
         Receive<UnregisterInterest>(msg => RemoveSubscriber(msg.Subscriber));
+        Receive<SourceRemoved>(OnSourceRemoved);
+        Receive<SweepReleased>(_ => OnSweep());
         Receive<DriverInstanceActor.AttributeValuePublished>(OnAttributeValuePublished);
         Receive<DriverInstanceActor.AttributeValueBatchPublished>(OnAttributeValueBatchPublished);
         Receive<Terminated>(msg => RemoveSubscriber(msg.ActorRef));
@@ -59,60 +161,125 @@ public sealed class DependencyMuxActor : ReceiveActor
 
     private void OnRegister(RegisterInterest msg)
     {
-        // Replace any prior interest set so re-registers on actor restart don't leak old refs.
-        if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs))
-        {
-            foreach (var r in priorRefs)
-            {
-                if (_byRef.TryGetValue(r, out var set))
-                {
-                    set.Remove(msg.Subscriber);
-                    if (set.Count == 0) _byRef.Remove(r);
-                }
-            }
-        }
+        // Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first
+        // may put a ref into grace; the re-attach below takes it straight back out.
+        Detach(msg.Subscriber);
 
         var refs = new HashSet<string>(msg.TagRefs, StringComparer.Ordinal);
-        _bySubscriber[msg.Subscriber] = refs;
+        _bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained);
+        List<VirtualTagActor.DependencyValueChanged>? replay = null;
         foreach (var r in refs)
         {
-            if (!_byRef.TryGetValue(r, out var set))
+            if (!_byRef.TryGetValue(r, out var entry))
             {
-                set = new HashSet<IActorRef>();
-                _byRef[r] = set;
+                entry = new RefEntry();
+                _byRef[r] = entry;
             }
-            set.Add(msg.Subscriber);
+            entry.Subscribers.Add(msg.Subscriber);
+            if (!msg.ReplayRetained) continue;
+            entry.Replayers++;
+            entry.ReleasedAtUtc = null;
+            if (entry.Last is { } retained)
+                (replay ??= []).Add(retained);
         }
         Context.Watch(msg.Subscriber);
-        _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count);
+        _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})",
+            msg.Subscriber, refs.Count, msg.ReplayRetained);
+
+        // One batch, after the registration is complete, in exactly the shape a driver burst arrives in —
+        // so the subscriber's hot path handles it with no special case.
+        if (replay is not null)
+        {
+            msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay));
+            _log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber);
+        }
     }
 
     private void RemoveSubscriber(IActorRef subscriber)
     {
-        if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return;
-        foreach (var r in refs)
-        {
-            if (_byRef.TryGetValue(r, out var set))
-            {
-                set.Remove(subscriber);
-                if (set.Count == 0) _byRef.Remove(r);
-            }
-        }
-        _bySubscriber.Remove(subscriber);
+        if (!Detach(subscriber)) return;
         Context.Unwatch(subscriber);
         _log.Debug("DependencyMux: subscriber {Sub} removed", subscriber);
     }
 
+    /// <summary>Drop <paramref name="subscriber"/>'s registration. A ref losing its last replay-opting
+    /// subscriber goes into grace; a ref left with no state at all is removed.</summary>
+    private bool Detach(IActorRef subscriber)
+    {
+        if (!_bySubscriber.Remove(subscriber, out var registration)) return false;
+        var now = DateTime.UtcNow;
+        var released = false;
+        foreach (var r in registration.Refs)
+        {
+            if (!_byRef.TryGetValue(r, out var entry)) continue;
+            entry.Subscribers.Remove(subscriber);
+            if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null)
+            {
+                entry.ReleasedAtUtc = now;
+                released = true;
+            }
+            if (entry.IsDead) _byRef.Remove(r);
+        }
+        if (released && !Timers.IsTimerActive(SweepTimerKey))
+            Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace);
+        return true;
+    }
+
+    /// <summary>Drop retained values whose grace has run out — with a periodic sweep at the grace interval a
+    /// released value lives between one and two grace periods — and stop the sweep once nothing is in grace.</summary>
+    private void OnSweep()
+    {
+        var cutoff = DateTime.UtcNow - RetentionGrace;
+        var stillInGrace = false;
+        List<string>? dead = null;
+        foreach (var (r, entry) in _byRef)
+        {
+            if (entry.ReleasedAtUtc is not { } releasedAt) continue;
+            if (releasedAt > cutoff)
+            {
+                stillInGrace = true;
+                continue;
+            }
+            entry.Forget();
+            if (entry.IsDead) (dead ??= []).Add(r);
+        }
+        if (dead is not null)
+            foreach (var r in dead) _byRef.Remove(r);
+        if (!stillInGrace) Timers.Cancel(SweepTimerKey);
+    }
+
+    private void OnSourceRemoved(SourceRemoved msg)
+    {
+        List<string>? dead = null;
+        var evicted = 0;
+        foreach (var (r, entry) in _byRef)
+        {
+            if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal))
+                continue;
+            entry.Forget();
+            evicted++;
+            if (entry.IsDead) (dead ??= []).Add(r);
+        }
+        if (dead is not null)
+            foreach (var r in dead) _byRef.Remove(r);
+        _log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted);
+    }
+
     private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg)
     {
-        if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0)
+        if (!_byRef.TryGetValue(msg.FullReference, out var entry))
         {
-            // No virtual tag cares about this ref — drop. Common in normal operation; the address
+            // Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address
             // space carries thousands of tags and only a fraction feed virtual-tag expressions.
             return;
         }
         var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode);
-        foreach (var sub in subscribers)
+        if (entry.Retains)
+        {
+            entry.Last = dep;
+            entry.LastSource = msg.DriverInstanceId;
+        }
+        foreach (var sub in entry.Subscribers)
         {
             // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs
             // only the batch receive on its hot path. One list per subscriber: batch ownership transfers
@@ -122,18 +289,9 @@ public sealed class DependencyMuxActor : ReceiveActor
         }
     }
 
-    /// <summary>Filters one batched driver production burst per item and delivers each subscriber's matched
-    /// slice as ONE <see cref="VirtualTagActor.DependencyValueBatch"/>. Items nobody registered for are
-    /// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a
-    /// burst of N values costs one inbound message plus one outbound message per genuinely interested
-    /// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what
-    /// lets <see cref="VirtualTagActor"/> evaluate its script once per burst instead of once per input.
-    /// <para>The registered set is NOT just virtual-tag inputs: <c>ContinuousHistorizationRecorder</c>
-    /// registers interest in every historized ref, so on a historizing node <see cref="_byRef"/> can be
-    /// large and a sizeable fraction of each burst resolves.</para></summary>
     private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg)
     {
-        // Nobody has registered anything — skip the whole burst rather than probing an empty map per item.
+        // Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item.
         if (_byRef.Count == 0) return;
 
         // Allocated lazily on the first matched item: a burst in which nothing is registered for — still
@@ -146,8 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor
         for (var i = 0; i < count; i++)
         {
             var item = msg.Items[i];
-            if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue;
+            if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue;
             var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode);
+            if (entry.Retains)
+            {
+                entry.Last = dep;
+                entry.LastSource = msg.DriverInstanceId;
+            }
+            var subscribers = entry.Subscribers;
+            if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now
             perSubscriber ??= new Dictionary<IActorRef, List<VirtualTagActor.DependencyValueChanged>>();
             foreach (var sub in subscribers)
             {

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs

@@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Commons.Types;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
 
 namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor
     private readonly IReadOnlyList<string> _dependencyRefs;
     private readonly IActorRef? _mux;
     private readonly ITelemetryLocalHub? _telemetryHub;
+    // R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS
+    // ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded
+    // telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary).
+    private readonly ScriptLogRateLimiter _failureLogLimiter;
     private readonly ILoggingAdapter _log = Context.GetLogger();
     // Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a
     // comms-lost input from a genuine Good null, and it is what the script reads through
@@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor
     /// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
     /// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
     /// emitted into (Phase 5); null (tests) simply skips the emit.</param>
+    /// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
+    /// the default 20 per minute + one summary.</param>
     /// <returns>A configured <see cref="Akka.Actor.Props"/> for creating the actor.</returns>
     public static Props Props(
         string virtualTagId,
@@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor
         Func<DPSPublisher>? publisherFactory = null,
         IReadOnlyList<string>? dependencyRefs = null,
         IActorRef? mux = null,
-        ITelemetryLocalHub? telemetryHub = null) =>
+        ITelemetryLocalHub? telemetryHub = null,
+        ScriptLogRateLimiter? failureLogLimiter = null) =>
         Akka.Actor.Props.Create(() => new VirtualTagActor(
             virtualTagId, expression,
             evaluator ?? NullVirtualTagEvaluator.Instance,
@@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor
             publisherFactory,
             dependencyRefs ?? Array.Empty<string>(),
             mux,
-            telemetryHub));
+            telemetryHub,
+            failureLogLimiter));
 
     /// <summary>Initializes a virtual tag actor with the given configuration and dependencies.</summary>
     /// <param name="virtualTagId">Unique identifier for the virtual tag.</param>
@@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor
     /// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
     /// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
     /// emitted into (Phase 5); null (tests) simply skips the emit.</param>
+    /// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
+    /// the default 20 per minute + one summary.</param>
     public VirtualTagActor(
         string virtualTagId,
         string expression,
@@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor
         Func<DPSPublisher>? publisherFactory,
         IReadOnlyList<string> dependencyRefs,
         IActorRef? mux,
-        ITelemetryLocalHub? telemetryHub = null)
+        ITelemetryLocalHub? telemetryHub = null,
+        ScriptLogRateLimiter? failureLogLimiter = null)
     {
         _virtualTagId = virtualTagId;
         _scriptId = scriptId;
@@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor
         _dependencyRefs = dependencyRefs;
         _mux = mux;
         _telemetryHub = telemetryHub;
+        _failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter();
 
         Receive<DependencyValueChanged>(OnDependencyChanged);
         Receive<DependencyValueBatch>(OnDependencyBatch);
@@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor
     {
         if (_mux is not null && _dependencyRefs.Count > 0)
         {
-            _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self));
+            // ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's
+            // last-known inputs at once instead of waiting for each to change again.
+            _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true));
         }
     }
 
@@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor
     private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc)
     {
         OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "fail"));
-        PublishLog(level, reason);
+
+        // Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are
+        // counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad
+        // publish below are not limited — only the cluster-wide log fan-out is.
+        var admission = _failureLogLimiter.Admit(_virtualTagId);
+        if (admission.SuppressedInPreviousWindow > 0)
+        {
+            PublishLog("Warning",
+                $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " +
+                $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}");
+        }
+        if (admission.Admitted) PublishLog(level, reason);
 
         if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived())
         {

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs

@@ -3,6 +3,7 @@ using Akka.Event;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
 using ZB.MOM.WW.OtOpcUa.OpcUaServer;
 using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
@@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor
             (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts();
         }
 
+        // Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the
+        // operator redeployed to retry" is one of the two documented ways out of a quarantine (the other,
+        // an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map).
+        (_evaluator as IScriptQuarantineOwner)?.ResetQuarantines();
+
         var desired = new HashSet<string>(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal);
 
         // Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its

src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs

@@ -0,0 +1,95 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Components.Authorization;
+using Microsoft.AspNetCore.Http;
+using ZB.MOM.WW.Auth.AspNetCore;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
+
+/// <summary>
+///     <see cref="IAuditActorAccessor"/> that reads the Blazor circuit's principal first (R03:2.5).
+/// </summary>
+/// <remarks>
+///     <para>
+///         <see cref="HttpAuditActorAccessor"/> reads <c>IHttpContextAccessor</c>, which is right for a
+///         minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no
+///         current <c>HttpContext</c> on the circuit's threads, so every audit row written from an interactive
+///         page would have been stamped <c>"system"</c>. The circuit's identity lives in the scoped
+///         <see cref="AuthenticationStateProvider"/>, so that is consulted first and the HTTP context is the
+///         fallback for non-circuit scopes (where the provider is not registered at all).
+///     </para>
+///     <para>
+///         <c>CurrentActor</c> is synchronous by contract; the server-side provider's
+///         <see cref="AuthenticationStateProvider.GetAuthenticationStateAsync"/> is already completed in a
+///         circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on.
+///     </para>
+/// </remarks>
+public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor
+{
+    private readonly IHttpContextAccessor _httpContextAccessor;
+    private readonly AuthenticationStateProvider? _authenticationState;
+
+    /// <summary>Initializes a new instance of the <see cref="AuthenticationStateAuditActorAccessor"/> class.</summary>
+    /// <param name="httpContextAccessor">The HTTP context accessor (minimal-API fallback).</param>
+    /// <param name="authenticationState">The circuit's authentication state, when resolved inside a Blazor circuit.</param>
+    public AuthenticationStateAuditActorAccessor(
+        IHttpContextAccessor httpContextAccessor,
+        AuthenticationStateProvider? authenticationState = null)
+    {
+        ArgumentNullException.ThrowIfNull(httpContextAccessor);
+        _httpContextAccessor = httpContextAccessor;
+        _authenticationState = authenticationState;
+    }
+
+    /// <inheritdoc />
+    public string? CurrentActor
+    {
+        get
+        {
+            if (_authenticationState is not null)
+            {
+                var actor = TryResolveCircuitActor(_authenticationState);
+                if (actor is not null) return actor;
+            }
+            return ResolveActor(_httpContextAccessor.HttpContext?.User);
+        }
+    }
+
+    /// <summary>
+    ///     The circuit's actor, or null when this scope has no circuit identity. The server-side provider
+    ///     is registered for EVERY scope on an AdminUI node, but only a circuit calls
+    ///     <c>SetAuthenticationState</c>; anywhere else (a minimal-API request, a hosted service's scope)
+    ///     <c>GetAuthenticationStateAsync</c> throws <see cref="InvalidOperationException"/> synchronously
+    ///     (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to
+    ///     the HTTP context.
+    /// </summary>
+    /// <param name="provider">The scoped provider.</param>
+    /// <returns>The actor, or null.</returns>
+    private static string? TryResolveCircuitActor(AuthenticationStateProvider provider)
+    {
+        Task<AuthenticationState> task;
+        try
+        {
+            task = provider.GetAuthenticationStateAsync();
+        }
+        catch (InvalidOperationException)
+        {
+            return null;
+        }
+        return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null;
+    }
+
+    /// <summary>The claim-resolution rule shared with <see cref="HttpAuditActorAccessor"/>.</summary>
+    /// <param name="user">The principal, or null.</param>
+    /// <returns>The actor string, or null when unauthenticated.</returns>
+    internal static string? ResolveActor(ClaimsPrincipal? user)
+    {
+        if (user?.Identity?.IsAuthenticated != true)
+            return null;
+
+        // Prefer the canonical login-name claim; fall back to the Name claim or
+        // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name).
+        return user.FindFirst(ZbClaimTypes.Username)?.Value
+               ?? user.FindFirst(ZbClaimTypes.Name)?.Value
+               ?? user.Identity.Name;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs

@@ -1,5 +1,4 @@
 using Microsoft.AspNetCore.Http;
-using ZB.MOM.WW.Auth.AspNetCore;
 
 namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
 
@@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
 ///   <see cref="System.Security.Principal.IIdentity.Name"/>, in that order.</item>
 /// </list>
 /// <para>
-/// Registered as <b>scoped</b> in <see cref="ZB.MOM.WW.OtOpcUa.Security.ServiceCollectionExtensions.AddOtOpcUaAuth"/>
-/// so that it correctly follows the request scope used by Blazor Server interactive components
-/// and minimal-API endpoints. <c>IHttpContextAccessor</c> is registered by
-/// <c>AddOtOpcUaAuth</c> via <c>services.AddHttpContextAccessor()</c>.
+/// No longer the registered implementation: <see cref="AuthenticationStateAuditActorAccessor"/> is, because
+/// a Blazor circuit has no <c>HttpContext</c> after its first render and this accessor would stamp every row
+/// written from an interactive page <c>"system"</c>. Kept as the explicit HTTP-only variant for non-circuit
+/// hosts and tests.
 /// </para>
 /// </remarks>
 public sealed class HttpAuditActorAccessor : IAuditActorAccessor
@@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor
     }
 
     /// <inheritdoc />
-    public string? CurrentActor
-    {
-        get
-        {
-            var user = _httpContextAccessor.HttpContext?.User;
-            if (user?.Identity?.IsAuthenticated != true)
-                return null;
-
-            // Prefer the canonical login-name claim; fall back to the Name claim or
-            // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name).
-            return user.FindFirst(ZbClaimTypes.Username)?.Value
-                   ?? user.FindFirst(ZbClaimTypes.Name)?.Value
-                   ?? user.Identity.Name;
-        }
-    }
+    public string? CurrentActor =>
+        AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User);
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs

@@ -0,0 +1,39 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
+using ZB.MOM.WW.OtOpcUa.Security.Jwt;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Auth;
+
+/// <summary>The authentication scheme names OtOpcUa registers.</summary>
+public static class OtOpcUaAuthSchemes
+{
+    /// <summary>
+    ///     The bearer scheme that validates the HS256 tokens <c>/auth/token</c> mints: the first-party
+    ///     <c>AddJwtBearer</c> handler over <see cref="JwtTokenService.BuildValidationParameters"/>.
+    ///     Registered only when <see cref="JwtOptions.IsConfigured"/>.
+    /// </summary>
+    public const string Bearer = "Bearer";
+
+    /// <summary>
+    ///     The default scheme: a policy scheme that forwards to <see cref="Bearer"/> when the request
+    ///     carries an <c>Authorization: Bearer</c> header and to the cookie scheme otherwise, so one
+    ///     <c>RequireAuthorization()</c> / policy accepts a browser session AND a machine token.
+    /// </summary>
+    public const string CookieOrBearer = "OtOpcUa.CookieOrBearer";
+
+    /// <summary>Whether the request presents a bearer token (the selector's whole rule).</summary>
+    /// <param name="context">The request.</param>
+    /// <returns><see langword="true"/> when an <c>Authorization: Bearer …</c> header is present.</returns>
+    public static bool HasBearerToken(HttpContext context) =>
+        TryGetBearerToken(context.Request.Headers.Authorization, out _);
+
+    internal static bool TryGetBearerToken(StringValues authorization, out string token)
+    {
+        token = string.Empty;
+        var header = authorization.ToString();
+        const string prefix = "Bearer ";
+        if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false;
+        token = header[prefix.Length..].Trim();
+        return token.Length > 0;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs

@@ -1,79 +0,0 @@
-using System.Net.Http.Json;
-using System.Security.Claims;
-using Microsoft.AspNetCore.Components.Authorization;
-using Microsoft.Extensions.Logging;
-
-namespace ZB.MOM.WW.OtOpcUa.Security.Blazor;
-
-/// <summary>
-/// Blazor Server <see cref="AuthenticationStateProvider"/> that snapshots the cookie-backed
-/// principal supplied at circuit boot and polls <c>/auth/ping</c> every 60 seconds to detect
-/// expiry. Mirrors ScadaLink's CentralUI implementation.
-/// </summary>
-public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable
-{
-    private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60);
-
-    private readonly HttpClient _http;
-    private readonly ILogger<CookieAuthenticationStateProvider> _logger;
-    private readonly CancellationTokenSource _cts = new();
-    private ClaimsPrincipal _current;
-    private Task? _pingLoop;
-
-    /// <summary>Initializes a new instance of the CookieAuthenticationStateProvider class.</summary>
-    /// <param name="initial">The initial claims principal from circuit boot.</param>
-    /// <param name="http">The HTTP client for authentication ping requests.</param>
-    /// <param name="logger">The logger for diagnostic messages.</param>
-    public CookieAuthenticationStateProvider(
-        ClaimsPrincipal initial,
-        HttpClient http,
-        ILogger<CookieAuthenticationStateProvider> logger)
-    {
-        _current = initial;
-        _http = http;
-        _logger = logger;
-    }
-
-    /// <inheritdoc />
-    public override Task<AuthenticationState> GetAuthenticationStateAsync()
-    {
-        _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token));
-        return Task.FromResult(new AuthenticationState(_current));
-    }
-
-    private async Task PingLoopAsync(CancellationToken ct)
-    {
-        try
-        {
-            while (!ct.IsCancellationRequested)
-            {
-                await Task.Delay(PingInterval, ct).ConfigureAwait(false);
-                var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false);
-                if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true)
-                {
-                    _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode);
-                    _current = new ClaimsPrincipal(new ClaimsIdentity());
-                    NotifyAuthenticationStateChanged(
-                        Task.FromResult(new AuthenticationState(_current)));
-                }
-            }
-        }
-        catch (OperationCanceledException) { /* expected on shutdown */ }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly");
-        }
-    }
-
-    /// <summary>Cancels the background ping loop, awaits its completion, and disposes the cancellation token source.</summary>
-    /// <returns>A task that represents the asynchronous operation.</returns>
-    public async ValueTask DisposeAsync()
-    {
-        _cts.Cancel();
-        if (_pingLoop is not null)
-        {
-            try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ }
-        }
-        _cts.Dispose();
-    }
-}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using ZB.MOM.WW.Auth.AspNetCore;
 using ZB.MOM.WW.Auth.Abstractions.Roles;
 using ZB.MOM.WW.OtOpcUa.Security.Jwt;
@@ -23,6 +24,17 @@ public static class AuthEndpoints
     /// <summary>Response for a token issue request.</summary>
     public sealed record TokenResponse(string Token);
 
+    /// <summary>
+    ///     The <c>/auth/token</c> policy: an authenticated principal from the COOKIE scheme. Naming the scheme
+    ///     makes the authorization middleware authenticate that scheme alone, so an <c>Authorization: Bearer</c>
+    ///     header is never consulted here (with login disabled the auto-login handler sits under the same
+    ///     scheme name, so the dev rig is unchanged).
+    /// </summary>
+    internal static readonly AuthorizationPolicy CookieSessionOnly =
+        new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme)
+            .RequireAuthenticatedUser()
+            .Build();
+
     /// <summary>Maps OtOpcUa authentication endpoints to the application route builder.</summary>
     /// <param name="app">The endpoint route builder.</param>
     /// <returns>The endpoint route builder for chaining.</returns>
@@ -36,7 +48,17 @@ public static class AuthEndpoints
         // AddOtOpcUaAuth fallback policy that would otherwise 401 the request.
         app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery();
         app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous();
-        app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization();
+        // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured
+        // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call.
+        if (app.ServiceProvider.GetRequiredService<IOptions<JwtOptions>>().Value.IsConfigured)
+            // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer
+            // that could mint a fresh token would renew itself forever — outliving its own expiry and the
+            // user's LDAP membership. A token is minted only for an interactive (cookie) session.
+            app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly);
+        else
+            app.ServiceProvider.GetService<ILoggerFactory>()?
+                .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints")
+                .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted");
         app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization();
         return app;
     }
@@ -73,8 +95,13 @@ public static class AuthEndpoints
         {
             result = await ldap.AuthenticateAsync(username, password, ct);
         }
-        catch (Exception)
+        catch (Exception ex) when (ex is not OperationCanceledException)
         {
+            // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the
+            // operator had to infer every 503 from the directory side.
+            http.RequestServices.GetService<ILoggerFactory>()?
+                .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints")
+                .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username);
             return Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
         }
 

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs

@@ -15,4 +15,26 @@ public sealed class JwtOptions
 
     /// <summary>Default token expiry. Mirrors ScadaLink (15 min).</summary>
     public int ExpiryMinutes { get; set; } = 15;
+
+    /// <summary>
+    ///     Whether the JWT feature is on: a non-blank <see cref="SigningKey"/>. When off, <c>/auth/token</c>
+    ///     is not mapped and no bearer scheme is registered, so an admin node that never configured a key
+    ///     keeps booting exactly as before; when on, the key is validated at host start
+    ///     (<c>ValidateOnStart</c>) instead of throwing HTTP 500 on the first token request.
+    /// </summary>
+    public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey);
+
+    /// <summary>The validation rule shared by the startup validator and the tests: why the options are invalid, or null.</summary>
+    /// <returns>A failure message, or <see langword="null"/> when valid.</returns>
+    public string? Validate()
+    {
+        if (!IsConfigured) return null;
+        var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey);
+        if (keyBytes < MinSigningKeyBytes)
+            return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s).";
+        if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank.";
+        if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank.";
+        if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0.";
+        return null;
+    }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs

@@ -0,0 +1,19 @@
+using Microsoft.Extensions.Options;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Jwt;
+
+/// <summary>
+///     Startup validator for <see cref="JwtOptions"/> (R03:2.4): when a signing key is configured it must
+///     be usable, and the host must learn that at start (<c>ValidateOnStart</c>) rather than from an HTTP 500
+///     on the first <c>/auth/token</c>. An unconfigured section (no key) is valid — the feature is simply off.
+/// </summary>
+public sealed class JwtOptionsValidator : IValidateOptions<JwtOptions>
+{
+    /// <inheritdoc />
+    public ValidateOptionsResult Validate(string? name, JwtOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        var failure = options.Validate();
+        return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure);
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs

@@ -25,24 +25,12 @@ public sealed class JwtTokenService
     /// <summary>
     /// Role claim type used in the JWT payload.
     /// <para>
-    /// <b>Issued-only / no internal JwtBearer scheme:</b> OtOpcUa uses a single Cookie
-    /// authentication scheme; the JWT is minted by the <c>/auth/token</c> endpoint and
-    /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no
-    /// <c>AddJwtBearer</c> pipeline in OtOpcUa — the cookie stores the
-    /// <see cref="System.Security.Claims.ClaimsPrincipal"/> directly. Because no internal
-    /// bearer validation path exists, the short "Role" key is intentionally used here rather
-    /// than the long <see cref="ClaimTypes.Role"/> URI; external consumers receive exactly the
-    /// key they expect.
-    /// </para>
-    /// <para>
-    /// <b>If a JwtBearer scheme is ever added:</b> the
-    /// <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/> passed to
-    /// <c>AddJwtBearer</c> MUST set <c>RoleClaimType = JwtTokenService.RoleClaimType</c> (and
-    /// <c>NameClaimType = JwtTokenService.UsernameClaimType</c>) so that
-    /// <c>[Authorize(Roles=...)]</c> and <c>ClaimsPrincipal.IsInRole</c> resolve correctly.
-    /// <see cref="BuildValidationParameters"/> is already wired to do this and MUST be used
-    /// rather than constructing <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/>
-    /// ad hoc.
+    /// The short "Role" key is what <see cref="Issue"/> mints and what external consumers of the token
+    /// expect. The internal bearer scheme (<c>AddJwtBearer</c> in <c>AddOtOpcUaAuth</c>) validates with
+    /// <see cref="BuildValidationParameters"/>, which sets <c>RoleClaimType = RoleClaimType</c> and
+    /// <c>NameClaimType = UsernameClaimType</c>, and turns <c>MapInboundClaims</c> off, so
+    /// <c>[Authorize(Roles=...)]</c> and <c>ClaimsPrincipal.IsInRole</c> resolve against this key. Never
+    /// construct <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/> ad hoc.
     /// </para>
     /// </summary>
     public const string RoleClaimType = "Role";
@@ -150,5 +138,9 @@ public sealed class JwtTokenService
         // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale.
         RoleClaimType = RoleClaimType,
         NameClaimType = UsernameClaimType,
+        // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a
+        // different HMAC variant over the same key is refused on both the bearer and TryValidate paths.
+        ValidAlgorithms = [SecurityAlgorithms.HmacSha256],
+        RequireSignedTokens = true,
     };
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs

@@ -1,5 +1,6 @@
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.Cookies;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.DataProtection;
 using Microsoft.AspNetCore.Http;
@@ -19,9 +20,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap;
 namespace ZB.MOM.WW.OtOpcUa.Security;
 
 /// <summary>
-///     DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the
-///     cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge
-///     structurally — see <c>docs/plans/2026-05-29-auth-alignment-design.md</c>.
+///     DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when
+///     <c>Security:Jwt:SigningKey</c> is configured a <see cref="OtOpcUaAuthSchemes.Bearer"/> scheme
+///     accepts the tokens <c>/auth/token</c> mints (R03:2.4), and the default scheme is a policy scheme
+///     that forwards to whichever the request presents — so <c>/api/*</c> and every
+///     <c>RequireAuthorization</c> endpoint take a browser session OR a machine token. Matches
+///     ScadaBridge structurally — see <c>docs/plans/2026-05-29-auth-alignment-design.md</c>.
 /// </summary>
 public static class ServiceCollectionExtensions
 {
@@ -36,7 +40,14 @@ public static class ServiceCollectionExtensions
     /// <returns>The same service collection, for chaining.</returns>
     public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration)
     {
-        services.AddOptions<JwtOptions>().Bind(configuration.GetSection(JwtOptions.SectionName));
+        // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme)
+        // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) —
+        // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500).
+        services.AddOptions<JwtOptions>()
+            .Bind(configuration.GetSection(JwtOptions.SectionName))
+            .ValidateOnStart();
+        services.AddSingleton<IValidateOptions<JwtOptions>, JwtOptionsValidator>();
+        var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get<JwtOptions>()?.IsConfigured ?? false;
         services.AddOptions<OtOpcUaCookieOptions>().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName));
         services.AddOptions<LdapOptions>().Bind(configuration.GetSection(LdapOptions.SectionName));
         services.AddOptions<AuthDisableLoginOptions>().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName));
@@ -53,7 +64,10 @@ public static class ServiceCollectionExtensions
         // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered
         // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API
         // endpoints.
-        services.TryAddScoped<IAuditActorAccessor, HttpAuditActorAccessor>();
+        // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext
+        // exists on a circuit's threads after the first render) and falls back to the HTTP context for
+        // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5).
+        services.TryAddScoped<IAuditActorAccessor, AuthenticationStateAuditActorAccessor>();
 
         // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it
         // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton
@@ -78,7 +92,39 @@ public static class ServiceCollectionExtensions
             .GetSection(AuthDisableLoginOptions.SectionName)
             .GetValue<bool>(nameof(AuthDisableLoginOptions.DisableLogin));
 
-        var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme);
+        // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is
+        // the only one that can issue a session. With JWT off the selector always picks the cookie.
+        var authBuilder = services.AddAuthentication(o =>
+        {
+            o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer;
+            o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+            o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+        });
+        authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o =>
+        {
+            o.ForwardDefaultSelector = ctx =>
+                jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx)
+                    ? OtOpcUaAuthSchemes.Bearer
+                    : CookieAuthenticationDefaults.AuthenticationScheme;
+        });
+        if (jwtConfigured)
+        {
+            // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone.
+            // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/
+            // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an
+            // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted
+            // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token
+            // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500.
+            authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { });
+            services.AddOptions<JwtBearerOptions>(OtOpcUaAuthSchemes.Bearer)
+                .Configure<JwtTokenService>((o, jwt) =>
+                {
+                    o.TokenValidationParameters = jwt.BuildValidationParameters();
+                    o.MapInboundClaims = false;
+                    o.SaveToken = false;
+                    o.IncludeErrorDetails = false;
+                });
+        }
         if (disableLogin)
         {
             // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered
@@ -142,8 +188,8 @@ public static class ServiceCollectionExtensions
 
         services.AddAuthorization(o =>
         {
-            o.FallbackPolicy = new AuthorizationPolicyBuilder(
-                    CookieAuthenticationDefaults.AuthenticationScheme)
+            // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too.
+            o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer)
                 .RequireAuthenticatedUser()
                 .Build();
 

src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj

@@ -10,6 +10,10 @@
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer"/>
+        <!-- Direct, to promote JwtBearer's 8.0.1 IdentityModel.Protocols dependencies to the central 8.11.0 pin. -->
+        <PackageReference Include="Microsoft.IdentityModel.Protocols"/>
+        <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect"/>
         <PackageReference Include="Microsoft.IdentityModel.Tokens"/>
         <PackageReference Include="System.IdentityModel.Tokens.Jwt"/>
         <PackageReference Include="ZB.MOM.WW.Auth.Abstractions"/>