Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -837,7 +837,7 @@ public class AlarmActor : ReceiveActor
                parentExecutionId,
                // Scheduler seam (#18): share this alarm's scheduler override with the
                // launched on-trigger script body (null = process-wide shared).
-                _scheduler);
+                _scheduler, Context.System.ActorSelection("/user/site-communication"));
        }
        catch (Exception ex)
        {
@@ -129,9 +129,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
    /// <summary>
    /// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
    /// the terminating actor ref. The buffered command is applied once <see cref="Terminated"/>
-    /// confirms the child has fully stopped.
+    /// confirms the child has fully stopped and its final persistence acknowledgement arrives.
    /// </summary>
    private readonly Dictionary<IActorRef, PendingRedeploy> _pendingRedeploys = new();
+    private readonly Dictionary<IActorRef, InstanceActor.StaticOverridesFlushed> _redeployFlushes = new();
+    private readonly HashSet<IActorRef> _redeployTerminated = new();
+    private readonly HashSet<IActorRef> _redeployStopsInFlight = new();
+    private readonly Queue<IActorRef> _redeployStopQueue = new();
+    private sealed record LifecycleStop(string Name, string CommandId, bool Delete, IActorRef ReplyTo);
+    private sealed record LifecycleStopPersisted(IActorRef Actor, Exception? Error);
+    private readonly Dictionary<IActorRef, LifecycleStop> _lifecycleStops = new();
+    private readonly HashSet<string> _lifecycleStopNames = new();
+
+

    /// <summary>
    /// Name → terminating actor ref shadow of <see cref="_pendingRedeploys"/>.
@@ -487,6 +497,31 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
        // Terminated signal — drains a buffered redeployment once the previous
        // Instance Actor has fully stopped.
        Receive<Terminated>(HandleTerminated);
+        Receive<LifecycleStopPersisted>(message =>
+        {
+            if (!_lifecycleStops.Remove(message.Actor, out var stop)) return;
+            _lifecycleStopNames.Remove(stop.Name);
+            _terminatingActorsByName.Remove(stop.Name);
+            BumpStoreWriteGeneration("lifecycle stop committed");
+            if (stop.Delete && message.Error is null)
+            {
+                _deployedInstanceNames.Remove(stop.Name);
+                _runningRevisions.Remove(stop.Name);
+                _sweepAttemptedRevisions.Remove(stop.Name);
+            }
+            UpdateInstanceCounts();
+            var action = stop.Delete ? "deleted" : "disabled";
+            LogDeploymentEvent(message.Error is null ? "Info" : "Error", stop.Name,
+                $"Instance {stop.Name} {action}", message.Error?.GetBaseException().Message);
+            stop.ReplyTo.Tell(new InstanceLifecycleResponse(stop.CommandId, stop.Name,
+                message.Error is null, message.Error?.GetBaseException().Message, DateTimeOffset.UtcNow));
+        });
+        Receive<InstanceActor.StaticOverridesFlushed>(message =>
+        {
+            if (!_pendingRedeploys.ContainsKey(message.Actor) && !_redeployStopsInFlight.Contains(message.Actor) && !_lifecycleStops.ContainsKey(message.Actor)) return;
+            _redeployFlushes[message.Actor] = message;
+            TryFinishRedeployStop(message.Actor);
+        });
    }

    /// <inheritdoc />
@@ -835,6 +870,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
    private void HandleDeploy(DeployInstanceCommand command, IActorRef replyTo)
    {
        var instanceName = command.InstanceUniqueName;
+        if (_lifecycleStopNames.Contains(instanceName))
+        {
+            replyTo.Tell(new DeploymentStatusResponse(command.DeploymentId, instanceName, DeploymentStatus.Failed,
+                "Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
+            return;
+        }
+

        // WP3.1 warm-then-gate. The S3 gate below must stay synchronous on the actor thread
        // (mailbox FIFO is what makes redeploy-supersede and delete-during-redeploy correct),
@@ -1029,7 +1071,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
            _pendingRedeploys[existing] = new PendingRedeploy(command, replyTo);
            _terminatingActorsByName[instanceName] = existing;
            Context.Watch(existing);
-            Context.Stop(existing);
+            _redeployStopQueue.Enqueue(existing);
+            StartQueuedRedeployStops();
            UpdateInstanceCounts();
            return;
        }
@@ -1061,6 +1104,74 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
        ApplyDeployment(command, replyTo, isRedeploy: false);
    }

+    private bool DeferLifecycleStop(string name, string commandId, bool delete, IActorRef replyTo)
+    {
+        if (_lifecycleStopNames.Contains(name))
+        {
+            replyTo.Tell(new InstanceLifecycleResponse(commandId, name, false,
+                "Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
+            return true;
+        }
+        if (!_terminatingActorsByName.TryGetValue(name, out var actor) && !_instanceActors.TryGetValue(name, out actor))
+            return false;
+        if (_pendingRedeploys.Remove(actor, out var pending))
+            pending.OriginalSender.Tell(new DeploymentStatusResponse(pending.Command.DeploymentId, name,
+                DeploymentStatus.Failed, "Superseded by instance lifecycle command.", DateTimeOffset.UtcNow));
+        _instanceActors.Remove(name);
+        _terminatingActorsByName[name] = actor;
+        _lifecycleStopNames.Add(name);
+        _lifecycleStops[actor] = new LifecycleStop(name, commandId, delete, replyTo);
+        BumpStoreWriteGeneration("lifecycle stop dispatch");
+        Context.Watch(actor);
+        Context.Stop(actor);
+        UpdateInstanceCounts();
+        // Termination and flush may both already have arrived for a canceled
+        // redeploy. Reconcile immediately as well as on either completion signal.
+        TryFinishRedeployStop(actor);
+        return true;
+    }
+
+    private void StartQueuedRedeployStops()
+    {
+        while (_redeployStopsInFlight.Count < 4 && _redeployStopQueue.TryDequeue(out var actor))
+        {
+            if (!_pendingRedeploys.ContainsKey(actor)) continue;
+            _redeployStopsInFlight.Add(actor);
+            Context.Stop(actor);
+        }
+    }
+
+    private void TryFinishRedeployStop(IActorRef actor)
+    {
+        if (!_redeployTerminated.Contains(actor) || !_redeployFlushes.Remove(actor, out var flush)) return;
+        _redeployTerminated.Remove(actor);
+        _redeployStopsInFlight.Remove(actor);
+        if (_lifecycleStops.TryGetValue(actor, out var lifecycle))
+        {
+            var persistence = Task.Run(async () =>
+            {
+                if (flush.Error is not null) throw new InvalidOperationException("Final static override persistence failed.", flush.Error);
+                if (lifecycle.Delete) await _storage.RemoveDeployedConfigAsync(lifecycle.Name).ConfigureAwait(false);
+                else await _storage.SetInstanceEnabledAsync(lifecycle.Name, false).ConfigureAwait(false);
+            });
+            persistence.PipeTo(Self, Self, () => new LifecycleStopPersisted(actor, null),
+                error => new LifecycleStopPersisted(actor, error));
+            StartQueuedRedeployStops();
+            return;
+        }
+        if (_pendingRedeploys.Remove(actor, out var pending))
+        {
+            _terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
+            if (flush.Error is null)
+                ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
+            else
+                pending.OriginalSender.Tell(new DeploymentStatusResponse(pending.Command.DeploymentId,
+                    pending.Command.InstanceUniqueName, DeploymentStatus.Failed,
+                    "Predecessor static override flush failed; redeployment was not applied.", DateTimeOffset.UtcNow));
+        }
+        StartQueuedRedeployStops();
+    }
+
    /// <summary>
    /// Handles an Instance Actor's <see cref="Terminated"/> signal. Two cases:
    /// <list type="number">
@@ -1075,16 +1186,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
    {
        // Case 1 — redeployment drain: the predecessor of a buffered redeploy has
        // fully stopped and freed its actor name.
-        if (_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
+        if (_pendingRedeploys.ContainsKey(terminated.ActorRef) || _redeployStopsInFlight.Contains(terminated.ActorRef) || _lifecycleStops.ContainsKey(terminated.ActorRef))
        {
-            // Drop the name-keyed shadow now that the predecessor has
-            // fully terminated and its actor name is free. Any deploy arriving after
-            // this point sees neither _instanceActors[name] (cleared when we stopped
-            // the predecessor) nor _terminatingActorsByName[name] (cleared here), so
-            // ApplyDeployment + Context.ActorOf below safely reuses the name.
-            _terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
-
-            ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
+            _redeployTerminated.Add(terminated.ActorRef);
+            TryFinishRedeployStop(terminated.ActorRef);
            return;
        }

@@ -1437,33 +1542,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
        // WP3.1 warm-then-gate: a deploy for this instance is mid-compile-warm; queue this
        // command so same-instance ordering is preserved (see TryBufferDuringDeployWarm).
        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
-
-        // A disable arriving mid-redeploy must cancel the buffered
-        // redeploy. Otherwise HandleTerminated re-creates the Instance Actor and
-        // re-stores its config with isEnabled: true when the predecessor terminates,
-        // silently reverting the operator's disable back to enabled. Mirror the
-        // last-write-wins handling in HandleDeploy/HandleDelete: drop the pending
-        // command (so HandleTerminated returns early), clear the shadow, and tell the
-        // displaced deployer it was superseded. The disable itself still persists
-        // is_enabled = false below, which becomes the durable state.
-        if (_terminatingActorsByName.TryGetValue(instanceName, out var terminatingRef))
-        {
-            if (_pendingRedeploys.Remove(terminatingRef, out var pending))
-            {
-                pending.OriginalSender.Tell(new DeploymentStatusResponse(
-                    pending.Command.DeploymentId,
-                    instanceName,
-                    DeploymentStatus.Failed,
-                    $"superseded by disable of {instanceName} before redeploy finished terminating",
-                    DateTimeOffset.UtcNow));
-            }
-            _terminatingActorsByName.Remove(instanceName);
-        }
-        else if (_instanceActors.TryGetValue(instanceName, out var actor))
-        {
-            Context.Stop(actor);
-            _instanceActors.Remove(instanceName);
-        }
+        if (DeferLifecycleStop(instanceName, command.CommandId, false, replyTo)) return;

        UpdateInstanceCounts();

@@ -1512,6 +1591,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
    private void HandleEnable(EnableInstanceCommand command, IActorRef replyTo)
    {
        var instanceName = command.InstanceUniqueName;
+        if (_lifecycleStopNames.Contains(instanceName))
+        {
+            replyTo.Tell(new InstanceLifecycleResponse(command.CommandId, instanceName, false,
+                "Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
+            return;
+        }
+

        // WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
@@ -1997,37 +2083,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers

        // WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
-
-        // A delete arriving while a redeploy is still terminating must
-        // be authoritative over the mid-redeploy bookkeeping. HandleDeploy already
-        // removed the instance from _instanceActors and buffered a PendingRedeploy
-        // keyed by the terminating ref. If we fall straight through to the
-        // _instanceActors miss + unconditional decrement, the buffered redeploy is
-        // left intact — so when Terminated fires, HandleTerminated calls
-        // ApplyDeployment(isRedeploy: true) and RESURRECTS the just-deleted instance,
-        // with the counter now inconsistent. Cancel the pending redeploy first.
-        if (_terminatingActorsByName.TryGetValue(instanceName, out var terminatingRef))
-        {
-            // Drop the buffered command so HandleTerminated's _pendingRedeploys.Remove
-            // misses and it returns early (no resurrection). Clear the shadow too.
-            if (_pendingRedeploys.Remove(terminatingRef, out var pending))
-            {
-                pending.OriginalSender.Tell(new DeploymentStatusResponse(
-                    pending.Command.DeploymentId,
-                    instanceName,
-                    DeploymentStatus.Failed,
-                    $"superseded by delete of {instanceName} before redeploy finished terminating",
-                    DateTimeOffset.UtcNow));
-            }
-            _terminatingActorsByName.Remove(instanceName);
-            // The terminating predecessor is already being stopped by HandleDeploy;
-            // no Context.Stop needed here.
-        }
-        else if (_instanceActors.TryGetValue(instanceName, out var actor))
-        {
-            Context.Stop(actor);
-            _instanceActors.Remove(instanceName);
-        }
+        if (DeferLifecycleStop(instanceName, command.CommandId, true, replyTo)) return;

        // The deployed count is derived from the authoritative set of
        // deployed config names, so removing the name here decrements it. Correct for a
@@ -120,15 +120,9 @@ public class InstanceActor : ReceiveActor
    /// </summary>
    private readonly Task? _staticStateResetGate;

-    /// <summary>
-    /// How long <see cref="PostStop"/> waits for the final coalesced static-override flush to
-    /// commit. The wait exists so a redeploy's clear is ordered AFTER this actor's last write
-    /// (arch-review 2026-08-19 P1-2 leg 2) — the Deployment Manager gates the clear on the
-    /// Terminated signal, which a fire-and-forget flush could outlive and durably resurrect
-    /// overrides the redeploy had just cleared. Bounded so a wedged SQLite write cannot stall
-    /// the redeploy indefinitely; exceeding it is logged and the stop proceeds.
-    /// </summary>
-    private static readonly TimeSpan FinalStaticOverrideFlushTimeout = TimeSpan.FromSeconds(5);
+    private Task _staticOverridePersistTail = Task.CompletedTask;
+
+    internal sealed record StaticOverridesFlushed(IActorRef Actor, Exception? Error);

    /// <summary>
    /// Cached string forms of <see cref="QualityCode"/> (perf remediation, arch-review
@@ -437,15 +431,13 @@ public class InstanceActor : ReceiveActor
            state.Timer?.Cancel();
        _tagSubscribeStates.Clear();

-        // Final flush of any coalesced static-override writes so a graceful stop does not lose
-        // the last <= flush-interval of writes (P4 pattern, mirrors NativeAlarmActor.PostStop).
-        // AWAITED, not fire-and-forget (arch-review 2026-08-19 P1-2 leg 2): a redeploy stops
-        // this actor, waits for Terminated, and only then clears the static overrides — so a
-        // detached final flush could commit up to one flush interval of the PREDECESSOR's dirty
-        // writes AFTER that clear and durably resurrect them. Blocking here is what puts the
-        // write provably before the Terminated signal the clear is gated on.
+        // SQLite's async APIs may block synchronously. Persist on the ordered
+        // thread-pool chain and acknowledge completion separately from Terminated.
        _staticOverrideFlushTimer?.Cancel();
-        WaitForFinalStaticOverrideFlush();
+        var stoppedActor = Self;
+        FlushDirtyStaticOverrides(final: true).PipeTo(Context.Parent, Self,
+            success: () => new StaticOverridesFlushed(stoppedActor, null),
+            failure: error => new StaticOverridesFlushed(stoppedActor, error));

        // Operational `instance_lifecycle` event — instance stopped. An
        // instance stops on disable, delete, redeployment, and graceful shutdown;
@@ -659,15 +651,23 @@ public class InstanceActor : ReceiveActor
    /// applied) remains authoritative.
    /// </summary>
    /// <returns>
-    /// The persist task, so the stop path can order the FINAL flush before termination
-    /// (<see cref="WaitForFinalStaticOverrideFlush"/>). Every other caller ignores it, keeping
+    /// The ordered persist task, acknowledged separately from termination so a redeploy
+    /// can wait for the final commit. Every other caller ignores it, keeping
    /// the fire-and-forget semantics unchanged. Completed when there was nothing dirty.
    /// </returns>
-    private Task FlushDirtyStaticOverrides()
+    private Task FlushDirtyStaticOverrides(bool final = false)
    {
        _staticOverrideFlushTimer = null;
        if (_dirtyStaticOverrides.Count == 0)
-            return Task.CompletedTask;
+            return _staticOverridePersistTail;
+        if (!final && !_staticOverridePersistTail.IsCompleted)
+        {
+            // Keep coalescing while SQLite is busy instead of accumulating an
+            // unbounded task/batch queue behind a stalled transaction.
+            _staticOverrideFlushTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(
+                _staticOverrideFlushInterval, Self, FlushStaticOverrides.Instance, Self);
+            return _staticOverridePersistTail;
+        }

        var batch = _dirtyStaticOverrides
            .Select(kv => (AttributeName: kv.Key, Value: kv.Value))
@@ -676,7 +676,14 @@ public class InstanceActor : ReceiveActor

        var instanceName = _instanceUniqueName;
        var logger = _logger;
-        var persist = _storage.SetStaticOverridesAsync(instanceName, batch);
+        var previous = _staticOverridePersistTail;
+        var persist = Task.Run(async () =>
+        {
+            try { await previous.ConfigureAwait(false); }
+            catch { /* The prior batch was logged; a transient failure must not poison future writes. */ }
+            await _storage.SetStaticOverridesAsync(instanceName, batch).ConfigureAwait(false);
+        });
+        _staticOverridePersistTail = persist;
        persist.ContinueWith(t => logger.LogWarning(
                t.Exception?.GetBaseException(),
                "Failed to persist {Count} static overrides for {Instance}; in-memory state is authoritative",
@@ -685,34 +692,6 @@ public class InstanceActor : ReceiveActor
        return persist;
    }

-    /// <summary>
-    /// Runs the stop-path static-override flush and waits for it to commit, bounded by
-    /// <see cref="FinalStaticOverrideFlushTimeout"/>. A fault is swallowed — the
-    /// OnlyOnFaulted continuation inside <see cref="FlushDirtyStaticOverrides"/> has already
-    /// logged it, and a persist failure must never turn a graceful stop into a crash.
-    /// </summary>
-    private void WaitForFinalStaticOverrideFlush()
-    {
-        var flush = FlushDirtyStaticOverrides();
-        if (flush.IsCompleted)
-            return;
-
-        try
-        {
-            if (!flush.Wait(FinalStaticOverrideFlushTimeout))
-            {
-                _logger.LogWarning(
-                    "Final static-override flush for {Instance} did not complete within {Timeout}; "
-                    + "stopping anyway — a redeploy's override reset may race it",
-                    _instanceUniqueName, FinalStaticOverrideFlushTimeout);
-            }
-        }
-        catch (Exception)
-        {
-            // Already logged by the flush's OnlyOnFaulted continuation.
-        }
-    }
-
    /// <summary>
    /// Data-sourced attribute write: forwards a write request to the DCL and pipes
    /// the device write result back to the caller. The in-memory value is left
@@ -86,6 +86,7 @@ public class NativeAlarmActor : ReceiveActor
    /// a logging failure never affects the mirror.</param>
    /// <param name="persistFlushInterval">How long native-alarm transitions coalesce before a
    /// batched SQLite persist flush (P4). Defaults to 100 ms.</param>
+    /// <param name="rehydrationRead">Optional prestarted persisted-state read; null reads from storage.</param>
    public NativeAlarmActor(
        ResolvedNativeAlarmSource source,
        string instanceName,
@@ -96,8 +97,10 @@ public class NativeAlarmActor : ReceiveActor
        ILogger logger,
        AlarmKind nativeKind = AlarmKind.NativeOpcUa,
        IServiceProvider? serviceProvider = null,
-        TimeSpan? persistFlushInterval = null)
+        TimeSpan? persistFlushInterval = null,
+        Task<IReadOnlyList<NativeAlarmRow>>? rehydrationRead = null)
    {
+        _rehydrationRead = rehydrationRead;
        _source = source;
        _instanceName = instanceName;
        _instanceActor = instanceActor;
@@ -124,7 +127,7 @@ public class NativeAlarmActor : ReceiveActor

        // Rehydrate last-known state from SQLite (non-blocking), then subscribe.
        // A fresh source snapshot will reconcile/replace this once it arrives.
-        _storage.GetNativeAlarmsAsync(_instanceName, _source.CanonicalName)
+        (_rehydrationRead ?? _storage.GetNativeAlarmsAsync(_instanceName, _source.CanonicalName))
            .PipeTo(Self, Self, rows => new RehydrationCompleted(rows));

        SendSubscribe();
@@ -149,32 +152,48 @@ public class NativeAlarmActor : ReceiveActor
        base.PostStop();
    }

+    private readonly Task<IReadOnlyList<NativeAlarmRow>>? _rehydrationRead;
+
+    private string? _subscribeCorrelation;
+    private DateTimeOffset _subscribeDeadline;
+    private int _subscribeDelayMs;
+    private bool _subscribeSucceeded;
+
    private void SendSubscribe()
    {
-        _dclManager.Tell(new SubscribeAlarmsRequest(
-            CorrelationId: Guid.NewGuid().ToString("N"),
-            InstanceUniqueName: _instanceName,
-            ConnectionName: _source.ConnectionName,
-            SourceReference: _source.SourceReference,
-            ConditionFilter: _source.ConditionFilter,
-            Timestamp: DateTimeOffset.UtcNow));
-
-        // Arm a response-timeout retry so a LOST subscribe response is re-sent, not
-        // only a failure reply (S4). Cancelled on a Success reply; a failure reply
-        // re-arms it at 1× (overriding this 2× response-timeout arm). 2× interval so
-        // a slow-but-arriving response is not pre-empted by a redundant re-send.
+        if (_subscribeSucceeded) return;
+        if (_subscribeCorrelation is not null && DateTimeOffset.UtcNow < _subscribeDeadline)
+        {
+            _retryTimer?.Cancel();
+            _retryTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(
+                TimeSpan.FromMilliseconds(Math.Max(1, (_subscribeDeadline - DateTimeOffset.UtcNow).TotalMilliseconds)),
+                Self, RetrySubscribe.Instance, Self);
+            return;
+        }
+        _subscribeDelayMs = _subscribeDelayMs == 0
+            ? Math.Clamp(_options.NativeAlarmRetryIntervalMs, 1, 60000) : Math.Min(60000, _subscribeDelayMs * 2);
+        _subscribeCorrelation = Guid.NewGuid().ToString("N");
+        var responseTimeout = TimeSpan.FromMilliseconds(Math.Min(120000, _subscribeDelayMs * 2));
+        _subscribeDeadline = DateTimeOffset.UtcNow + responseTimeout;
+        _dclManager.Tell(new SubscribeAlarmsRequest(_subscribeCorrelation, _instanceName,
+            _source.ConnectionName, _source.SourceReference, _source.ConditionFilter, DateTimeOffset.UtcNow));
        _retryTimer?.Cancel();
-        _retryTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(
-            TimeSpan.FromMilliseconds(_options.NativeAlarmRetryIntervalMs * 2),
+        _retryTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(responseTimeout,
            Self, RetrySubscribe.Instance, Self);
    }

+    private readonly HashSet<string> _liveTouched = new();
+    private bool _authoritativeSnapshotReceived;
+    private bool _rehydrationCompleted;
+
    private void HandleRehydration(RehydrationCompleted msg)
    {
+        if (_authoritativeSnapshotReceived || _rehydrationCompleted) return;
+        _rehydrationCompleted = true;
        foreach (var row in msg.Rows)
        {
            // A live transition that arrived before the read completed wins.
-            if (_alarms.ContainsKey(row.SourceReference))
+            if (_alarms.ContainsKey(row.SourceReference) || _liveTouched.Contains(row.SourceReference))
            {
                continue;
            }
@@ -227,8 +246,12 @@ public class NativeAlarmActor : ReceiveActor
            // event (it is not a live transition).
            Emit(t, t.Condition, logSiteEvent: false);
        }
+        _liveTouched.Clear();
    }

+    private bool _snapshotInProgress;
+    private readonly Dictionary<string, NativeAlarmTransition> _liveDuringSnapshot = new();
+
    private void HandleTransition(NativeAlarmTransitionUpdate update)
    {
        var t = update.Transition;
@@ -237,15 +260,18 @@ public class NativeAlarmActor : ReceiveActor
        switch (t.Kind)
        {
            case AlarmTransitionKind.SnapshotStart:
-                if (t.SnapshotId is null || t.SnapshotId == _lastCompletedSnapshotId || t.SnapshotId == _snapshotId)
+                if (t.SnapshotId is not null && (t.SnapshotId == _lastCompletedSnapshotId || t.SnapshotId == _snapshotId))
                    return;
                _snapshotId = t.SnapshotId;
+                _snapshotInProgress = true;
                _snapshotBuffer.Clear();
+                _liveDuringSnapshot.Clear();
                break;

            case AlarmTransitionKind.Snapshot:
                if (t.SnapshotId != _snapshotId)
                    return;
+                _snapshotInProgress = true;
                _snapshotBuffer[t.SourceReference] = t;
                break;

@@ -270,6 +296,16 @@ public class NativeAlarmActor : ReceiveActor
    /// </summary>
    private void ApplySnapshotSwap()
    {
+        foreach (var (source, live) in _liveDuringSnapshot)
+        {
+            if (_snapshotBuffer.TryGetValue(source, out var snapshot) && snapshot.TransitionTime > live.TransitionTime) continue;
+            if (!live.Condition.Active && live.Condition.Acknowledged) _snapshotBuffer.Remove(source);
+            else _snapshotBuffer[source] = live;
+        }
+        _liveDuringSnapshot.Clear();
+        _snapshotInProgress = false;
+        _authoritativeSnapshotReceived = true;
+        _liveTouched.Clear();
        foreach (var (sourceRef, prior) in _alarms)
        {
            if (!_snapshotBuffer.ContainsKey(sourceRef))
@@ -304,12 +340,18 @@ public class NativeAlarmActor : ReceiveActor

    private void ApplyLiveTransition(NativeAlarmTransition t)
    {
+        if (!_authoritativeSnapshotReceived && !_rehydrationCompleted) _liveTouched.Add(t.SourceReference);
+        // Retain timestamp ordering for a clear removed from the visible mirror
+        // while a refresh is still assembling.
+        if (_liveDuringSnapshot.TryGetValue(t.SourceReference, out var recent) && t.TransitionTime < recent.TransitionTime)
+            return;
        // Ignore stale (out-of-order) transitions for a condition we already hold.
        if (_alarms.TryGetValue(t.SourceReference, out var existing) && t.TransitionTime < existing.TransitionTime)
        {
            return;
        }

+        if (_snapshotInProgress) _liveDuringSnapshot[t.SourceReference] = t;
        _alarms[t.SourceReference] = t;
        MarkDirtyUpsert(t);
        Emit(t, t.Condition);
@@ -333,6 +375,8 @@ public class NativeAlarmActor : ReceiveActor
        // An interrupted refresh is not a snapshot. Never merge its partial rows
        // into the next connection's replay.
        _snapshotBuffer.Clear();
+        _liveDuringSnapshot.Clear();
+        _snapshotInProgress = false;
        _snapshotId = null;
        // Keep last-known conditions (uncertain) rather than clearing them; the
        // reconnect snapshot reconciles state. No emit — avoids a flap.
@@ -343,8 +387,12 @@ public class NativeAlarmActor : ReceiveActor

    private void HandleSubscribeResponse(SubscribeAlarmsResponse resp)
    {
+        if (resp.CorrelationId != _subscribeCorrelation) return;
+        _subscribeCorrelation = null;
        if (resp.Success)
        {
+            _subscribeSucceeded = true;
+            _subscribeDelayMs = 0;
            // Success — cancel the response-timeout retry armed in SendSubscribe.
            _retryTimer?.Cancel();
            _logger.LogInformation("Native alarm subscription established for {Source} on {Instance}",
@@ -358,7 +406,7 @@ public class NativeAlarmActor : ReceiveActor

        _retryTimer?.Cancel();
        _retryTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(
-            TimeSpan.FromMilliseconds(_options.NativeAlarmRetryIntervalMs),
+            TimeSpan.FromMilliseconds(_subscribeDelayMs),
            Self, RetrySubscribe.Instance, Self);
    }

@@ -111,14 +111,13 @@ internal sealed class AuditingDbConnection : DbConnection
    internal DbConnection Inner => _inner;

    /// <inheritdoc />
-    // ConnectionString is settable on DbConnection — forward both halves.
+    // The configured target is the audit identity; scripts cannot retarget it or
+    // retrieve connection credentials through the raw ADO surface.
    public override string ConnectionString
    {
-        // Some providers throw on get when the connection hasn't been opened
-        // with a string set explicitly. The wrapper has no opinion — forward.
-#pragma warning disable CS8765 // nullability of overridden member parameter — base setter accepts null in practice
-        get => _inner.ConnectionString;
-        set => _inner.ConnectionString = value;
+#pragma warning disable CS8765
+        get => string.Empty;
+        set => throw new NotSupportedException("Audited connections cannot be retargeted.");
 #pragma warning restore CS8765
    }

@@ -135,7 +134,7 @@ internal sealed class AuditingDbConnection : DbConnection
    public override ConnectionState State => _inner.State;

    /// <inheritdoc />
-    public override void ChangeDatabase(string databaseName) => _inner.ChangeDatabase(databaseName);
+    public override void ChangeDatabase(string databaseName) => throw new NotSupportedException("Audited connections cannot be retargeted.");

    /// <inheritdoc />
    public override void Close() => _inner.Close();
@@ -165,6 +165,7 @@ internal static class ScriptRunLauncher
    /// (external data has no spawning execution) — that on-trigger run is a tree ROOT.
    /// </param>
    /// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
+    /// <param name="siteCommunicationActor">Site transport for notification status queries, resolved on the actor thread.</param>
    public static void LaunchAlarmScript(
        string alarmName,
        string instanceName,
@@ -182,7 +183,8 @@ internal static class ScriptRunLauncher
        long runId,
        int? executionTimeoutSeconds,
        Guid? parentExecutionId,
-        ScriptExecutionScheduler? scheduler)
+        ScriptExecutionScheduler? scheduler,
+        ICanTell? siteCommunicationActor = null)
    {
        var timeout = ResolveTimeout(executionTimeoutSeconds, options);
        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
@@ -196,7 +198,7 @@ internal static class ScriptRunLauncher
                    alarmName, instanceName, level, priority, message, compiledScript,
                    instanceActor, sharedScriptLibrary, options, completionTarget, logger,
                    healthCollector, serviceProvider, runId, parentExecutionId, timeout,
-                    executionScheduler, cts),
+                    executionScheduler, cts, siteCommunicationActor),
                CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
                .Unwrap();
        }
@@ -659,7 +661,8 @@ internal static class ScriptRunLauncher
        Guid? parentExecutionId,
        TimeSpan timeout,
        ScriptExecutionScheduler executionScheduler,
-        CancellationTokenSource cts)
+        CancellationTokenSource cts,
+        ICanTell? siteCommunicationActor)
    {
        // arch-review 2026-08-19 P1-3: singletons resolved from the root provider up front,
        // exactly like RunScriptAsync, so they are available to the catch blocks regardless
@@ -696,6 +699,9 @@ internal static class ScriptRunLauncher
            // run it late — the alarm it would react to is already stale.
            cts.Token.ThrowIfCancellationRequested();

+            using var serviceScope = serviceProvider?.CreateScope();
+            var services = serviceScope?.ServiceProvider;
+
            // An alarm on-trigger run can call Instance.CallScript()
            // via the ScriptRuntimeContext injected into globals
            var context = new ScriptRuntimeContext(
@@ -711,6 +717,15 @@ internal static class ScriptRunLauncher
                // it every Log.* line from an alarm script was attributed to the INSTANCE, not the
                // alarm. `scriptName` above is already the "AlarmActor:{alarmName}" source key, so
                // it is passed as-is (re-prefixing it would leave "AlarmActor:" in the target).
+                externalSystemClient: services?.GetService<IExternalSystemClient>(),
+                databaseGateway: services?.GetService<IDatabaseGateway>(),
+                storeAndForward: services?.GetService<StoreAndForwardService>(),
+                siteCommunicationActor: siteCommunicationActor,
+                siteId: services?.GetService<ISiteIdentityProvider>()?.SiteId ?? string.Empty,
+                auditWriter: services?.GetService<IAuditWriter>(),
+                operationTrackingStore: services?.GetService<IOperationTrackingStore>(),
+                cachedForwarder: services?.GetService<ICachedCallTelemetryForwarder>(),
+                sourceNode: services?.GetService<INodeIdentityProvider>()?.NodeName,
                sourceScript: scriptName,
                // ParentExecutionId tag-cascade: the
                // alarm on-trigger run mints its own fresh ExecutionId (the
@@ -612,30 +612,11 @@ public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable,
    /// Synchronously disposes the tracking store and its SQLite connection.
    /// </summary>
    /// <remarks>
-    /// This path does NOT bridge to async via
-    /// <c>.AsTask().GetAwaiter().GetResult()</c>. Sync-over-async on a SemaphoreSlim
-    /// can deadlock when invoked from a non-reentrant SyncContext (e.g. host
-    /// shutdown continuations observed on the host sync context). In-flight writes
-    /// at the moment of <see cref="Dispose"/> will fail their next operation
-    /// against the disposed connection with <see cref="ObjectDisposedException"/> —
-    /// the caller's responsibility is to ensure no concurrent operations during
-    /// the synchronous dispose. Use <see cref="DisposeAsync"/> if you need to
-    /// drain in-flight writes before close.
+    /// The pump runs on the thread pool and its drain does not capture a caller's
+    /// synchronization context. Join it before releasing SQLite resources, exactly
+    /// as the asynchronous path does; never close a live writer transaction.
    /// </remarks>
-    public void Dispose()
-    {
-        if (Interlocked.Exchange(ref _disposeState, 1) != 0)
-        {
-            return;
-        }
-
-        // Stop accepting work; whatever is still queued fails on the pump with
-        // ObjectDisposedException (see ApplyBatch) — this path does not drain.
-        _queue.Writer.TryComplete();
-        _writeConnection.Dispose();
-        _writeGate.Dispose();
-        GC.SuppressFinalize(this);
-    }
+    public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult();

    /// <summary>
    /// Asynchronously disposes the tracking store and its SQLite connection.