Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -2899,6 +2899,87 @@ public class DataConnectionActor : UntypedActor, IWithTimers

    // ── Native alarm subscriptions ──

+    private sealed class AlarmReplayState
+    {
+        public Dictionary<string, NativeAlarmTransition> Current { get; set; } = new();
+        public Dictionary<string, NativeAlarmTransition> Snapshot { get; } = new();
+        public Dictionary<string, NativeAlarmTransition> LiveDuringSnapshot { get; } = new();
+        public bool InSnapshot { get; set; }
+        public bool Ready { get; set; }
+        public Guid? Epoch { get; set; }
+    }
+    private readonly Dictionary<string, AlarmReplayState> _alarmReplay = new();
+
+    private void CacheAlarmTransition(string source, NativeAlarmTransition transition)
+    {
+        if (!_alarmReplay.TryGetValue(source, out var state))
+            _alarmReplay[source] = state = new AlarmReplayState();
+        switch (transition.Kind)
+        {
+            case AlarmTransitionKind.SnapshotStart:
+                if (transition.SnapshotId is not null && transition.SnapshotId == state.Epoch) return;
+                state.Snapshot.Clear();
+                state.LiveDuringSnapshot.Clear();
+                state.InSnapshot = true;
+                state.Epoch = transition.SnapshotId;
+                break;
+            case AlarmTransitionKind.Snapshot:
+                if (transition.SnapshotId != state.Epoch || (transition.SnapshotId is not null && !state.InSnapshot)) return;
+                state.InSnapshot = true;
+                state.Snapshot[transition.SourceReference] = transition;
+                break;
+            case AlarmTransitionKind.SnapshotComplete:
+                if (transition.SnapshotId != state.Epoch || (transition.SnapshotId is not null && !state.InSnapshot)) return;
+                state.Current = SnapshotWithLiveChanges(state);
+                state.Snapshot.Clear();
+                state.LiveDuringSnapshot.Clear();
+                state.InSnapshot = false;
+                state.Ready = true;
+                break;
+            default:
+                if (state.LiveDuringSnapshot.TryGetValue(transition.SourceReference, out var recent)
+                    && transition.TransitionTime < recent.TransitionTime) return;
+                if (state.Current.TryGetValue(transition.SourceReference, out var prior)
+                    && transition.TransitionTime < prior.TransitionTime) return;
+                if (state.InSnapshot) state.LiveDuringSnapshot[transition.SourceReference] = transition;
+                if (!transition.Condition.Active && transition.Condition.Acknowledged)
+                    state.Current.Remove(transition.SourceReference);
+                else
+                    state.Current[transition.SourceReference] = transition;
+                break;
+        }
+    }
+
+    private static Dictionary<string, NativeAlarmTransition> SnapshotWithLiveChanges(AlarmReplayState state)
+    {
+        var result = new Dictionary<string, NativeAlarmTransition>(state.Snapshot);
+        foreach (var (key, live) in state.LiveDuringSnapshot)
+        {
+            if (result.TryGetValue(key, out var snapshot) && snapshot.TransitionTime > live.TransitionTime) continue;
+            if (!live.Condition.Active && live.Condition.Acknowledged) result.Remove(key);
+            else result[key] = live;
+        }
+        return result;
+    }
+
+    private void ReplayAlarmsTo(string source, IActorRef subscriber)
+    {
+        if (!_alarmReplay.TryGetValue(source, out var state) || (!state.Ready && !state.InSnapshot)) return;
+        var epoch = state.InSnapshot ? state.Epoch : Guid.NewGuid();
+        var boundary = new NativeAlarmTransition("", source, "", AlarmTransitionKind.SnapshotStart,
+            new AlarmConditionState(false, true, null, AlarmShelveState.Unshelved, false, 0),
+            "", "", "", "", "", null, DateTimeOffset.UtcNow, "", "", SnapshotSourceReference: source, SnapshotId: epoch);
+        // Legacy Mx snapshots have no start or epoch. Mid-frame subscribers get
+        // the prefix already received and then share the remaining live frame.
+        if (epoch is not null)
+            subscriber.Tell(new NativeAlarmTransitionUpdate(_connectionName, boundary));
+        foreach (var transition in (state.InSnapshot ? SnapshotWithLiveChanges(state) : state.Current).Values)
+            subscriber.Tell(new NativeAlarmTransitionUpdate(_connectionName, transition with
+            { Kind = AlarmTransitionKind.Snapshot, SnapshotSourceReference = source, SnapshotId = epoch }));
+        if (!state.InSnapshot)
+            subscriber.Tell(new NativeAlarmTransitionUpdate(_connectionName, boundary with { Kind = AlarmTransitionKind.SnapshotComplete }));
+    }
+
    private void HandleSubscribeAlarms(SubscribeAlarmsRequest request)
    {
        var subscriber = Sender;
@@ -2920,7 +3001,7 @@ public class DataConnectionActor : UntypedActor, IWithTimers
            _alarmSourceSubscribers[request.SourceReference] = subs;
            IndexAlarmSource(request.SourceReference);
        }
-        subs.Add(subscriber);
+        var newlyAdded = subs.Add(subscriber);
        Context.Watch(subscriber);
        // S10: the adapter feed carries a single shared condition filter per source.
        // A second subscriber with a different filter silently overwrites the first
@@ -2943,6 +3024,7 @@ public class DataConnectionActor : UntypedActor, IWithTimers
        if (_alarmSubscriptionIds.ContainsKey(request.SourceReference) ||
            _alarmSubscribesInFlight.Contains(request.SourceReference))
        {
+            if (newlyAdded) ReplayAlarmsTo(request.SourceReference, subscriber);
            subscriber.Tell(new SubscribeAlarmsResponse(
                request.CorrelationId, request.InstanceUniqueName, true, null, now));
            return;
@@ -3081,12 +3163,15 @@ public class DataConnectionActor : UntypedActor, IWithTimers
            var scope = transition.SnapshotSourceReference
                ?? (string.IsNullOrEmpty(transition.SourceObjectReference) ? null : transition.SourceObjectReference);
            var targets = scope is null
-                ? _alarmSourceSubscribers.Values
-                : _alarmSourceSubscribers.Where(kv => kv.Key == scope).Select(kv => kv.Value);
-            foreach (var subs in targets)
-                foreach (var sub in subs)
+                ? _alarmSourceSubscribers
+                : _alarmSourceSubscribers.Where(kv => kv.Key == scope);
+            foreach (var (source, subscribers) in targets)
+            {
+                CacheAlarmTransition(source, transition);
+                foreach (var sub in subscribers)
                    if (notified.Add(sub))
                        sub.Tell(new NativeAlarmTransitionUpdate(_connectionName, transition));
+            }
            return;
        }

@@ -3110,6 +3195,7 @@ public class DataConnectionActor : UntypedActor, IWithTimers
                !predicate.IsAllowed(transition))
                continue;

+            CacheAlarmTransition(sourceRef, transition);
            foreach (var sub in subs)
            {
                if (notified.Add(sub))
@@ -3215,6 +3301,7 @@ public class DataConnectionActor : UntypedActor, IWithTimers

        // No subscribers remain for this source — tear down the adapter feed.
        _alarmSourceSubscribers.Remove(sourceReference);
+        _alarmReplay.Remove(sourceReference);
        UnindexAlarmSource(sourceReference);
        _alarmSourceFilter.Remove(sourceReference);
        _alarmSourceFilterPredicate.Remove(sourceReference);
@@ -3237,6 +3324,7 @@ public class DataConnectionActor : UntypedActor, IWithTimers
            return;

        _alarmSubscriptionIds.Clear();
+        _alarmReplay.Clear();
        _alarmSubscribesInFlight.Clear();
        var self = Self;
        var generation = _adapterGeneration;
@@ -319,8 +319,8 @@ public class MxGatewayDataConnection
            // MxGateway has no server-side CONDITION filter either, so conditionFilter is
            // still not forwarded: the DataConnectionActor's per-source + per-condition-type
            // gate stays authoritative, uniform with the OPC UA WhereClause stance.
-            if (_alarmCts != null && AlarmFilterPrefix.Covers(_alarmStreamPrefix, sourceReference))
-                return Task.FromResult(subscriptionId);
+            // Even a covered new source needs a standing-state snapshot. Reopen
+            // the shared stream; DCL separately replays same-source co-subscribers.

            // A new source outside the live prefix (or the first subscriber): (re)open the
            // stream on the recomputed union. The source replays a fresh snapshot, which
@@ -336,7 +336,23 @@ public class MxGatewayDataConnection
            var token = _alarmCts.Token;
            var client = _client!;
            var prefix = _alarmStreamPrefix.Length == 0 ? null : _alarmStreamPrefix;
-            _ = Task.Run(() => client.RunAlarmStreamAsync(prefix, t => callback(t), token), token);
+            var epoch = Guid.NewGuid();
+            void Forward(Commons.Types.Alarms.NativeAlarmTransition transition)
+            {
+                lock (_alarmLock)
+                {
+                    if (token.IsCancellationRequested) return;
+                    if (transition.Kind == AlarmTransitionKind.SnapshotStart) epoch = Guid.NewGuid();
+                    callback(transition.Kind is AlarmTransitionKind.SnapshotStart or AlarmTransitionKind.Snapshot or AlarmTransitionKind.SnapshotComplete
+                        ? transition with { SnapshotId = epoch } : transition);
+                }
+            }
+            _ = Task.Run(async () =>
+            {
+                Forward(MxGatewayAlarmMapper.SnapshotComplete() with { Kind = AlarmTransitionKind.SnapshotStart });
+                if (token.IsCancellationRequested) return;
+                await client.RunAlarmStreamAsync(prefix, Forward, token);
+            }, token);
        }

        return Task.FromResult(subscriptionId);
@@ -427,6 +427,9 @@ public sealed class RealMxGatewayClient : IMxGatewayClient
                    AlarmFilterPrefix = alarmFilterPrefix ?? string.Empty,
                };

+                // Every transport reopen produces a fresh authoritative frame,
+                // including an empty snapshot after the previous stream faulted.
+                onTransition(MxGatewayAlarmMapper.SnapshotComplete() with { Kind = Commons.Types.Enums.AlarmTransitionKind.SnapshotStart });
                await foreach (var message in _client!.StreamAlarmsAsync(request, ct)
                    .WithCancellation(ct).ConfigureAwait(false))
                {
@@ -96,6 +96,22 @@ public class RealOpcUaClient : IOpcUaClient
    /// <summary>Raised when the OPC UA connection is lost.</summary>
    public event Action? ConnectionLost;

+    /// <summary>Selects exactly the requested protection mode; protected discovery failures fail closed.</summary>
+    internal static EndpointDescription DiscoverEndpoint(string url, MessageSecurityMode requested,
+        Func<EndpointDescriptionCollection> discover)
+    {
+        EndpointDescriptionCollection endpoints;
+        try { endpoints = discover(); }
+        catch (Exception ex)
+        {
+            if (requested != MessageSecurityMode.None)
+                throw new InvalidOperationException($"OPC UA endpoint discovery failed; requested security mode {requested} cannot be verified.", ex);
+            return new EndpointDescription(url);
+        }
+        return endpoints.FirstOrDefault(e => e.SecurityMode == requested)
+            ?? throw new InvalidOperationException($"OPC UA requested security mode {requested} is unavailable. Available modes: {string.Join(", ", endpoints.Select(e => e.SecurityMode).Distinct())}.");
+    }
+
    /// <inheritdoc />
    public async Task ConnectAsync(string endpointUrl, OpcUaConnectionOptions? options = null, CancellationToken cancellationToken = default)
    {
@@ -140,24 +156,13 @@ public class RealOpcUaClient : IOpcUaClient
        }

        // Discover endpoints from the server, pick the preferred security mode
-        EndpointDescription? endpoint;
-        try
+        var endpoint = DiscoverEndpoint(endpointUrl, preferredSecurityMode, () =>
        {
 #pragma warning disable CS0618
            using var discoveryClient = DiscoveryClient.Create(new Uri(endpointUrl));
+            return discoveryClient.GetEndpoints(null);
 #pragma warning restore CS0618
-#pragma warning disable CS0618
-            var endpoints = discoveryClient.GetEndpoints(null);
-#pragma warning restore CS0618
-            endpoint = endpoints
-                .Where(e => e.SecurityMode == preferredSecurityMode)
-                .FirstOrDefault() ?? endpoints.FirstOrDefault();
-        }
-        catch
-        {
-            // Fallback: construct endpoint description manually
-            endpoint = new EndpointDescription(endpointUrl);
-        }
+        });

        // The server advertises its own base address, which is frequently unroutable from here
        // (a 0.0.0.0 wildcard bind, or an internal container/NAT hostname). Swap it back to the
@@ -377,21 +382,13 @@ public class RealOpcUaClient : IOpcUaClient
            await appConfig.ValidateAsync(ApplicationType.Client);

            // Discover endpoints, pick the preferred security mode (same logic as ConnectAsync).
-            EndpointDescription? endpoint;
-            try
+            var endpoint = DiscoverEndpoint(endpointUrl, preferredSecurityMode, () =>
            {
 #pragma warning disable CS0618
                using var discoveryClient = DiscoveryClient.Create(new Uri(endpointUrl));
-                var endpoints = discoveryClient.GetEndpoints(null);
+                return discoveryClient.GetEndpoints(null);
 #pragma warning restore CS0618
-                endpoint = endpoints
-                    .Where(ep => ep.SecurityMode == preferredSecurityMode)
-                    .FirstOrDefault() ?? endpoints.FirstOrDefault();
-            }
-            catch
-            {
-                endpoint = new EndpointDescription(endpointUrl);
-            }
+            });

            // Same reachability rewrite as ConnectAsync — a probe against a server advertising a
            // 0.0.0.0 / NAT base address must dial the reachable host, not the advertised one.