Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -244,4 +244,102 @@ public class HighReviewRegressionTests : TestKit
        b.ExpectMsg<NativeAlarmTransitionUpdate>();
        a.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
    }
+    [Theory]
+    [InlineData("OpcUa")]
+    [InlineData("MxGateway")]
+    public void DC5_SecondSubscriberReceivesStandingAlarmSnapshot(string protocol)
+    {
+        var adapter = Adapter();
+        AlarmTransitionCallback? callback = null;
+        ((IAlarmSubscribableConnection)adapter).SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(),
+            Arg.Do<AlarmTransitionCallback>(c => callback = c), Arg.Any<CancellationToken>()).Returns("alarm-handle");
+        var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor("conn", adapter, new DataConnectionOptions(),
+            _health, Substitute.For<IDataConnectionFactory>(), protocol)));
+        var a = CreateTestProbe(); var b = CreateTestProbe();
+        actor.Tell(new SubscribeAlarmsRequest("a", "a", "conn", "A", null, DateTimeOffset.UtcNow), a.Ref);
+        a.ExpectMsg<SubscribeAlarmsResponse>();
+        var alarm = new NativeAlarmTransition("A.Hi", "A", "Limit", AlarmTransitionKind.Snapshot,
+            new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), "", "", "", "", "", null, DateTimeOffset.UtcNow, "", "");
+        if (protocol == "OpcUa")
+        {
+            alarm = alarm with { SnapshotSourceReference = "A", SnapshotId = Guid.NewGuid() };
+            callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotStart });
+            a.ExpectMsg<NativeAlarmTransitionUpdate>();
+        }
+        callback!(alarm);
+        callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotComplete });
+        a.ExpectMsg<NativeAlarmTransitionUpdate>(); a.ExpectMsg<NativeAlarmTransitionUpdate>();
+        if (protocol == "OpcUa")
+        {
+            // A duplicate end must not erase the DCL replay cache either.
+            callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotComplete });
+            a.ExpectMsg<NativeAlarmTransitionUpdate>();
+        }
+        actor.Tell(new SubscribeAlarmsRequest("b", "b", "conn", "A", null, DateTimeOffset.UtcNow), b.Ref);
+        var received = b.ReceiveN(4, TimeSpan.FromSeconds(2));
+        Assert.Contains(received.OfType<NativeAlarmTransitionUpdate>(), m => m.Transition.Kind == AlarmTransitionKind.Snapshot && m.Transition.SourceReference == "A.Hi");
+        Assert.Contains(received.OfType<NativeAlarmTransitionUpdate>(), m => m.Transition.Kind == AlarmTransitionKind.SnapshotComplete);
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void DC5_MidSnapshotSubscriberReceivesAlreadySeenPrefix(bool scoped)
+    {
+        var adapter = Adapter();
+        AlarmTransitionCallback? callback = null;
+        ((IAlarmSubscribableConnection)adapter).SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(),
+            Arg.Do<AlarmTransitionCallback>(c => callback = c), Arg.Any<CancellationToken>()).Returns("alarm-handle");
+        var actor = Actor(adapter);
+        var a = CreateTestProbe(); var b = CreateTestProbe();
+        actor.Tell(new SubscribeAlarmsRequest("a", "a", "conn", "A", null, DateTimeOffset.UtcNow), a.Ref);
+        a.ExpectMsg<SubscribeAlarmsResponse>();
+        var alarm = new NativeAlarmTransition("A.Hi", "A", "Limit", AlarmTransitionKind.Snapshot,
+            new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), "", "", "", "", "", null, DateTimeOffset.UtcNow, "", "",
+            SnapshotSourceReference: scoped ? "A" : null, SnapshotId: scoped ? Guid.NewGuid() : null);
+        if (scoped)
+        {
+            callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotStart });
+            a.ExpectMsg<NativeAlarmTransitionUpdate>();
+        }
+        callback!(alarm);
+        a.ExpectMsg<NativeAlarmTransitionUpdate>();
+        actor.Tell(new SubscribeAlarmsRequest("b", "b", "conn", "A", null, DateTimeOffset.UtcNow), b.Ref);
+        var prefix = b.ReceiveN(scoped ? 3 : 2, TimeSpan.FromSeconds(2));
+        Assert.Contains(prefix.OfType<NativeAlarmTransitionUpdate>(), m => m.Transition.SourceReference == "A.Hi");
+        callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotComplete });
+        var end = b.ExpectMsg<NativeAlarmTransitionUpdate>();
+        Assert.Equal(AlarmTransitionKind.SnapshotComplete, end.Transition.Kind);
+        Assert.Equal(alarm.SnapshotId, end.Transition.SnapshotId);
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void DC5_ReplayRejectsOldEpochRowsAndPreservesNewerLiveClear(bool staleEpoch)
+    {
+        var adapter = Adapter();
+        AlarmTransitionCallback? callback = null;
+        ((IAlarmSubscribableConnection)adapter).SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(),
+            Arg.Do<AlarmTransitionCallback>(c => callback = c), Arg.Any<CancellationToken>()).Returns("alarm-handle");
+        var actor = Actor(adapter);
+        var a = CreateTestProbe(); var b = CreateTestProbe();
+        actor.Tell(new SubscribeAlarmsRequest("a", "a", "conn", "A", null, DateTimeOffset.UtcNow), a.Ref);
+        a.ExpectMsg<SubscribeAlarmsResponse>();
+        var alarm = new NativeAlarmTransition("A.Hi", "A", "Limit", AlarmTransitionKind.Snapshot,
+            new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), "", "", "", "", "", null, DateTimeOffset.UtcNow, "", "",
+            SnapshotSourceReference: "A", SnapshotId: Guid.NewGuid());
+        callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotStart });
+        if (staleEpoch) callback!(alarm with { SourceReference = "A.Old", SnapshotId = Guid.NewGuid() });
+        callback!(alarm);
+        if (!staleEpoch) callback!(alarm with { Kind = AlarmTransitionKind.Clear, Condition = alarm.Condition with { Active = false, Acknowledged = true }, TransitionTime = alarm.TransitionTime.AddSeconds(1) });
+        callback!(alarm with { SourceReference = "", Kind = AlarmTransitionKind.SnapshotComplete });
+        a.ReceiveN(4, TimeSpan.FromSeconds(2));
+        actor.Tell(new SubscribeAlarmsRequest("b", "b", "conn", "A", null, DateTimeOffset.UtcNow), b.Ref);
+        var replay = b.ReceiveN(staleEpoch ? 4 : 3, TimeSpan.FromSeconds(2));
+        var rows = replay.OfType<NativeAlarmTransitionUpdate>().Where(m => m.Transition.Kind == AlarmTransitionKind.Snapshot).ToList();
+        if (staleEpoch) Assert.Equal("A.Hi", Assert.Single(rows).Transition.SourceReference);
+        else Assert.Empty(rows);
+    }
+
 }
@@ -101,7 +101,7 @@ public class MxGatewayBatchSeamTests
    }

    [Fact]
-    public async Task AlarmStream_OpensOnTheLongestCommonPrefix_AndOnlyRestartsForAnEscapingSource()
+    public async Task AlarmStream_UsesLongestCommonPrefix_AndRefreshesEveryNewSource()
    {
        var fake = new FakeMxGatewayClient();
        var adapter = NewAdapter(fake);
@@ -119,15 +119,15 @@ public class MxGatewayBatchSeamTests
        await WaitForPrefixCountAsync(fake, 2);
        Assert.Equal("Area1.Tank", fake.AlarmStreamPrefixes[1]);

-        // A source ALREADY covered by the live prefix must NOT restart the stream.
+        // A covered new source still needs its standing-state snapshot.
        await adapter.SubscribeAlarmsAsync("Area1.Tank3.Sub", null, NoTransition);
-        await Task.Delay(100);
-        Assert.Equal(2, fake.AlarmStreamPrefixes.Count);
+        await WaitForPrefixCountAsync(fake, 3);
+        Assert.Equal("Area1.Tank", fake.AlarmStreamPrefixes[2]);

        // A source outside the prefix widens it again — here down to gateway-wide.
        await adapter.SubscribeAlarmsAsync("Zone9.Pump", null, NoTransition);
-        await WaitForPrefixCountAsync(fake, 3);
-        Assert.Null(fake.AlarmStreamPrefixes[2]); // "" → gateway-wide
+        await WaitForPrefixCountAsync(fake, 4);
+        Assert.Null(fake.AlarmStreamPrefixes[3]); // "" → gateway-wide
    }

    [Fact]
@@ -25,6 +25,46 @@ public class MxGatewayDataConnectionAlarmReconnectTests
        ["ApiKey"] = "key",
    };

+
+
+    [Fact]
+    public async Task DC5_EachTransportRefreshHasItsOwnConnectionWideEpoch()
+    {
+        var clients = new List<PerInstanceFakeMxGatewayClient>();
+        var adapter = new MxGatewayDataConnection(new RecordingFactory(clients), NullLogger<MxGatewayDataConnection>.Instance);
+        var received = new System.Collections.Concurrent.ConcurrentQueue<NativeAlarmTransition>();
+        await adapter.ConnectAsync(Details());
+        await adapter.SubscribeAlarmsAsync("Area1", null, received.Enqueue);
+        await WaitUntilAsync(() => clients[0].AlarmStreamOpens.Count == 1);
+        clients[0].EmitAlarm(MakeTransition(AlarmTransitionKind.SnapshotComplete));
+        clients[0].EmitAlarm(MakeTransition(AlarmTransitionKind.SnapshotStart));
+        clients[0].EmitAlarm(MakeTransition(AlarmTransitionKind.SnapshotComplete));
+        var starts = received.Where(t => t.Kind == AlarmTransitionKind.SnapshotStart).ToArray();
+        var ends = received.Where(t => t.Kind == AlarmTransitionKind.SnapshotComplete).ToArray();
+        Assert.Equal(2, starts.Length);
+        Assert.Equal(2, ends.Length);
+        Assert.NotNull(starts[0].SnapshotId);
+        Assert.NotEqual(starts[0].SnapshotId, starts[1].SnapshotId);
+        Assert.Equal(starts[0].SnapshotId, ends[0].SnapshotId);
+        Assert.Equal(starts[1].SnapshotId, ends[1].SnapshotId);
+        Assert.All(starts, t => Assert.Null(t.SnapshotSourceReference));
+        await adapter.DisconnectAsync();
+    }
+
+    [Fact]
+    public async Task DC5_CoveredNewSourceRequestsFreshStandingSnapshot()
+    {
+        var clients = new List<PerInstanceFakeMxGatewayClient>();
+        var adapter = new MxGatewayDataConnection(new RecordingFactory(clients), NullLogger<MxGatewayDataConnection>.Instance);
+        await adapter.ConnectAsync(Details());
+        await adapter.SubscribeAlarmsAsync("Area1", null, _ => { });
+        await WaitUntilAsync(() => clients[0].AlarmStreamOpens.Count == 1);
+        await adapter.SubscribeAlarmsAsync("Area1.Tank1", null, _ => { });
+        await WaitUntilAsync(() => clients[0].AlarmStreamOpens.Count == 2);
+        Assert.True(clients[0].AlarmStreamTokens[0].IsCancellationRequested);
+        await adapter.DisconnectAsync();
+    }
+
    [Fact]
    public async Task Reconnect_ThenSubscribeAlarms_OpensNewStreamOnNewClient()
    {
@@ -78,7 +118,7 @@ public class MxGatewayDataConnectionAlarmReconnectTests

        var received = new List<NativeAlarmTransition>();
        await adapter.ConnectAsync(Details());
-        await adapter.SubscribeAlarmsAsync("Area1.Tank1", null, t => received.Add(t));
+        await adapter.SubscribeAlarmsAsync("Area1.Tank1", null, t => { if (t.Kind != AlarmTransitionKind.SnapshotStart) received.Add(t); });
        await WaitUntilAsync(() => clients[0].AlarmStreamOpens.Count == 1);

        // Prove the feed works pre-reconnect.
@@ -86,7 +126,7 @@ public class MxGatewayDataConnectionAlarmReconnectTests
        await WaitUntilAsync(() => received.Count == 1);

        await adapter.ConnectAsync(Details()); // reconnect
-        await adapter.SubscribeAlarmsAsync("Area1.Tank1", null, t => received.Add(t));
+        await adapter.SubscribeAlarmsAsync("Area1.Tank1", null, t => { if (t.Kind != AlarmTransitionKind.SnapshotStart) received.Add(t); });
        await WaitUntilAsync(() => clients[1].AlarmStreamOpens.Count == 1);

        // A transition on the OLD (disposed, orphaned) client must never reach the
@@ -0,0 +1,23 @@
+using Opc.Ua;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
+public class OpcUaSecurityDowngradeTests
+{
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void DC4_RequestedProtectionNeverFallsBackToNone(bool discoveryFails)
+    {
+        Assert.Throws<InvalidOperationException>(() => RealOpcUaClient.DiscoverEndpoint("opc.tcp://test:4840",
+            MessageSecurityMode.SignAndEncrypt, () => discoveryFails
+                ? throw new Exception("discovery unavailable")
+                : new EndpointDescriptionCollection { new("opc.tcp://test:4840") { SecurityMode = MessageSecurityMode.None } }));
+    }
+    [Fact]
+    public void DC4_ExactRequestedModeIsSelected()
+    {
+        var encrypted = new EndpointDescription("opc.tcp://test:4840") { SecurityMode = MessageSecurityMode.SignAndEncrypt };
+        Assert.Same(encrypted, RealOpcUaClient.DiscoverEndpoint("opc.tcp://test:4840", MessageSecurityMode.SignAndEncrypt,
+            () => new EndpointDescriptionCollection { new("opc.tcp://test:4840"), encrypted }));
+    }
+}