Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -0,0 +1,35 @@
+using Microsoft.Extensions.DependencyInjection;
+using ZB.MOM.WW.ScadaBridge.CentralUI.ScriptAnalysis;
+namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.ScriptAnalysis;
+[Collection("Sandbox worker processes")]
+public class BoundedSandboxOutputTests
+{
+    [Theory]
+    [InlineData("block")]
+    [InlineData("new { Data = block }")]
+    public async Task Worker_OversizedConsoleAndReturn_AreExplicitlyBounded(string returnExpression)
+    {
+        using var services = new Microsoft.Extensions.DependencyInjection.ServiceCollection().BuildServiceProvider();
+        using var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
+        var runner = new ScriptAnalysisService(NSubstitute.Substitute.For<ISharedScriptCatalog>(), cache, services);
+        var result = await runner.RunInSandboxAsync(new SandboxRunRequest(
+            "var block = new string('x', 100000); for (int i=0; i<1000; i++) System.Console.Write(block); return " + returnExpression + ";", null, 10), CancellationToken.None);
+        Assert.True(result.Success, result.Error);
+        Assert.True(result.ConsoleOutput.Length < 32100);
+        Assert.Contains("truncated", result.ConsoleOutput);
+        Assert.Contains("truncated", result.ReturnValueJson!);
+        Assert.True(System.Text.Encoding.UTF8.GetByteCount(result.ReturnValueJson!) < 32000);
+    }
+
+    [Fact]
+    public void ConcurrentRepeatedLargeWrites_RetainOnlyBudget()
+    {
+        var writer = new BoundedSandboxWriter(32000);
+        var block = new string('x', 100000);
+        Parallel.For(0, 1000, _ => writer.WriteLine(block));
+        Assert.Equal(32000, writer.RetainedCharacters);
+        Assert.True(writer.DroppedCharacters >= 99968000);
+        Assert.True(writer.ToString().Length < 32100);
+        Assert.Contains("truncated", writer.ToString());
+    }
+}
@@ -0,0 +1,86 @@
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.DependencyInjection;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.CentralUI.ScriptAnalysis;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Types;
+
+namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.ScriptAnalysis;
+
+[Collection("Sandbox worker processes")]
+public class SandboxWorkerIsolationTests
+{
+    [Fact]
+    public async Task FourBusyWorkers_RejectQueuedFifthWithoutStartingAnotherProcess()
+    {
+        var allEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var release = new TaskCompletionSource<ExternalCallResult>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var count = 0;
+        var external = Substitute.For<IExternalSystemClient>();
+        external.CallAsync("probe", "blocked", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
+            .Returns(_ => { if (Interlocked.Increment(ref count) == 4) allEntered.TrySetResult(); return release.Task; });
+        using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
+        var catalog = Substitute.For<ISharedScriptCatalog>();
+        var workers = Enumerable.Range(0, 4).Select(_ => SandboxWorkerProtocol.RunAsync(new SandboxRunRequest(
+            "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services, catalog, CancellationToken.None)).ToArray();
+        try
+        {
+            await allEntered.Task.WaitAsync(TimeSpan.FromSeconds(8));
+            var fifthStarted = false;
+            var fifth = await SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest("return 1;", null, 1),
+                services, catalog, CancellationToken.None, _ => fifthStarted = true);
+            Assert.Equal(SandboxErrorKind.Timeout, fifth.ErrorKind);
+            Assert.False(fifthStarted);
+        }
+        finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); }
+        foreach (var result in await Task.WhenAll(workers)) Assert.True(result.Success, result.Error);
+    }
+
+    [Fact]
+    public async Task UncooperativeBroker_StillKillsChildAtDeadline_AndKeepsLifetimeUntilBrokerCompletes()
+    {
+        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var release = new TaskCompletionSource<ExternalCallResult>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var external = Substitute.For<IExternalSystemClient>();
+        external.CallAsync("probe", "blocked", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
+            .Returns(_ => { entered.TrySetResult(); return release.Task; });
+        using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
+        System.Diagnostics.Process? child = null;
+        var run = SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest(
+            "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services,
+            Substitute.For<ISharedScriptCatalog>(), CancellationToken.None,
+            id => child = System.Diagnostics.Process.GetProcessById(id));
+        try
+        {
+            await entered.Task.WaitAsync(TimeSpan.FromSeconds(10));
+            Assert.NotNull(child);
+            await child.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(12));
+            Assert.False(run.IsCompleted); // Do not dispose the broker's request scope or recycle its charged slot.
+        }
+        finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); }
+        var result = await run.WaitAsync(TimeSpan.FromSeconds(5));
+        Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind);
+        child?.Dispose();
+    }
+
+    [Fact]
+    public async Task NoncooperativeWorker_IsTerminated_AndCapacityCanBeReused()
+    {
+        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var external = Substitute.For<IExternalSystemClient>();
+        external.CallAsync("probe", "entered", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
+            .Returns(_ => { entered.TrySetResult(); return new ExternalCallResult(true, "null", null); });
+        var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
+        using var cache = new MemoryCache(new MemoryCacheOptions());
+        var runner = new ScriptAnalysisService(Substitute.For<ISharedScriptCatalog>(), cache, services);
+        var run = runner.RunInSandboxAsync(new SandboxRunRequest(
+            "await ExternalSystem.Call(\"probe\", \"entered\"); while (true) { }", null, 10), CancellationToken.None);
+        await entered.Task.WaitAsync(TimeSpan.FromSeconds(10));
+        var result = await run.WaitAsync(TimeSpan.FromSeconds(15));
+        Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind);
+        var next = await runner.RunInSandboxAsync(new SandboxRunRequest("return 42;", null, 10), CancellationToken.None);
+        Assert.True(next.Success, next.Error);
+        Assert.Equal("42", next.ReturnValueJson);
+        Assert.Single(external.ReceivedCalls());
+    }
+}
@@ -4,6 +4,7 @@ using ZB.MOM.WW.ScadaBridge.CentralUI.ScriptAnalysis;

 namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.ScriptAnalysis;

+[Collection("Sandbox worker processes")]
 public class ScriptAnalysisServiceTests
 {
    private readonly ISharedScriptCatalog _catalog = Substitute.For<ISharedScriptCatalog>();
@@ -651,9 +652,9 @@ public class ScriptAnalysisServiceTests

        var longTask = RunLong();
        var shortTasks = new List<Task<string>>();
-        for (var round = 0; round < 12; round++)
+        for (var round = 0; round < 1; round++)
        {
-            for (var k = 0; k < 4; k++)
+            for (var k = 0; k < 3; k++)
                shortTasks.Add(RunShort(round * 4 + k));
            await Task.Yield();
        }