Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.

Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.

Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.

Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.

Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.

Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.

Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
This commit is contained in:
Joseph Doherty
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -1,3 +1,4 @@
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
@@ -34,8 +35,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// <para>
/// The fix dispatches the reset before the successor exists and hands it to the actor as a
/// gate, and makes the predecessor's final flush complete before its Terminated signal — which
/// the Deployment Manager already waits for.
/// gate, and waits for both predecessor termination and its separate final-flush acknowledgement.
/// </para>
/// </summary>
public class RedeployStaticOverrideResetTests : TestKit, IDisposable
@@ -220,22 +220,11 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
}
// ── Leg 2: the predecessor's final flush must precede its Terminated signal ──
/// <summary>
/// The Deployment Manager gates a redeploy's override clear on the predecessor's
/// <c>Terminated</c> signal, so PostStop's final flush is only safe if it has COMMITTED by
/// the time that signal is observed. This pins exactly that ordering: a dirty override that
/// no timer has flushed yet must be on disk the instant Terminated arrives.
///
/// <para>
/// <b>Fails before the fix</b> (racily — the write is a small local SQLite commit): the
/// flush was fire-and-forget, so Terminated could be observed with nothing written, and in
/// a redeploy that write would then land after the clear.
/// </para>
/// </summary>
// Leg 2: final persistence is acknowledged independently of termination.
// The manager waits for both before resetting state; a standalone actor still
// persists its final dirty batch asynchronously after stopping.
[Fact]
public async Task InstanceActorStop_CommitsTheFinalOverrideFlush_BeforeTerminated()
public async Task InstanceActorStop_CompletesFinalOverrideFlushAsynchronously()
{
const string instanceName = "FlushOrderPump";
@@ -265,9 +254,8 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
// No delay, no polling: the commit must already have happened.
var overrides = await _storage.GetStaticOverridesAsync(instanceName);
Assert.Equal("77.7", overrides["Temperature"]);
AwaitAssert(() => Assert.Equal("77.7",
_storage.GetStaticOverridesAsync(instanceName).GetAwaiter().GetResult()["Temperature"]));
}
/// <summary>
@@ -330,4 +318,105 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
"corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
Assert.Equal("123.4", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
}
[Fact]
public void SR5_StopDoesNotBlockDispatcherWhileSqliteWriterIsBusy()
{
const string name = "BusyFlushPump";
var actor = ActorOf(Props.Create(() => new InstanceActor(name, MakeConfigJson(name), _storage,
_compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger<InstanceActor>.Instance,
null, null, null, TimeSpan.FromMinutes(5))));
actor.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
blocker.Open();
using var transaction = blocker.BeginTransaction();
Watch(actor);
Sys.Stop(actor);
// The outstanding persistence is acknowledged separately; actor termination
// must not wait for a database lock on the shared dispatcher.
ExpectTerminated(actor, TimeSpan.FromMilliseconds(500));
}
[Fact]
public async Task SR5_RedeployWaitsForFlushAcknowledgementAfterTermination()
{
const string name = "AckOrderPump";
var manager = CreateDeploymentManager();
Deploy(manager, name, "first");
var old = await ResolveInstanceChildAsync(manager, name);
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
blocker.Open();
using (var transaction = blocker.BeginTransaction())
{
old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
Watch(old);
manager.Tell(new DeployInstanceCommand("second", name, "sha256:second", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
ExpectTerminated(old, TimeSpan.FromSeconds(2));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
transaction.Commit();
}
Assert.Equal(DeploymentStatus.Success, ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10)).Status);
Assert.Empty(await _storage.GetStaticOverridesAsync(name));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SR5_CancelQueuedRedeployStopsItsPredecessor(bool delete)
{
var manager = CreateDeploymentManager();
var actors = new List<IActorRef>();
for (var i = 0; i < 5; i++)
{
Deploy(manager, "Queued" + i, "initial" + i);
actors.Add(await ResolveInstanceChildAsync(manager, "Queued" + i));
}
var pending = CreateTestProbe();
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
blocker.Open();
using var transaction = blocker.BeginTransaction();
for (var i = 0; i < 5; i++)
{
var name = "Queued" + i;
actors[i].Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
manager.Tell(new DeployInstanceCommand("next" + i, name, "sha256:next", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow), pending.Ref);
}
Watch(actors[4]);
ExpectNoMsg(TimeSpan.FromMilliseconds(100)); // fifth predecessor waits for one of four flush slots
var cancellation = CreateTestProbe();
if (delete) manager.Tell(new DeleteInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
else manager.Tell(new DisableInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
ExpectTerminated(actors[4], TimeSpan.FromSeconds(2));
transaction.Commit();
}
[Fact]
public async Task SR5_DeleteWaitsForFinalFlushAndRejectsAnOverlappingStart()
{
const string name = "DeleteFlushPump";
var manager = CreateDeploymentManager();
Deploy(manager, name, "first");
var old = await ResolveInstanceChildAsync(manager, name);
var deletion = CreateTestProbe();
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
blocker.Open();
using (var transaction = blocker.BeginTransaction())
{
old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
Watch(old);
manager.Tell(new DeleteInstanceCommand("delete", name, DateTimeOffset.UtcNow), deletion.Ref);
ExpectTerminated(old, TimeSpan.FromSeconds(2));
deletion.ExpectNoMsg(TimeSpan.FromMilliseconds(100));
manager.Tell(new DeployInstanceCommand("overlap", name, "sha256:overlap", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
Assert.Equal(DeploymentStatus.Failed, ExpectMsg<DeploymentStatusResponse>().Status);
transaction.Commit();
}
Assert.True(deletion.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(10)).Success);
Assert.Empty(await _storage.GetStaticOverridesAsync(name));
Deploy(manager, name, "after-delete");
}
}