Resolve all 33 medium-severity architecture review findings
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.
Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.
Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.
Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.
Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.
Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.
Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
This commit is contained in:
Joseph Doherty
@@ -1,3 +1,4 @@
|
||||
using Moq;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
@@ -86,4 +87,30 @@ public class AlarmScriptLogTargetTests : TestKit, IDisposable
|
||||
},
|
||||
TimeSpan.FromSeconds(15));
|
||||
}
|
||||
[Fact]
|
||||
public void SR2_AlarmRunResolvesAndDisposesScopedIntegrationServices()
|
||||
{
|
||||
var store = new Moq.Mock<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.IOperationTrackingStore>();
|
||||
store.Setup(s => s.GetStatusAsync(Moq.It.IsAny<ZB.MOM.WW.ScadaBridge.Commons.Types.TrackedOperationId>(), Moq.It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((ZB.MOM.WW.ScadaBridge.Commons.Types.TrackingStatusSnapshot?)null);
|
||||
store.As<IDisposable>().Setup(s => s.Dispose());
|
||||
using var provider = Microsoft.Extensions.DependencyInjection.ServiceCollectionContainerBuilderExtensions.BuildServiceProvider(
|
||||
Microsoft.Extensions.DependencyInjection.ServiceCollectionServiceExtensions.AddScoped<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.IOperationTrackingStore>(
|
||||
new Microsoft.Extensions.DependencyInjection.ServiceCollection(), _ => store.Object));
|
||||
var options = ScriptOptions.Default.WithReferences(typeof(object).Assembly, typeof(ScriptGlobals).Assembly,
|
||||
typeof(ZB.MOM.WW.ScadaBridge.Commons.Types.TrackedOperationId).Assembly)
|
||||
.WithImports("ZB.MOM.WW.ScadaBridge.Commons.Types");
|
||||
var compiled = CSharpScript.Create<object?>("await Instance.Tracking.Status(TrackedOperationId.New()); return null;", options, typeof(ScriptGlobals));
|
||||
Assert.DoesNotContain(compiled.Compile(), d => d.Severity == Microsoft.CodeAnalysis.DiagnosticSeverity.Error);
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var done = CreateTestProbe();
|
||||
var logger = new RecordingLogger<AlarmActor>();
|
||||
ScriptRunLauncher.LaunchAlarmScript("alarm", "instance", ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmLevel.High,
|
||||
1, "alarm", compiled, CreateTestProbe().Ref, _sharedLibrary, new SiteRuntimeOptions(), done.Ref,
|
||||
logger, null, provider, 1, 10, null, scheduler);
|
||||
Assert.True(done.ExpectMsg<AlarmActor.AlarmExecutionCompleted>(TimeSpan.FromSeconds(10)).Success, string.Join(";", logger.Entries.Select(e => e.Message)));
|
||||
store.Verify(s => s.GetStatusAsync(Moq.It.IsAny<ZB.MOM.WW.ScadaBridge.Commons.Types.TrackedOperationId>(), Moq.It.IsAny<CancellationToken>()), Moq.Times.Once);
|
||||
AwaitAssert(() => store.As<IDisposable>().Verify(s => s.Dispose(), Moq.Times.Once));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -604,4 +604,70 @@ public class NativeAlarmActorTests : TestKit, IDisposable
|
||||
// Swallow the response entirely — the actor must still re-subscribe.
|
||||
dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task SR12_LateRehydrationCannotResurrectResolvedOrSnapshotRemovedAlarm(bool snapshot)
|
||||
{
|
||||
var instance = CreateTestProbe();
|
||||
var dcl = CreateTestProbe();
|
||||
var read = new TaskCompletionSource<IReadOnlyList<NativeAlarmRow>>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var actor = ActorOf(Props.Create(() => new NativeAlarmActor(Source(), "inst", instance.Ref, dcl.Ref, _storage, _options,
|
||||
NullLogger.Instance, AlarmKind.NativeOpcUa, null, null, read.Task)));
|
||||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||||
await _storage.UpsertNativeAlarmAsync("saved", "Pressure", "T01.Hi", JsonSerializer.Serialize(condition), DateTimeOffset.UtcNow);
|
||||
var rows = await _storage.GetNativeAlarmsAsync("saved", "Pressure");
|
||||
if (snapshot)
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotComplete, condition)));
|
||||
else
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("T01.Hi", AlarmTransitionKind.Clear, condition with { Active = false, Acknowledged = true })));
|
||||
read.SetResult(rows);
|
||||
var messages = instance.ReceiveWhile<object>(o => o, TimeSpan.FromMilliseconds(300));
|
||||
Assert.DoesNotContain(messages.OfType<AlarmStateChanged>(), e => e.State == AlarmState.Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SR3_StaleSuccessCannotCancelCurrentSubscribeRetry()
|
||||
{
|
||||
var dcl = CreateTestProbe();
|
||||
var actor = Spawn(CreateTestProbe().Ref, dcl.Ref, new SiteRuntimeOptions { NativeAlarmRetryIntervalMs = 100 });
|
||||
var first = dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||||
var second = dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(2));
|
||||
Assert.NotEqual(first.CorrelationId, second.CorrelationId);
|
||||
actor.Tell(new SubscribeAlarmsResponse(first.CorrelationId, "inst", true, null, DateTimeOffset.UtcNow));
|
||||
var third = dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(3));
|
||||
actor.Tell(new SubscribeAlarmsResponse(third.CorrelationId, "inst", true, null, DateTimeOffset.UtcNow));
|
||||
dcl.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DC5_ConnectionWideRestartDiscardsInterruptedMxSnapshot()
|
||||
{
|
||||
var parent = CreateTestProbe(); var dcl = CreateTestProbe();
|
||||
var actor = Spawn(parent.Ref, dcl.Ref);
|
||||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("Old.Hi", AlarmTransitionKind.Snapshot, condition)));
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotStart, condition)));
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotComplete, condition)));
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DC5_LiveClearDuringSnapshotCannotBeResurrectedBySnapshotEnd()
|
||||
{
|
||||
var parent = CreateTestProbe(); var dcl = CreateTestProbe();
|
||||
var actor = Spawn(parent.Ref, dcl.Ref);
|
||||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||||
var snapshot = Transition("A.Hi", AlarmTransitionKind.Snapshot, condition) with { SnapshotId = Guid.NewGuid(), SnapshotSourceReference = Source().SourceReference };
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.SnapshotStart }));
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot));
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.Clear, Condition = condition with { Active = false, Acknowledged = true }, TransitionTime = snapshot.TransitionTime.AddSeconds(1) }));
|
||||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.SnapshotComplete }));
|
||||
var messages = parent.ReceiveWhile<object>(o => o, TimeSpan.FromMilliseconds(300));
|
||||
Assert.DoesNotContain(messages.OfType<AlarmStateChanged>(), m => m.State == AlarmState.Active);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+109
-20
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
@@ -34,8 +35,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
///
|
||||
/// <para>
|
||||
/// The fix dispatches the reset before the successor exists and hands it to the actor as a
|
||||
/// gate, and makes the predecessor's final flush complete before its Terminated signal — which
|
||||
/// the Deployment Manager already waits for.
|
||||
/// gate, and waits for both predecessor termination and its separate final-flush acknowledgement.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class RedeployStaticOverrideResetTests : TestKit, IDisposable
|
||||
@@ -220,22 +220,11 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
|
||||
Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
|
||||
}
|
||||
|
||||
// ── Leg 2: the predecessor's final flush must precede its Terminated signal ──
|
||||
|
||||
/// <summary>
|
||||
/// The Deployment Manager gates a redeploy's override clear on the predecessor's
|
||||
/// <c>Terminated</c> signal, so PostStop's final flush is only safe if it has COMMITTED by
|
||||
/// the time that signal is observed. This pins exactly that ordering: a dirty override that
|
||||
/// no timer has flushed yet must be on disk the instant Terminated arrives.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Fails before the fix</b> (racily — the write is a small local SQLite commit): the
|
||||
/// flush was fire-and-forget, so Terminated could be observed with nothing written, and in
|
||||
/// a redeploy that write would then land after the clear.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
// Leg 2: final persistence is acknowledged independently of termination.
|
||||
// The manager waits for both before resetting state; a standalone actor still
|
||||
// persists its final dirty batch asynchronously after stopping.
|
||||
[Fact]
|
||||
public async Task InstanceActorStop_CommitsTheFinalOverrideFlush_BeforeTerminated()
|
||||
public async Task InstanceActorStop_CompletesFinalOverrideFlushAsynchronously()
|
||||
{
|
||||
const string instanceName = "FlushOrderPump";
|
||||
|
||||
@@ -265,9 +254,8 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
|
||||
|
||||
// No delay, no polling: the commit must already have happened.
|
||||
var overrides = await _storage.GetStaticOverridesAsync(instanceName);
|
||||
Assert.Equal("77.7", overrides["Temperature"]);
|
||||
AwaitAssert(() => Assert.Equal("77.7",
|
||||
_storage.GetStaticOverridesAsync(instanceName).GetAwaiter().GetResult()["Temperature"]));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -330,4 +318,105 @@ public class RedeployStaticOverrideResetTests : TestKit, IDisposable
|
||||
"corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
|
||||
Assert.Equal("123.4", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
|
||||
}
|
||||
[Fact]
|
||||
public void SR5_StopDoesNotBlockDispatcherWhileSqliteWriterIsBusy()
|
||||
{
|
||||
const string name = "BusyFlushPump";
|
||||
var actor = ActorOf(Props.Create(() => new InstanceActor(name, MakeConfigJson(name), _storage,
|
||||
_compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger<InstanceActor>.Instance,
|
||||
null, null, null, TimeSpan.FromMinutes(5))));
|
||||
actor.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
|
||||
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
|
||||
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
|
||||
blocker.Open();
|
||||
using var transaction = blocker.BeginTransaction();
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
// The outstanding persistence is acknowledged separately; actor termination
|
||||
// must not wait for a database lock on the shared dispatcher.
|
||||
ExpectTerminated(actor, TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SR5_RedeployWaitsForFlushAcknowledgementAfterTermination()
|
||||
{
|
||||
const string name = "AckOrderPump";
|
||||
var manager = CreateDeploymentManager();
|
||||
Deploy(manager, name, "first");
|
||||
var old = await ResolveInstanceChildAsync(manager, name);
|
||||
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
|
||||
blocker.Open();
|
||||
using (var transaction = blocker.BeginTransaction())
|
||||
{
|
||||
old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
|
||||
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
|
||||
Watch(old);
|
||||
manager.Tell(new DeployInstanceCommand("second", name, "sha256:second", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
|
||||
ExpectTerminated(old, TimeSpan.FromSeconds(2));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
transaction.Commit();
|
||||
}
|
||||
Assert.Equal(DeploymentStatus.Success, ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10)).Status);
|
||||
Assert.Empty(await _storage.GetStaticOverridesAsync(name));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task SR5_CancelQueuedRedeployStopsItsPredecessor(bool delete)
|
||||
{
|
||||
var manager = CreateDeploymentManager();
|
||||
var actors = new List<IActorRef>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
Deploy(manager, "Queued" + i, "initial" + i);
|
||||
actors.Add(await ResolveInstanceChildAsync(manager, "Queued" + i));
|
||||
}
|
||||
var pending = CreateTestProbe();
|
||||
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
|
||||
blocker.Open();
|
||||
using var transaction = blocker.BeginTransaction();
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
var name = "Queued" + i;
|
||||
actors[i].Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77", DateTimeOffset.UtcNow));
|
||||
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
|
||||
manager.Tell(new DeployInstanceCommand("next" + i, name, "sha256:next", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow), pending.Ref);
|
||||
}
|
||||
Watch(actors[4]);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(100)); // fifth predecessor waits for one of four flush slots
|
||||
var cancellation = CreateTestProbe();
|
||||
if (delete) manager.Tell(new DeleteInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
|
||||
else manager.Tell(new DisableInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
|
||||
ExpectTerminated(actors[4], TimeSpan.FromSeconds(2));
|
||||
transaction.Commit();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SR5_DeleteWaitsForFinalFlushAndRejectsAnOverlappingStart()
|
||||
{
|
||||
const string name = "DeleteFlushPump";
|
||||
var manager = CreateDeploymentManager();
|
||||
Deploy(manager, name, "first");
|
||||
var old = await ResolveInstanceChildAsync(manager, name);
|
||||
var deletion = CreateTestProbe();
|
||||
using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
|
||||
blocker.Open();
|
||||
using (var transaction = blocker.BeginTransaction())
|
||||
{
|
||||
old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
|
||||
Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
|
||||
Watch(old);
|
||||
manager.Tell(new DeleteInstanceCommand("delete", name, DateTimeOffset.UtcNow), deletion.Ref);
|
||||
ExpectTerminated(old, TimeSpan.FromSeconds(2));
|
||||
deletion.ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
||||
manager.Tell(new DeployInstanceCommand("overlap", name, "sha256:overlap", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
|
||||
Assert.Equal(DeploymentStatus.Failed, ExpectMsg<DeploymentStatusResponse>().Status);
|
||||
transaction.Commit();
|
||||
}
|
||||
Assert.True(deletion.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(10)).Success);
|
||||
Assert.Empty(await _storage.GetStaticOverridesAsync(name));
|
||||
Deploy(manager, name, "after-delete");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -52,4 +52,16 @@ public class AuditingDbTransactionTests
|
||||
using var command = connection.CreateCommand();
|
||||
Assert.Same(connection, command.Connection);
|
||||
}
|
||||
[Fact]
|
||||
public void SR4_ConnectionCannotBeRetargetedOrRevealCredentials()
|
||||
{
|
||||
using var inner = new SqliteConnection("Data Source=:memory:;Password=secret-value");
|
||||
using var connection = new AuditingDbConnection(inner, Mock.Of<IAuditWriter>(), "MES-Prod", "site", "instance",
|
||||
"script", NullLogger.Instance, Guid.NewGuid());
|
||||
Assert.DoesNotContain("secret-value", connection.ConnectionString);
|
||||
Assert.Throws<NotSupportedException>(() => connection.ConnectionString = "Data Source=other.db");
|
||||
Assert.Throws<NotSupportedException>(() => connection.ChangeDatabase("other"));
|
||||
Assert.Equal("Data Source=:memory:;Password=secret-value", inner.ConnectionString);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -984,4 +984,24 @@ public class OperationTrackingStoreTests : IDisposable
|
||||
cmd.ExecuteNonQuery();
|
||||
return boundary;
|
||||
}
|
||||
[Fact]
|
||||
public async Task ORCH1_SyncDisposeDoesNotCloseConnectionWhileWriterGateIsHeld()
|
||||
{
|
||||
var (store, _) = CreateStore(nameof(ORCH1_SyncDisposeDoesNotCloseConnectionWhileWriterGateIsHeld));
|
||||
var flags = System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic;
|
||||
var gate = (SemaphoreSlim)typeof(OperationTrackingStore).GetField("_writeGate", flags)!.GetValue(store)!;
|
||||
var connection = (SqliteConnection)typeof(OperationTrackingStore).GetField("_writeConnection", flags)!.GetValue(store)!;
|
||||
await gate.WaitAsync();
|
||||
var disposal = Task.Run(store.Dispose);
|
||||
try
|
||||
{
|
||||
await Task.Delay(100);
|
||||
Assert.Equal(System.Data.ConnectionState.Open, connection.State);
|
||||
Assert.False(disposal.IsCompleted);
|
||||
}
|
||||
finally { gate.Release(); }
|
||||
await disposal.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(System.Data.ConnectionState.Closed, connection.State);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user