dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve all 33 medium-severity architecture review findings

Browse Source 
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.

Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.

Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.

Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.

Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.

Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.

Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
This commit is contained in:
Joseph Doherty
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+125 -69
View File
@@ -129,9 +129,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// <summary>
/// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
/// the terminating actor ref. The buffered command is applied once <see cref="Terminated"/>
/// confirms the child has fully stopped.
/// confirms the child has fully stopped and its final persistence acknowledgement arrives.
/// </summary>
private readonly Dictionary<IActorRef, PendingRedeploy> _pendingRedeploys = new();
private readonly Dictionary<IActorRef, InstanceActor.StaticOverridesFlushed> _redeployFlushes = new();
private readonly HashSet<IActorRef> _redeployTerminated = new();
private readonly HashSet<IActorRef> _redeployStopsInFlight = new();
private readonly Queue<IActorRef> _redeployStopQueue = new();
private sealed record LifecycleStop(string Name, string CommandId, bool Delete, IActorRef ReplyTo);
private sealed record LifecycleStopPersisted(IActorRef Actor, Exception? Error);
private readonly Dictionary<IActorRef, LifecycleStop> _lifecycleStops = new();
private readonly HashSet<string> _lifecycleStopNames = new();
/// <summary>
/// Name → terminating actor ref shadow of <see cref="_pendingRedeploys"/>.
@@ -487,6 +497,31 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// Terminated signal — drains a buffered redeployment once the previous
// Instance Actor has fully stopped.
Receive<Terminated>(HandleTerminated);
Receive<LifecycleStopPersisted>(message =>
{
if (!_lifecycleStops.Remove(message.Actor, out var stop)) return;
_lifecycleStopNames.Remove(stop.Name);
_terminatingActorsByName.Remove(stop.Name);
BumpStoreWriteGeneration("lifecycle stop committed");
if (stop.Delete && message.Error is null)
{
_deployedInstanceNames.Remove(stop.Name);
_runningRevisions.Remove(stop.Name);
_sweepAttemptedRevisions.Remove(stop.Name);
}
UpdateInstanceCounts();
var action = stop.Delete ? "deleted" : "disabled";
LogDeploymentEvent(message.Error is null ? "Info" : "Error", stop.Name,
$"Instance {stop.Name} {action}", message.Error?.GetBaseException().Message);
stop.ReplyTo.Tell(new InstanceLifecycleResponse(stop.CommandId, stop.Name,
message.Error is null, message.Error?.GetBaseException().Message, DateTimeOffset.UtcNow));
});
Receive<InstanceActor.StaticOverridesFlushed>(message =>
{
if (!_pendingRedeploys.ContainsKey(message.Actor) && !_redeployStopsInFlight.Contains(message.Actor) && !_lifecycleStops.ContainsKey(message.Actor)) return;
_redeployFlushes[message.Actor] = message;
TryFinishRedeployStop(message.Actor);
});
}
/// <inheritdoc />
@@ -835,6 +870,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
private void HandleDeploy(DeployInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
if (_lifecycleStopNames.Contains(instanceName))
{
replyTo.Tell(new DeploymentStatusResponse(command.DeploymentId, instanceName, DeploymentStatus.Failed,
"Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
return;
}
// WP3.1 warm-then-gate. The S3 gate below must stay synchronous on the actor thread
// (mailbox FIFO is what makes redeploy-supersede and delete-during-redeploy correct),
@@ -1029,7 +1071,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_pendingRedeploys[existing] = new PendingRedeploy(command, replyTo);
_terminatingActorsByName[instanceName] = existing;
Context.Watch(existing);
Context.Stop(existing);
_redeployStopQueue.Enqueue(existing);
StartQueuedRedeployStops();
UpdateInstanceCounts();
return;
}
@@ -1061,6 +1104,74 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
ApplyDeployment(command, replyTo, isRedeploy: false);
}
private bool DeferLifecycleStop(string name, string commandId, bool delete, IActorRef replyTo)
{
if (_lifecycleStopNames.Contains(name))
{
replyTo.Tell(new InstanceLifecycleResponse(commandId, name, false,
"Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
return true;
}
if (!_terminatingActorsByName.TryGetValue(name, out var actor) && !_instanceActors.TryGetValue(name, out actor))
return false;
if (_pendingRedeploys.Remove(actor, out var pending))
pending.OriginalSender.Tell(new DeploymentStatusResponse(pending.Command.DeploymentId, name,
DeploymentStatus.Failed, "Superseded by instance lifecycle command.", DateTimeOffset.UtcNow));
_instanceActors.Remove(name);
_terminatingActorsByName[name] = actor;
_lifecycleStopNames.Add(name);
_lifecycleStops[actor] = new LifecycleStop(name, commandId, delete, replyTo);
BumpStoreWriteGeneration("lifecycle stop dispatch");
Context.Watch(actor);
Context.Stop(actor);
UpdateInstanceCounts();
// Termination and flush may both already have arrived for a canceled
// redeploy. Reconcile immediately as well as on either completion signal.
TryFinishRedeployStop(actor);
return true;
}
private void StartQueuedRedeployStops()
{
while (_redeployStopsInFlight.Count < 4 && _redeployStopQueue.TryDequeue(out var actor))
{
if (!_pendingRedeploys.ContainsKey(actor)) continue;
_redeployStopsInFlight.Add(actor);
Context.Stop(actor);
}
}
private void TryFinishRedeployStop(IActorRef actor)
{
if (!_redeployTerminated.Contains(actor) || !_redeployFlushes.Remove(actor, out var flush)) return;
_redeployTerminated.Remove(actor);
_redeployStopsInFlight.Remove(actor);
if (_lifecycleStops.TryGetValue(actor, out var lifecycle))
{
var persistence = Task.Run(async () =>
{
if (flush.Error is not null) throw new InvalidOperationException("Final static override persistence failed.", flush.Error);
if (lifecycle.Delete) await _storage.RemoveDeployedConfigAsync(lifecycle.Name).ConfigureAwait(false);
else await _storage.SetInstanceEnabledAsync(lifecycle.Name, false).ConfigureAwait(false);
});
persistence.PipeTo(Self, Self, () => new LifecycleStopPersisted(actor, null),
error => new LifecycleStopPersisted(actor, error));
StartQueuedRedeployStops();
return;
}
if (_pendingRedeploys.Remove(actor, out var pending))
{
_terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
if (flush.Error is null)
ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
else
pending.OriginalSender.Tell(new DeploymentStatusResponse(pending.Command.DeploymentId,
pending.Command.InstanceUniqueName, DeploymentStatus.Failed,
"Predecessor static override flush failed; redeployment was not applied.", DateTimeOffset.UtcNow));
}
StartQueuedRedeployStops();
}
/// <summary>
/// Handles an Instance Actor's <see cref="Terminated"/> signal. Two cases:
/// <list type="number">
@@ -1075,16 +1186,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
// Case 1 — redeployment drain: the predecessor of a buffered redeploy has
// fully stopped and freed its actor name.
if (_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
if (_pendingRedeploys.ContainsKey(terminated.ActorRef) || _redeployStopsInFlight.Contains(terminated.ActorRef) || _lifecycleStops.ContainsKey(terminated.ActorRef))
{
// Drop the name-keyed shadow now that the predecessor has
// fully terminated and its actor name is free. Any deploy arriving after
// this point sees neither _instanceActors[name] (cleared when we stopped
// the predecessor) nor _terminatingActorsByName[name] (cleared here), so
// ApplyDeployment + Context.ActorOf below safely reuses the name.
_terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
_redeployTerminated.Add(terminated.ActorRef);
TryFinishRedeployStop(terminated.ActorRef);
return;
}
@@ -1437,33 +1542,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// WP3.1 warm-then-gate: a deploy for this instance is mid-compile-warm; queue this
// command so same-instance ordering is preserved (see TryBufferDuringDeployWarm).
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
// A disable arriving mid-redeploy must cancel the buffered
// redeploy. Otherwise HandleTerminated re-creates the Instance Actor and
// re-stores its config with isEnabled: true when the predecessor terminates,
// silently reverting the operator's disable back to enabled. Mirror the
// last-write-wins handling in HandleDeploy/HandleDelete: drop the pending
// command (so HandleTerminated returns early), clear the shadow, and tell the
// displaced deployer it was superseded. The disable itself still persists
// is_enabled = false below, which becomes the durable state.
if (_terminatingActorsByName.TryGetValue(instanceName, out var terminatingRef))
{
if (_pendingRedeploys.Remove(terminatingRef, out var pending))
{
pending.OriginalSender.Tell(new DeploymentStatusResponse(
pending.Command.DeploymentId,
instanceName,
DeploymentStatus.Failed,
$"superseded by disable of {instanceName} before redeploy finished terminating",
DateTimeOffset.UtcNow));
}
_terminatingActorsByName.Remove(instanceName);
}
else if (_instanceActors.TryGetValue(instanceName, out var actor))
{
Context.Stop(actor);
_instanceActors.Remove(instanceName);
}
if (DeferLifecycleStop(instanceName, command.CommandId, false, replyTo)) return;
UpdateInstanceCounts();
@@ -1512,6 +1591,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
private void HandleEnable(EnableInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
if (_lifecycleStopNames.Contains(instanceName))
{
replyTo.Tell(new InstanceLifecycleResponse(command.CommandId, instanceName, false,
"Instance lifecycle persistence is in progress; retry after it completes.", DateTimeOffset.UtcNow));
return;
}
// WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
@@ -1997,37 +2083,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
// A delete arriving while a redeploy is still terminating must
// be authoritative over the mid-redeploy bookkeeping. HandleDeploy already
// removed the instance from _instanceActors and buffered a PendingRedeploy
// keyed by the terminating ref. If we fall straight through to the
// _instanceActors miss + unconditional decrement, the buffered redeploy is
// left intact — so when Terminated fires, HandleTerminated calls
// ApplyDeployment(isRedeploy: true) and RESURRECTS the just-deleted instance,
// with the counter now inconsistent. Cancel the pending redeploy first.
if (_terminatingActorsByName.TryGetValue(instanceName, out var terminatingRef))
{
// Drop the buffered command so HandleTerminated's _pendingRedeploys.Remove
// misses and it returns early (no resurrection). Clear the shadow too.
if (_pendingRedeploys.Remove(terminatingRef, out var pending))
{
pending.OriginalSender.Tell(new DeploymentStatusResponse(
pending.Command.DeploymentId,
instanceName,
DeploymentStatus.Failed,
$"superseded by delete of {instanceName} before redeploy finished terminating",
DateTimeOffset.UtcNow));
}
_terminatingActorsByName.Remove(instanceName);
// The terminating predecessor is already being stopped by HandleDeploy;
// no Context.Stop needed here.
}
else if (_instanceActors.TryGetValue(instanceName, out var actor))
{
Context.Stop(actor);
_instanceActors.Remove(instanceName);
}
if (DeferLifecycleStop(instanceName, command.CommandId, true, replyTo)) return;
// The deployed count is derived from the authoritative set of
// deployed config names, so removing the name here decrements it. Correct for a