dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(site-runtime): artifact-apply watchdog + queued-apply failure on stop, boot-skip site event, enable revert, batch Ask slack (review 2026-10-21 lane A L1-L4)

Browse Source 
L1: an artifact apply holding the slot past 10 min (ctor-overridable) logs an
Error site event naming it without releasing the slot; PostStop answers
queued applies Failed. L2: a boot-batch instance whose actor cannot be created
emits an Error site event and leaves the deployed set. L3: a failed enable
reverts the committed is_enabled (generation-bumped both ends). L4: the routed
SetAttributes batch Ask is the 30 s per-write bound plus 10 s mailbox slack.
This commit is contained in:
Joseph Doherty
2026-10-22 09:06:28 -04:00
parent 099b2221d6
commit 6f6534ac7e
4 changed files with 158 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+89 -7
View File
@@ -115,6 +115,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
private readonly Func<Task<List<DeployedConfigMetadata>>> _configMetadataLoader;
/// <summary>
/// How long one artifact apply may hold the single apply slot before an Error site event names
/// it (lane A L1). The slot is NOT released — a still-running apply must not be overlapped.
/// </summary>
private readonly TimeSpan _artifactApplyWatchdog;
/// <summary>
/// Reads the durable <c>data_connection_definitions</c> rows (deferred register rows 89 + 90).
/// Defaults to <see cref="SiteStorageService.GetAllDataConnectionDefinitionsAsync"/>;
/// overridable via the ctor so a test can drive a read failure.
@@ -323,6 +328,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// ensure-started heal uses; defaults to reading from <paramref name="storage"/>. Primarily
/// for tests that need to hold the read open mid-flight to drive the freshness race.
/// </param>
/// <param name="artifactApplyWatchdog">
/// Optional override (default 10 minutes) for the stuck-artifact-apply watchdog; primarily for tests.
/// </param>
/// <param name="configMetadataLoader">
/// Optional override for the instance reconcile sweep's metadata-only read (01/SR-P1);
/// defaults to reading from <paramref name="storage"/>.
@@ -352,8 +360,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Func<Task<List<StoredDataConnectionDefinition>>>? connectionDefinitionLoader = null,
Func<string, Task<DeployedInstance?>>? deployedConfigLoader = null,
DeployedSecretsMirror? secretsMirror = null,
Func<Task<List<DeployedConfigMetadata>>>? configMetadataLoader = null)
Func<Task<List<DeployedConfigMetadata>>>? configMetadataLoader = null,
TimeSpan? artifactApplyWatchdog = null)
{
_artifactApplyWatchdog = artifactApplyWatchdog ?? TimeSpan.FromMinutes(10);
_scriptScheduler = scriptScheduler;
_secretsMirror = secretsMirror;
_storage = storage;
@@ -437,6 +447,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Receive<DeployArtifactsCommand>(HandleDeployArtifacts);
// Review 2026-10-21 01/SR-C2: releases the single artifact-apply slot.
Receive<ArtifactApplyCompleted>(_ => HandleArtifactApplyCompleted());
Receive<ArtifactApplyStuck>(HandleArtifactApplyStuck);
// Artifact-deploy DCL push, dispatched back from the
// off-thread persistence task so the hash-cache mutation stays
@@ -679,6 +690,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
protected override void PostStop()
{
_healthCollector?.SetActiveNode(false);
// Lane A L1: queued artifact deploys will never run on this incarnation — answer them
// instead of leaving central's Ask to time out.
while (_queuedArtifactApplies.TryDequeue(out var queued))
{
queued.ReplyTo.Tell(new ArtifactDeploymentResponse(
queued.Command.DeploymentId, "", false,
"Deployment Manager stopped before this queued artifact deployment could be applied",
DateTimeOffset.UtcNow));
}
if (ClusterAvailable(Context.System))
{
Cluster.Get(Context.System).Unsubscribe(Self);
@@ -922,9 +944,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// refused actor name is logged inside and skips just that instance (01/SR-C1). The
// warm's parsed config is reused rather than deserialized again (01/SR-P5).
var parsed = warmed.Parsed is { } all && i - startIdx < all.Length ? all[i - startIdx] : null;
CreateInstanceActor(
config.InstanceUniqueName, config.ConfigJson, config.RevisionHash, out _,
parsed: parsed);
if (!CreateInstanceActor(
config.InstanceUniqueName, config.ConfigJson, config.RevisionHash, out var createError,
parsed: parsed))
{
// Lane A L2: surface the skip and stop counting the instance as deployed-but-
// disabled — it is neither running nor startable until it is redeployed.
_deployedInstanceNames.Remove(config.InstanceUniqueName);
LogDeploymentEvent("Error", config.InstanceUniqueName,
$"Instance {config.InstanceUniqueName} could not be started at boot", createError);
}
}
UpdateInstanceCounts();
@@ -1820,6 +1849,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// row whose name Akka refuses (deployed before the gate checked names) lands here.
// Report it instead of letting ActorOf restart the singleton.
LogDeploymentEvent("Error", instanceName, $"Instance {instanceName} enable failed", createError);
// Lane A L3: is_enabled=1 has already committed; put it back so the durable row
// agrees with the runtime (and the sweep does not keep re-trying it). Row 101 bumps at
// both ends, as for every other mutation of deployed_configurations.
BumpStoreWriteGeneration("enable revert dispatch");
var revert = _storage.SetInstanceEnabledAsync(instanceName, false);
revert.ContinueWith(t =>
{
if (!t.IsCompletedSuccessfully)
_logger.LogError(t.Exception?.GetBaseException(),
"Failed to revert is_enabled for {Instance} after a failed enable", instanceName);
});
revert.ContinueWith(_ => new StoreWriteCompleted("enable revert")).PipeTo(Self);
result.OriginalSender.Tell(new InstanceLifecycleResponse(
result.Command.CommandId, instanceName, false, createError, DateTimeOffset.UtcNow));
return;
@@ -3494,11 +3536,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
.ToList();
// Each data-sourced write inside the batch is bounded by its own 30 s DCL Ask, and the
// writes run concurrently; the slack keeps a device timeout surfacing as that attribute's
// own failure rather than as a whole-batch Ask timeout.
// writes run concurrently, so the batch answers within one write timeout of the Instance
// Actor dequeuing it. The outer Ask is that per-write bound plus mailbox slack, so a
// device timeout (or a busy instance mailbox) surfaces as that attribute's own failure
// rather than as a whole-batch timeout over writes that were in fact applied.
instanceActor.Ask<InstanceSetAttributesResponse>(
new InstanceSetAttributesRequest(correlationId, writes),
TimeSpan.FromSeconds(35))
RoutedWritePerAttributeTimeout + RoutedBatchMailboxSlack)
.ContinueWith(t =>
{
if (!t.IsCompletedSuccessfully)
@@ -3528,6 +3572,36 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// <summary>True while one artifact apply's off-thread task is running.</summary>
private bool _artifactApplyInFlight;
/// <summary>Deployment id of the apply holding the slot, for the watchdog (lane A L1).</summary>
private string? _artifactApplyInFlightId;
private const string ArtifactApplyWatchdogKey = "artifact-apply-watchdog";
/// <summary>
/// Lane A L1: the in-flight artifact apply has held the slot past the watchdog. Surfaced as an
/// Error site event naming it (queued deploys are waiting behind it); the slot is deliberately
/// kept — releasing it would let the next apply overlap a still-running one (01/SR-C2).
/// </summary>
private void HandleArtifactApplyStuck(ArtifactApplyStuck stuck)
{
if (!_artifactApplyInFlight || _artifactApplyInFlightId != stuck.DeploymentId)
return;
_logger.LogError(
"Artifact deployment {DeploymentId} has been applying for over {Watchdog}; {Queued} queued deployment(s) are waiting behind it",
stuck.DeploymentId, _artifactApplyWatchdog, _queuedArtifactApplies.Count);
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"deployment", "Error", null, "DeploymentManagerActor",
$"Artifact deployment {stuck.DeploymentId} is stuck — still applying after {_artifactApplyWatchdog}",
$"{_queuedArtifactApplies.Count} queued artifact deployment(s) are waiting behind it.");
}
/// <summary>Per-write DCL bound inside a routed SetAttributes batch (matches InstanceActor's DCL Ask).</summary>
private static readonly TimeSpan RoutedWritePerAttributeTimeout = TimeSpan.FromSeconds(30);
/// <summary>Allowance for the Instance Actor's mailbox ahead of a routed batch (lane A L4).</summary>
private static readonly TimeSpan RoutedBatchMailboxSlack = TimeSpan.FromSeconds(10);
/// <summary>
/// Handles system-wide artifact deployment (shared scripts, external systems, etc.).
/// Persists artifacts to SiteStorageService and recompiles shared scripts.
@@ -3564,6 +3638,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
private void HandleArtifactApplyCompleted()
{
_artifactApplyInFlight = false;
_artifactApplyInFlightId = null;
Timers.Cancel(ArtifactApplyWatchdogKey);
if (_queuedArtifactApplies.TryDequeue(out var next))
StartArtifactApply(next.Command, next.ReplyTo);
}
@@ -3572,6 +3648,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
private void StartArtifactApply(DeployArtifactsCommand command, IActorRef sender)
{
_artifactApplyInFlight = true;
_artifactApplyInFlightId = command.DeploymentId;
Timers.StartSingleTimer(
ArtifactApplyWatchdogKey, new ArtifactApplyStuck(command.DeploymentId), _artifactApplyWatchdog);
_logger.LogInformation(
"Deploying system artifacts, deploymentId={DeploymentId}", command.DeploymentId);
@@ -4289,6 +4368,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// Review 2026-10-21 01/SR-C2: piped to self when an artifact apply's task has completed, so
/// the next queued <see cref="DeployArtifactsCommand"/> may start.
/// </summary>
/// <summary>Lane A L1: the stuck-apply watchdog fired for <paramref name="DeploymentId"/>.</summary>
internal sealed record ArtifactApplyStuck(string DeploymentId);
internal sealed record ArtifactApplyCompleted
{
/// <summary>The shared singleton instance.</summary>