fix(site-runtime): serialise DeployArtifactsCommand applies per site so an overlapping apply cannot delete a live artifact (review 2026-10-21 01/SR-C2)

Each apply upserts its full set then deletes everything else, off the actor
thread; two overlapping commands interleaved and deleted artifacts central
never removed. The Deployment Manager now runs one apply at a time, queuing
the rest in arrival order; the slot is released by a self-message that lands
behind the apply's own DCL push.
This commit is contained in:
Joseph Doherty
2026-10-22 06:33:41 -04:00
parent e765700eac
commit 35ffd025c3
3 changed files with 170 additions and 3 deletions
@@ -419,6 +419,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// Handle system-wide artifact deployment
Receive<DeployArtifactsCommand>(HandleDeployArtifacts);
// Review 2026-10-21 01/SR-C2: releases the single artifact-apply slot.
Receive<ArtifactApplyCompleted>(_ => HandleArtifactApplyCompleted());
// Artifact-deploy DCL push, dispatched back from the
// off-thread persistence task so the hash-cache mutation stays
@@ -3267,16 +3269,62 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
}).PipeTo(sender);
}
/// <summary>
/// Artifact applies waiting for the one in flight to finish (review 2026-10-21 01/SR-C2), in
/// arrival order, each with the sender to answer. See <see cref="HandleDeployArtifacts"/>.
/// </summary>
private readonly Queue<(DeployArtifactsCommand Command, IActorRef ReplyTo)> _queuedArtifactApplies = new();
/// <summary>True while one artifact apply's off-thread task is running.</summary>
private bool _artifactApplyInFlight;
/// <summary>
/// Handles system-wide artifact deployment (shared scripts, external systems, etc.).
/// Persists artifacts to SiteStorageService and recompiles shared scripts.
///
/// <para>
/// <b>Applies are serialised, one at a time, in arrival order (review 2026-10-21
/// 01/SR-C2).</b> Each apply is "upsert my complete set, then delete every stored row not in
/// my set", and it runs off the actor thread. Two commands that overlapped — central's per-site
/// retry racing a fresh deploy, or two operators clicking <i>Deploy Artifacts</i> — used to run
/// CONCURRENTLY, so the interleaving "B upserts Z → A deletes-except(A's set)" deleted Z (and
/// unregistered it from the shared-script library, or soft-removed a deployed secret) although
/// central never removed it, and the site stayed wrong until the next artifact deploy. Queued
/// here, the final state is always exactly one command's set: the last one to arrive.
/// </para>
/// </summary>
private void HandleDeployArtifacts(DeployArtifactsCommand command)
{
if (_artifactApplyInFlight)
{
_queuedArtifactApplies.Enqueue((command, Sender));
_logger.LogInformation(
"Artifact deployment {DeploymentId} queued behind the apply in flight ({Queued} waiting)",
command.DeploymentId, _queuedArtifactApplies.Count);
return;
}
StartArtifactApply(command, Sender);
}
/// <summary>
/// The in-flight artifact apply has finished (its reply and any DCL push are already on
/// their way); start the next queued one, if any.
/// </summary>
private void HandleArtifactApplyCompleted()
{
_artifactApplyInFlight = false;
if (_queuedArtifactApplies.TryDequeue(out var next))
StartArtifactApply(next.Command, next.ReplyTo);
}
/// <summary>Runs one artifact apply off the actor thread; see <see cref="HandleDeployArtifacts"/>.</summary>
private void StartArtifactApply(DeployArtifactsCommand command, IActorRef sender)
{
_artifactApplyInFlight = true;
_logger.LogInformation(
"Deploying system artifacts, deploymentId={DeploymentId}", command.DeploymentId);
var sender = Sender;
// Capture Self before entering Task.Run: the Self/Sender/Context properties
// are backed by the ambient ActorCell, which is null on a thread-pool thread,
// so reading Self *inside* the lambda throws "no active ActorContext". The
@@ -3289,7 +3337,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// pre-apply table and be treated as current.
BumpStoreWriteGeneration("artifact apply dispatch");
Task.Run(async () =>
var apply = Task.Run(async () =>
{
// The DCL push for this deploy's data connections is captured
// here and sent only AFTER the shipped secrets are written and mirrored (in the
@@ -3508,7 +3556,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
self.Tell(pendingDclApply);
}
}
}).PipeTo(sender);
});
apply.PipeTo(sender);
// Releases the single apply slot. Told after the task completes, so it is enqueued behind
// this apply's own ApplyArtifactDataConnectionsToDcl (sent from the finally above) — the
// next apply never starts before this one's DCL push has been processed.
apply.ContinueWith(_ => ArtifactApplyCompleted.Instance).PipeTo(self);
}
/// <summary>
@@ -3916,6 +3970,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// carries the reconcile-deleted definitions so the hash-cache eviction and
/// live DCL connection stop also happen actor-thread-confined.
/// </summary>
/// <summary>
/// Review 2026-10-21 01/SR-C2: piped to self when an artifact apply's task has completed, so
/// the next queued <see cref="DeployArtifactsCommand"/> may start.
/// </summary>
internal sealed record ArtifactApplyCompleted
{
/// <summary>The shared singleton instance.</summary>
public static readonly ArtifactApplyCompleted Instance = new();
private ArtifactApplyCompleted() { }
}
internal record ApplyArtifactDataConnectionsToDcl(
IReadOnlyList<Commons.Messages.Artifacts.DataConnectionArtifact> DataConnections,
IReadOnlyList<string> RemovedConnectionNames);