fix(site-runtime): serialise DeployArtifactsCommand applies per site so an overlapping apply cannot delete a live artifact (review 2026-10-21 01/SR-C2)
Each apply upserts its full set then deletes everything else, off the actor thread; two overlapping commands interleaved and deleted artifacts central never removed. The Deployment Manager now runs one apply at a time, queuing the rest in arrival order; the slot is released by a self-message that lands behind the apply's own DCL push.
This commit is contained in:
@@ -419,6 +419,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
// Handle system-wide artifact deployment
|
||||
Receive<DeployArtifactsCommand>(HandleDeployArtifacts);
|
||||
// Review 2026-10-21 01/SR-C2: releases the single artifact-apply slot.
|
||||
Receive<ArtifactApplyCompleted>(_ => HandleArtifactApplyCompleted());
|
||||
|
||||
// Artifact-deploy DCL push, dispatched back from the
|
||||
// off-thread persistence task so the hash-cache mutation stays
|
||||
@@ -3267,16 +3269,62 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
}).PipeTo(sender);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Artifact applies waiting for the one in flight to finish (review 2026-10-21 01/SR-C2), in
|
||||
/// arrival order, each with the sender to answer. See <see cref="HandleDeployArtifacts"/>.
|
||||
/// </summary>
|
||||
private readonly Queue<(DeployArtifactsCommand Command, IActorRef ReplyTo)> _queuedArtifactApplies = new();
|
||||
|
||||
/// <summary>True while one artifact apply's off-thread task is running.</summary>
|
||||
private bool _artifactApplyInFlight;
|
||||
|
||||
/// <summary>
|
||||
/// Handles system-wide artifact deployment (shared scripts, external systems, etc.).
|
||||
/// Persists artifacts to SiteStorageService and recompiles shared scripts.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Applies are serialised, one at a time, in arrival order (review 2026-10-21
|
||||
/// 01/SR-C2).</b> Each apply is "upsert my complete set, then delete every stored row not in
|
||||
/// my set", and it runs off the actor thread. Two commands that overlapped — central's per-site
|
||||
/// retry racing a fresh deploy, or two operators clicking <i>Deploy Artifacts</i> — used to run
|
||||
/// CONCURRENTLY, so the interleaving "B upserts Z → A deletes-except(A's set)" deleted Z (and
|
||||
/// unregistered it from the shared-script library, or soft-removed a deployed secret) although
|
||||
/// central never removed it, and the site stayed wrong until the next artifact deploy. Queued
|
||||
/// here, the final state is always exactly one command's set: the last one to arrive.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void HandleDeployArtifacts(DeployArtifactsCommand command)
|
||||
{
|
||||
if (_artifactApplyInFlight)
|
||||
{
|
||||
_queuedArtifactApplies.Enqueue((command, Sender));
|
||||
_logger.LogInformation(
|
||||
"Artifact deployment {DeploymentId} queued behind the apply in flight ({Queued} waiting)",
|
||||
command.DeploymentId, _queuedArtifactApplies.Count);
|
||||
return;
|
||||
}
|
||||
|
||||
StartArtifactApply(command, Sender);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The in-flight artifact apply has finished (its reply and any DCL push are already on
|
||||
/// their way); start the next queued one, if any.
|
||||
/// </summary>
|
||||
private void HandleArtifactApplyCompleted()
|
||||
{
|
||||
_artifactApplyInFlight = false;
|
||||
if (_queuedArtifactApplies.TryDequeue(out var next))
|
||||
StartArtifactApply(next.Command, next.ReplyTo);
|
||||
}
|
||||
|
||||
/// <summary>Runs one artifact apply off the actor thread; see <see cref="HandleDeployArtifacts"/>.</summary>
|
||||
private void StartArtifactApply(DeployArtifactsCommand command, IActorRef sender)
|
||||
{
|
||||
_artifactApplyInFlight = true;
|
||||
_logger.LogInformation(
|
||||
"Deploying system artifacts, deploymentId={DeploymentId}", command.DeploymentId);
|
||||
|
||||
var sender = Sender;
|
||||
// Capture Self before entering Task.Run: the Self/Sender/Context properties
|
||||
// are backed by the ambient ActorCell, which is null on a thread-pool thread,
|
||||
// so reading Self *inside* the lambda throws "no active ActorContext". The
|
||||
@@ -3289,7 +3337,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// pre-apply table and be treated as current.
|
||||
BumpStoreWriteGeneration("artifact apply dispatch");
|
||||
|
||||
Task.Run(async () =>
|
||||
var apply = Task.Run(async () =>
|
||||
{
|
||||
// The DCL push for this deploy's data connections is captured
|
||||
// here and sent only AFTER the shipped secrets are written and mirrored (in the
|
||||
@@ -3508,7 +3556,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
self.Tell(pendingDclApply);
|
||||
}
|
||||
}
|
||||
}).PipeTo(sender);
|
||||
});
|
||||
apply.PipeTo(sender);
|
||||
|
||||
// Releases the single apply slot. Told after the task completes, so it is enqueued behind
|
||||
// this apply's own ApplyArtifactDataConnectionsToDcl (sent from the finally above) — the
|
||||
// next apply never starts before this one's DCL push has been processed.
|
||||
apply.ContinueWith(_ => ArtifactApplyCompleted.Instance).PipeTo(self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -3916,6 +3970,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// carries the reconcile-deleted definitions so the hash-cache eviction and
|
||||
/// live DCL connection stop also happen actor-thread-confined.
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 01/SR-C2: piped to self when an artifact apply's task has completed, so
|
||||
/// the next queued <see cref="DeployArtifactsCommand"/> may start.
|
||||
/// </summary>
|
||||
internal sealed record ArtifactApplyCompleted
|
||||
{
|
||||
/// <summary>The shared singleton instance.</summary>
|
||||
public static readonly ArtifactApplyCompleted Instance = new();
|
||||
private ArtifactApplyCompleted() { }
|
||||
}
|
||||
|
||||
internal record ApplyArtifactDataConnectionsToDcl(
|
||||
IReadOnlyList<Commons.Messages.Artifacts.DataConnectionArtifact> DataConnections,
|
||||
IReadOnlyList<string> RemovedConnectionNames);
|
||||
|
||||
Reference in New Issue
Block a user