fix(siteruntime): guard the DCL arbitration overlay against stale store snapshots — register row 101
An off-thread snapshot read racing an artifact apply could land after the apply populated the overlay; its stale absences then tombstoned and stopped just-applied LIVE connections, which nothing re-creates until an instance redeploy (the removal evicts the hash slot the overlay push is gated on). Each read now captures a definitions-write generation at dispatch; a stale result is discarded whole (additions too — a stale row would reconfigure a live connection backward). Deterministic repro fails before the fix; a fresh-generation guard test pins that row-90 disappearance still fires.
This commit is contained in:
Joseph Doherty
@@ -498,10 +498,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private void LoadDeployedConfigs()
|
||||
{
|
||||
// Row 101: capture on the actor thread BEFORE the off-thread read starts, so a landing
|
||||
// handler can tell whether an artifact apply mutated the overlay while it was in flight.
|
||||
var generationAtRead = _definitionsWriteGeneration;
|
||||
ReadStoreSnapshotAsync().ContinueWith(t =>
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
return new StartupConfigsLoaded(t.Result.Configs, null, t.Result.Definitions);
|
||||
return new StartupConfigsLoaded(
|
||||
t.Result.Configs, null, t.Result.Definitions, generationAtRead);
|
||||
return new StartupConfigsLoaded([], t.Exception?.GetBaseException().Message);
|
||||
}).PipeTo(Self);
|
||||
}
|
||||
@@ -606,7 +610,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// on the peer node rebuilds the whole DCL from flattened snapshots, and without the
|
||||
// overlay in place first it would rebuild them STALE and only correct itself a sweep
|
||||
// tick later — a needless create-then-diverge on every failover.
|
||||
ApplyConnectionDefinitionOverlay(msg.ConnectionDefinitions);
|
||||
ApplyConnectionDefinitionOverlayIfFresh(
|
||||
msg.ConnectionDefinitions, msg.DefinitionsGenerationAtRead, "boot load");
|
||||
|
||||
var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList();
|
||||
_deployedInstanceNames.Clear();
|
||||
@@ -1629,8 +1634,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
_instanceSweepInFlight = true;
|
||||
|
||||
// Row 101: capture on the actor thread BEFORE the off-thread read starts.
|
||||
var generationAtRead = _definitionsWriteGeneration;
|
||||
ReadStoreSnapshotAsync().ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? new InstanceSweepLoaded(t.Result.Configs, null, t.Result.Definitions)
|
||||
? new InstanceSweepLoaded(t.Result.Configs, null, t.Result.Definitions, generationAtRead)
|
||||
: new InstanceSweepLoaded([], t.Exception?.GetBaseException().Message))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
@@ -1661,7 +1668,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// convergence path for a definition row that arrived by LocalDb CDC replication: no new
|
||||
// cross-node message exists (or is needed) — whichever node runs the singleton reads the
|
||||
// row out of its own file on the next tick.
|
||||
ApplyConnectionDefinitionOverlay(msg.ConnectionDefinitions);
|
||||
ApplyConnectionDefinitionOverlayIfFresh(
|
||||
msg.ConnectionDefinitions, msg.DefinitionsGenerationAtRead, "instance sweep");
|
||||
|
||||
foreach (var config in msg.Configs)
|
||||
{
|
||||
@@ -2219,6 +2227,48 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private readonly HashSet<string> _centrallyRemovedConnections = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Monotonic count of actor-thread definition mutations from artifact applies (deferred
|
||||
/// register row 101). A store snapshot is read OFF the actor thread and compared against
|
||||
/// the overlay only when its piped message lands — so a read that started before an
|
||||
/// artifact apply's store writes can land AFTER <see
|
||||
/// cref="HandleApplyArtifactDataConnectionsToDcl"/> populated the overlay, and its stale
|
||||
/// data would then "prove" the just-applied names disappeared, tombstoning and stopping
|
||||
/// live connections that nothing re-creates until an instance redeploy. Each snapshot read
|
||||
/// captures this value on the actor thread at dispatch; a result carrying a stale
|
||||
/// generation skips the overlay apply entirely (additions too — a stale row pushed via
|
||||
/// <see cref="EnsureDclConnection"/> would reconfigure a live connection BACKWARD and set
|
||||
/// up a flip-flop with the next fresh read). The next tick reads fresh and converges.
|
||||
/// </summary>
|
||||
private long _definitionsWriteGeneration;
|
||||
|
||||
/// <summary>
|
||||
/// Row 101 guard in front of <see cref="ApplyConnectionDefinitionOverlay"/>: applies the
|
||||
/// snapshot only when no artifact apply mutated the definitions while the read was in
|
||||
/// flight. A stale snapshot is discarded WHOLE — its absences would spuriously tombstone
|
||||
/// and stop just-applied live connections (which nothing re-creates until an instance
|
||||
/// redeploy), and its rows would push superseded configs onto live connections. Discarding
|
||||
/// is safe because the artifact apply already updated the overlay itself and the next
|
||||
/// tick's read starts after the write, so convergence is one tick away.
|
||||
/// </summary>
|
||||
private void ApplyConnectionDefinitionOverlayIfFresh(
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? definitions,
|
||||
long generationAtRead,
|
||||
string source)
|
||||
{
|
||||
if (generationAtRead != _definitionsWriteGeneration)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Discarding the {Source} data-connection definitions snapshot — an artifact "
|
||||
+ "apply changed the definitions while the read was in flight (generation "
|
||||
+ "{ReadGeneration} vs {CurrentGeneration}); the next sweep tick reads fresh",
|
||||
source, generationAtRead, _definitionsWriteGeneration);
|
||||
return;
|
||||
}
|
||||
|
||||
ApplyConnectionDefinitionOverlay(definitions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refreshes <see cref="_connectionDefinitions"/> from a store read and reconciles the live
|
||||
/// DCL against it (deferred register rows 89 + 90).
|
||||
@@ -2239,6 +2289,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// connections inline" — so it is left to the flattened config. That asymmetry is the
|
||||
/// documented residual of rows 89/90, not an oversight.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// Reached only through <see cref="ApplyConnectionDefinitionOverlayIfFresh"/> (row 101) —
|
||||
/// never with a snapshot proven stale.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void ApplyConnectionDefinitionOverlay(
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? definitions)
|
||||
@@ -3066,6 +3121,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private void HandleApplyArtifactDataConnectionsToDcl(ApplyArtifactDataConnectionsToDcl msg)
|
||||
{
|
||||
// Row 101: any snapshot read dispatched before this point carries data that predates
|
||||
// these mutations — mark it stale so its overlay half is discarded when it lands.
|
||||
_definitionsWriteGeneration++;
|
||||
|
||||
foreach (var dc in msg.DataConnections)
|
||||
{
|
||||
// Rows 89 + 90: the rows have just been committed to data_connection_definitions, so
|
||||
@@ -3157,10 +3216,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// is the deploy-only site shape (see <see cref="ApplyConnectionDefinitionOverlay"/>).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="DefinitionsGenerationAtRead">
|
||||
/// Row 101: <see cref="_definitionsWriteGeneration"/> captured on the actor thread when the
|
||||
/// read was dispatched. A mismatch on landing means an artifact apply mutated the overlay
|
||||
/// while this read was in flight — the snapshot's definitions half is stale and is skipped.
|
||||
/// </param>
|
||||
internal record StartupConfigsLoaded(
|
||||
List<DeployedInstance> Configs,
|
||||
string? Error,
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? ConnectionDefinitions = null);
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? ConnectionDefinitions = null,
|
||||
long DefinitionsGenerationAtRead = 0);
|
||||
|
||||
/// <summary>
|
||||
/// Internal singleton signal that re-attempts the startup deployed-config load
|
||||
@@ -3253,10 +3318,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// contract as <see cref="StartupConfigsLoaded"/>: null = read failed, keep the overlay.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="DefinitionsGenerationAtRead">
|
||||
/// Row 101: same stale-snapshot guard as <see cref="StartupConfigsLoaded"/> — see there.
|
||||
/// </param>
|
||||
internal record InstanceSweepLoaded(
|
||||
List<DeployedInstance> Configs,
|
||||
string? Error,
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? ConnectionDefinitions = null);
|
||||
IReadOnlyList<StoredDataConnectionDefinition>? ConnectionDefinitions = null,
|
||||
long DefinitionsGenerationAtRead = 0);
|
||||
|
||||
internal record DeployPersistenceResult(
|
||||
string DeploymentId, string InstanceName, bool Success, string? Error,
|
||||
|
||||
Reference in New Issue
Block a user