fix(siteruntime): stamp the ensure-started heal with the store-write generation + order the redeploy override reset ahead of the successor (arch-review 2026-08-19 P1-1 + P1-2)
P1-1 — the row-67 ensure-started self-heal re-read the durable row off-thread but captured no _storeWriteGeneration and never ran IsStoreSnapshotStale, unlike the boot load and the reconcile sweep. A Disable or Delete committing inside that window left the result carrying an enabled config that passed every guard (neither command sets any of the three pipeline markers ShouldAttemptEnsureStart re-checks), so the heal started an instance the operator had just stopped — PERMANENTLY, since the sweep iterates store rows only and has no runtime→store stop leg. The generation is now captured at dispatch, threaded on EnsureInstanceStartedResult and checked on landing, and the comment claiming the stored row already covered this is corrected. The heal reads through _configLoader, the same seam the boot load and sweep use (identical production call), so a test can hold the read open across a committed disable/delete. P1-2 — the "static overrides reset on redeployment" invariant was racy in both directions. ApplyDeployment created the successor Instance Actor first and only then dispatched the clear, queued behind a full config-JSON write, so the successor's PreStart override load routinely won and applied the PREDECESSOR's overrides — keys absent from the new config included — to in-memory state for the actor's lifetime while the disk ended cleared; a later restart then silently flipped the values. The clear is now dispatched BEFORE CreateInstanceActor and handed to the actor as a gate its override load chains behind; the persistence task joins the same task, so a failed clear still fails the deploy exactly as before. Leg 2: the predecessor's PostStop flush was fire-and-forget and could commit up to one coalescing window AFTER the clear (the Deployment Manager gates the clear on Terminated), durably resurrecting overrides — PostStop now waits for it, bounded at 5s with a Warning on overrun. Tests: new RedeployStaticOverrideResetTests (5) pins both legs plus the gate mechanism and first-deploy behaviour; three new cases in ReconcileStartsInstanceTests pin the discarded disable/delete heals and that an unraced heal still starts. InstanceActorTests' StaticOverride_ResetOnRedeployment comment claimed to simulate an ordering the production path did not have — corrected.
This commit is contained in:
Joseph Doherty
@@ -111,6 +111,25 @@ public class InstanceActor : ReceiveActor
|
||||
private ICancelable? _staticOverrideFlushTimer;
|
||||
private readonly TimeSpan _staticOverrideFlushInterval;
|
||||
|
||||
/// <summary>
|
||||
/// The deploy path's static-override / native-alarm reset, supplied by
|
||||
/// <see cref="DeploymentManagerActor.CreateInstanceActor"/> on a (re)deploy and
|
||||
/// <see langword="null"/> on every other start (arch-review 2026-08-19 P1-2). When set,
|
||||
/// <see cref="PreStart"/> starts the override load only once it has completed, so a
|
||||
/// redeploy can never load the PREDECESSOR's overrides into the successor's memory.
|
||||
/// </summary>
|
||||
private readonly Task? _staticStateResetGate;
|
||||
|
||||
/// <summary>
|
||||
/// How long <see cref="PostStop"/> waits for the final coalesced static-override flush to
|
||||
/// commit. The wait exists so a redeploy's clear is ordered AFTER this actor's last write
|
||||
/// (arch-review 2026-08-19 P1-2 leg 2) — the Deployment Manager gates the clear on the
|
||||
/// Terminated signal, which a fire-and-forget flush could outlive and durably resurrect
|
||||
/// overrides the redeploy had just cleared. Bounded so a wedged SQLite write cannot stall
|
||||
/// the redeploy indefinitely; exceeding it is logged and the stop proceeds.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan FinalStaticOverrideFlushTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Cached string forms of <see cref="QualityCode"/> (perf remediation, arch-review
|
||||
/// WP1.5) — the TagValueUpdate ingest path calls this once or twice per update, so a
|
||||
@@ -182,6 +201,11 @@ public class InstanceActor : ReceiveActor
|
||||
/// <param name="dclManager">Optional Data Connection Layer manager actor reference.</param>
|
||||
/// <param name="healthCollector">Optional health collector for reporting metrics.</param>
|
||||
/// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
|
||||
/// <param name="staticOverrideFlushInterval">Coalescing window for buffered static-override writes.</param>
|
||||
/// <param name="staticStateResetGate">
|
||||
/// Optional (re)deploy static-override / native-alarm reset to order the PreStart override
|
||||
/// load behind — see <see cref="_staticStateResetGate"/>.
|
||||
/// </param>
|
||||
public InstanceActor(
|
||||
string instanceUniqueName,
|
||||
string configJson,
|
||||
@@ -194,7 +218,8 @@ public class InstanceActor : ReceiveActor
|
||||
IActorRef? dclManager = null,
|
||||
ISiteHealthCollector? healthCollector = null,
|
||||
IServiceProvider? serviceProvider = null,
|
||||
TimeSpan? staticOverrideFlushInterval = null)
|
||||
TimeSpan? staticOverrideFlushInterval = null,
|
||||
Task? staticStateResetGate = null)
|
||||
{
|
||||
_instanceUniqueName = instanceUniqueName;
|
||||
_storage = storage;
|
||||
@@ -208,6 +233,7 @@ public class InstanceActor : ReceiveActor
|
||||
_serviceProvider = serviceProvider;
|
||||
// Defaults to 100 ms, mirroring NativeAlarmActor's persistFlushInterval (P4).
|
||||
_staticOverrideFlushInterval = staticOverrideFlushInterval ?? TimeSpan.FromMilliseconds(100);
|
||||
_staticStateResetGate = staticStateResetGate;
|
||||
|
||||
// Deserialize the flattened configuration
|
||||
_configuration = JsonSerializer.Deserialize<FlattenedConfiguration>(configJson);
|
||||
@@ -326,9 +352,20 @@ public class InstanceActor : ReceiveActor
|
||||
// the actor), and on failover/restart; this single point covers them all.
|
||||
LogLifecycleEvent($"Instance {_instanceUniqueName} started");
|
||||
|
||||
// Asynchronously load static overrides from SQLite and pipe to self
|
||||
// Asynchronously load static overrides from SQLite and pipe to self. On a (re)deploy
|
||||
// the read is chained behind the reset the Deployment Manager dispatched before this
|
||||
// actor existed, so it can only ever observe the CLEARED table (arch-review
|
||||
// 2026-08-19 P1-2). A faulted reset does not block the load — the deploy itself fails
|
||||
// on that fault through the persistence join, which stops this actor anyway.
|
||||
var self = Self;
|
||||
_storage.GetStaticOverridesAsync(_instanceUniqueName).ContinueWith(t =>
|
||||
var instanceName = _instanceUniqueName;
|
||||
var storage = _storage;
|
||||
var load = _staticStateResetGate == null
|
||||
? storage.GetStaticOverridesAsync(instanceName)
|
||||
: _staticStateResetGate
|
||||
.ContinueWith(_ => storage.GetStaticOverridesAsync(instanceName))
|
||||
.Unwrap();
|
||||
load.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
return new LoadOverridesResult(t.Result, null);
|
||||
@@ -356,12 +393,15 @@ public class InstanceActor : ReceiveActor
|
||||
timer.Cancel();
|
||||
_tagSubscribeRetryTimers.Clear();
|
||||
|
||||
// Best-effort final flush of any coalesced static-override writes so a graceful
|
||||
// stop does not lose the last <= flush-interval of writes (P4 pattern, mirrors
|
||||
// NativeAlarmActor.PostStop). Fire-and-forget — the in-memory attribute value
|
||||
// (already applied) remains authoritative even if this final persist is lost.
|
||||
// Final flush of any coalesced static-override writes so a graceful stop does not lose
|
||||
// the last <= flush-interval of writes (P4 pattern, mirrors NativeAlarmActor.PostStop).
|
||||
// AWAITED, not fire-and-forget (arch-review 2026-08-19 P1-2 leg 2): a redeploy stops
|
||||
// this actor, waits for Terminated, and only then clears the static overrides — so a
|
||||
// detached final flush could commit up to one flush interval of the PREDECESSOR's dirty
|
||||
// writes AFTER that clear and durably resurrect them. Blocking here is what puts the
|
||||
// write provably before the Terminated signal the clear is gated on.
|
||||
_staticOverrideFlushTimer?.Cancel();
|
||||
FlushDirtyStaticOverrides();
|
||||
WaitForFinalStaticOverrideFlush();
|
||||
|
||||
// Operational `instance_lifecycle` event — instance stopped. An
|
||||
// instance stops on disable, delete, redeployment, and graceful shutdown;
|
||||
@@ -571,11 +611,16 @@ public class InstanceActor : ReceiveActor
|
||||
/// failed write never blocks the actor; the in-memory attribute value (already
|
||||
/// applied) remains authoritative.
|
||||
/// </summary>
|
||||
private void FlushDirtyStaticOverrides()
|
||||
/// <returns>
|
||||
/// The persist task, so the stop path can order the FINAL flush before termination
|
||||
/// (<see cref="WaitForFinalStaticOverrideFlush"/>). Every other caller ignores it, keeping
|
||||
/// the fire-and-forget semantics unchanged. Completed when there was nothing dirty.
|
||||
/// </returns>
|
||||
private Task FlushDirtyStaticOverrides()
|
||||
{
|
||||
_staticOverrideFlushTimer = null;
|
||||
if (_dirtyStaticOverrides.Count == 0)
|
||||
return;
|
||||
return Task.CompletedTask;
|
||||
|
||||
var batch = _dirtyStaticOverrides
|
||||
.Select(kv => (AttributeName: kv.Key, Value: kv.Value))
|
||||
@@ -584,12 +629,41 @@ public class InstanceActor : ReceiveActor
|
||||
|
||||
var instanceName = _instanceUniqueName;
|
||||
var logger = _logger;
|
||||
_storage.SetStaticOverridesAsync(instanceName, batch)
|
||||
.ContinueWith(t => logger.LogWarning(
|
||||
var persist = _storage.SetStaticOverridesAsync(instanceName, batch);
|
||||
persist.ContinueWith(t => logger.LogWarning(
|
||||
t.Exception?.GetBaseException(),
|
||||
"Failed to persist {Count} static overrides for {Instance}; in-memory state is authoritative",
|
||||
batch.Count, instanceName),
|
||||
TaskContinuationOptions.OnlyOnFaulted);
|
||||
return persist;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the stop-path static-override flush and waits for it to commit, bounded by
|
||||
/// <see cref="FinalStaticOverrideFlushTimeout"/>. A fault is swallowed — the
|
||||
/// OnlyOnFaulted continuation inside <see cref="FlushDirtyStaticOverrides"/> has already
|
||||
/// logged it, and a persist failure must never turn a graceful stop into a crash.
|
||||
/// </summary>
|
||||
private void WaitForFinalStaticOverrideFlush()
|
||||
{
|
||||
var flush = FlushDirtyStaticOverrides();
|
||||
if (flush.IsCompleted)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
if (!flush.Wait(FinalStaticOverrideFlushTimeout))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Final static-override flush for {Instance} did not complete within {Timeout}; "
|
||||
+ "stopping anyway — a redeploy's override reset may race it",
|
||||
_instanceUniqueName, FinalStaticOverrideFlushTimeout);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Already logged by the flush's OnlyOnFaulted continuation.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user