Merge rows 78+72: DeploymentManager singleton instance sweep — starts replication-seeded enabled instances and converges stale revisions through the real HandleDeploy path; one repair attempt per (instance, revision)
This commit is contained in:
@@ -159,6 +159,58 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private readonly ScriptExecutionScheduler? _scriptScheduler;
|
||||
|
||||
/// <summary>
|
||||
/// Deferred register row 72: the config revision hash each RUNNING Instance Actor was
|
||||
/// created from, keyed by instance name. Written by <see cref="CreateInstanceActor"/> —
|
||||
/// the single creation funnel — and compared against the stored row by the instance
|
||||
/// reconcile sweep to detect an actor still serving a config the durable store has since
|
||||
/// replaced.
|
||||
/// <para>
|
||||
/// Reads are always guarded by <c>_instanceActors.ContainsKey</c>, so a leftover entry for
|
||||
/// an instance that has stopped is never consulted, and a stop-then-recreate overwrites it.
|
||||
/// An instance with no entry (only reachable if a future creation path bypasses the funnel)
|
||||
/// is treated as "revision unknown" and is deliberately never restarted.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, string> _runningRevisions = new();
|
||||
|
||||
/// <summary>
|
||||
/// The stored revision hash the instance reconcile sweep has already ACTED on for an
|
||||
/// instance, keyed by instance name — <b>one repair attempt per (instance, revision)
|
||||
/// pair</b>, for both the row-78 start and the row-72 restart.
|
||||
/// <para>
|
||||
/// This is what keeps a periodic self-heal from becoming a restart loop. An instance whose
|
||||
/// Instance Actor dies during init leaves its config row in place, so without this the
|
||||
/// sweep would recreate it — and re-announce it — every tick forever; likewise a stored
|
||||
/// config the site compile gate rejects would be re-attempted forever. The divergence is
|
||||
/// still surfaced either way: the attempt itself emits a Warning site event, and a failed
|
||||
/// restart emits a second one. A genuinely NEW revision (the normal case: central redeploys
|
||||
/// something better) is a different key and is always attempted.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, string> _sweepAttemptedRevisions = new();
|
||||
|
||||
/// <summary>
|
||||
/// Interval of the instance reconcile sweep (rows 78 + 72);
|
||||
/// <see cref="TimeSpan.Zero"/> disables it. See
|
||||
/// <see cref="SiteRuntimeOptions.InstanceSweepIntervalSeconds"/>.
|
||||
/// </summary>
|
||||
private readonly TimeSpan _instanceSweepInterval;
|
||||
|
||||
/// <summary>
|
||||
/// True once boot-time Instance Actor creation has finished (every staggered batch created,
|
||||
/// or there were no enabled configs to create). The sweep is inert until then: a tick landing
|
||||
/// between two staggered batches would otherwise "heal" instances the boot load is already
|
||||
/// about to create, emitting a Warning site event for a node that is merely still starting.
|
||||
/// </summary>
|
||||
private bool _startupInstanceCreationComplete;
|
||||
|
||||
/// <summary>
|
||||
/// True while a sweep's off-thread storage read is outstanding. A slow read must not let
|
||||
/// ticks stack up into overlapping reads (and duplicate repairs) on a large site.
|
||||
/// </summary>
|
||||
private bool _instanceSweepInFlight;
|
||||
|
||||
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
@@ -224,6 +276,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
_serviceProvider = serviceProvider;
|
||||
_configFetcher = configFetcher;
|
||||
_startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5);
|
||||
_instanceSweepInterval = options.InstanceSweepIntervalSeconds > 0
|
||||
? TimeSpan.FromSeconds(options.InstanceSweepIntervalSeconds)
|
||||
: TimeSpan.Zero;
|
||||
_configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
|
||||
_logger = logger;
|
||||
// Reuse a single logger factory for all Instance Actors.
|
||||
@@ -262,6 +317,18 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
Receive<EnsureInstanceStartedCommand>(HandleEnsureInstanceStarted);
|
||||
Receive<EnsureInstanceStartedResult>(HandleEnsureInstanceStartedResult);
|
||||
|
||||
// Instance reconcile sweep (deferred register rows 78 + 72). Periodically re-reads the
|
||||
// durable rows and repairs the two ways the runtime can silently diverge from them:
|
||||
// an enabled config with no running actor (row 78), and a running actor whose revision
|
||||
// no longer matches the stored one (row 72).
|
||||
Receive<InstanceSweepTick>(_ => HandleInstanceSweepTick());
|
||||
Receive<InstanceSweepLoaded>(HandleInstanceSweepLoaded);
|
||||
// Terminal reply of a sweep-driven stale-revision restart. Every OTHER deploy path
|
||||
// replies to central (or to a test probe); only the sweep makes this actor its own
|
||||
// deployer, so a response landing here is always its own and is surfaced, never
|
||||
// dead-lettered.
|
||||
Receive<DeploymentStatusResponse>(HandleSweepRestartResult);
|
||||
|
||||
// Query-the-site-before-redeploy idempotency.
|
||||
// Central asks for the instance's currently-applied deployment identity
|
||||
// before re-sending a deployment whose prior record is stuck InProgress
|
||||
@@ -380,6 +447,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
typeof(ClusterEvent.MemberUp));
|
||||
}
|
||||
|
||||
// Instance reconcile sweep (rows 78 + 72). Armed here rather than after the startup
|
||||
// load so it survives a load that never succeeds; the handler itself is inert until
|
||||
// boot-time Instance Actor creation has finished.
|
||||
if (_instanceSweepInterval > TimeSpan.Zero)
|
||||
{
|
||||
Timers.StartPeriodicTimer(
|
||||
"instance-sweep", InstanceSweepTick.Instance,
|
||||
_instanceSweepInterval, _instanceSweepInterval);
|
||||
}
|
||||
|
||||
LoadDeployedConfigs();
|
||||
}
|
||||
|
||||
@@ -491,7 +568,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
msg.CompiledCount, msg.TotalCount);
|
||||
|
||||
if (msg.EnabledConfigs.Count == 0)
|
||||
{
|
||||
// Nothing to create, so boot-time creation is trivially complete and the instance
|
||||
// reconcile sweep may run. This is exactly the row-78 live case: the node booted
|
||||
// with an empty deployed_configurations table and CDC replication seeds it moments
|
||||
// later, with no boot load left to notice.
|
||||
_startupInstanceCreationComplete = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Start the first batch immediately
|
||||
var batchState = new BatchState(msg.EnabledConfigs, 0);
|
||||
@@ -563,7 +647,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
var config = state.Configs[i];
|
||||
EnsureDclConnections(config.ConfigJson);
|
||||
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson);
|
||||
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson, config.RevisionHash);
|
||||
}
|
||||
|
||||
UpdateInstanceCounts();
|
||||
@@ -581,6 +665,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"All {Count} Instance Actors created", state.Configs.Count);
|
||||
// Boot-time creation is done — the instance reconcile sweep may now run without
|
||||
// racing the staggered batches it would otherwise mistake for a divergence.
|
||||
_startupInstanceCreationComplete = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -993,7 +1080,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
EnsureDclConnections(command.FlattenedConfigurationJson);
|
||||
|
||||
// Create the Instance Actor immediately
|
||||
CreateInstanceActor(instanceName, command.FlattenedConfigurationJson);
|
||||
CreateInstanceActor(instanceName, command.FlattenedConfigurationJson, command.RevisionHash);
|
||||
if (!isRedeploy)
|
||||
_deployedInstanceNames.Add(instanceName);
|
||||
UpdateInstanceCounts();
|
||||
@@ -1277,7 +1364,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (!_instanceActors.ContainsKey(instanceName))
|
||||
{
|
||||
CreateInstanceActor(instanceName, result.Config.ConfigJson);
|
||||
CreateInstanceActor(instanceName, result.Config.ConfigJson, result.Config.RevisionHash);
|
||||
}
|
||||
UpdateInstanceCounts();
|
||||
|
||||
@@ -1335,6 +1422,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
.PipeTo(Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when a deploy/redeploy/delete pipeline currently OWNS the instance — a compile warm
|
||||
/// is in flight, a predecessor is still terminating, or a deployment is awaiting its
|
||||
/// persistence/init join. Every advisory self-heal (row 67's ensure-started and the row
|
||||
/// 78/72 sweep) defers to those paths: they are authoritative, they will create or replace
|
||||
/// the actor themselves, and stepping in front of them would break the deploy gate's
|
||||
/// ordering guarantees and the all-or-nothing-per-instance rule.
|
||||
/// </summary>
|
||||
private bool InstanceIsOwnedByADeployPipeline(string instanceName) =>
|
||||
_deployWarms.ContainsKey(instanceName) ||
|
||||
_terminatingActorsByName.ContainsKey(instanceName) ||
|
||||
_inFlightDeployments.ContainsKey(instanceName);
|
||||
|
||||
/// <summary>
|
||||
/// Shared guard for the two halves of the ensure-started round trip. Re-evaluated after
|
||||
/// the storage read because the instance's state can change while that read is in flight
|
||||
@@ -1342,9 +1442,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private bool ShouldAttemptEnsureStart(string instanceName) =>
|
||||
!_instanceActors.ContainsKey(instanceName) &&
|
||||
!_deployWarms.ContainsKey(instanceName) &&
|
||||
!_terminatingActorsByName.ContainsKey(instanceName) &&
|
||||
!_inFlightDeployments.ContainsKey(instanceName);
|
||||
!InstanceIsOwnedByADeployPipeline(instanceName);
|
||||
|
||||
/// <summary>
|
||||
/// Actor-thread half of the reconcile self-heal: creates the Instance Actor for a config
|
||||
@@ -1396,21 +1494,261 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
CreateInstanceActor(instanceName, result.Config.ConfigJson);
|
||||
StartStoredInstance(result.Config, reason,
|
||||
"Config was missing at node boot and was restored by reconciliation; "
|
||||
+ "the Instance Actor was not running and has now been started.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The one place an advisory self-heal turns a stored, enabled config into a running
|
||||
/// Instance Actor — shared by the row-67 ensure-started handler and the row-78 sweep so
|
||||
/// both produce identical bookkeeping and identical operator-facing surfacing. Callers
|
||||
/// are responsible for the guards; this method assumes they have passed.
|
||||
/// </summary>
|
||||
/// <param name="config">The durable row to start from — the authority on config and revision.</param>
|
||||
/// <param name="reason">Provenance, carried verbatim into the log line and the site event.</param>
|
||||
/// <param name="details">Longer explanation attached to the site event.</param>
|
||||
private void StartStoredInstance(DeployedInstance config, string reason, string details)
|
||||
{
|
||||
var instanceName = config.InstanceUniqueName;
|
||||
|
||||
CreateInstanceActor(instanceName, config.ConfigJson, config.RevisionHash);
|
||||
_deployedInstanceNames.Add(instanceName);
|
||||
UpdateInstanceCounts();
|
||||
|
||||
// Surface the heal. Warning (not Info) on purpose: reaching here means the node
|
||||
// booted WITHOUT a config for an enabled, successfully-deployed instance — the
|
||||
// condition is repaired, but it is abnormal and worth an operator's attention.
|
||||
// Surface the heal. Warning (not Info) on purpose: reaching here means an enabled,
|
||||
// successfully-deployed instance was NOT running while its config sat in the store —
|
||||
// the condition is repaired, but it is abnormal and worth an operator's attention.
|
||||
_logger.LogWarning(
|
||||
"Instance {Instance} started by {Reason} self-heal — its config was missing at boot "
|
||||
+ "and has been restored; the instance was NOT running until now",
|
||||
"Instance {Instance} started by {Reason} self-heal — its config is present in the "
|
||||
+ "deployed-config store but no Instance Actor was running until now",
|
||||
instanceName, reason);
|
||||
LogDeploymentEvent("Warning", instanceName,
|
||||
$"Instance {instanceName} started by {reason} self-heal",
|
||||
"Config was missing at node boot and was restored by reconciliation; "
|
||||
+ "the Instance Actor was not running and has now been started.");
|
||||
$"Instance {instanceName} started by {reason} self-heal", details);
|
||||
}
|
||||
|
||||
/// <summary>Provenance string carried on every repair the instance reconcile sweep makes.</summary>
|
||||
private const string SweepReason = "deployment-manager sweep";
|
||||
|
||||
/// <summary>
|
||||
/// Instance reconcile sweep, tick half (deferred register rows 78 + 72). Kicks off an
|
||||
/// off-thread read of the durable <c>deployed_configurations</c> rows; the comparison and
|
||||
/// every repair happen on the actor thread in
|
||||
/// <see cref="HandleInstanceSweepLoaded"/>.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Why a periodic sweep and not a LocalDb change hook.</b> Row 78's live symptom was a
|
||||
/// config that arrived by CDC <i>replication</i>: the sync applier writes
|
||||
/// <c>deployed_configurations</c> rows straight into the file, which is invisible to the
|
||||
/// boot-time load (already run) and to the startup reconcile (central reports no gap,
|
||||
/// because central's view is correct — the site really does have the config). A hook on the
|
||||
/// replication apply would cover exactly that one arrival path, and only for the half of
|
||||
/// the fleet that runs with replication enabled. It is also not available: <c>ILocalDb</c>
|
||||
/// exposes no change-notification or applied-rows callback at all, so a hook would mean a
|
||||
/// new upstream library capability plus an in-process bus from the sync applier into this
|
||||
/// singleton. A sweep over the durable rows is the arrival-path-agnostic form of the same
|
||||
/// check — replication seeding, a post-boot snapshot resync, an operator restoring a file,
|
||||
/// and any future path all converge on "the store says X, the runtime says Y" — and it
|
||||
/// inherits row 67's guards unchanged.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Inert until boot-time creation finishes.</b> A tick landing between two staggered
|
||||
/// startup batches would see instances the boot load is about to create and "heal" them,
|
||||
/// announcing a fault on a node that is merely still starting.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void HandleInstanceSweepTick()
|
||||
{
|
||||
if (!_startupInstanceCreationComplete || _instanceSweepInFlight)
|
||||
return;
|
||||
|
||||
_instanceSweepInFlight = true;
|
||||
|
||||
_configLoader().ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? new InstanceSweepLoaded(t.Result, null)
|
||||
: new InstanceSweepLoaded([], t.Exception?.GetBaseException().Message))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Instance reconcile sweep, actor-thread half: compares the durable rows against the
|
||||
/// running Instance Actors and repairs the two divergences the register recorded.
|
||||
///
|
||||
/// <para>
|
||||
/// A sweep that finds nothing is <b>completely silent</b> — no log line above Trace, no site
|
||||
/// event — because that is the case on every tick of a healthy site.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void HandleInstanceSweepLoaded(InstanceSweepLoaded msg)
|
||||
{
|
||||
_instanceSweepInFlight = false;
|
||||
|
||||
if (msg.Error != null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Instance reconcile sweep could not read the deployed configs: {Error}", msg.Error);
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var config in msg.Configs)
|
||||
{
|
||||
var instanceName = config.InstanceUniqueName;
|
||||
|
||||
// A disabled instance legitimately has a config row and no running actor; that is
|
||||
// its correct steady state, not a divergence.
|
||||
if (!config.IsEnabled)
|
||||
continue;
|
||||
|
||||
// Defer to any deploy pipeline that owns the instance right now. It is
|
||||
// authoritative and will create or replace the actor itself.
|
||||
if (InstanceIsOwnedByADeployPipeline(instanceName))
|
||||
continue;
|
||||
|
||||
if (!_instanceActors.ContainsKey(instanceName))
|
||||
{
|
||||
// Row 78 — an enabled config with no actor.
|
||||
if (!TryClaimSweepAttempt(instanceName, config.RevisionHash))
|
||||
continue;
|
||||
|
||||
StartStoredInstance(config, SweepReason,
|
||||
"An enabled instance had a deployed config in the site store but no running "
|
||||
+ "Instance Actor. Configs that arrive by LocalDb replication (a standby "
|
||||
+ "seeded from its peer, or a snapshot resync) are written after the "
|
||||
+ "boot-time load and are invisible to the startup reconcile, so nothing "
|
||||
+ "else would have started it.");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Row 72 — a running actor serving a revision the store has replaced.
|
||||
TryRestartStaleInstance(config);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Claims the single sweep repair attempt for an (instance, revision) pair. Returns
|
||||
/// <see langword="false"/> when this exact revision has already been attempted — see
|
||||
/// <see cref="_sweepAttemptedRevisions"/> for why a periodic self-heal must not retry the
|
||||
/// same repair forever.
|
||||
/// </summary>
|
||||
private bool TryClaimSweepAttempt(string instanceName, string revisionHash)
|
||||
{
|
||||
if (_sweepAttemptedRevisions.TryGetValue(instanceName, out var attempted) &&
|
||||
string.Equals(attempted, revisionHash, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_sweepAttemptedRevisions[instanceName] = revisionHash;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deferred register row 72 — <b>the running Instance Actor is serving a config the durable
|
||||
/// store has replaced.</b>
|
||||
///
|
||||
/// <para>
|
||||
/// The startup reconcile refreshes a STALE row via <c>StoreDeployedConfigIfNewerAsync</c>
|
||||
/// (and a CDC-replicated deploy lands the same way), which heals the file while the running
|
||||
/// actor keeps executing the previously compiled config until the next restart or redeploy.
|
||||
/// Nothing surfaced the difference.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>The decision (2026-08-15, <c>docs/plans/2026-08-15-instance-sweep-self-heal-design.md</c>):
|
||||
/// converge, do not merely warn.</b> The stored row is what central successfully deployed;
|
||||
/// the running actor is executing a config central believes was replaced. Serving it
|
||||
/// indefinitely is the actual fault. The stored config is therefore applied through the
|
||||
/// EXACTLY the same path a central redeploy uses — <see cref="HandleDeploy"/>, so the
|
||||
/// compile gate, the warm-then-gate ordering, the stop/watch/recreate handshake, the
|
||||
/// static-override and native-alarm reset, and all-or-nothing-per-instance are the ones
|
||||
/// already in production, not a second implementation. A stop still does not cancel an
|
||||
/// in-flight script run, exactly as for a central redeploy.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Safety.</b> The restart is attempted once per stored revision, so a config the
|
||||
/// compile gate rejects (which leaves the running actor untouched — the gate returns before
|
||||
/// anything is stopped) is announced but never looped on. The sweep is also skipped
|
||||
/// entirely while a deploy pipeline owns the instance, so it can never interleave with a
|
||||
/// real redeploy.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void TryRestartStaleInstance(DeployedInstance config)
|
||||
{
|
||||
var instanceName = config.InstanceUniqueName;
|
||||
var storedRevision = config.RevisionHash;
|
||||
|
||||
// No stored revision to compare against — nothing can be concluded, so do nothing.
|
||||
if (string.IsNullOrEmpty(storedRevision))
|
||||
return;
|
||||
|
||||
// No recorded running revision. Unreachable while CreateInstanceActor is the single
|
||||
// creation funnel; treated as "unknown", never as "stale", so a future path that
|
||||
// bypasses the funnel degrades to no-op rather than to spurious restarts.
|
||||
if (!_runningRevisions.TryGetValue(instanceName, out var runningRevision) ||
|
||||
string.IsNullOrEmpty(runningRevision))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.Equals(runningRevision, storedRevision, StringComparison.Ordinal))
|
||||
return;
|
||||
|
||||
if (!TryClaimSweepAttempt(instanceName, storedRevision))
|
||||
return;
|
||||
|
||||
_logger.LogWarning(
|
||||
"Instance {Instance} is running stale revision {RunningRevision} while the deployed-config "
|
||||
+ "store holds {StoredRevision} — applying the stored config through the redeploy path "
|
||||
+ "({Reason} self-heal)",
|
||||
instanceName, runningRevision, storedRevision, SweepReason);
|
||||
LogDeploymentEvent("Warning", instanceName,
|
||||
$"Instance {instanceName} restarted by {SweepReason} self-heal — "
|
||||
+ $"running revision {runningRevision} is stale, applying stored revision {storedRevision}",
|
||||
"The site's deployed-config store was refreshed out of band (startup reconcile "
|
||||
+ "refreshing a stale row, or a replicated deploy) while the Instance Actor kept "
|
||||
+ "serving the previously compiled config. The stored config is what central "
|
||||
+ "successfully deployed, so it is applied through the normal redeploy path.");
|
||||
|
||||
// Reply to Self: this actor is its own deployer here, and HandleSweepRestartResult
|
||||
// surfaces the outcome. Replying to NoSender would dead-letter it, and dead letters are
|
||||
// a monitored site health metric.
|
||||
HandleDeploy(
|
||||
new DeployInstanceCommand(
|
||||
config.DeploymentId,
|
||||
instanceName,
|
||||
storedRevision,
|
||||
config.ConfigJson,
|
||||
$"{SweepReason} self-heal",
|
||||
DateTimeOffset.UtcNow),
|
||||
Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Terminal outcome of a sweep-driven stale-revision restart. Only the sweep makes this
|
||||
/// actor its own deployer, so any <see cref="DeploymentStatusResponse"/> arriving here is
|
||||
/// its own. A failure is surfaced and NOT retried (see
|
||||
/// <see cref="_sweepAttemptedRevisions"/>); the instance keeps serving whatever config it
|
||||
/// had, which for a compile-gate rejection is the old one, untouched.
|
||||
/// </summary>
|
||||
private void HandleSweepRestartResult(DeploymentStatusResponse response)
|
||||
{
|
||||
if (response.Status == DeploymentStatus.Success)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Instance {Instance} re-applied its stored config after a stale-revision "
|
||||
+ "{Reason} self-heal (deploymentId={DeploymentId})",
|
||||
response.InstanceUniqueName, SweepReason, response.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Stale-revision {Reason} self-heal for {Instance} did not apply: {Error}",
|
||||
SweepReason, response.InstanceUniqueName, response.ErrorMessage);
|
||||
LogDeploymentEvent("Error", response.InstanceUniqueName,
|
||||
$"Instance {response.InstanceUniqueName} stale-revision {SweepReason} self-heal failed",
|
||||
response.ErrorMessage ?? "no error detail");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1462,6 +1800,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// removes nothing and leaves the count unchanged. Supersedes the earlier
|
||||
// map-presence check, which leaked the count on disabled-instance deletes.
|
||||
_deployedInstanceNames.Remove(instanceName);
|
||||
// Delete is the one lifecycle event after which the instance name never comes back on
|
||||
// its own, so drop the sweep's per-instance bookkeeping with it rather than letting it
|
||||
// accumulate for the process lifetime.
|
||||
_runningRevisions.Remove(instanceName);
|
||||
_sweepAttemptedRevisions.Remove(instanceName);
|
||||
UpdateInstanceCounts();
|
||||
|
||||
var sender = replyTo;
|
||||
@@ -2373,7 +2716,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
/// <param name="instanceName">The unique name of the instance to create an actor for.</param>
|
||||
/// <param name="configJson">The JSON-serialized flattened configuration for the instance.</param>
|
||||
internal void CreateInstanceActor(string instanceName, string configJson)
|
||||
/// <param name="revisionHash">
|
||||
/// Revision hash of <paramref name="configJson"/>, recorded in <see cref="_runningRevisions"/>
|
||||
/// so the instance reconcile sweep can tell an actor serving the CURRENT stored config from
|
||||
/// one still serving a revision the store has replaced (deferred register row 72). This is
|
||||
/// the single creation funnel, so recording it here covers every path that can ever start an
|
||||
/// Instance Actor.
|
||||
/// </param>
|
||||
internal void CreateInstanceActor(string instanceName, string configJson, string revisionHash)
|
||||
{
|
||||
if (_instanceActors.ContainsKey(instanceName))
|
||||
{
|
||||
@@ -2398,6 +2748,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
var actorRef = Context.ActorOf(props, instanceName);
|
||||
_instanceActors[instanceName] = actorRef;
|
||||
_runningRevisions[instanceName] = revisionHash;
|
||||
// Watch every Instance Actor so an unexpected termination (init failure or
|
||||
// crash) delivers a Terminated signal that evicts the dead ref from routing
|
||||
// (S6). Redeploy already double-watches; a double Watch is idempotent in
|
||||
@@ -2557,6 +2908,24 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
internal record EnsureInstanceStartedResult(
|
||||
EnsureInstanceStartedCommand Command, DeployedInstance? Config, string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// Periodic instance reconcile-sweep tick (deferred register rows 78 + 72). Interval comes
|
||||
/// from <see cref="SiteRuntimeOptions.InstanceSweepIntervalSeconds"/>.
|
||||
/// </summary>
|
||||
internal sealed record InstanceSweepTick
|
||||
{
|
||||
/// <summary>The singleton tick instance — the message carries no state.</summary>
|
||||
public static readonly InstanceSweepTick Instance = new();
|
||||
private InstanceSweepTick() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Carries one sweep's off-thread read of the durable deployed-config rows back onto the
|
||||
/// actor thread, where every comparison and repair happens.
|
||||
/// </summary>
|
||||
internal record InstanceSweepLoaded(List<DeployedInstance> Configs, string? Error);
|
||||
|
||||
internal record DeployPersistenceResult(
|
||||
string DeploymentId, string InstanceName, bool Success, string? Error,
|
||||
IActorRef OriginalSender, bool IsRedeploy);
|
||||
|
||||
@@ -181,4 +181,25 @@ public class SiteRuntimeOptions
|
||||
/// script run under the legacy per-run behaviour.
|
||||
/// </summary>
|
||||
public int ScriptRunSummaryIntervalSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Interval (seconds) of the Deployment Manager singleton's <b>instance reconcile sweep</b>
|
||||
/// (deferred register rows 78 + 72). Every tick the singleton re-reads the durable
|
||||
/// <c>deployed_configurations</c> rows and repairs two silent divergences between the
|
||||
/// stored configs and the running Instance Actors:
|
||||
/// <list type="bullet">
|
||||
/// <item>an ENABLED instance with a stored config and NO running actor is started
|
||||
/// (row 78 — configs that arrive by LocalDb CDC replication are written straight into the
|
||||
/// file by the sync applier, so they are invisible to both the boot-time load, which has
|
||||
/// already run, and the startup reconcile, which reports no gap because central's view is
|
||||
/// correct);</item>
|
||||
/// <item>a running instance whose stored revision hash no longer matches the revision it
|
||||
/// was started from is re-applied through the ordinary redeploy path (row 72 — the startup
|
||||
/// reconcile refreshes a stale config row but leaves the running actor serving the config
|
||||
/// central believes it replaced).</item>
|
||||
/// </list>
|
||||
/// The sweep is silent when there is nothing to repair. <c>0</c> disables it entirely
|
||||
/// (the row-67 <c>EnsureInstanceStartedCommand</c> path is unaffected). Default: 60.
|
||||
/// </summary>
|
||||
public int InstanceSweepIntervalSeconds { get; set; } = 60;
|
||||
}
|
||||
|
||||
@@ -91,5 +91,12 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
|
||||
$"ScadaBridge:SiteRuntime:ScriptRunSummaryIntervalSeconds must be >= 0 " +
|
||||
$"(was {options.ScriptRunSummaryIntervalSeconds}); it is the ScriptRunSummaryRecorder " +
|
||||
"flush interval and a negative value throws inside its delay. 0 disables the flush.");
|
||||
|
||||
builder.RequireThat(options.InstanceSweepIntervalSeconds >= 0,
|
||||
$"ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds must be >= 0 " +
|
||||
$"(was {options.InstanceSweepIntervalSeconds}); it is the Deployment Manager's " +
|
||||
"instance reconcile-sweep interval and a negative value throws inside the Akka timer. " +
|
||||
"0 disables the sweep, which reopens the replication-seeded-config and stale-revision " +
|
||||
"self-heals (deferred register rows 78 and 72).");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user