Merge rows 78+72: DeploymentManager singleton instance sweep — starts replication-seeded enabled instances and converges stale revisions through the real HandleDeploy path; one repair attempt per (instance, revision)

This commit is contained in:
Joseph Doherty
2026-08-16 04:17:45 -04:00
7 changed files with 1186 additions and 16 deletions
@@ -159,6 +159,58 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
private readonly ScriptExecutionScheduler? _scriptScheduler;
/// <summary>
/// Deferred register row 72: the config revision hash each RUNNING Instance Actor was
/// created from, keyed by instance name. Written by <see cref="CreateInstanceActor"/> —
/// the single creation funnel — and compared against the stored row by the instance
/// reconcile sweep to detect an actor still serving a config the durable store has since
/// replaced.
/// <para>
/// Reads are always guarded by <c>_instanceActors.ContainsKey</c>, so a leftover entry for
/// an instance that has stopped is never consulted, and a stop-then-recreate overwrites it.
/// An instance with no entry (only reachable if a future creation path bypasses the funnel)
/// is treated as "revision unknown" and is deliberately never restarted.
/// </para>
/// </summary>
private readonly Dictionary<string, string> _runningRevisions = new();
/// <summary>
/// The stored revision hash the instance reconcile sweep has already ACTED on for an
/// instance, keyed by instance name — <b>one repair attempt per (instance, revision)
/// pair</b>, for both the row-78 start and the row-72 restart.
/// <para>
/// This is what keeps a periodic self-heal from becoming a restart loop. An instance whose
/// Instance Actor dies during init leaves its config row in place, so without this the
/// sweep would recreate it — and re-announce it — every tick forever; likewise a stored
/// config the site compile gate rejects would be re-attempted forever. The divergence is
/// still surfaced either way: the attempt itself emits a Warning site event, and a failed
/// restart emits a second one. A genuinely NEW revision (the normal case: central redeploys
/// something better) is a different key and is always attempted.
/// </para>
/// </summary>
private readonly Dictionary<string, string> _sweepAttemptedRevisions = new();
/// <summary>
/// Interval of the instance reconcile sweep (rows 78 + 72);
/// <see cref="TimeSpan.Zero"/> disables it. See
/// <see cref="SiteRuntimeOptions.InstanceSweepIntervalSeconds"/>.
/// </summary>
private readonly TimeSpan _instanceSweepInterval;
/// <summary>
/// True once boot-time Instance Actor creation has finished (every staggered batch created,
/// or there were no enabled configs to create). The sweep is inert until then: a tick landing
/// between two staggered batches would otherwise "heal" instances the boot load is already
/// about to create, emitting a Warning site event for a node that is merely still starting.
/// </summary>
private bool _startupInstanceCreationComplete;
/// <summary>
/// True while a sweep's off-thread storage read is outstanding. A slow read must not let
/// ticks stack up into overlapping reads (and duplicate repairs) on a large site.
/// </summary>
private bool _instanceSweepInFlight;
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -224,6 +276,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_serviceProvider = serviceProvider;
_configFetcher = configFetcher;
_startupLoadRetryInterval = startupLoadRetryInterval ?? TimeSpan.FromSeconds(5);
_instanceSweepInterval = options.InstanceSweepIntervalSeconds > 0
? TimeSpan.FromSeconds(options.InstanceSweepIntervalSeconds)
: TimeSpan.Zero;
_configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
_logger = logger;
// Reuse a single logger factory for all Instance Actors.
@@ -262,6 +317,18 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Receive<EnsureInstanceStartedCommand>(HandleEnsureInstanceStarted);
Receive<EnsureInstanceStartedResult>(HandleEnsureInstanceStartedResult);
// Instance reconcile sweep (deferred register rows 78 + 72). Periodically re-reads the
// durable rows and repairs the two ways the runtime can silently diverge from them:
// an enabled config with no running actor (row 78), and a running actor whose revision
// no longer matches the stored one (row 72).
Receive<InstanceSweepTick>(_ => HandleInstanceSweepTick());
Receive<InstanceSweepLoaded>(HandleInstanceSweepLoaded);
// Terminal reply of a sweep-driven stale-revision restart. Every OTHER deploy path
// replies to central (or to a test probe); only the sweep makes this actor its own
// deployer, so a response landing here is always its own and is surfaced, never
// dead-lettered.
Receive<DeploymentStatusResponse>(HandleSweepRestartResult);
// Query-the-site-before-redeploy idempotency.
// Central asks for the instance's currently-applied deployment identity
// before re-sending a deployment whose prior record is stuck InProgress
@@ -380,6 +447,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
typeof(ClusterEvent.MemberUp));
}
// Instance reconcile sweep (rows 78 + 72). Armed here rather than after the startup
// load so it survives a load that never succeeds; the handler itself is inert until
// boot-time Instance Actor creation has finished.
if (_instanceSweepInterval > TimeSpan.Zero)
{
Timers.StartPeriodicTimer(
"instance-sweep", InstanceSweepTick.Instance,
_instanceSweepInterval, _instanceSweepInterval);
}
LoadDeployedConfigs();
}
@@ -491,7 +568,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
msg.CompiledCount, msg.TotalCount);
if (msg.EnabledConfigs.Count == 0)
{
// Nothing to create, so boot-time creation is trivially complete and the instance
// reconcile sweep may run. This is exactly the row-78 live case: the node booted
// with an empty deployed_configurations table and CDC replication seeds it moments
// later, with no boot load left to notice.
_startupInstanceCreationComplete = true;
return;
}
// Start the first batch immediately
var batchState = new BatchState(msg.EnabledConfigs, 0);
@@ -563,7 +647,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
var config = state.Configs[i];
EnsureDclConnections(config.ConfigJson);
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson);
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson, config.RevisionHash);
}
UpdateInstanceCounts();
@@ -581,6 +665,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
_logger.LogInformation(
"All {Count} Instance Actors created", state.Configs.Count);
// Boot-time creation is done — the instance reconcile sweep may now run without
// racing the staggered batches it would otherwise mistake for a divergence.
_startupInstanceCreationComplete = true;
}
}
@@ -993,7 +1080,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
EnsureDclConnections(command.FlattenedConfigurationJson);
// Create the Instance Actor immediately
CreateInstanceActor(instanceName, command.FlattenedConfigurationJson);
CreateInstanceActor(instanceName, command.FlattenedConfigurationJson, command.RevisionHash);
if (!isRedeploy)
_deployedInstanceNames.Add(instanceName);
UpdateInstanceCounts();
@@ -1277,7 +1364,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
if (!_instanceActors.ContainsKey(instanceName))
{
CreateInstanceActor(instanceName, result.Config.ConfigJson);
CreateInstanceActor(instanceName, result.Config.ConfigJson, result.Config.RevisionHash);
}
UpdateInstanceCounts();
@@ -1335,6 +1422,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
.PipeTo(Self);
}
/// <summary>
/// True when a deploy/redeploy/delete pipeline currently OWNS the instance — a compile warm
/// is in flight, a predecessor is still terminating, or a deployment is awaiting its
/// persistence/init join. Every advisory self-heal (row 67's ensure-started and the row
/// 78/72 sweep) defers to those paths: they are authoritative, they will create or replace
/// the actor themselves, and stepping in front of them would break the deploy gate's
/// ordering guarantees and the all-or-nothing-per-instance rule.
/// </summary>
private bool InstanceIsOwnedByADeployPipeline(string instanceName) =>
_deployWarms.ContainsKey(instanceName) ||
_terminatingActorsByName.ContainsKey(instanceName) ||
_inFlightDeployments.ContainsKey(instanceName);
/// <summary>
/// Shared guard for the two halves of the ensure-started round trip. Re-evaluated after
/// the storage read because the instance's state can change while that read is in flight
@@ -1342,9 +1442,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
private bool ShouldAttemptEnsureStart(string instanceName) =>
!_instanceActors.ContainsKey(instanceName) &&
!_deployWarms.ContainsKey(instanceName) &&
!_terminatingActorsByName.ContainsKey(instanceName) &&
!_inFlightDeployments.ContainsKey(instanceName);
!InstanceIsOwnedByADeployPipeline(instanceName);
/// <summary>
/// Actor-thread half of the reconcile self-heal: creates the Instance Actor for a config
@@ -1396,21 +1494,261 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
return;
}
CreateInstanceActor(instanceName, result.Config.ConfigJson);
StartStoredInstance(result.Config, reason,
"Config was missing at node boot and was restored by reconciliation; "
+ "the Instance Actor was not running and has now been started.");
}
/// <summary>
/// The one place an advisory self-heal turns a stored, enabled config into a running
/// Instance Actor — shared by the row-67 ensure-started handler and the row-78 sweep so
/// both produce identical bookkeeping and identical operator-facing surfacing. Callers
/// are responsible for the guards; this method assumes they have passed.
/// </summary>
/// <param name="config">The durable row to start from — the authority on config and revision.</param>
/// <param name="reason">Provenance, carried verbatim into the log line and the site event.</param>
/// <param name="details">Longer explanation attached to the site event.</param>
private void StartStoredInstance(DeployedInstance config, string reason, string details)
{
var instanceName = config.InstanceUniqueName;
CreateInstanceActor(instanceName, config.ConfigJson, config.RevisionHash);
_deployedInstanceNames.Add(instanceName);
UpdateInstanceCounts();
// Surface the heal. Warning (not Info) on purpose: reaching here means the node
// booted WITHOUT a config for an enabled, successfully-deployed instance — the
// condition is repaired, but it is abnormal and worth an operator's attention.
// Surface the heal. Warning (not Info) on purpose: reaching here means an enabled,
// successfully-deployed instance was NOT running while its config sat in the store —
// the condition is repaired, but it is abnormal and worth an operator's attention.
_logger.LogWarning(
"Instance {Instance} started by {Reason} self-heal — its config was missing at boot "
+ "and has been restored; the instance was NOT running until now",
"Instance {Instance} started by {Reason} self-heal — its config is present in the "
+ "deployed-config store but no Instance Actor was running until now",
instanceName, reason);
LogDeploymentEvent("Warning", instanceName,
$"Instance {instanceName} started by {reason} self-heal",
"Config was missing at node boot and was restored by reconciliation; "
+ "the Instance Actor was not running and has now been started.");
$"Instance {instanceName} started by {reason} self-heal", details);
}
/// <summary>Provenance string carried on every repair the instance reconcile sweep makes.</summary>
private const string SweepReason = "deployment-manager sweep";
/// <summary>
/// Instance reconcile sweep, tick half (deferred register rows 78 + 72). Kicks off an
/// off-thread read of the durable <c>deployed_configurations</c> rows; the comparison and
/// every repair happen on the actor thread in
/// <see cref="HandleInstanceSweepLoaded"/>.
///
/// <para>
/// <b>Why a periodic sweep and not a LocalDb change hook.</b> Row 78's live symptom was a
/// config that arrived by CDC <i>replication</i>: the sync applier writes
/// <c>deployed_configurations</c> rows straight into the file, which is invisible to the
/// boot-time load (already run) and to the startup reconcile (central reports no gap,
/// because central's view is correct — the site really does have the config). A hook on the
/// replication apply would cover exactly that one arrival path, and only for the half of
/// the fleet that runs with replication enabled. It is also not available: <c>ILocalDb</c>
/// exposes no change-notification or applied-rows callback at all, so a hook would mean a
/// new upstream library capability plus an in-process bus from the sync applier into this
/// singleton. A sweep over the durable rows is the arrival-path-agnostic form of the same
/// check — replication seeding, a post-boot snapshot resync, an operator restoring a file,
/// and any future path all converge on "the store says X, the runtime says Y" — and it
/// inherits row 67's guards unchanged.
/// </para>
///
/// <para>
/// <b>Inert until boot-time creation finishes.</b> A tick landing between two staggered
/// startup batches would see instances the boot load is about to create and "heal" them,
/// announcing a fault on a node that is merely still starting.
/// </para>
/// </summary>
private void HandleInstanceSweepTick()
{
if (!_startupInstanceCreationComplete || _instanceSweepInFlight)
return;
_instanceSweepInFlight = true;
_configLoader().ContinueWith(t => t.IsCompletedSuccessfully
? new InstanceSweepLoaded(t.Result, null)
: new InstanceSweepLoaded([], t.Exception?.GetBaseException().Message))
.PipeTo(Self);
}
/// <summary>
/// Instance reconcile sweep, actor-thread half: compares the durable rows against the
/// running Instance Actors and repairs the two divergences the register recorded.
///
/// <para>
/// A sweep that finds nothing is <b>completely silent</b> — no log line above Trace, no site
/// event — because that is the case on every tick of a healthy site.
/// </para>
/// </summary>
private void HandleInstanceSweepLoaded(InstanceSweepLoaded msg)
{
_instanceSweepInFlight = false;
if (msg.Error != null)
{
_logger.LogWarning(
"Instance reconcile sweep could not read the deployed configs: {Error}", msg.Error);
return;
}
foreach (var config in msg.Configs)
{
var instanceName = config.InstanceUniqueName;
// A disabled instance legitimately has a config row and no running actor; that is
// its correct steady state, not a divergence.
if (!config.IsEnabled)
continue;
// Defer to any deploy pipeline that owns the instance right now. It is
// authoritative and will create or replace the actor itself.
if (InstanceIsOwnedByADeployPipeline(instanceName))
continue;
if (!_instanceActors.ContainsKey(instanceName))
{
// Row 78 — an enabled config with no actor.
if (!TryClaimSweepAttempt(instanceName, config.RevisionHash))
continue;
StartStoredInstance(config, SweepReason,
"An enabled instance had a deployed config in the site store but no running "
+ "Instance Actor. Configs that arrive by LocalDb replication (a standby "
+ "seeded from its peer, or a snapshot resync) are written after the "
+ "boot-time load and are invisible to the startup reconcile, so nothing "
+ "else would have started it.");
continue;
}
// Row 72 — a running actor serving a revision the store has replaced.
TryRestartStaleInstance(config);
}
}
/// <summary>
/// Claims the single sweep repair attempt for an (instance, revision) pair. Returns
/// <see langword="false"/> when this exact revision has already been attempted — see
/// <see cref="_sweepAttemptedRevisions"/> for why a periodic self-heal must not retry the
/// same repair forever.
/// </summary>
private bool TryClaimSweepAttempt(string instanceName, string revisionHash)
{
if (_sweepAttemptedRevisions.TryGetValue(instanceName, out var attempted) &&
string.Equals(attempted, revisionHash, StringComparison.Ordinal))
{
return false;
}
_sweepAttemptedRevisions[instanceName] = revisionHash;
return true;
}
/// <summary>
/// Deferred register row 72 — <b>the running Instance Actor is serving a config the durable
/// store has replaced.</b>
///
/// <para>
/// The startup reconcile refreshes a STALE row via <c>StoreDeployedConfigIfNewerAsync</c>
/// (and a CDC-replicated deploy lands the same way), which heals the file while the running
/// actor keeps executing the previously compiled config until the next restart or redeploy.
/// Nothing surfaced the difference.
/// </para>
///
/// <para>
/// <b>The decision (2026-08-15, <c>docs/plans/2026-08-15-instance-sweep-self-heal-design.md</c>):
/// converge, do not merely warn.</b> The stored row is what central successfully deployed;
/// the running actor is executing a config central believes was replaced. Serving it
/// indefinitely is the actual fault. The stored config is therefore applied through the
/// EXACTLY the same path a central redeploy uses — <see cref="HandleDeploy"/>, so the
/// compile gate, the warm-then-gate ordering, the stop/watch/recreate handshake, the
/// static-override and native-alarm reset, and all-or-nothing-per-instance are the ones
/// already in production, not a second implementation. A stop still does not cancel an
/// in-flight script run, exactly as for a central redeploy.
/// </para>
///
/// <para>
/// <b>Safety.</b> The restart is attempted once per stored revision, so a config the
/// compile gate rejects (which leaves the running actor untouched — the gate returns before
/// anything is stopped) is announced but never looped on. The sweep is also skipped
/// entirely while a deploy pipeline owns the instance, so it can never interleave with a
/// real redeploy.
/// </para>
/// </summary>
private void TryRestartStaleInstance(DeployedInstance config)
{
var instanceName = config.InstanceUniqueName;
var storedRevision = config.RevisionHash;
// No stored revision to compare against — nothing can be concluded, so do nothing.
if (string.IsNullOrEmpty(storedRevision))
return;
// No recorded running revision. Unreachable while CreateInstanceActor is the single
// creation funnel; treated as "unknown", never as "stale", so a future path that
// bypasses the funnel degrades to no-op rather than to spurious restarts.
if (!_runningRevisions.TryGetValue(instanceName, out var runningRevision) ||
string.IsNullOrEmpty(runningRevision))
{
return;
}
if (string.Equals(runningRevision, storedRevision, StringComparison.Ordinal))
return;
if (!TryClaimSweepAttempt(instanceName, storedRevision))
return;
_logger.LogWarning(
"Instance {Instance} is running stale revision {RunningRevision} while the deployed-config "
+ "store holds {StoredRevision} — applying the stored config through the redeploy path "
+ "({Reason} self-heal)",
instanceName, runningRevision, storedRevision, SweepReason);
LogDeploymentEvent("Warning", instanceName,
$"Instance {instanceName} restarted by {SweepReason} self-heal — "
+ $"running revision {runningRevision} is stale, applying stored revision {storedRevision}",
"The site's deployed-config store was refreshed out of band (startup reconcile "
+ "refreshing a stale row, or a replicated deploy) while the Instance Actor kept "
+ "serving the previously compiled config. The stored config is what central "
+ "successfully deployed, so it is applied through the normal redeploy path.");
// Reply to Self: this actor is its own deployer here, and HandleSweepRestartResult
// surfaces the outcome. Replying to NoSender would dead-letter it, and dead letters are
// a monitored site health metric.
HandleDeploy(
new DeployInstanceCommand(
config.DeploymentId,
instanceName,
storedRevision,
config.ConfigJson,
$"{SweepReason} self-heal",
DateTimeOffset.UtcNow),
Self);
}
/// <summary>
/// Terminal outcome of a sweep-driven stale-revision restart. Only the sweep makes this
/// actor its own deployer, so any <see cref="DeploymentStatusResponse"/> arriving here is
/// its own. A failure is surfaced and NOT retried (see
/// <see cref="_sweepAttemptedRevisions"/>); the instance keeps serving whatever config it
/// had, which for a compile-gate rejection is the old one, untouched.
/// </summary>
private void HandleSweepRestartResult(DeploymentStatusResponse response)
{
if (response.Status == DeploymentStatus.Success)
{
_logger.LogInformation(
"Instance {Instance} re-applied its stored config after a stale-revision "
+ "{Reason} self-heal (deploymentId={DeploymentId})",
response.InstanceUniqueName, SweepReason, response.DeploymentId);
return;
}
_logger.LogWarning(
"Stale-revision {Reason} self-heal for {Instance} did not apply: {Error}",
SweepReason, response.InstanceUniqueName, response.ErrorMessage);
LogDeploymentEvent("Error", response.InstanceUniqueName,
$"Instance {response.InstanceUniqueName} stale-revision {SweepReason} self-heal failed",
response.ErrorMessage ?? "no error detail");
}
/// <summary>
@@ -1462,6 +1800,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// removes nothing and leaves the count unchanged. Supersedes the earlier
// map-presence check, which leaked the count on disabled-instance deletes.
_deployedInstanceNames.Remove(instanceName);
// Delete is the one lifecycle event after which the instance name never comes back on
// its own, so drop the sweep's per-instance bookkeeping with it rather than letting it
// accumulate for the process lifetime.
_runningRevisions.Remove(instanceName);
_sweepAttemptedRevisions.Remove(instanceName);
UpdateInstanceCounts();
var sender = replyTo;
@@ -2373,7 +2716,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
/// <param name="instanceName">The unique name of the instance to create an actor for.</param>
/// <param name="configJson">The JSON-serialized flattened configuration for the instance.</param>
internal void CreateInstanceActor(string instanceName, string configJson)
/// <param name="revisionHash">
/// Revision hash of <paramref name="configJson"/>, recorded in <see cref="_runningRevisions"/>
/// so the instance reconcile sweep can tell an actor serving the CURRENT stored config from
/// one still serving a revision the store has replaced (deferred register row 72). This is
/// the single creation funnel, so recording it here covers every path that can ever start an
/// Instance Actor.
/// </param>
internal void CreateInstanceActor(string instanceName, string configJson, string revisionHash)
{
if (_instanceActors.ContainsKey(instanceName))
{
@@ -2398,6 +2748,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
var actorRef = Context.ActorOf(props, instanceName);
_instanceActors[instanceName] = actorRef;
_runningRevisions[instanceName] = revisionHash;
// Watch every Instance Actor so an unexpected termination (init failure or
// crash) delivers a Terminated signal that evicts the dead ref from routing
// (S6). Redeploy already double-watches; a double Watch is idempotent in
@@ -2557,6 +2908,24 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
internal record EnsureInstanceStartedResult(
EnsureInstanceStartedCommand Command, DeployedInstance? Config, string? Error);
/// <summary>
/// Periodic instance reconcile-sweep tick (deferred register rows 78 + 72). Interval comes
/// from <see cref="SiteRuntimeOptions.InstanceSweepIntervalSeconds"/>.
/// </summary>
internal sealed record InstanceSweepTick
{
/// <summary>The singleton tick instance — the message carries no state.</summary>
public static readonly InstanceSweepTick Instance = new();
private InstanceSweepTick() { }
}
/// <summary>
/// Carries one sweep's off-thread read of the durable deployed-config rows back onto the
/// actor thread, where every comparison and repair happens.
/// </summary>
internal record InstanceSweepLoaded(List<DeployedInstance> Configs, string? Error);
internal record DeployPersistenceResult(
string DeploymentId, string InstanceName, bool Success, string? Error,
IActorRef OriginalSender, bool IsRedeploy);
@@ -181,4 +181,25 @@ public class SiteRuntimeOptions
/// script run under the legacy per-run behaviour.
/// </summary>
public int ScriptRunSummaryIntervalSeconds { get; set; } = 300;
/// <summary>
/// Interval (seconds) of the Deployment Manager singleton's <b>instance reconcile sweep</b>
/// (deferred register rows 78 + 72). Every tick the singleton re-reads the durable
/// <c>deployed_configurations</c> rows and repairs two silent divergences between the
/// stored configs and the running Instance Actors:
/// <list type="bullet">
/// <item>an ENABLED instance with a stored config and NO running actor is started
/// (row 78 — configs that arrive by LocalDb CDC replication are written straight into the
/// file by the sync applier, so they are invisible to both the boot-time load, which has
/// already run, and the startup reconcile, which reports no gap because central's view is
/// correct);</item>
/// <item>a running instance whose stored revision hash no longer matches the revision it
/// was started from is re-applied through the ordinary redeploy path (row 72 — the startup
/// reconcile refreshes a stale config row but leaves the running actor serving the config
/// central believes it replaced).</item>
/// </list>
/// The sweep is silent when there is nothing to repair. <c>0</c> disables it entirely
/// (the row-67 <c>EnsureInstanceStartedCommand</c> path is unaffected). Default: 60.
/// </summary>
public int InstanceSweepIntervalSeconds { get; set; } = 60;
}
@@ -91,5 +91,12 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
$"ScadaBridge:SiteRuntime:ScriptRunSummaryIntervalSeconds must be >= 0 " +
$"(was {options.ScriptRunSummaryIntervalSeconds}); it is the ScriptRunSummaryRecorder " +
"flush interval and a negative value throws inside its delay. 0 disables the flush.");
builder.RequireThat(options.InstanceSweepIntervalSeconds >= 0,
$"ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds must be >= 0 " +
$"(was {options.InstanceSweepIntervalSeconds}); it is the Deployment Manager's " +
"instance reconcile-sweep interval and a negative value throws inside the Akka timer. " +
"0 disables the sweep, which reopens the replication-seeded-config and stale-revision " +
"self-heals (deferred register rows 78 and 72).");
}
}