fix(site-runtime): post-boot settle sweep so replicated rows start in seconds, not a sweep interval (#135)

When a replicated site pair cold-starts together, the node that wins the
Deployment Manager singleton loads its own (stale) store, and the peer's
rows arrive by LocalDb sync about a second later. The boot load has already
run and the startup reconcile rightly reports no gap (the rows are in the
store by the time it asks central), so only the instance sweep starts those
instances - at its plain 60 s cadence. Reproduced on the primary rig: boot
load without the instance at 05:03:16, sweep start at 05:04:16.

The env2 drill read this as "never started" because the operator
redeployed at +49 s, before the first sweep tick at +60 s.

Fix: when boot-time Instance Actor creation finishes, the singleton runs
one sweep at once, then sweeps every InstanceSweepSettleIntervalSeconds
(default 5) for InstanceSweepSettleWindowSeconds (default 120), then falls
back to InstanceSweepIntervalSeconds. Startup never waits for the peer, so
a site whose peer is down for good still starts alone from its own store.
LocalDb 0.3.7 exposes no "snapshot applied" signal, so the fix rides the
arrival-path-agnostic sweep with all its existing guards.

Tests: PostBootSettleSweepTests (4) + validator theory. The two positive
tests fail with the fix disabled; the window-close test fails when the
settle timer is never cancelled.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -271,6 +271,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// </summary>
     private readonly TimeSpan _instanceSweepInterval;
 
+    /// <summary>
+    /// Fast sweep cadence for the post-boot settle window (ScadaBridge#135);
+    /// <see cref="TimeSpan.Zero"/> when the settle phase is off. See
+    /// <see cref="SiteRuntimeOptions.InstanceSweepSettleIntervalSeconds"/>.
+    /// </summary>
+    private readonly TimeSpan _instanceSweepSettleInterval;
+
+    /// <summary>
+    /// Length of the post-boot settle window (ScadaBridge#135). See
+    /// <see cref="SiteRuntimeOptions.InstanceSweepSettleWindowSeconds"/>.
+    /// </summary>
+    private readonly TimeSpan _instanceSweepSettleWindow;
+
     /// <summary>
     /// True once boot-time Instance Actor creation has finished (every staggered batch created,
     /// or there were no enabled configs to create). The sweep is inert until then: a tick landing
@@ -383,6 +396,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _instanceSweepInterval = options.InstanceSweepIntervalSeconds > 0
             ? TimeSpan.FromSeconds(options.InstanceSweepIntervalSeconds)
             : TimeSpan.Zero;
+        _instanceSweepSettleInterval = options.InstanceSweepSettleIntervalSeconds > 0
+            ? TimeSpan.FromSeconds(options.InstanceSweepSettleIntervalSeconds)
+            : TimeSpan.Zero;
+        _instanceSweepSettleWindow = options.InstanceSweepSettleWindowSeconds > 0
+            ? TimeSpan.FromSeconds(options.InstanceSweepSettleWindowSeconds)
+            : TimeSpan.Zero;
         _configLoader = configLoader ?? (() => _storage.GetAllDeployedConfigsAsync());
         _deployedConfigLoader = deployedConfigLoader ?? (name => _storage.GetDeployedConfigAsync(name));
         _configMetadataLoader = configMetadataLoader ?? (() => _storage.GetAllDeployedConfigMetadataAsync());
@@ -430,6 +449,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         // an enabled config with no running actor (row 78), and a running actor whose revision
         // no longer matches the stored one (row 72).
         Receive<InstanceSweepTick>(_ => HandleInstanceSweepTick());
+        Receive<InstanceSweepSettleEnded>(_ => Timers.Cancel(InstanceSweepSettleTimerKey));
         Receive<InstanceSweepLoaded>(HandleInstanceSweepLoaded);
         // 01/SR-P1: the full rows of the instances a sweep decided to start/restart.
         Receive<InstanceSweepRepairsLoaded>(HandleInstanceSweepRepairsLoaded);
@@ -842,7 +862,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
             // reconcile sweep may run. This is exactly the live case: the node booted
             // with an empty deployed_configurations table and CDC replication seeds it moments
             // later, with no boot load left to notice.
-            _startupInstanceCreationComplete = true;
+            MarkStartupInstanceCreationComplete();
             return;
         }
 
@@ -998,7 +1018,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
                 "All {Count} Instance Actors created", state.Configs.Count);
             // Boot-time creation is done — the instance reconcile sweep may now run without
             // racing the staggered batches it would otherwise mistake for a divergence.
-            _startupInstanceCreationComplete = true;
+            MarkStartupInstanceCreationComplete();
         }
     }
 
@@ -2103,6 +2123,58 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
             $"Instance {instanceName} started by {reason} self-heal", details);
     }
 
+    /// <summary>Timer key of the fast post-boot settle sweep (ScadaBridge#135).</summary>
+    private const string InstanceSweepSettleTimerKey = "instance-sweep-settle";
+
+    /// <summary>
+    /// Ends boot-time Instance Actor creation and opens the post-boot settle window
+    /// (ScadaBridge#135).
+    ///
+    /// <para>
+    /// <b>The race this closes.</b> On a replicated pair that cold-starts together, the node that
+    /// wins the singleton loads its OWN store, and the peer's rows arrive by LocalDb sync about a
+    /// second later — a first enable on diverged stores, or a node that was down while its peer
+    /// took deploys. The boot load has already run, and the startup reconcile reports no gap
+    /// because the rows ARE in the store by the time it asks central. Only the sweep starts those
+    /// instances, and at its plain cadence (60 s by default) the site collected nothing for up to
+    /// a full interval after a healthy-looking boot. LocalDb exposes no "snapshot applied" signal,
+    /// so the fix rides the arrival-path-agnostic sweep: one sweep NOW (rows that landed while the
+    /// boot batches were being created), then a fast cadence for a bounded window, then the plain
+    /// interval.
+    /// </para>
+    ///
+    /// <para>
+    /// <b>Startup never waits.</b> The node has already started everything its own store holds, so
+    /// a peer that is down for good costs only a few cheap metadata reads. Every guard the sweep
+    /// already carries (deploy-pipeline deferral, per-revision claims, the store-generation check)
+    /// applies unchanged, and a sweep that finds nothing stays silent.
+    /// </para>
+    /// </summary>
+    private void MarkStartupInstanceCreationComplete()
+    {
+        if (_startupInstanceCreationComplete)
+            return;
+
+        _startupInstanceCreationComplete = true;
+
+        if (_instanceSweepInterval <= TimeSpan.Zero)
+            return;
+
+        Self.Tell(InstanceSweepTick.Instance);
+
+        if (_instanceSweepSettleInterval > TimeSpan.Zero &&
+            _instanceSweepSettleWindow > TimeSpan.Zero &&
+            _instanceSweepSettleInterval < _instanceSweepInterval)
+        {
+            Timers.StartPeriodicTimer(
+                InstanceSweepSettleTimerKey, InstanceSweepTick.Instance,
+                _instanceSweepSettleInterval, _instanceSweepSettleInterval);
+            Timers.StartSingleTimer(
+                "instance-sweep-settle-end", InstanceSweepSettleEnded.Instance,
+                _instanceSweepSettleWindow);
+        }
+    }
+
     /// <summary>Provenance string carried on every repair the instance reconcile sweep makes.</summary>
     private const string SweepReason = "deployment-manager sweep";
 
@@ -4304,6 +4376,17 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         private InstanceSweepTick() { }
     }
 
+    /// <summary>
+    /// Closes the post-boot settle window (ScadaBridge#135): the fast settle sweep timer is
+    /// cancelled and the sweep continues at <see cref="SiteRuntimeOptions.InstanceSweepIntervalSeconds"/>.
+    /// </summary>
+    internal sealed record InstanceSweepSettleEnded
+    {
+        /// <summary>The singleton instance — the message carries no state.</summary>
+        public static readonly InstanceSweepSettleEnded Instance = new();
+        private InstanceSweepSettleEnded() { }
+    }
+
     /// <summary>
     /// Carries one sweep's off-thread read of the durable deployed-config rows back onto the
     /// actor thread, where every comparison and repair happens.

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs

@@ -242,6 +242,36 @@ public class SiteRuntimeOptions
     /// </summary>
     public int InstanceSweepIntervalSeconds { get; set; } = 60;
 
+    /// <summary>
+    /// Fast cadence (seconds) of the instance reconcile sweep during the <b>post-boot settle
+    /// window</b> (ScadaBridge#135). When the Deployment Manager singleton finishes its boot-time
+    /// Instance Actor creation it runs one sweep at once, then sweeps every
+    /// <see cref="InstanceSweepSettleIntervalSeconds"/> for
+    /// <see cref="InstanceSweepSettleWindowSeconds"/> before falling back to
+    /// <see cref="InstanceSweepIntervalSeconds"/>.
+    /// <para>
+    /// Why: on a replicated pair the node that boots active loads its OWN store, and the peer's
+    /// rows arrive by LocalDb sync a moment later (a first enable on diverged stores, or a node
+    /// that was down while its peer took deploys). Those rows are invisible to the boot load and to
+    /// the startup reconcile (central rightly sees no gap), so only the sweep starts them — and at
+    /// the plain 60 s cadence that left the site collecting nothing for up to a full minute after a
+    /// cold start. Startup is never delayed: the node starts from its own store at once and a peer
+    /// that never answers costs nothing but a few cheap metadata reads.
+    /// </para>
+    /// <c>0</c> disables the settle phase (the immediate post-boot sweep still runs). It only
+    /// applies when the sweep is enabled and is faster than <see cref="InstanceSweepIntervalSeconds"/>.
+    /// Default: 5.
+    /// </summary>
+    public int InstanceSweepSettleIntervalSeconds { get; set; } = 5;
+
+    /// <summary>
+    /// Length (seconds) of the post-boot settle window, measured from the end of boot-time
+    /// Instance Actor creation, during which the sweep runs at
+    /// <see cref="InstanceSweepSettleIntervalSeconds"/> (ScadaBridge#135). <c>0</c> disables
+    /// the settle phase. Default: 120.
+    /// </summary>
+    public int InstanceSweepSettleWindowSeconds { get; set; } = 120;
+
     /// <summary>
     /// Flow Debugger trace arm window (design 2026-09-05 §5.3). When a TRACED script write
     /// targets a data-sourced attribute, the Instance Actor remembers

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs

@@ -113,6 +113,18 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
             "0 disables the sweep, which reopens the replication-seeded-config and stale-revision " +
             "self-heals (deferred register rows 78 and 72).");
 
+        builder.RequireThat(options.InstanceSweepSettleIntervalSeconds >= 0,
+            $"ScadaBridge:SiteRuntime:InstanceSweepSettleIntervalSeconds must be >= 0 " +
+            $"(was {options.InstanceSweepSettleIntervalSeconds}); it is the instance sweep's fast " +
+            "post-boot cadence and a negative value throws inside the Akka timer. 0 disables the " +
+            "settle phase (ScadaBridge#135).");
+
+        builder.RequireThat(options.InstanceSweepSettleWindowSeconds >= 0,
+            $"ScadaBridge:SiteRuntime:InstanceSweepSettleWindowSeconds must be >= 0 " +
+            $"(was {options.InstanceSweepSettleWindowSeconds}); it is how long the instance sweep " +
+            "runs at its fast post-boot cadence and a negative value throws inside the Akka timer. " +
+            "0 disables the settle phase (ScadaBridge#135).");
+
         builder.RequireThat(options.TraceArmWindowSeconds > 0,
             $"ScadaBridge:SiteRuntime:TraceArmWindowSeconds must be greater than 0 " +
             $"(was {options.TraceArmWindowSeconds}); it is how long the Instance Actor waits for the " +