fix(site-runtime): post-boot settle sweep so replicated rows start in seconds, not a sweep interval (#135)
When a replicated site pair cold-starts together, the node that wins the Deployment Manager singleton loads its own (stale) store, and the peer's rows arrive by LocalDb sync about a second later. The boot load has already run and the startup reconcile rightly reports no gap (the rows are in the store by the time it asks central), so only the instance sweep starts those instances - at its plain 60 s cadence. Reproduced on the primary rig: boot load without the instance at 05:03:16, sweep start at 05:04:16. The env2 drill read this as "never started" because the operator redeployed at +49 s, before the first sweep tick at +60 s. Fix: when boot-time Instance Actor creation finishes, the singleton runs one sweep at once, then sweeps every InstanceSweepSettleIntervalSeconds (default 5) for InstanceSweepSettleWindowSeconds (default 120), then falls back to InstanceSweepIntervalSeconds. Startup never waits for the peer, so a site whose peer is down for good still starts alone from its own store. LocalDb 0.3.7 exposes no "snapshot applied" signal, so the fix rides the arrival-path-agnostic sweep with all its existing guards. Tests: PostBootSettleSweepTests (4) + validator theory. The two positive tests fail with the fix disabled; the window-close test fails when the settle timer is never cancelled. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,240 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Regression tests for <b>ScadaBridge#135</b>: a replicated site pair cold-starts together, the
|
||||
/// node that wins the Deployment Manager singleton loads its OWN (stale) store, and the peer's
|
||||
/// rows arrive by LocalDb sync about a second later. Those rows are invisible to the boot load
|
||||
/// (already run) and to the startup reconcile (central sees no gap — the rows ARE in the store by
|
||||
/// then), so only the instance sweep starts them. At its plain 60 s cadence the site ran none of
|
||||
/// the replicated instances for a full interval after a healthy-looking boot — reproduced on the
|
||||
/// primary rig 2026-11-13 (boot load 05:03:16 without the instance, sweep start at 05:04:16).
|
||||
///
|
||||
/// <para>
|
||||
/// The fix opens a post-boot settle window: one sweep the moment boot-time creation finishes,
|
||||
/// then a fast cadence for a bounded window, then the plain interval. Startup itself never waits
|
||||
/// for the peer. Every test here pins the plain interval at 60 s or more, so a pass can only come
|
||||
/// from the settle path — a plain-cadence sweep cannot land inside the assertion budgets.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class PostBootSettleSweepTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
/// <summary>The plain sweep interval: far longer than any assertion budget below.</summary>
|
||||
private const int PlainSweepSeconds = 60;
|
||||
|
||||
/// <summary>Well inside one plain interval, comfortably more than a few settle ticks.</summary>
|
||||
private static readonly TimeSpan HealBudget = TimeSpan.FromSeconds(8);
|
||||
|
||||
/// <summary>Creates a temporary LocalDb-backed site store and the compilation services for each test.</summary>
|
||||
public PostBootSettleSweepTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("post-boot-settle-sweep-test");
|
||||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedScriptLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
|
||||
Shutdown();
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read
|
||||
/// storage (its first instance-count report), so any row written afterwards is invisible to
|
||||
/// the boot load — the replication arrival this issue is about.
|
||||
/// </summary>
|
||||
private IActorRef CreateBootedDeploymentManager(
|
||||
SiteRuntimeOptions options,
|
||||
Func<Task<List<DeployedInstance>>>? configLoader = null)
|
||||
{
|
||||
var health = new DeploymentManagerStartupProbe();
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage,
|
||||
_compilationService,
|
||||
_sharedScriptLibrary,
|
||||
null,
|
||||
options,
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
null,
|
||||
health,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
configLoader,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null)));
|
||||
|
||||
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
|
||||
TimeSpan.FromMilliseconds(25));
|
||||
return dm;
|
||||
}
|
||||
|
||||
private static string MakeConfigJson(string instanceName) =>
|
||||
JsonSerializer.Serialize(new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
Attributes =
|
||||
[
|
||||
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
|
||||
]
|
||||
});
|
||||
|
||||
private async Task<bool> InstanceIsRunningAsync(IActorRef dm, string instanceName) =>
|
||||
!(await dm.Ask<DebugViewSnapshot>(
|
||||
new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()),
|
||||
TimeSpan.FromSeconds(5))).InstanceNotFound;
|
||||
|
||||
/// <summary>
|
||||
/// THE #135 reproduction. The node boots on a stale store holding one instance; the peer's
|
||||
/// snapshot then adds a second enabled instance as a bare storage write (the sync applier's
|
||||
/// only side effect). The stale instance runs from the boot load, and the replicated one must
|
||||
/// start within seconds — not one full plain sweep interval later.
|
||||
///
|
||||
/// <para><b>Fails before the fix</b>: with only the 60 s plain sweep, the replicated instance
|
||||
/// is still not running when the budget expires.</para>
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReplicatedRowArrivingJustAfterBoot_StartsWithinTheSettleWindow()
|
||||
{
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"StaleLocal", MakeConfigJson("StaleLocal"), "dep-local", "rev-local", true);
|
||||
|
||||
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
|
||||
{
|
||||
InstanceSweepIntervalSeconds = PlainSweepSeconds,
|
||||
InstanceSweepSettleIntervalSeconds = 1,
|
||||
InstanceSweepSettleWindowSeconds = 30,
|
||||
});
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
Assert.True(await InstanceIsRunningAsync(dm, "StaleLocal")), TimeSpan.FromSeconds(10));
|
||||
|
||||
// The peer's snapshot lands about a second after the boot load.
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"FromPeer", MakeConfigJson("FromPeer"), "dep-peer", "rev-peer", true);
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
Assert.True(
|
||||
await InstanceIsRunningAsync(dm, "FromPeer"),
|
||||
"A replicated config that arrived just after boot was not started within the "
|
||||
+ "settle window — the site would run without it for a full sweep interval (#135)"),
|
||||
HealBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A snapshot that lands while the boot batches are still being created (after the boot read)
|
||||
/// is caught by the one sweep run the moment boot-time creation finishes — even with the fast
|
||||
/// settle cadence switched off. The config loader writes the peer's row right after it has
|
||||
/// read the store, which pins the arrival deterministically into that window.
|
||||
///
|
||||
/// <para><b>Fails before the fix</b>: nothing sweeps until the 60 s plain tick.</para>
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReplicatedRowArrivingDuringBootCreation_IsStartedByTheImmediatePostBootSweep()
|
||||
{
|
||||
var dm = CreateBootedDeploymentManager(
|
||||
new SiteRuntimeOptions
|
||||
{
|
||||
InstanceSweepIntervalSeconds = PlainSweepSeconds,
|
||||
InstanceSweepSettleIntervalSeconds = 0,
|
||||
},
|
||||
configLoader: async () =>
|
||||
{
|
||||
var bootView = await _storage.GetAllDeployedConfigsAsync();
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"MidBootPeer", MakeConfigJson("MidBootPeer"), "dep-mid", "rev-mid", true);
|
||||
return bootView;
|
||||
});
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
Assert.True(
|
||||
await InstanceIsRunningAsync(dm, "MidBootPeer"),
|
||||
"A config replicated in while boot-time creation was running was not started "
|
||||
+ "by the post-boot sweep (#135)"),
|
||||
HealBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The settle window is bounded: once it closes the sweep is back on its plain interval, so a
|
||||
/// row arriving afterwards is NOT picked up at the fast cadence. Guards against a settle timer
|
||||
/// that is never cancelled (a permanent 1 s sweep on every site).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AfterTheSettleWindowCloses_TheSweepFallsBackToThePlainInterval()
|
||||
{
|
||||
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
|
||||
{
|
||||
InstanceSweepIntervalSeconds = PlainSweepSeconds,
|
||||
InstanceSweepSettleIntervalSeconds = 1,
|
||||
InstanceSweepSettleWindowSeconds = 2,
|
||||
});
|
||||
|
||||
// Let the window close.
|
||||
await Task.Delay(TimeSpan.FromSeconds(4));
|
||||
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"LatePeer", MakeConfigJson("LatePeer"), "dep-late", "rev-late", true);
|
||||
|
||||
// Several fast-cadence ticks' worth of time.
|
||||
await Task.Delay(TimeSpan.FromSeconds(4));
|
||||
|
||||
Assert.False(await InstanceIsRunningAsync(dm, "LatePeer"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>InstanceSweepIntervalSeconds = 0</c> keeps meaning "no sweep at all": the settle
|
||||
/// window must not resurrect a sweep an operator switched off.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DisabledSweep_IsNotReenabledByTheSettleWindow()
|
||||
{
|
||||
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
|
||||
{
|
||||
InstanceSweepIntervalSeconds = 0,
|
||||
InstanceSweepSettleIntervalSeconds = 1,
|
||||
InstanceSweepSettleWindowSeconds = 30,
|
||||
});
|
||||
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"NoSweepPeer", MakeConfigJson("NoSweepPeer"), "dep-n", "rev-n", true);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(4));
|
||||
|
||||
Assert.False(await InstanceIsRunningAsync(dm, "NoSweepPeer"));
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,35 @@ public class SiteRuntimeOptionsValidatorTests
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#135: the post-boot settle sweep cadence and window feed Akka timers, so a
|
||||
/// negative value must be rejected with a key-naming message; 0 (disable) is valid.
|
||||
/// </summary>
|
||||
/// <param name="interval">InstanceSweepSettleIntervalSeconds.</param>
|
||||
/// <param name="window">InstanceSweepSettleWindowSeconds.</param>
|
||||
/// <param name="expectedKey">The key the failure must name, or null when valid.</param>
|
||||
[Theory]
|
||||
[InlineData(-1, 120, "InstanceSweepSettleIntervalSeconds")]
|
||||
[InlineData(5, -1, "InstanceSweepSettleWindowSeconds")]
|
||||
[InlineData(0, 0, null)]
|
||||
public void InstanceSweepSettleOptions_RejectNegativeValues(int interval, int window, string? expectedKey)
|
||||
{
|
||||
var result = Validate(new SiteRuntimeOptions
|
||||
{
|
||||
InstanceSweepSettleIntervalSeconds = interval,
|
||||
InstanceSweepSettleWindowSeconds = window,
|
||||
});
|
||||
|
||||
if (expectedKey == null)
|
||||
{
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains(expectedKey, result.FailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that zero startup batch size is rejected.</summary>
|
||||
[Fact]
|
||||
public void ZeroStartupBatchSize_IsRejected()
|
||||
|
||||
Reference in New Issue
Block a user