using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// Regression tests for ScadaBridge#135: a replicated site pair cold-starts together, the
/// node that wins the Deployment Manager singleton loads its OWN (stale) store, and the peer's
/// rows arrive by LocalDb sync about a second later. Those rows are invisible to the boot load
/// (already run) and to the startup reconcile (central sees no gap — the rows ARE in the store by
/// then), so only the instance sweep starts them. At its plain 60 s cadence the site ran none of
/// the replicated instances for a full interval after a healthy-looking boot — reproduced on the
/// primary rig 2026-11-13 (boot load 05:03:16 without the instance, sweep start at 05:04:16).
///
///
/// The fix opens a post-boot settle window: one sweep the moment boot-time creation finishes,
/// then a fast cadence for a bounded window, then the plain interval. Startup itself never waits
/// for the peer. Every test here pins the plain interval at 60 s or more, so a pass can only come
/// from the settle path — a plain-cadence sweep cannot land inside the assertion budgets.
///
///
public class PostBootSettleSweepTests : TestKit, IDisposable
{
private readonly SiteStorageService _storage;
private readonly ScriptCompilationService _compilationService;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly TestLocalDb _localDb;
/// The plain sweep interval: far longer than any assertion budget below.
private const int PlainSweepSeconds = 60;
/// Well inside one plain interval, comfortably more than a few settle ticks.
private static readonly TimeSpan HealBudget = TimeSpan.FromSeconds(8);
/// Creates a temporary LocalDb-backed site store and the compilation services for each test.
public PostBootSettleSweepTests()
{
_localDb = TestLocalDb.CreateTemp("post-boot-settle-sweep-test");
_storage = new SiteStorageService(_localDb.Db, NullLogger.Instance);
_storage.InitializeAsync().GetAwaiter().GetResult();
_compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedScriptLibrary = new SharedScriptLibrary(
_compilationService, NullLogger.Instance);
}
///
void IDisposable.Dispose()
{
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
Shutdown();
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
///
/// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read
/// storage (its first instance-count report), so any row written afterwards is invisible to
/// the boot load — the replication arrival this issue is about.
///
private IActorRef CreateBootedDeploymentManager(
SiteRuntimeOptions options,
Func>>? configLoader = null)
{
var health = new DeploymentManagerStartupProbe();
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
_storage,
_compilationService,
_sharedScriptLibrary,
null,
options,
NullLogger.Instance,
null,
health,
null,
null,
null,
null,
configLoader,
null,
null,
null,
null,
null,
null,
null)));
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
TimeSpan.FromMilliseconds(25));
return dm;
}
private static string MakeConfigJson(string instanceName) =>
JsonSerializer.Serialize(new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
]
});
private async Task InstanceIsRunningAsync(IActorRef dm, string instanceName) =>
!(await dm.Ask(
new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()),
TimeSpan.FromSeconds(5))).InstanceNotFound;
///
/// THE #135 reproduction. The node boots on a stale store holding one instance; the peer's
/// snapshot then adds a second enabled instance as a bare storage write (the sync applier's
/// only side effect). The stale instance runs from the boot load, and the replicated one must
/// start within seconds — not one full plain sweep interval later.
///
/// Fails before the fix: with only the 60 s plain sweep, the replicated instance
/// is still not running when the budget expires.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task ReplicatedRowArrivingJustAfterBoot_StartsWithinTheSettleWindow()
{
await _storage.StoreDeployedConfigAsync(
"StaleLocal", MakeConfigJson("StaleLocal"), "dep-local", "rev-local", true);
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
{
InstanceSweepIntervalSeconds = PlainSweepSeconds,
InstanceSweepSettleIntervalSeconds = 1,
InstanceSweepSettleWindowSeconds = 30,
});
await AwaitAssertAsync(async () =>
Assert.True(await InstanceIsRunningAsync(dm, "StaleLocal")), TimeSpan.FromSeconds(10));
// The peer's snapshot lands about a second after the boot load.
await _storage.StoreDeployedConfigAsync(
"FromPeer", MakeConfigJson("FromPeer"), "dep-peer", "rev-peer", true);
await AwaitAssertAsync(async () =>
Assert.True(
await InstanceIsRunningAsync(dm, "FromPeer"),
"A replicated config that arrived just after boot was not started within the "
+ "settle window — the site would run without it for a full sweep interval (#135)"),
HealBudget);
}
///
/// A snapshot that lands while the boot batches are still being created (after the boot read)
/// is caught by the one sweep run the moment boot-time creation finishes — even with the fast
/// settle cadence switched off. The config loader writes the peer's row right after it has
/// read the store, which pins the arrival deterministically into that window.
///
/// Fails before the fix: nothing sweeps until the 60 s plain tick.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task ReplicatedRowArrivingDuringBootCreation_IsStartedByTheImmediatePostBootSweep()
{
var dm = CreateBootedDeploymentManager(
new SiteRuntimeOptions
{
InstanceSweepIntervalSeconds = PlainSweepSeconds,
InstanceSweepSettleIntervalSeconds = 0,
},
configLoader: async () =>
{
var bootView = await _storage.GetAllDeployedConfigsAsync();
await _storage.StoreDeployedConfigAsync(
"MidBootPeer", MakeConfigJson("MidBootPeer"), "dep-mid", "rev-mid", true);
return bootView;
});
await AwaitAssertAsync(async () =>
Assert.True(
await InstanceIsRunningAsync(dm, "MidBootPeer"),
"A config replicated in while boot-time creation was running was not started "
+ "by the post-boot sweep (#135)"),
HealBudget);
}
///
/// The settle window is bounded: once it closes the sweep is back on its plain interval, so a
/// row arriving afterwards is NOT picked up at the fast cadence. Guards against a settle timer
/// that is never cancelled (a permanent 1 s sweep on every site).
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task AfterTheSettleWindowCloses_TheSweepFallsBackToThePlainInterval()
{
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
{
InstanceSweepIntervalSeconds = PlainSweepSeconds,
InstanceSweepSettleIntervalSeconds = 1,
InstanceSweepSettleWindowSeconds = 2,
});
// Let the window close.
await Task.Delay(TimeSpan.FromSeconds(4));
await _storage.StoreDeployedConfigAsync(
"LatePeer", MakeConfigJson("LatePeer"), "dep-late", "rev-late", true);
// Several fast-cadence ticks' worth of time.
await Task.Delay(TimeSpan.FromSeconds(4));
Assert.False(await InstanceIsRunningAsync(dm, "LatePeer"));
}
///
/// InstanceSweepIntervalSeconds = 0 keeps meaning "no sweep at all": the settle
/// window must not resurrect a sweep an operator switched off.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task DisabledSweep_IsNotReenabledByTheSettleWindow()
{
var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions
{
InstanceSweepIntervalSeconds = 0,
InstanceSweepSettleIntervalSeconds = 1,
InstanceSweepSettleWindowSeconds = 30,
});
await _storage.StoreDeployedConfigAsync(
"NoSweepPeer", MakeConfigJson("NoSweepPeer"), "dep-n", "rev-n", true);
await Task.Delay(TimeSpan.FromSeconds(4));
Assert.False(await InstanceIsRunningAsync(dm, "NoSweepPeer"));
}
}