using System.Text.Json; using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; using ZB.MOM.WW.ScadaBridge.TestSupport; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// Regression tests for ScadaBridge#135: a replicated site pair cold-starts together, the /// node that wins the Deployment Manager singleton loads its OWN (stale) store, and the peer's /// rows arrive by LocalDb sync about a second later. Those rows are invisible to the boot load /// (already run) and to the startup reconcile (central sees no gap — the rows ARE in the store by /// then), so only the instance sweep starts them. At its plain 60 s cadence the site ran none of /// the replicated instances for a full interval after a healthy-looking boot — reproduced on the /// primary rig 2026-11-13 (boot load 05:03:16 without the instance, sweep start at 05:04:16). /// /// /// The fix opens a post-boot settle window: one sweep the moment boot-time creation finishes, /// then a fast cadence for a bounded window, then the plain interval. Startup itself never waits /// for the peer. Every test here pins the plain interval at 60 s or more, so a pass can only come /// from the settle path — a plain-cadence sweep cannot land inside the assertion budgets. /// /// public class PostBootSettleSweepTests : TestKit, IDisposable { private readonly SiteStorageService _storage; private readonly ScriptCompilationService _compilationService; private readonly SharedScriptLibrary _sharedScriptLibrary; private readonly TestLocalDb _localDb; /// The plain sweep interval: far longer than any assertion budget below. private const int PlainSweepSeconds = 60; /// Well inside one plain interval, comfortably more than a few settle ticks. private static readonly TimeSpan HealBudget = TimeSpan.FromSeconds(8); /// Creates a temporary LocalDb-backed site store and the compilation services for each test. public PostBootSettleSweepTests() { _localDb = TestLocalDb.CreateTemp("post-boot-settle-sweep-test"); _storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); _storage.InitializeAsync().GetAwaiter().GetResult(); _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedScriptLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); } /// void IDisposable.Dispose() { // Shut the actors down BEFORE the ILocalDb they write through is disposed. Shutdown(); var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } /// /// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read /// storage (its first instance-count report), so any row written afterwards is invisible to /// the boot load — the replication arrival this issue is about. /// private IActorRef CreateBootedDeploymentManager( SiteRuntimeOptions options, Func>>? configLoader = null) { var health = new DeploymentManagerStartupProbe(); var dm = ActorOf(Props.Create(() => new DeploymentManagerActor( _storage, _compilationService, _sharedScriptLibrary, null, options, NullLogger.Instance, null, health, null, null, null, null, configLoader, null, null, null, null, null, null, null))); AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25)); return dm; } private static string MakeConfigJson(string instanceName) => JsonSerializer.Serialize(new FlattenedConfiguration { InstanceUniqueName = instanceName, Attributes = [ new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" } ] }); private async Task InstanceIsRunningAsync(IActorRef dm, string instanceName) => !(await dm.Ask( new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()), TimeSpan.FromSeconds(5))).InstanceNotFound; /// /// THE #135 reproduction. The node boots on a stale store holding one instance; the peer's /// snapshot then adds a second enabled instance as a bare storage write (the sync applier's /// only side effect). The stale instance runs from the boot load, and the replicated one must /// start within seconds — not one full plain sweep interval later. /// /// Fails before the fix: with only the 60 s plain sweep, the replicated instance /// is still not running when the budget expires. /// /// A task that represents the asynchronous operation. [Fact] public async Task ReplicatedRowArrivingJustAfterBoot_StartsWithinTheSettleWindow() { await _storage.StoreDeployedConfigAsync( "StaleLocal", MakeConfigJson("StaleLocal"), "dep-local", "rev-local", true); var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions { InstanceSweepIntervalSeconds = PlainSweepSeconds, InstanceSweepSettleIntervalSeconds = 1, InstanceSweepSettleWindowSeconds = 30, }); await AwaitAssertAsync(async () => Assert.True(await InstanceIsRunningAsync(dm, "StaleLocal")), TimeSpan.FromSeconds(10)); // The peer's snapshot lands about a second after the boot load. await _storage.StoreDeployedConfigAsync( "FromPeer", MakeConfigJson("FromPeer"), "dep-peer", "rev-peer", true); await AwaitAssertAsync(async () => Assert.True( await InstanceIsRunningAsync(dm, "FromPeer"), "A replicated config that arrived just after boot was not started within the " + "settle window — the site would run without it for a full sweep interval (#135)"), HealBudget); } /// /// A snapshot that lands while the boot batches are still being created (after the boot read) /// is caught by the one sweep run the moment boot-time creation finishes — even with the fast /// settle cadence switched off. The config loader writes the peer's row right after it has /// read the store, which pins the arrival deterministically into that window. /// /// Fails before the fix: nothing sweeps until the 60 s plain tick. /// /// A task that represents the asynchronous operation. [Fact] public async Task ReplicatedRowArrivingDuringBootCreation_IsStartedByTheImmediatePostBootSweep() { var dm = CreateBootedDeploymentManager( new SiteRuntimeOptions { InstanceSweepIntervalSeconds = PlainSweepSeconds, InstanceSweepSettleIntervalSeconds = 0, }, configLoader: async () => { var bootView = await _storage.GetAllDeployedConfigsAsync(); await _storage.StoreDeployedConfigAsync( "MidBootPeer", MakeConfigJson("MidBootPeer"), "dep-mid", "rev-mid", true); return bootView; }); await AwaitAssertAsync(async () => Assert.True( await InstanceIsRunningAsync(dm, "MidBootPeer"), "A config replicated in while boot-time creation was running was not started " + "by the post-boot sweep (#135)"), HealBudget); } /// /// The settle window is bounded: once it closes the sweep is back on its plain interval, so a /// row arriving afterwards is NOT picked up at the fast cadence. Guards against a settle timer /// that is never cancelled (a permanent 1 s sweep on every site). /// /// A task that represents the asynchronous operation. [Fact] public async Task AfterTheSettleWindowCloses_TheSweepFallsBackToThePlainInterval() { var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions { InstanceSweepIntervalSeconds = PlainSweepSeconds, InstanceSweepSettleIntervalSeconds = 1, InstanceSweepSettleWindowSeconds = 2, }); // Let the window close. await Task.Delay(TimeSpan.FromSeconds(4)); await _storage.StoreDeployedConfigAsync( "LatePeer", MakeConfigJson("LatePeer"), "dep-late", "rev-late", true); // Several fast-cadence ticks' worth of time. await Task.Delay(TimeSpan.FromSeconds(4)); Assert.False(await InstanceIsRunningAsync(dm, "LatePeer")); } /// /// InstanceSweepIntervalSeconds = 0 keeps meaning "no sweep at all": the settle /// window must not resurrect a sweep an operator switched off. /// /// A task that represents the asynchronous operation. [Fact] public async Task DisabledSweep_IsNotReenabledByTheSettleWindow() { var dm = CreateBootedDeploymentManager(new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0, InstanceSweepSettleIntervalSeconds = 1, InstanceSweepSettleWindowSeconds = 30, }); await _storage.StoreDeployedConfigAsync( "NoSweepPeer", MakeConfigJson("NoSweepPeer"), "dep-n", "rev-n", true); await Task.Delay(TimeSpan.FromSeconds(4)); Assert.False(await InstanceIsRunningAsync(dm, "NoSweepPeer")); } }