using System.Text.Json; using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Instance; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; using ZB.MOM.WW.ScadaBridge.TestSupport; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// Regression tests for arch-review 2026-08-19 P1-2 — "static overrides are reset on /// redeployment" was racy in both directions. /// /// /// The reset used to run inside the deploy's persistence task, i.e. AFTER the successor /// Instance Actor had already been created and QUEUED BEHIND a full config-JSON write. Two /// legs fell out of that ordering: /// /// /// The successor's PreStart override load typically won the race and applied the /// PREDECESSOR's overrides — including keys the new config no longer declares — to in-memory /// state for the actor's whole lifetime, while the disk ended up cleared. The instance then /// silently disagreed with its own store, and a restart flipped the values back. /// The predecessor's PostStop flush was fire-and-forget, so up to one coalescing window /// of its dirty overrides could COMMIT after the clear and durably resurrect them. /// /// /// /// The fix dispatches the reset before the successor exists and hands it to the actor as a /// gate, and makes the predecessor's final flush complete before its Terminated signal — which /// the Deployment Manager already waits for. /// /// public class RedeployStaticOverrideResetTests : TestKit, IDisposable { private readonly SiteStorageService _storage; private readonly ScriptCompilationService _compilationService; private readonly SharedScriptLibrary _sharedScriptLibrary; private readonly TestLocalDb _localDb; public RedeployStaticOverrideResetTests() { _localDb = TestLocalDb.CreateTemp("redeploy-override-test"); _storage = new SiteStorageService( _localDb.Db, NullLogger.Instance); _storage.InitializeAsync().GetAwaiter().GetResult(); _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedScriptLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); } void IDisposable.Dispose() { // Shut the actors down BEFORE the ILocalDb they write through is disposed. Shutdown(); var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } /// /// The sweep is disabled so nothing but the deploy path can create or replace an Instance /// Actor while a test is mid-redeploy. /// private IActorRef CreateDeploymentManager() { var health = new DeploymentManagerStartupProbe(); var dm = ActorOf(Props.Create(() => new DeploymentManagerActor( _storage, _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 }, NullLogger.Instance, dclManager: null, healthCollector: health, serviceProvider: null, loggerFactory: null, configFetcher: null))); AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25)); return dm; } private static string MakeConfigJson(string instanceName) => JsonSerializer.Serialize(new FlattenedConfiguration { InstanceUniqueName = instanceName, Attributes = [ new ResolvedAttribute { CanonicalName = "Temperature", Value = "98.6", DataType = "Double" } ] }); private async Task ResolveInstanceChildAsync(IActorRef dm, string instanceName) => await Sys.ActorSelection(dm.Path / instanceName).ResolveOne(TimeSpan.FromSeconds(5)); private void Deploy(IActorRef dm, string instanceName, string deploymentId) { dm.Tell(new DeployInstanceCommand( deploymentId, instanceName, "sha256:" + deploymentId, MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow)); Assert.Equal( DeploymentStatus.Success, ExpectMsg(TimeSpan.FromSeconds(10)).Status); } // ── Leg 1: the successor must never load the predecessor's overrides ── /// /// A redeploy must leave the successor on its config defaults. The override written before /// the redeploy is cleared from the store, so an actor that still serves it is an actor /// disagreeing with its own durable state — and a later restart would silently flip the /// value back to the default. /// /// /// Fails before the fix at the attribute assert — but only at production timings: the /// unfixed clear is queued behind the config-JSON write, and a unit-test config is written in /// well under a millisecond, so the clear wins the race here anyway. Injecting a 200 ms delay /// ahead of the unfixed clear (modelling a real config write) fails this test and the late-flush /// test below, while the fixed code passes both under the same injected delay. The deterministic /// pin of the mechanism itself is /// . /// /// [Fact] public async Task Redeploy_DoesNotApplyPreRedeployOverridesToTheNewActor() { const string instanceName = "OverrideRedeployPump"; var dm = CreateDeploymentManager(); Deploy(dm, instanceName, "dep-1"); // Write a static override and wait until it is DURABLE, so the redeploy below races the // successor's load against the clear — not against the predecessor's flush (leg 2, which // the next tests cover). var first = await ResolveInstanceChildAsync(dm, instanceName); first.Tell(new SetStaticAttributeCommand( "corr-set", instanceName, "Temperature", "200.0", DateTimeOffset.UtcNow)); Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success); await AwaitAssertAsync(async () => Assert.Equal("200.0", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]), TimeSpan.FromSeconds(5)); Deploy(dm, instanceName, "dep-2"); // Settle: the successor's override load is asynchronous, so reading immediately could // pass simply by outrunning it. Give the (unfixed) load every chance to land first. await Task.Delay(1500); var second = await ResolveInstanceChildAsync(dm, instanceName); second.Tell(new GetAttributeRequest( "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow)); var response = ExpectMsg(TimeSpan.FromSeconds(5)); Assert.True(response.Found); Assert.Equal("98.6", response.Value?.ToString()); // And the store agrees — in-memory state and disk must not diverge. Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName)); } /// /// The deterministic pin of leg 1's mechanism, at the Instance Actor's own seam: given a /// reset that has not committed yet, the actor must NOT have read static overrides — no /// matter how long it has been running. Holding the reset open and clearing the table only /// after it is released proves the read is ordered behind the reset rather than merely /// losing a race to it. /// [Fact] public async Task RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted() { const string instanceName = "GatedLoadPump"; // The predecessor's override, still on disk exactly as it is when a redeploy starts. await _storage.SetStaticOverrideAsync(instanceName, "Temperature", "200.0"); var reset = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var actor = ActorOf(Props.Create(() => new InstanceActor( instanceName, MakeConfigJson(instanceName), _storage, _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger.Instance, null, null, null, null, reset.Task))); // While the reset is outstanding the actor stays on its config default, even though the // override is sitting in the table it would otherwise have read. await Task.Delay(750); actor.Tell(new GetAttributeRequest( "corr-gated", instanceName, "Temperature", DateTimeOffset.UtcNow)); Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString()); // Complete the reset the way the deploy path does — clear, then release. await _storage.ClearStaticOverridesAsync(instanceName); reset.SetResult(); await Task.Delay(750); actor.Tell(new GetAttributeRequest( "corr-gated-2", instanceName, "Temperature", DateTimeOffset.UtcNow)); Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString()); } // ── Leg 2: the predecessor's final flush must precede its Terminated signal ── /// /// The Deployment Manager gates a redeploy's override clear on the predecessor's /// Terminated signal, so PostStop's final flush is only safe if it has COMMITTED by /// the time that signal is observed. This pins exactly that ordering: a dirty override that /// no timer has flushed yet must be on disk the instant Terminated arrives. /// /// /// Fails before the fix (racily — the write is a small local SQLite commit): the /// flush was fire-and-forget, so Terminated could be observed with nothing written, and in /// a redeploy that write would then land after the clear. /// /// [Fact] public async Task InstanceActorStop_CommitsTheFinalOverrideFlush_BeforeTerminated() { const string instanceName = "FlushOrderPump"; // A flush interval far longer than the test: nothing but PostStop can persist the write. var actor = ActorOf(Props.Create(() => new InstanceActor( instanceName, MakeConfigJson(instanceName), _storage, _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger.Instance, null, null, null, TimeSpan.FromMinutes(5)))); actor.Tell(new SetStaticAttributeCommand( "corr-set", instanceName, "Temperature", "77.7", DateTimeOffset.UtcNow)); Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success); // Still only in the coalescing buffer. Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName)); Watch(actor); Sys.Stop(actor); ExpectTerminated(actor, TimeSpan.FromSeconds(10)); // No delay, no polling: the commit must already have happened. var overrides = await _storage.GetStaticOverridesAsync(instanceName); Assert.Equal("77.7", overrides["Temperature"]); } /// /// The end-to-end statement of leg 2: a redeploy that stops an instance with unflushed /// static overrides must leave the store EMPTY. A detached final flush could commit after /// the clear and durably resurrect the predecessor's values — invisible until the next /// restart loaded them. /// [Fact] public async Task Redeploy_LateFlushFromTheStoppedActor_CannotResurrectClearedOverrides() { const string instanceName = "LateFlushPump"; var dm = CreateDeploymentManager(); Deploy(dm, instanceName, "dep-1"); // Redeploy while the write is still buffered — no wait for the coalescing timer. var first = await ResolveInstanceChildAsync(dm, instanceName); first.Tell(new SetStaticAttributeCommand( "corr-set", instanceName, "Temperature", "300.0", DateTimeOffset.UtcNow)); Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success); Deploy(dm, instanceName, "dep-2"); // Settle well past the predecessor's coalescing window so a late flush would have // landed by now. await Task.Delay(1500); Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName)); var second = await ResolveInstanceChildAsync(dm, instanceName); second.Tell(new GetAttributeRequest( "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow)); Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString()); } /// /// A FIRST deploy has no predecessor and no reset to wait on: the instance must still come /// up on its config defaults, and an override written after it must persist normally. Pins /// that the reset gate did not turn the ordinary deploy path into a no-op. /// [Fact] public async Task FirstDeploy_StillLoadsAndPersistsStaticOverridesNormally() { const string instanceName = "FreshPump"; var dm = CreateDeploymentManager(); Deploy(dm, instanceName, "dep-1"); var actor = await ResolveInstanceChildAsync(dm, instanceName); actor.Tell(new SetStaticAttributeCommand( "corr-set", instanceName, "Temperature", "123.4", DateTimeOffset.UtcNow)); Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success); await AwaitAssertAsync(async () => Assert.Equal("123.4", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]), TimeSpan.FromSeconds(5)); actor.Tell(new GetAttributeRequest( "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow)); Assert.Equal("123.4", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString()); } }