using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Instance;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// Regression tests for arch-review 2026-08-19 P1-2 — "static overrides are reset on
/// redeployment" was racy in both directions.
///
///
/// The reset used to run inside the deploy's persistence task, i.e. AFTER the successor
/// Instance Actor had already been created and QUEUED BEHIND a full config-JSON write. Two
/// legs fell out of that ordering:
///
///
/// - The successor's PreStart override load typically won the race and applied the
/// PREDECESSOR's overrides — including keys the new config no longer declares — to in-memory
/// state for the actor's whole lifetime, while the disk ended up cleared. The instance then
/// silently disagreed with its own store, and a restart flipped the values back.
/// - The predecessor's PostStop flush was fire-and-forget, so up to one coalescing window
/// of its dirty overrides could COMMIT after the clear and durably resurrect them.
///
///
///
/// The fix dispatches the reset before the successor exists and hands it to the actor as a
/// gate, and makes the predecessor's final flush complete before its Terminated signal — which
/// the Deployment Manager already waits for.
///
///
public class RedeployStaticOverrideResetTests : TestKit, IDisposable
{
private readonly SiteStorageService _storage;
private readonly ScriptCompilationService _compilationService;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly TestLocalDb _localDb;
public RedeployStaticOverrideResetTests()
{
_localDb = TestLocalDb.CreateTemp("redeploy-override-test");
_storage = new SiteStorageService(
_localDb.Db, NullLogger.Instance);
_storage.InitializeAsync().GetAwaiter().GetResult();
_compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedScriptLibrary = new SharedScriptLibrary(
_compilationService, NullLogger.Instance);
}
void IDisposable.Dispose()
{
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
Shutdown();
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
///
/// The sweep is disabled so nothing but the deploy path can create or replace an Instance
/// Actor while a test is mid-redeploy.
///
private IActorRef CreateDeploymentManager()
{
var health = new DeploymentManagerStartupProbe();
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
_storage,
_compilationService,
_sharedScriptLibrary,
null,
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
NullLogger.Instance,
dclManager: null,
healthCollector: health,
serviceProvider: null,
loggerFactory: null,
configFetcher: null)));
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
TimeSpan.FromMilliseconds(25));
return dm;
}
private static string MakeConfigJson(string instanceName) =>
JsonSerializer.Serialize(new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute
{
CanonicalName = "Temperature", Value = "98.6", DataType = "Double"
}
]
});
private async Task ResolveInstanceChildAsync(IActorRef dm, string instanceName) =>
await Sys.ActorSelection(dm.Path / instanceName).ResolveOne(TimeSpan.FromSeconds(5));
private void Deploy(IActorRef dm, string instanceName, string deploymentId)
{
dm.Tell(new DeployInstanceCommand(
deploymentId, instanceName, "sha256:" + deploymentId,
MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
Assert.Equal(
DeploymentStatus.Success,
ExpectMsg(TimeSpan.FromSeconds(10)).Status);
}
// ── Leg 1: the successor must never load the predecessor's overrides ──
///
/// A redeploy must leave the successor on its config defaults. The override written before
/// the redeploy is cleared from the store, so an actor that still serves it is an actor
/// disagreeing with its own durable state — and a later restart would silently flip the
/// value back to the default.
///
///
/// Fails before the fix at the attribute assert — but only at production timings: the
/// unfixed clear is queued behind the config-JSON write, and a unit-test config is written in
/// well under a millisecond, so the clear wins the race here anyway. Injecting a 200 ms delay
/// ahead of the unfixed clear (modelling a real config write) fails this test and the late-flush
/// test below, while the fixed code passes both under the same injected delay. The deterministic
/// pin of the mechanism itself is
/// .
///
///
[Fact]
public async Task Redeploy_DoesNotApplyPreRedeployOverridesToTheNewActor()
{
const string instanceName = "OverrideRedeployPump";
var dm = CreateDeploymentManager();
Deploy(dm, instanceName, "dep-1");
// Write a static override and wait until it is DURABLE, so the redeploy below races the
// successor's load against the clear — not against the predecessor's flush (leg 2, which
// the next tests cover).
var first = await ResolveInstanceChildAsync(dm, instanceName);
first.Tell(new SetStaticAttributeCommand(
"corr-set", instanceName, "Temperature", "200.0", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success);
await AwaitAssertAsync(async () =>
Assert.Equal("200.0", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
TimeSpan.FromSeconds(5));
Deploy(dm, instanceName, "dep-2");
// Settle: the successor's override load is asynchronous, so reading immediately could
// pass simply by outrunning it. Give the (unfixed) load every chance to land first.
await Task.Delay(1500);
var second = await ResolveInstanceChildAsync(dm, instanceName);
second.Tell(new GetAttributeRequest(
"corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
var response = ExpectMsg(TimeSpan.FromSeconds(5));
Assert.True(response.Found);
Assert.Equal("98.6", response.Value?.ToString());
// And the store agrees — in-memory state and disk must not diverge.
Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));
}
///
/// The deterministic pin of leg 1's mechanism, at the Instance Actor's own seam: given a
/// reset that has not committed yet, the actor must NOT have read static overrides — no
/// matter how long it has been running. Holding the reset open and clearing the table only
/// after it is released proves the read is ordered behind the reset rather than merely
/// losing a race to it.
///
[Fact]
public async Task RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted()
{
const string instanceName = "GatedLoadPump";
// The predecessor's override, still on disk exactly as it is when a redeploy starts.
await _storage.SetStaticOverrideAsync(instanceName, "Temperature", "200.0");
var reset = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var actor = ActorOf(Props.Create(() => new InstanceActor(
instanceName,
MakeConfigJson(instanceName),
_storage,
_compilationService,
_sharedScriptLibrary,
null,
new SiteRuntimeOptions(),
NullLogger.Instance,
null,
null,
null,
null,
reset.Task)));
// While the reset is outstanding the actor stays on its config default, even though the
// override is sitting in the table it would otherwise have read.
await Task.Delay(750);
actor.Tell(new GetAttributeRequest(
"corr-gated", instanceName, "Temperature", DateTimeOffset.UtcNow));
Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString());
// Complete the reset the way the deploy path does — clear, then release.
await _storage.ClearStaticOverridesAsync(instanceName);
reset.SetResult();
await Task.Delay(750);
actor.Tell(new GetAttributeRequest(
"corr-gated-2", instanceName, "Temperature", DateTimeOffset.UtcNow));
Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString());
}
// ── Leg 2: the predecessor's final flush must precede its Terminated signal ──
///
/// The Deployment Manager gates a redeploy's override clear on the predecessor's
/// Terminated signal, so PostStop's final flush is only safe if it has COMMITTED by
/// the time that signal is observed. This pins exactly that ordering: a dirty override that
/// no timer has flushed yet must be on disk the instant Terminated arrives.
///
///
/// Fails before the fix (racily — the write is a small local SQLite commit): the
/// flush was fire-and-forget, so Terminated could be observed with nothing written, and in
/// a redeploy that write would then land after the clear.
///
///
[Fact]
public async Task InstanceActorStop_CommitsTheFinalOverrideFlush_BeforeTerminated()
{
const string instanceName = "FlushOrderPump";
// A flush interval far longer than the test: nothing but PostStop can persist the write.
var actor = ActorOf(Props.Create(() => new InstanceActor(
instanceName,
MakeConfigJson(instanceName),
_storage,
_compilationService,
_sharedScriptLibrary,
null,
new SiteRuntimeOptions(),
NullLogger.Instance,
null,
null,
null,
TimeSpan.FromMinutes(5))));
actor.Tell(new SetStaticAttributeCommand(
"corr-set", instanceName, "Temperature", "77.7", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success);
// Still only in the coalescing buffer.
Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
// No delay, no polling: the commit must already have happened.
var overrides = await _storage.GetStaticOverridesAsync(instanceName);
Assert.Equal("77.7", overrides["Temperature"]);
}
///
/// The end-to-end statement of leg 2: a redeploy that stops an instance with unflushed
/// static overrides must leave the store EMPTY. A detached final flush could commit after
/// the clear and durably resurrect the predecessor's values — invisible until the next
/// restart loaded them.
///
[Fact]
public async Task Redeploy_LateFlushFromTheStoppedActor_CannotResurrectClearedOverrides()
{
const string instanceName = "LateFlushPump";
var dm = CreateDeploymentManager();
Deploy(dm, instanceName, "dep-1");
// Redeploy while the write is still buffered — no wait for the coalescing timer.
var first = await ResolveInstanceChildAsync(dm, instanceName);
first.Tell(new SetStaticAttributeCommand(
"corr-set", instanceName, "Temperature", "300.0", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success);
Deploy(dm, instanceName, "dep-2");
// Settle well past the predecessor's coalescing window so a late flush would have
// landed by now.
await Task.Delay(1500);
Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));
var second = await ResolveInstanceChildAsync(dm, instanceName);
second.Tell(new GetAttributeRequest(
"corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
Assert.Equal("98.6", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString());
}
///
/// A FIRST deploy has no predecessor and no reset to wait on: the instance must still come
/// up on its config defaults, and an override written after it must persist normally. Pins
/// that the reset gate did not turn the ordinary deploy path into a no-op.
///
[Fact]
public async Task FirstDeploy_StillLoadsAndPersistsStaticOverridesNormally()
{
const string instanceName = "FreshPump";
var dm = CreateDeploymentManager();
Deploy(dm, instanceName, "dep-1");
var actor = await ResolveInstanceChildAsync(dm, instanceName);
actor.Tell(new SetStaticAttributeCommand(
"corr-set", instanceName, "Temperature", "123.4", DateTimeOffset.UtcNow));
Assert.True(ExpectMsg(TimeSpan.FromSeconds(5)).Success);
await AwaitAssertAsync(async () =>
Assert.Equal("123.4", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
TimeSpan.FromSeconds(5));
actor.Tell(new GetAttributeRequest(
"corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
Assert.Equal("123.4", ExpectMsg(TimeSpan.FromSeconds(5)).Value?.ToString());
}
}