ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/RedeployStaticOverrideResetTests.cs

using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Instance;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Regression tests for arch-review 2026-08-19 P1-2 — <b>"static overrides are reset on
/// redeployment" was racy in both directions</b>.
///
/// <para>
/// The reset used to run inside the deploy's persistence task, i.e. AFTER the successor
/// Instance Actor had already been created and QUEUED BEHIND a full config-JSON write. Two
/// legs fell out of that ordering:
/// </para>
/// <list type="number">
/// <item>The successor's PreStart override load typically won the race and applied the
/// PREDECESSOR's overrides — including keys the new config no longer declares — to in-memory
/// state for the actor's whole lifetime, while the disk ended up cleared. The instance then
/// silently disagreed with its own store, and a restart flipped the values back.</item>
/// <item>The predecessor's PostStop flush was fire-and-forget, so up to one coalescing window
/// of its dirty overrides could COMMIT after the clear and durably resurrect them.</item>
/// </list>
///
/// <para>
/// The fix dispatches the reset before the successor exists and hands it to the actor as a
/// gate, and makes the predecessor's final flush complete before its Terminated signal — which
/// the Deployment Manager already waits for.
/// </para>
/// </summary>
public class RedeployStaticOverrideResetTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    public RedeployStaticOverrideResetTests()
    {
        _localDb = TestLocalDb.CreateTemp("redeploy-override-test");
        _storage = new SiteStorageService(
            _localDb.Db, NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    void IDisposable.Dispose()
    {
        // Shut the actors down BEFORE the ILocalDb they write through is disposed.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    /// <summary>
    /// The sweep is disabled so nothing but the deploy path can create or replace an Instance
    /// Actor while a test is mid-redeploy.
    /// </summary>
    private IActorRef CreateDeploymentManager()
    {
        var health = new DeploymentManagerStartupProbe();
        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager: null,
            healthCollector: health,
            serviceProvider: null,
            loggerFactory: null,
            configFetcher: null)));

        AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
            TimeSpan.FromMilliseconds(25));
        return dm;
    }

    private static string MakeConfigJson(string instanceName) =>
        JsonSerializer.Serialize(new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute
                {
                    CanonicalName = "Temperature", Value = "98.6", DataType = "Double"
                }
            ]
        });

    private async Task<IActorRef> ResolveInstanceChildAsync(IActorRef dm, string instanceName) =>
        await Sys.ActorSelection(dm.Path / instanceName).ResolveOne(TimeSpan.FromSeconds(5));

    private void Deploy(IActorRef dm, string instanceName, string deploymentId)
    {
        dm.Tell(new DeployInstanceCommand(
            deploymentId, instanceName, "sha256:" + deploymentId,
            MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
        Assert.Equal(
            DeploymentStatus.Success,
            ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10)).Status);
    }

    // ── Leg 1: the successor must never load the predecessor's overrides ──

    /// <summary>
    /// A redeploy must leave the successor on its config defaults. The override written before
    /// the redeploy is cleared from the store, so an actor that still serves it is an actor
    /// disagreeing with its own durable state — and a later restart would silently flip the
    /// value back to the default.
    ///
    /// <para>
    /// <b>Fails before the fix</b> at the attribute assert — but only at production timings: the
    /// unfixed clear is queued behind the config-JSON write, and a unit-test config is written in
    /// well under a millisecond, so the clear wins the race here anyway. Injecting a 200 ms delay
    /// ahead of the unfixed clear (modelling a real config write) fails this test and the late-flush
    /// test below, while the fixed code passes both under the same injected delay. The deterministic
    /// pin of the mechanism itself is
    /// <see cref="RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted"/>.
    /// </para>
    /// </summary>
    [Fact]
    public async Task Redeploy_DoesNotApplyPreRedeployOverridesToTheNewActor()
    {
        const string instanceName = "OverrideRedeployPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        // Write a static override and wait until it is DURABLE, so the redeploy below races the
        // successor's load against the clear — not against the predecessor's flush (leg 2, which
        // the next tests cover).
        var first = await ResolveInstanceChildAsync(dm, instanceName);
        first.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "200.0", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);
        await AwaitAssertAsync(async () =>
            Assert.Equal("200.0", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
            TimeSpan.FromSeconds(5));

        Deploy(dm, instanceName, "dep-2");

        // Settle: the successor's override load is asynchronous, so reading immediately could
        // pass simply by outrunning it. Give the (unfixed) load every chance to land first.
        await Task.Delay(1500);

        var second = await ResolveInstanceChildAsync(dm, instanceName);
        second.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        var response = ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5));

        Assert.True(response.Found);
        Assert.Equal("98.6", response.Value?.ToString());

        // And the store agrees — in-memory state and disk must not diverge.
        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));
    }

    /// <summary>
    /// The deterministic pin of leg 1's mechanism, at the Instance Actor's own seam: given a
    /// reset that has not committed yet, the actor must NOT have read static overrides — no
    /// matter how long it has been running. Holding the reset open and clearing the table only
    /// after it is released proves the read is ordered behind the reset rather than merely
    /// losing a race to it.
    /// </summary>
    [Fact]
    public async Task RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted()
    {
        const string instanceName = "GatedLoadPump";

        // The predecessor's override, still on disk exactly as it is when a redeploy starts.
        await _storage.SetStaticOverrideAsync(instanceName, "Temperature", "200.0");

        var reset = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var actor = ActorOf(Props.Create(() => new InstanceActor(
            instanceName,
            MakeConfigJson(instanceName),
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions(),
            NullLogger<InstanceActor>.Instance,
            null,
            null,
            null,
            null,
            reset.Task)));

        // While the reset is outstanding the actor stays on its config default, even though the
        // override is sitting in the table it would otherwise have read.
        await Task.Delay(750);
        actor.Tell(new GetAttributeRequest(
            "corr-gated", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());

        // Complete the reset the way the deploy path does — clear, then release.
        await _storage.ClearStaticOverridesAsync(instanceName);
        reset.SetResult();

        await Task.Delay(750);
        actor.Tell(new GetAttributeRequest(
            "corr-gated-2", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }

    // ── Leg 2: the predecessor's final flush must precede its Terminated signal ──

    /// <summary>
    /// The Deployment Manager gates a redeploy's override clear on the predecessor's
    /// <c>Terminated</c> signal, so PostStop's final flush is only safe if it has COMMITTED by
    /// the time that signal is observed. This pins exactly that ordering: a dirty override that
    /// no timer has flushed yet must be on disk the instant Terminated arrives.
    ///
    /// <para>
    /// <b>Fails before the fix</b> (racily — the write is a small local SQLite commit): the
    /// flush was fire-and-forget, so Terminated could be observed with nothing written, and in
    /// a redeploy that write would then land after the clear.
    /// </para>
    /// </summary>
    [Fact]
    public async Task InstanceActorStop_CommitsTheFinalOverrideFlush_BeforeTerminated()
    {
        const string instanceName = "FlushOrderPump";

        // A flush interval far longer than the test: nothing but PostStop can persist the write.
        var actor = ActorOf(Props.Create(() => new InstanceActor(
            instanceName,
            MakeConfigJson(instanceName),
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions(),
            NullLogger<InstanceActor>.Instance,
            null,
            null,
            null,
            TimeSpan.FromMinutes(5))));

        actor.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "77.7", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        // Still only in the coalescing buffer.
        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));

        Watch(actor);
        Sys.Stop(actor);
        ExpectTerminated(actor, TimeSpan.FromSeconds(10));

        // No delay, no polling: the commit must already have happened.
        var overrides = await _storage.GetStaticOverridesAsync(instanceName);
        Assert.Equal("77.7", overrides["Temperature"]);
    }

    /// <summary>
    /// The end-to-end statement of leg 2: a redeploy that stops an instance with unflushed
    /// static overrides must leave the store EMPTY. A detached final flush could commit after
    /// the clear and durably resurrect the predecessor's values — invisible until the next
    /// restart loaded them.
    /// </summary>
    [Fact]
    public async Task Redeploy_LateFlushFromTheStoppedActor_CannotResurrectClearedOverrides()
    {
        const string instanceName = "LateFlushPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        // Redeploy while the write is still buffered — no wait for the coalescing timer.
        var first = await ResolveInstanceChildAsync(dm, instanceName);
        first.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "300.0", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        Deploy(dm, instanceName, "dep-2");

        // Settle well past the predecessor's coalescing window so a late flush would have
        // landed by now.
        await Task.Delay(1500);

        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));

        var second = await ResolveInstanceChildAsync(dm, instanceName);
        second.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }

    /// <summary>
    /// A FIRST deploy has no predecessor and no reset to wait on: the instance must still come
    /// up on its config defaults, and an override written after it must persist normally. Pins
    /// that the reset gate did not turn the ordinary deploy path into a no-op.
    /// </summary>
    [Fact]
    public async Task FirstDeploy_StillLoadsAndPersistsStaticOverridesNormally()
    {
        const string instanceName = "FreshPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        var actor = await ResolveInstanceChildAsync(dm, instanceName);
        actor.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "123.4", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        await AwaitAssertAsync(async () =>
            Assert.Equal("123.4", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
            TimeSpan.FromSeconds(5));

        actor.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("123.4", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }
}