ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/RedeployStaticOverrideResetTests.cs

using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Instance;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Regression tests for arch-review 2026-08-19 P1-2 — <b>"static overrides are reset on
/// redeployment" was racy in both directions</b>.
///
/// <para>
/// The reset used to run inside the deploy's persistence task, i.e. AFTER the successor
/// Instance Actor had already been created and QUEUED BEHIND a full config-JSON write. Two
/// legs fell out of that ordering:
/// </para>
/// <list type="number">
/// <item>The successor's PreStart override load typically won the race and applied the
/// PREDECESSOR's overrides — including keys the new config no longer declares — to in-memory
/// state for the actor's whole lifetime, while the disk ended up cleared. The instance then
/// silently disagreed with its own store, and a restart flipped the values back.</item>
/// <item>The predecessor's PostStop flush was fire-and-forget, so up to one coalescing window
/// of its dirty overrides could COMMIT after the clear and durably resurrect them.</item>
/// </list>
///
/// <para>
/// The fix dispatches the reset before the successor exists and hands it to the actor as a
/// gate, and waits for both predecessor termination and its separate final-flush acknowledgement.
/// </para>
/// </summary>
public class RedeployStaticOverrideResetTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    public RedeployStaticOverrideResetTests()
    {
        _localDb = TestLocalDb.CreateTemp("redeploy-override-test");
        _storage = new SiteStorageService(
            _localDb.Db, NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    void IDisposable.Dispose()
    {
        // Shut the actors down BEFORE the ILocalDb they write through is disposed.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    /// <summary>
    /// The sweep is disabled so nothing but the deploy path can create or replace an Instance
    /// Actor while a test is mid-redeploy.
    /// </summary>
    private IActorRef CreateDeploymentManager()
    {
        var health = new DeploymentManagerStartupProbe();
        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager: null,
            healthCollector: health,
            serviceProvider: null,
            loggerFactory: null,
            configFetcher: null)));

        AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
            TimeSpan.FromMilliseconds(25));
        return dm;
    }

    private static string MakeConfigJson(string instanceName) =>
        JsonSerializer.Serialize(new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute
                {
                    CanonicalName = "Temperature", Value = "98.6", DataType = "Double"
                }
            ]
        });

    private async Task<IActorRef> ResolveInstanceChildAsync(IActorRef dm, string instanceName) =>
        await Sys.ActorSelection(dm.Path / instanceName).ResolveOne(TimeSpan.FromSeconds(5));

    private void Deploy(IActorRef dm, string instanceName, string deploymentId)
    {
        dm.Tell(new DeployInstanceCommand(
            deploymentId, instanceName, "sha256:" + deploymentId,
            MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
        Assert.Equal(
            DeploymentStatus.Success,
            ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10)).Status);
    }

    // ── Leg 1: the successor must never load the predecessor's overrides ──

    /// <summary>
    /// A redeploy must leave the successor on its config defaults. The override written before
    /// the redeploy is cleared from the store, so an actor that still serves it is an actor
    /// disagreeing with its own durable state — and a later restart would silently flip the
    /// value back to the default.
    ///
    /// <para>
    /// <b>Fails before the fix</b> at the attribute assert — but only at production timings: the
    /// unfixed clear is queued behind the config-JSON write, and a unit-test config is written in
    /// well under a millisecond, so the clear wins the race here anyway. Injecting a 200 ms delay
    /// ahead of the unfixed clear (modelling a real config write) fails this test and the late-flush
    /// test below, while the fixed code passes both under the same injected delay. The deterministic
    /// pin of the mechanism itself is
    /// <see cref="RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted"/>.
    /// </para>
    /// </summary>
    [Fact]
    public async Task Redeploy_DoesNotApplyPreRedeployOverridesToTheNewActor()
    {
        const string instanceName = "OverrideRedeployPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        // Write a static override and wait until it is DURABLE, so the redeploy below races the
        // successor's load against the clear — not against the predecessor's flush (leg 2, which
        // the next tests cover).
        var first = await ResolveInstanceChildAsync(dm, instanceName);
        first.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "200.0", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);
        await AwaitAssertAsync(async () =>
            Assert.Equal("200.0", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
            TimeSpan.FromSeconds(5));

        Deploy(dm, instanceName, "dep-2");

        // Settle: the successor's override load is asynchronous, so reading immediately could
        // pass simply by outrunning it. Give the (unfixed) load every chance to land first.
        await Task.Delay(1500);

        var second = await ResolveInstanceChildAsync(dm, instanceName);
        second.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        var response = ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5));

        Assert.True(response.Found);
        Assert.Equal("98.6", response.Value?.ToString());

        // And the store agrees — in-memory state and disk must not diverge.
        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));
    }

    /// <summary>
    /// The deterministic pin of leg 1's mechanism, at the Instance Actor's own seam: given a
    /// reset that has not committed yet, the actor must NOT have read static overrides — no
    /// matter how long it has been running. Holding the reset open and clearing the table only
    /// after it is released proves the read is ordered behind the reset rather than merely
    /// losing a race to it.
    /// </summary>
    [Fact]
    public async Task RedeployResetGate_HoldsTheOverrideLoadUntilTheResetHasCommitted()
    {
        const string instanceName = "GatedLoadPump";

        // The predecessor's override, still on disk exactly as it is when a redeploy starts.
        await _storage.SetStaticOverrideAsync(instanceName, "Temperature", "200.0");

        var reset = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var actor = ActorOf(Props.Create(() => new InstanceActor(
            instanceName,
            MakeConfigJson(instanceName),
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions(),
            NullLogger<InstanceActor>.Instance,
            null,
            null,
            null,
            null,
            reset.Task)));

        // While the reset is outstanding the actor stays on its config default, even though the
        // override is sitting in the table it would otherwise have read.
        await Task.Delay(750);
        actor.Tell(new GetAttributeRequest(
            "corr-gated", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());

        // Complete the reset the way the deploy path does — clear, then release.
        await _storage.ClearStaticOverridesAsync(instanceName);
        reset.SetResult();

        await Task.Delay(750);
        actor.Tell(new GetAttributeRequest(
            "corr-gated-2", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }

    // Leg 2: final persistence is acknowledged independently of termination.
    // The manager waits for both before resetting state; a standalone actor still
    // persists its final dirty batch asynchronously after stopping.
    [Fact]
    public async Task InstanceActorStop_CompletesFinalOverrideFlushAsynchronously()
    {
        const string instanceName = "FlushOrderPump";

        // A flush interval far longer than the test: nothing but PostStop can persist the write.
        var actor = ActorOf(Props.Create(() => new InstanceActor(
            instanceName,
            MakeConfigJson(instanceName),
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions(),
            NullLogger<InstanceActor>.Instance,
            null,
            null,
            null,
            TimeSpan.FromMinutes(5))));

        actor.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "77.7", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        // Still only in the coalescing buffer.
        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));

        Watch(actor);
        Sys.Stop(actor);
        ExpectTerminated(actor, TimeSpan.FromSeconds(10));

        AwaitAssert(() => Assert.Equal("77.7",
            _storage.GetStaticOverridesAsync(instanceName).GetAwaiter().GetResult()["Temperature"]));
    }

    /// <summary>
    /// The end-to-end statement of leg 2: a redeploy that stops an instance with unflushed
    /// static overrides must leave the store EMPTY. A detached final flush could commit after
    /// the clear and durably resurrect the predecessor's values — invisible until the next
    /// restart loaded them.
    /// </summary>
    [Fact]
    public async Task Redeploy_LateFlushFromTheStoppedActor_CannotResurrectClearedOverrides()
    {
        const string instanceName = "LateFlushPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        // Redeploy while the write is still buffered — no wait for the coalescing timer.
        var first = await ResolveInstanceChildAsync(dm, instanceName);
        first.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "300.0", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        Deploy(dm, instanceName, "dep-2");

        // Settle well past the predecessor's coalescing window so a late flush would have
        // landed by now.
        await Task.Delay(1500);

        Assert.Empty(await _storage.GetStaticOverridesAsync(instanceName));

        var second = await ResolveInstanceChildAsync(dm, instanceName);
        second.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("98.6", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }

    /// <summary>
    /// A FIRST deploy has no predecessor and no reset to wait on: the instance must still come
    /// up on its config defaults, and an override written after it must persist normally. Pins
    /// that the reset gate did not turn the ordinary deploy path into a no-op.
    /// </summary>
    [Fact]
    public async Task FirstDeploy_StillLoadsAndPersistsStaticOverridesNormally()
    {
        const string instanceName = "FreshPump";
        var dm = CreateDeploymentManager();

        Deploy(dm, instanceName, "dep-1");

        var actor = await ResolveInstanceChildAsync(dm, instanceName);
        actor.Tell(new SetStaticAttributeCommand(
            "corr-set", instanceName, "Temperature", "123.4", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>(TimeSpan.FromSeconds(5)).Success);

        await AwaitAssertAsync(async () =>
            Assert.Equal("123.4", (await _storage.GetStaticOverridesAsync(instanceName))["Temperature"]),
            TimeSpan.FromSeconds(5));

        actor.Tell(new GetAttributeRequest(
            "corr-get", instanceName, "Temperature", DateTimeOffset.UtcNow));
        Assert.Equal("123.4", ExpectMsg<GetAttributeResponse>(TimeSpan.FromSeconds(5)).Value?.ToString());
    }
    [Fact]
    public void SR5_StopDoesNotBlockDispatcherWhileSqliteWriterIsBusy()
    {
        const string name = "BusyFlushPump";
        var actor = ActorOf(Props.Create(() => new InstanceActor(name, MakeConfigJson(name), _storage,
            _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger<InstanceActor>.Instance,
            null, null, null, TimeSpan.FromMinutes(5))));
        actor.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
        using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
        blocker.Open();
        using var transaction = blocker.BeginTransaction();
        Watch(actor);
        Sys.Stop(actor);
        // The outstanding persistence is acknowledged separately; actor termination
        // must not wait for a database lock on the shared dispatcher.
        ExpectTerminated(actor, TimeSpan.FromMilliseconds(500));
    }

    [Fact]
    public async Task SR5_RedeployWaitsForFlushAcknowledgementAfterTermination()
    {
        const string name = "AckOrderPump";
        var manager = CreateDeploymentManager();
        Deploy(manager, name, "first");
        var old = await ResolveInstanceChildAsync(manager, name);
        using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
        blocker.Open();
        using (var transaction = blocker.BeginTransaction())
        {
            old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
            Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
            Watch(old);
            manager.Tell(new DeployInstanceCommand("second", name, "sha256:second", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
            ExpectTerminated(old, TimeSpan.FromSeconds(2));
            ExpectNoMsg(TimeSpan.FromMilliseconds(200));
            transaction.Commit();
        }
        Assert.Equal(DeploymentStatus.Success, ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10)).Status);
        Assert.Empty(await _storage.GetStaticOverridesAsync(name));
    }

    [Theory]
    [InlineData(false)]
    [InlineData(true)]
    public async Task SR5_CancelQueuedRedeployStopsItsPredecessor(bool delete)
    {
        var manager = CreateDeploymentManager();
        var actors = new List<IActorRef>();
        for (var i = 0; i < 5; i++)
        {
            Deploy(manager, "Queued" + i, "initial" + i);
            actors.Add(await ResolveInstanceChildAsync(manager, "Queued" + i));
        }
        var pending = CreateTestProbe();
        using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
        blocker.Open();
        using var transaction = blocker.BeginTransaction();
        for (var i = 0; i < 5; i++)
        {
            var name = "Queued" + i;
            actors[i].Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77", DateTimeOffset.UtcNow));
            Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
            manager.Tell(new DeployInstanceCommand("next" + i, name, "sha256:next", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow), pending.Ref);
        }
        Watch(actors[4]);
        ExpectNoMsg(TimeSpan.FromMilliseconds(100)); // fifth predecessor waits for one of four flush slots
        var cancellation = CreateTestProbe();
        if (delete) manager.Tell(new DeleteInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
        else manager.Tell(new DisableInstanceCommand("cancel", "Queued4", DateTimeOffset.UtcNow), cancellation.Ref);
        ExpectTerminated(actors[4], TimeSpan.FromSeconds(2));
        transaction.Commit();
    }

    [Fact]
    public async Task SR5_DeleteWaitsForFinalFlushAndRejectsAnOverlappingStart()
    {
        const string name = "DeleteFlushPump";
        var manager = CreateDeploymentManager();
        Deploy(manager, name, "first");
        var old = await ResolveInstanceChildAsync(manager, name);
        var deletion = CreateTestProbe();
        using var blocker = new Microsoft.Data.Sqlite.SqliteConnection("Data Source=" + _localDb.Path);
        blocker.Open();
        using (var transaction = blocker.BeginTransaction())
        {
            old.Tell(new SetStaticAttributeCommand("set", name, "Temperature", "77.7", DateTimeOffset.UtcNow));
            Assert.True(ExpectMsg<SetStaticAttributeResponse>().Success);
            Watch(old);
            manager.Tell(new DeleteInstanceCommand("delete", name, DateTimeOffset.UtcNow), deletion.Ref);
            ExpectTerminated(old, TimeSpan.FromSeconds(2));
            deletion.ExpectNoMsg(TimeSpan.FromMilliseconds(100));
            manager.Tell(new DeployInstanceCommand("overlap", name, "sha256:overlap", MakeConfigJson(name), "admin", DateTimeOffset.UtcNow));
            Assert.Equal(DeploymentStatus.Failed, ExpectMsg<DeploymentStatusResponse>().Status);
            transaction.Commit();
        }
        Assert.True(deletion.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(10)).Success);
        Assert.Empty(await _storage.GetStaticOverridesAsync(name));
        Deploy(manager, name, "after-delete");
    }

}