ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ReconcileStartsInstanceTests.cs

using System.Collections.Concurrent;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Regression tests for deferred register row 67 — <b>startup reconcile healed the FILE but not
/// the RUNTIME</b>.
///
/// <para>
/// Instance Actors are created from exactly two paths: the Deployment Manager's boot-time SQLite
/// load, and an explicit deploy/enable command. <see cref="SiteReconciliationActor"/>'s
/// node-STARTUP fetch runs AFTER that boot load has already read storage, so a config it healed
/// into the file started nothing — the enabled, successfully-deployed instance stayed dead until
/// the NEXT restart, whose load finally picked it up. Proven live on site-a 2026-08-15:
/// "Reconcile: fetched + stored config" with no "InstanceActor started" after it, and a debug
/// snapshot answering <c>instanceNotFound</c>.
/// </para>
///
/// <para>
/// The fix is an <see cref="EnsureInstanceStartedCommand"/> raised by the reconcile pass and
/// handled by the Deployment Manager singleton. These tests pin both halves plus the seam
/// between them.
/// </para>
/// </summary>
public class ReconcileStartsInstanceTests : TestKit, IDisposable
{
    private const string SiteIdentifier = "site-1";
    private const string NodeId = "node-a";

    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    public ReconcileStartsInstanceTests()
    {
        _localDb = TestLocalDb.CreateTemp("reconcile-start-test");
        _storage = new SiteStorageService(
            _localDb.Db, NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    void IDisposable.Dispose()
    {
        // Shut the actors down BEFORE the ILocalDb they write through is disposed.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    private IActorRef CreateDeploymentManager(IServiceProvider? serviceProvider = null) =>
        ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions(),
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager: null,
            healthCollector: null,
            serviceProvider: serviceProvider,
            loggerFactory: null,
            configFetcher: null)));

    private IActorRef CreateReconciliationActor(
        IActorRef siteCommunicationActor,
        IDeploymentConfigFetcher fetcher,
        IActorRef? deploymentManager = null) =>
        ActorOf(Props.Create(() => new SiteReconciliationActor(
            _storage,
            fetcher,
            siteCommunicationActor,
            SiteIdentifier,
            NodeId,
            NullLogger<SiteReconciliationActor>.Instance,
            TimeSpan.FromMilliseconds(50),   // initial delay — fast for tests
            TimeSpan.FromSeconds(5),         // ask timeout
            deploymentManager)));

    private static string MakeConfigJson(string instanceName)
    {
        var config = new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
            ]
        };
        return JsonSerializer.Serialize(config);
    }

    /// <summary>
    /// Asks the Deployment Manager for a debug snapshot — the exact probe that answered
    /// <c>instanceNotFound</c> on the live rig. <c>InstanceNotFound == false</c> means an
    /// Instance Actor is running and routable.
    /// </summary>
    private async Task<bool> InstanceIsRunningAsync(IActorRef deploymentManager, string instanceName)
    {
        var snapshot = await deploymentManager.Ask<DebugViewSnapshot>(
            new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()),
            TimeSpan.FromSeconds(5));
        return !snapshot.InstanceNotFound;
    }

    // ── The reproduction ──

    /// <summary>
    /// THE row-67 reproduction, end to end over one shared storage file: the node boots with no
    /// config for an enabled instance (so the Deployment Manager's startup load starts nothing),
    /// then the startup reconcile pass fetches and stores that config. The instance must be
    /// RUNNING afterwards — not merely present in SQLite waiting for a second restart.
    ///
    /// <para>
    /// <b>Fails before the fix</b> at the final assert: the config row is stored, but the debug
    /// snapshot still answers <c>InstanceNotFound = true</c> because nothing ever created the
    /// Instance Actor.
    /// </para>
    /// </summary>
    [Fact]
    public async Task StartupReconcile_StoringConfigForEnabledInstance_StartsTheInstanceActor()
    {
        // Boot the Deployment Manager against EMPTY storage — this is the missing-config boot
        // (lost volume / first boot after a restore / rehomed data dir).
        var dm = CreateDeploymentManager();
        await Task.Delay(500);

        // Baseline: the instance is genuinely not running.
        Assert.False(await InstanceIsRunningAsync(dm, "HealedPump"));

        // Now the startup reconcile pass runs and central reports the instance as a gap.
        const string instanceName = "HealedPump";
        var configJson = MakeConfigJson(instanceName);
        var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson));
        var commProbe = CreateTestProbe();

        CreateReconciliationActor(commProbe, fetcher, dm);

        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
        commProbe.Reply(new ReconcileSiteResponse(
            [new ReconcileGapItem(instanceName, "dep-healed", "rev-healed", true, "tok-healed")],
            [],
            "http://central:9000"));

        // The file heals...
        await AwaitAssertAsync(async () =>
        {
            var configs = await _storage.GetAllDeployedConfigsAsync();
            Assert.Contains(configs, c => c.InstanceUniqueName == instanceName);
        }, TimeSpan.FromSeconds(5));

        // ...and so must the RUNTIME. This is the assertion that fails pre-fix.
        await AwaitAssertAsync(async () =>
            Assert.True(
                await InstanceIsRunningAsync(dm, instanceName),
                "Reconcile stored the config but the InstanceActor was never started "
                + "— the instance would stay dead until the next restart (register row 67)"),
            TimeSpan.FromSeconds(10));
    }

    /// <summary>
    /// The heal must not be silent: starting a reconcile-healed instance emits an operational
    /// <c>deployment</c> site event naming the instance, so an operator can see that a node
    /// booted without a config and self-repaired.
    /// </summary>
    [Fact]
    public async Task ReconcileStartedInstance_EmitsASiteEvent()
    {
        var siteLog = new FakeSiteEventLogger();
        var dm = CreateDeploymentManager(new SingleServiceProvider(siteLog));
        await Task.Delay(500);

        const string instanceName = "SurfacedPump";
        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-s", "rev-s", true);

        dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));

        await AwaitAssertAsync(() =>
        {
            Assert.Contains(siteLog.OfType("deployment"), r =>
                r.InstanceId == instanceName &&
                r.Source == "DeploymentManagerActor" &&
                r.Message.Contains("self-heal", StringComparison.OrdinalIgnoreCase));
            return Task.CompletedTask;
        }, TimeSpan.FromSeconds(10));
    }

    // ── The reconcile half: which instances get a start request ──

    /// <summary>
    /// An ENABLED instance healed by the pass produces an ensure-started request to the
    /// Deployment Manager proxy.
    /// </summary>
    [Fact]
    public void ReconcilePass_RequestsStart_ForHealedEnabledInstance()
    {
        var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"B\"}"));
        var commProbe = CreateTestProbe();
        var dmProbe = CreateTestProbe();

        CreateReconciliationActor(commProbe, fetcher, dmProbe);

        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
        commProbe.Reply(new ReconcileSiteResponse(
            [new ReconcileGapItem("B", "depB", "rev2", true, "tok-b")],
            [],
            "http://central:9000"));

        var ensure = dmProbe.ExpectMsg<EnsureInstanceStartedCommand>(TimeSpan.FromSeconds(10));
        Assert.Equal("B", ensure.InstanceUniqueName);
        Assert.Contains("reconcile", ensure.Reason, StringComparison.OrdinalIgnoreCase);
    }

    /// <summary>
    /// A DISABLED instance healed by the pass must NOT be started — a disabled instance
    /// legitimately has a config row and no running actor.
    /// </summary>
    [Fact]
    public void ReconcilePass_DoesNotRequestStart_ForHealedDisabledInstance()
    {
        var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"D\"}"));
        var commProbe = CreateTestProbe();
        var dmProbe = CreateTestProbe();

        CreateReconciliationActor(commProbe, fetcher, dmProbe);

        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
        commProbe.Reply(new ReconcileSiteResponse(
            [new ReconcileGapItem("D", "depD", "revD", false, "tok-d")],
            [],
            "http://central:9000"));

        dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2));
    }

    /// <summary>
    /// An item that failed to fetch was never stored, so it must not be started either.
    /// </summary>
    [Fact]
    public void ReconcilePass_DoesNotRequestStart_ForFailedFetch()
    {
        var fetcher = new FakeConfigFetcher(_ =>
            Task.FromException<string>(new DeploymentConfigFetchException("boom", isSuperseded: false)));
        var commProbe = CreateTestProbe();
        var dmProbe = CreateTestProbe();

        CreateReconciliationActor(commProbe, fetcher, dmProbe);

        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
        commProbe.Reply(new ReconcileSiteResponse(
            [new ReconcileGapItem("Bad", "depBad", "revBad", true, "tok-bad")],
            [],
            "http://central:9000"));

        dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2));
    }

    // ── The Deployment Manager half: the ensure-started contract ──

    /// <summary>
    /// The command is idempotent: both site nodes' reconcile actors legitimately raise it for
    /// the same instance, and a repeat must not disturb (or replace) the running actor.
    /// </summary>
    [Fact]
    public async Task EnsureInstanceStarted_IsIdempotent_WhenAlreadyRunning()
    {
        var dm = CreateDeploymentManager();
        await Task.Delay(500);

        const string instanceName = "IdempotentPump";
        dm.Tell(new DeployInstanceCommand(
            "dep-idem", instanceName, "sha256:abc",
            MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000);

        var before = await ResolveInstanceChildAsync(dm, instanceName);

        // Two redundant ensure-started requests, as the two nodes would send.
        dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
        dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
        await Task.Delay(1000);

        // Same actor ref — nothing was restarted or replaced, and it is still routable.
        var after = await ResolveInstanceChildAsync(dm, instanceName);
        Assert.Equal(before, after);
        Assert.True(await InstanceIsRunningAsync(dm, instanceName));
    }

    /// <summary>
    /// The durable row is the authority on enabled/disabled state: a DISABLED instance stays
    /// stopped even if an ensure-started request arrives for it (e.g. a disable landed between
    /// the reconcile fetch and the start request).
    /// </summary>
    [Fact]
    public async Task EnsureInstanceStarted_DisabledInstance_StaysStopped()
    {
        var dm = CreateDeploymentManager();
        await Task.Delay(500);

        const string instanceName = "DisabledPump";
        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-d", "rev-d", false);

        dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
        await Task.Delay(1500);

        Assert.False(await InstanceIsRunningAsync(dm, instanceName));
    }

    /// <summary>
    /// An ensure-started request for an instance with no stored config on THIS node is a quiet
    /// no-op — the normal case on an unreplicated site whose peer node healed its own file.
    /// The Deployment Manager must not crash or invent an instance.
    /// </summary>
    [Fact]
    public async Task EnsureInstanceStarted_UnknownInstance_IsAQuietNoOp()
    {
        var dm = CreateDeploymentManager();
        await Task.Delay(500);

        dm.Tell(new EnsureInstanceStartedCommand("NeverHeardOf", "startup reconcile"));
        await Task.Delay(1500);

        Assert.False(await InstanceIsRunningAsync(dm, "NeverHeardOf"));

        // The manager is still alive and serving other work.
        dm.Tell(new DeployInstanceCommand(
            "dep-after", "AfterPump", "sha256:abc",
            MakeConfigJson("AfterPump"), "admin", DateTimeOffset.UtcNow));
        Assert.Equal(
            DeploymentStatus.Success,
            ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5)).Status);
    }

    /// <summary>
    /// Resolves the Instance Actor child of the Deployment Manager by path — the direct
    /// "the InstanceActor exists" check, complementing the debug-snapshot probe.
    /// </summary>
    private async Task<IActorRef> ResolveInstanceChildAsync(IActorRef deploymentManager, string instanceName) =>
        await Sys.ActorSelection(deploymentManager.Path / instanceName)
            .ResolveOne(TimeSpan.FromSeconds(5));

    /// <summary>
    /// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior and
    /// records every call thread-safely (the fetch runs on a pool thread).
    /// </summary>
    private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
    {
        private readonly Func<string, Task<string>> _behavior;
        public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();

        public FakeConfigFetcher(Func<string, Task<string>> behavior) => _behavior = behavior;

        public async Task<string> FetchAsync(
            string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
        {
            Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
            await Task.Yield();
            return await _behavior(deploymentId);
        }
    }
}