ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentWarmThenGateTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Text.Json;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// WP3.1 test group 9 — warm-then-gate deploys and startup batch pre-warm.
///
/// <para>The site-side compile gate (S3) must stay synchronous on the Deployment Manager's
/// thread, because redeploy-supersede and delete-during-redeploy both depend on strict mailbox
/// FIFO. But the Roslyn compile it performs used to hold the singleton for the whole
/// compilation, stalling every OTHER instance's commands behind one instance's scripts. WP3.1
/// warms the compile off-thread first and then re-runs the gate as pure cache hits, with a
/// per-instance in-flight guard preserving same-instance ordering.</para>
///
/// <para>These tests pin the ordering contract, not the timing: a command for the SAME instance
/// arriving during a warm must be queued and applied after the deploy, a superseded deploy must
/// answer its deployer instead of leaving it to Ask-timeout, and commands for DIFFERENT
/// instances must not block each other.</para>
///
/// <para>Shares the <c>SiteScriptCompileCache</c> collection because the batch pre-warm test
/// asserts on that process-wide cache's hit counter.</para>
/// </summary>
[Collection("SiteScriptCompileCache")]
public class DeploymentWarmThenGateTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    public DeploymentWarmThenGateTests()
    {
        _localDb = TestLocalDb.CreateTemp("dm-warm-gate-test");
        _storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    void IDisposable.Dispose()
    {
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null) =>
        ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage, _compilationService, _sharedScriptLibrary, null,
            new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance, null,
            healthCollector)));

    /// <summary>
    /// Register row 40: starts a Deployment Manager and blocks until it has PROVABLY finished
    /// its startup config load, by waiting for its first instance-count report. Replaces a bare
    /// <c>await Task.Delay(500)</c>, which merely hoped 500 ms was enough and — for the empty-site
    /// assertion — could not tell "startup ran and found nothing" from "startup had not run yet".
    /// </summary>
    private (IActorRef Dm, DeployedCountCollector Health) CreateStartedDeploymentManager()
    {
        var health = new DeployedCountCollector();
        var dm = CreateDeploymentManager(health);
        AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
            TimeSpan.FromMilliseconds(25));
        return (dm, health);
    }

    /// <summary>
    /// Captures the deployed-instance count the Deployment Manager reports. The count is
    /// mutated only on the actor thread — <c>HandleDeploy</c> adds the instance name,
    /// <c>HandleDelete</c> removes it — so it is an exact, storage-race-free record of the
    /// order in which the two commands were APPLIED.
    /// </summary>
    private sealed class DeployedCountCollector : ISiteHealthCollector
    {
        private int _reports;

        public int LastDeployedCount { get; private set; }

        /// <summary>
        /// Register row 40: how many times the Deployment Manager has reported instance
        /// counts. The manager calls <see cref="SetInstanceCounts"/> unconditionally the
        /// moment its startup config load completes (<c>HandleStartupConfigsLoaded</c>) —
        /// including for an empty site — so a non-zero value here is the explicit
        /// observable edge for "startup has finished", replacing a wall-clock sleep.
        /// Without it, <c>LastDeployedCount == 0</c> is satisfied by the field's initial
        /// value and proves nothing about the manager having started at all.
        /// </summary>
        public int ReportCount => Volatile.Read(ref _reports);
        public void IncrementScriptError() { }
        public void IncrementAlarmError() { }
        public void IncrementDeadLetter() { }
        public void IncrementSiteAuditWriteFailures() { }
        public void IncrementAuditRedactionFailure() { }
        public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
        public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
        public void RemoveConnection(string connectionName) { }
        public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
        public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
        public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
        public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
        public void SetInstanceCounts(int deployed, int enabled, int disabled)
        {
            LastDeployedCount = deployed;
            Interlocked.Increment(ref _reports);
        }

        public void SetParkedMessageCount(int count) { }
        public void SetNodeHostname(string hostname) { }
        public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
        public void SetActiveNode(bool isActive) { }
        public bool IsActiveNode => true;
        public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId)
            => throw new NotSupportedException();
    }

    private static string ConfigJson(string instanceName, string? scriptCode = null) =>
        JsonSerializer.Serialize(new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
            ],
            Scripts = scriptCode is null
                ? []
                : [new ResolvedScript { CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call" }]
        });

    [Fact]
    public async Task DeleteArrivingDuringTheCompileWarm_IsQueuedAndAppliedAfterTheDeploy()
    {
        // The startup load has demonstrably completed (it reported counts at least once), so
        // the zero below is the manager's own observation of an empty site rather than an
        // untouched field.
        var (dm, health) = CreateStartedDeploymentManager();
        Assert.Equal(0, health.LastDeployedCount);

        var deployProbe = CreateTestProbe();
        var deleteProbe = CreateTestProbe();

        // Back-to-back on the mailbox: the delete lands while the deploy's compile warm is
        // still in flight, so it must be queued rather than racing ahead of the deploy.
        dm.Tell(new DeployInstanceCommand(
            "dep-1", "WarmPump", "h1", ConfigJson("WarmPump", "return 1;"), "admin", DateTimeOffset.UtcNow),
            deployProbe.Ref);
        dm.Tell(new DeleteInstanceCommand("del-1", "WarmPump", DateTimeOffset.UtcNow), deleteProbe.Ref);

        var deploy = deployProbe.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
        Assert.Equal(DeploymentStatus.Success, deploy.Status);

        var delete = deleteProbe.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));
        Assert.True(delete.Success);

        // Terminal in-memory state: the deploy applied FIRST (adding the instance) and the
        // delete applied SECOND (removing it), leaving the count at 0. Had the delete raced
        // ahead of the warm it would have removed nothing and the deploy would have left the
        // count at 1. This is the ordering signal rather than the SQLite row, because the
        // deploy's store and the delete's remove are independent background tasks whose
        // completion order the actor has never guaranteed (true before WP3.1 as well).
        AwaitAssert(() => Assert.Equal(0, health.LastDeployedCount), TimeSpan.FromSeconds(10));
    }

    [Fact]
    public async Task SecondDeployDuringTheWarm_SupersedesTheFirst_AndAnswersItsDeployer()
    {
        var (dm, _) = CreateStartedDeploymentManager();

        var first = CreateTestProbe();
        var second = CreateTestProbe();

        dm.Tell(new DeployInstanceCommand(
            "dep-a", "SupersedePump", "h1", ConfigJson("SupersedePump", "return 1;"), "admin", DateTimeOffset.UtcNow),
            first.Ref);
        dm.Tell(new DeployInstanceCommand(
            "dep-b", "SupersedePump", "h2", ConfigJson("SupersedePump", "return 2;"), "admin", DateTimeOffset.UtcNow),
            second.Ref);

        // The displaced deployer is answered rather than left to time out its Ask.
        var superseded = first.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
        Assert.Equal("dep-a", superseded.DeploymentId);
        Assert.Equal(DeploymentStatus.Failed, superseded.Status);
        Assert.Contains("superseded", superseded.ErrorMessage!, StringComparison.OrdinalIgnoreCase);

        var winner = second.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
        Assert.Equal("dep-b", winner.DeploymentId);
        Assert.Equal(DeploymentStatus.Success, winner.Status);

        // Exactly one row, carrying the winning revision hash.
        var configs = await _storage.GetAllDeployedConfigsAsync();
        var row = Assert.Single(configs, c => c.InstanceUniqueName == "SupersedePump");
        Assert.Equal("h2", row.RevisionHash);
    }

    [Fact]
    public async Task DeploysForDifferentInstances_DoNotBlockEachOther()
    {
        var (dm, _) = CreateStartedDeploymentManager();

        var probeX = CreateTestProbe();
        var probeY = CreateTestProbe();

        dm.Tell(new DeployInstanceCommand(
            "dep-x", "PumpX", "hx", ConfigJson("PumpX", "return 1;"), "admin", DateTimeOffset.UtcNow),
            probeX.Ref);
        dm.Tell(new DeployInstanceCommand(
            "dep-y", "PumpY", "hy", ConfigJson("PumpY", "return 2;"), "admin", DateTimeOffset.UtcNow),
            probeY.Ref);

        // Both apply; the per-instance warm guard scopes to the instance, so a warm for X
        // never queues a command for Y.
        Assert.Equal(DeploymentStatus.Success,
            probeX.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
        Assert.Equal(DeploymentStatus.Success,
            probeY.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);

        var configs = await _storage.GetAllDeployedConfigsAsync();
        Assert.Contains(configs, c => c.InstanceUniqueName == "PumpX");
        Assert.Contains(configs, c => c.InstanceUniqueName == "PumpY");
    }

    [Fact]
    public async Task StaggeredStartup_PreWarmsEachBatchSoInstanceActorPreStartCompilesAreCacheHits()
    {
        // Two instances sharing one script body. The batch pre-warm compiles it once; the
        // second config's warm and BOTH Instance Actors' PreStart compiles are then hits.
        // Before WP3.1 every Instance Actor Roslyn-compiled its own scripts inside PreStart,
        // serialising a site's whole failover recovery behind compilation.
        const string sharedCode = "return 41 + 1;";
        await _storage.StoreDeployedConfigAsync(
            "BatchOne", ConfigJson("BatchOne", sharedCode), "d1", "h1", true);
        await _storage.StoreDeployedConfigAsync(
            "BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true);

        SiteScriptCompileCache.Clear();
        // Baseline rather than Assert.Equal(0, Hits): Hits is a process-global counter and test
        // classes outside this collection compile scripts in parallel, so a foreign hit can land
        // between the Clear() and the read (deferred-work register row 91). Everything below is
        // asserted as a delta over this baseline.
        var hitsBefore = SiteScriptCompileCache.Hits;

        CreateDeploymentManager();

        AwaitAssert(() =>
        {
            // One compile, then repeated hits: the second pre-warm plus both PreStarts.
            Assert.True(SiteScriptCompileCache.Hits - hitsBefore >= 3,
                $"expected the pre-warmed body to be served from cache, saw " +
                $"{SiteScriptCompileCache.Hits - hitsBefore} hits over baseline {hitsBefore}");
        }, TimeSpan.FromSeconds(20));
    }
}