ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Text.Json;
using System.Threading;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Tests for DeploymentManagerActor: startup from SQLite, staggered batching,
/// lifecycle commands, and supervision strategy.
/// </summary>
public class DeploymentManagerActorTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    public DeploymentManagerActorTests()
    {
        _localDb = TestLocalDb.CreateTemp("dm-test");
        _storage = new SiteStorageService(
            _localDb.Db,
            NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    void IDisposable.Dispose()
    {
        // TestKit teardown first, so no in-flight actor can reach a disposed ILocalDb;
        // then dispose the database before deleting — the master connection anchors the WAL.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    private IActorRef CreateDeploymentManager(
        SiteRuntimeOptions? options = null, IServiceProvider? serviceProvider = null,
        IDeploymentConfigFetcher? configFetcher = null)
    {
        options ??= new SiteRuntimeOptions();
        return ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null, // no stream manager in tests
            options,
            NullLogger<DeploymentManagerActor>.Instance,
            // Named from here on. These trailing parameters are all optional and several
            // share a type, so a positional list silently binds the wrong argument when the
            // signature changes — which is exactly what removing replicationActor did.
            dclManager: null,
            healthCollector: null,
            serviceProvider: serviceProvider,
            loggerFactory: null,
            configFetcher: configFetcher)));
    }

    private static string MakeConfigJson(string instanceName)
    {
        var config = new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
            ]
        };
        return JsonSerializer.Serialize(config);
    }

    /// <summary>
    /// Builds a config carrying a single STATIC List attribute whose canonical
    /// JSON-array default the Instance Actor decodes into a typed <c>List&lt;int&gt;</c>
    /// in memory (InstanceActor.PreStart / DecodeAttributeValue). Used to drive a
    /// routed wait whose matched value is a collection — the shape WS-4 normalizes
    /// for cross-process transport.
    /// </summary>
    private static string MakeConfigWithListAttributeJson(string instanceName)
    {
        var config = new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute
                {
                    CanonicalName = "Setpoints", Value = "[10,20,30]",
                    DataType = "List", ElementDataType = "Int32"
                }
            ]
        };
        return JsonSerializer.Serialize(config);
    }

    /// <summary>
    /// Builds a config carrying BOTH a static List attribute (whose JSON-array default
    /// the Instance Actor decodes into a typed <c>List&lt;int&gt;</c>) and a static scalar
    /// attribute. Used to drive a routed <see cref="RouteToGetAttributesRequest"/> that
    /// exercises #162: the List value must be normalized for cross-process transport,
    /// while the scalar must pass through the normalizer unchanged.
    /// </summary>
    private static string MakeConfigWithListAndScalarAttributesJson(string instanceName)
    {
        var config = new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute
                {
                    CanonicalName = "Setpoints", Value = "[10,20,30]",
                    DataType = "List", ElementDataType = "Int32"
                },
                new ResolvedAttribute { CanonicalName = "Mode", Value = "Auto", DataType = "String" }
            ]
        };
        return JsonSerializer.Serialize(config);
    }

    /// <summary>
    /// Builds a config carrying a single callable (no-trigger) script that
    /// returns a constant — enough for an inbound <see cref="RouteToCallRequest"/>
    /// to be routed end-to-end through the Instance/Script/ScriptExecution actors.
    /// </summary>
    private static string MakeConfigWithScriptJson(string instanceName, string scriptName)
    {
        var config = new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
            ],
            Scripts =
            [
                new ResolvedScript { CanonicalName = scriptName, Code = "return 7;" }
            ]
        };
        return JsonSerializer.Serialize(config);
    }

    [Fact]
    public async Task DeploymentManager_CreatesInstanceActors_FromStoredConfigs()
    {
        // Pre-populate SQLite with deployed configs
        await _storage.StoreDeployedConfigAsync("Pump1", MakeConfigJson("Pump1"), "d1", "h1", true);
        await _storage.StoreDeployedConfigAsync("Pump2", MakeConfigJson("Pump2"), "d2", "h2", true);

        var actor = CreateDeploymentManager(
            new SiteRuntimeOptions { StartupBatchSize = 100, StartupBatchDelayMs = 10 });

        // Allow time for async startup (load configs + create actors)
        await Task.Delay(2000);

        // Verify by deploying -- if actors already exist, we'd get a warning
        // Instead, verify by checking we can send lifecycle commands
        actor.Tell(new DisableInstanceCommand("cmd-1", "Pump1", DateTimeOffset.UtcNow));
        var response = ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
        Assert.True(response.Success);
        Assert.Equal("Pump1", response.InstanceUniqueName);
    }

    [Fact]
    public async Task DeploymentManager_SkipsDisabledInstances_OnStartup()
    {
        await _storage.StoreDeployedConfigAsync("Active1", MakeConfigJson("Active1"), "d1", "h1", true);
        await _storage.StoreDeployedConfigAsync("Disabled1", MakeConfigJson("Disabled1"), "d2", "h2", false);

        var actor = CreateDeploymentManager(
            new SiteRuntimeOptions { StartupBatchSize = 100, StartupBatchDelayMs = 10 });

        await Task.Delay(2000);

        // The disabled instance should NOT have an actor running
        // Try to disable it -- it should succeed (no actor to stop, but SQLite update works)
        actor.Tell(new DisableInstanceCommand("cmd-2", "Disabled1", DateTimeOffset.UtcNow));
        var response = ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
        Assert.True(response.Success);
    }

    [Fact]
    public async Task DeploymentManager_StaggeredBatchCreation()
    {
        // Create more instances than the batch size
        for (int i = 0; i < 5; i++)
        {
            var name = $"Batch{i}";
            await _storage.StoreDeployedConfigAsync(name, MakeConfigJson(name), $"d{i}", $"h{i}", true);
        }

        // Use a small batch size to force multiple batches
        var actor = CreateDeploymentManager(
            new SiteRuntimeOptions { StartupBatchSize = 2, StartupBatchDelayMs = 50 });

        // Wait for all batches to complete (3 batches with 50ms delay = ~150ms + processing)
        await Task.Delay(3000);

        // Verify all instances are running by disabling them
        for (int i = 0; i < 5; i++)
        {
            actor.Tell(new DisableInstanceCommand($"cmd-{i}", $"Batch{i}", DateTimeOffset.UtcNow));
            var response = ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
            Assert.True(response.Success);
        }
    }

    [Fact]
    public async Task DeploymentManager_Deploy_CreatesNewInstance()
    {
        var actor = CreateDeploymentManager();

        await Task.Delay(500); // Wait for empty startup

        actor.Tell(new DeployInstanceCommand(
            "dep-100", "NewPump", "sha256:xyz", MakeConfigJson("NewPump"), "admin", DateTimeOffset.UtcNow));

        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Success, response.Status);
        Assert.Equal("NewPump", response.InstanceUniqueName);
    }

    // ── S3: site-side compile failure rejects the deployment (no partial state) ──

    private static string MakeConfigWithBrokenScriptJson(string instanceName) =>
        JsonSerializer.Serialize(new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes = [new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }],
            Scripts = [new ResolvedScript { CanonicalName = "S1", Code = "this is not C# ~~~" }]
        });

    [Fact]
    public async Task Deploy_WithNonCompilingScript_ReportsFailed_AndCreatesNoInstanceActor()
    {
        var actor = CreateDeploymentManager();
        await Task.Delay(500); // wait for empty startup

        actor.Tell(new DeployInstanceCommand(
            "dep-bad", "bad-instance", "r1",
            MakeConfigWithBrokenScriptJson("bad-instance"), "admin", DateTimeOffset.UtcNow));

        var resp = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal(DeploymentStatus.Failed, resp.Status);
        Assert.NotNull(resp.ErrorMessage);
        Assert.Contains("compil", resp.ErrorMessage!, StringComparison.OrdinalIgnoreCase);

        // No partial state applied: instance actor must not exist …
        Sys.ActorSelection(actor.Path / "bad-instance").Tell(new Identify(1));
        Assert.Null(ExpectMsg<ActorIdentity>().Subject);

        // … and no config must have been persisted.
        var configs = await _storage.GetAllDeployedConfigsAsync();
        Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "bad-instance");
    }

    // ── S5: startup deployed-config load is retried on transient failure ──

    [Fact]
    public void StartupLoadFailure_IsRetried_UntilSuccess()
    {
        // The first startup load throws (transient SQLite failure); the second
        // succeeds and returns one enabled config. The DM must retry rather than
        // stall as a silent zero-instance node — so the Instance Actor eventually
        // exists.
        var calls = 0;
        Func<Task<List<DeployedInstance>>> loader = () =>
            Interlocked.Increment(ref calls) == 1
                ? Task.FromException<List<DeployedInstance>>(new IOException("db locked"))
                : Task.FromResult(new List<DeployedInstance>
                {
                    new()
                    {
                        InstanceUniqueName = "inst-1",
                        ConfigJson = MakeConfigJson("inst-1"),
                        DeploymentId = "d1",
                        RevisionHash = "h1",
                        IsEnabled = true
                    }
                });

        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage, _compilationService, _sharedScriptLibrary, null,
            new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance,
            // dclManager, healthCollector, serviceProvider, loggerFactory, configFetcher.
            // Props.Create builds an expression tree, which rejects named arguments that
            // are out of position, so the optional tail has to be padded positionally.
            null, null, null, null, null,
            startupLoadRetryInterval: TimeSpan.FromMilliseconds(200), configLoader: loader)));

        AwaitAssert(() =>
        {
            Assert.True(Volatile.Read(ref calls) >= 2,
                $"expected the load to be retried; calls={Volatile.Read(ref calls)}");
            Sys.ActorSelection(dm.Path / "inst-1").Tell(new Identify(1));
            Assert.NotNull(ExpectMsg<ActorIdentity>(TimeSpan.FromSeconds(1)).Subject);
        }, TimeSpan.FromSeconds(10));
    }

    // ── M1.6: site event log `deployment` category ─────────────────────────

    [Fact]
    public async Task DeploymentManager_DeploySuccess_EmitsDeploymentSiteEvent()
    {
        var siteLog = new FakeSiteEventLogger();
        var actor = CreateDeploymentManager(serviceProvider: new SingleServiceProvider(siteLog));

        await Task.Delay(500); // wait for empty startup

        actor.Tell(new DeployInstanceCommand(
            "dep-evt-1", "AuditedPump", "sha256:xyz",
            MakeConfigJson("AuditedPump"), "admin", DateTimeOffset.UtcNow));
        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Success, response.Status);

        AwaitAssert(() =>
        {
            var rows = siteLog.OfType("deployment");
            Assert.Contains(rows, r =>
                r.Severity == "Info" &&
                r.InstanceId == "AuditedPump" &&
                r.Source == "DeploymentManagerActor" &&
                r.Message.Contains("deploy", StringComparison.OrdinalIgnoreCase));
        }, TimeSpan.FromSeconds(2));
    }

    [Fact]
    public async Task DeploymentManager_DisableEnableDelete_EmitDeploymentSiteEvents()
    {
        var siteLog = new FakeSiteEventLogger();
        var actor = CreateDeploymentManager(serviceProvider: new SingleServiceProvider(siteLog));

        await Task.Delay(500);

        actor.Tell(new DeployInstanceCommand(
            "dep-evt-2", "EvtPump", "sha256:abc",
            MakeConfigJson("EvtPump"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000);

        // The deployment site events are emitted fire-and-forget off the actor
        // thread (LogDeploymentEvent runs in a ContinueWith continuation), so
        // poll for each event with AwaitAssert rather than a bare Task.Delay —
        // a fixed sleep is racy under CI load.
        actor.Tell(new DisableInstanceCommand("cmd-de1", "EvtPump", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5)).Success);
        AwaitAssert(() => Assert.Contains(siteLog.OfType("deployment"),
            r => r.Message.Contains("disabled", StringComparison.OrdinalIgnoreCase)),
            TimeSpan.FromSeconds(2));

        actor.Tell(new EnableInstanceCommand("cmd-en1", "EvtPump", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5)).Success);
        AwaitAssert(() => Assert.Contains(siteLog.OfType("deployment"),
            r => r.Message.Contains("enabled", StringComparison.OrdinalIgnoreCase)),
            TimeSpan.FromSeconds(2));

        actor.Tell(new DeleteInstanceCommand("cmd-del-evt", "EvtPump", DateTimeOffset.UtcNow));
        Assert.True(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5)).Success);
        AwaitAssert(() => Assert.Contains(siteLog.OfType("deployment"),
            r => r.Message.Contains("deleted", StringComparison.OrdinalIgnoreCase)),
            TimeSpan.FromSeconds(2));
    }

    [Fact]
    public async Task DeploymentManager_Lifecycle_DisableEnableDelete()
    {
        var actor = CreateDeploymentManager();

        await Task.Delay(500);

        // Deploy
        actor.Tell(new DeployInstanceCommand(
            "dep-200", "LifecyclePump", "sha256:abc",
            MakeConfigJson("LifecyclePump"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));

        // Wait for the async deploy persistence (PipeTo) to complete
        await Task.Delay(1000);

        // Disable
        actor.Tell(new DisableInstanceCommand("cmd-d1", "LifecyclePump", DateTimeOffset.UtcNow));
        var disableResp = ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
        Assert.True(disableResp.Success);

        // Verify disabled in storage
        await Task.Delay(500);
        var configs = await _storage.GetAllDeployedConfigsAsync();
        var pump = configs.FirstOrDefault(c => c.InstanceUniqueName == "LifecyclePump");
        Assert.NotNull(pump);
        Assert.False(pump.IsEnabled);

        // Delete
        actor.Tell(new DeleteInstanceCommand("cmd-del1", "LifecyclePump", DateTimeOffset.UtcNow));
        var deleteResp = ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
        Assert.True(deleteResp.Success);

        // Verify removed from storage
        await Task.Delay(500);
        configs = await _storage.GetAllDeployedConfigsAsync();
        Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "LifecyclePump");
    }

    // ── DeploymentManager-006: query-the-site-before-redeploy ──

    [Fact]
    public async Task DeploymentStateQuery_DeployedInstance_ReturnsAppliedIdentity()
    {
        // A deployed instance must report its currently-applied deployment ID
        // and revision hash so central can reconcile before a re-deploy.
        await _storage.StoreDeployedConfigAsync(
            "QueriedPump", MakeConfigJson("QueriedPump"), "dep-applied", "sha256:applied", true);

        var actor = CreateDeploymentManager();
        await Task.Delay(2000); // allow startup to load configs

        actor.Tell(new DeploymentStateQueryRequest("corr-q1", "QueriedPump", DateTimeOffset.UtcNow));

        var response = ExpectMsg<DeploymentStateQueryResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal("corr-q1", response.CorrelationId);
        Assert.Equal("QueriedPump", response.InstanceUniqueName);
        Assert.True(response.IsDeployed);
        Assert.Equal("dep-applied", response.AppliedDeploymentId);
        Assert.Equal("sha256:applied", response.AppliedRevisionHash);
    }

    [Fact]
    public async Task DeploymentStateQuery_UnknownInstance_ReturnsNotDeployed()
    {
        // An instance the site has never received a deployment for must report
        // IsDeployed=false with null applied identity.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new DeploymentStateQueryRequest("corr-q2", "NeverDeployed", DateTimeOffset.UtcNow));

        var response = ExpectMsg<DeploymentStateQueryResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal("corr-q2", response.CorrelationId);
        Assert.False(response.IsDeployed);
        Assert.Null(response.AppliedDeploymentId);
        Assert.Null(response.AppliedRevisionHash);
    }

    [Fact]
    public void DeploymentManager_SupervisionStrategy_ResumesOnException()
    {
        var actor = CreateDeploymentManager();

        // The actor exists and is responsive -- supervision is configured
        actor.Tell(new DeployInstanceCommand(
            "dep-sup", "SupervisedPump", "sha256:sup",
            MakeConfigJson("SupervisedPump"), "admin", DateTimeOffset.UtcNow));

        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Success, response.Status);
    }

    // ── Audit Log #23 (ParentExecutionId, Task 4): inbound-API routing ──

    [Fact]
    public async Task RouteInboundApiCall_WithParentExecutionId_RoutesToScriptSuccessfully()
    {
        // A RouteToCallRequest carrying a ParentExecutionId (the inbound
        // request's ExecutionId) must be mapped to a ScriptCallRequest and
        // routed end-to-end through the Instance/Script/ScriptExecution actors.
        // The additive ParentExecutionId field must not break that routing.
        var actor = CreateDeploymentManager();
        await Task.Delay(500); // empty startup

        actor.Tell(new DeployInstanceCommand(
            "dep-route", "RoutedPump", "sha256:route",
            MakeConfigWithScriptJson("RoutedPump", "DoWork"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000); // let the InstanceActor + ScriptActor spin up

        var parentExecutionId = Guid.NewGuid();
        actor.Tell(new RouteToCallRequest(
            "route-corr-1", "RoutedPump", "DoWork",
            Parameters: null, DateTimeOffset.UtcNow, ParentExecutionId: parentExecutionId));

        var response = ExpectMsg<RouteToCallResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("route-corr-1", response.CorrelationId);
        Assert.True(response.Success, $"Routed call failed: {response.ErrorMessage}");
        Assert.Equal(7, Convert.ToInt32(response.ReturnValue));
    }

    [Fact]
    public async Task RouteInboundApiCall_WithoutParentExecutionId_StillRoutes()
    {
        // A routed call with no ParentExecutionId (e.g. the Central UI sandbox)
        // is the additive-default path — it must route exactly as before.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new DeployInstanceCommand(
            "dep-route2", "RoutedPump2", "sha256:route2",
            MakeConfigWithScriptJson("RoutedPump2", "DoWork"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000);

        // No ParentExecutionId argument — exercises the additive `= null` default.
        actor.Tell(new RouteToCallRequest(
            "route-corr-2", "RoutedPump2", "DoWork",
            Parameters: null, DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToCallResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("route-corr-2", response.CorrelationId);
        Assert.True(response.Success, $"Routed call failed: {response.ErrorMessage}");
    }

    // ── Spec §6 (WD-2b): routed RouteToWaitForAttributeRequest → InstanceActor ──

    [Fact]
    public async Task RouteInboundApiWaitForAttribute_AttributeAlreadyAtTarget_RepliesMatched()
    {
        // A routed wait whose target equals the instance's current (static)
        // attribute value must satisfy the InstanceActor fast-path and come back
        // Success:true, Matched:true with the matched value/quality.
        var actor = CreateDeploymentManager();
        await Task.Delay(500); // empty startup

        // MakeConfigJson seeds a scalar static attribute "TestAttr" = "42" (Good).
        actor.Tell(new DeployInstanceCommand(
            "dep-wait", "WaitPump", "sha256:wait",
            MakeConfigJson("WaitPump"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000); // let the InstanceActor spin up + load static attrs

        // Encode the target the same way the InstanceActor encodes the current
        // value for its codec-equality match (value-equality only across the wire).
        var encodedTarget = AttributeValueCodec.Encode("42");
        actor.Tell(new RouteToWaitForAttributeRequest(
            "wait-corr-1", "WaitPump", "TestAttr", encodedTarget,
            TimeSpan.FromSeconds(5), DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToWaitForAttributeResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("wait-corr-1", response.CorrelationId);
        Assert.True(response.Success, $"Routed wait failed: {response.ErrorMessage}");
        Assert.True(response.Matched, "Expected fast-path match (attribute already at target).");
        Assert.False(response.TimedOut);
        Assert.Equal("42", response.Value);
        Assert.Equal("Good", response.Quality);
    }

    [Fact]
    public async Task RouteInboundApiWaitForAttribute_RequireGoodQuality_IsThreadedToInstanceActor()
    {
        // #14 (full fidelity): the additive RouteToWaitForAttributeRequest.RequireGoodQuality
        // flag must reach the InstanceActor's wait. A static attribute is Good quality, so a
        // quality-gated wait whose target equals the current value still matches — proving the
        // flag threads through (and does not spuriously reject a Good value).
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new DeployInstanceCommand(
            "dep-wait-gq", "WaitPumpGq", "sha256:wait-gq",
            MakeConfigJson("WaitPumpGq"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000);

        var encodedTarget = AttributeValueCodec.Encode("42");
        actor.Tell(new RouteToWaitForAttributeRequest(
            "wait-corr-gq", "WaitPumpGq", "TestAttr", encodedTarget,
            TimeSpan.FromSeconds(5), DateTimeOffset.UtcNow,
            ParentExecutionId: null, RequireGoodQuality: true));

        var response = ExpectMsg<RouteToWaitForAttributeResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("wait-corr-gq", response.CorrelationId);
        Assert.True(response.Success, $"Routed quality-gated wait failed: {response.ErrorMessage}");
        Assert.True(response.Matched, "A Good-quality attribute at target must satisfy a RequireGoodQuality wait.");
        Assert.Equal("Good", response.Quality);
    }

    [Fact]
    public async Task RouteInboundApiWaitForAttribute_UnknownInstance_RepliesNotFound()
    {
        // A routed wait for an instance that was never deployed to this site must
        // come back Success:false with a not-found message (routing-level outcome),
        // mirroring the other RouteTo* unknown-instance paths.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new RouteToWaitForAttributeRequest(
            "wait-corr-2", "NeverDeployedWait", "TestAttr",
            AttributeValueCodec.Encode("42"), TimeSpan.FromSeconds(5), DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToWaitForAttributeResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal("wait-corr-2", response.CorrelationId);
        Assert.False(response.Success);
        Assert.False(response.Matched);
        Assert.NotNull(response.ErrorMessage);
        Assert.Contains("not found", response.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
    }

    [Fact]
    public async Task RouteInboundApiWaitForAttribute_ListValuedMatch_ReturnsSerializerSafeValue()
    {
        // WS-4: a routed wait whose matched attribute is a List-typed value comes
        // back from the Instance Actor as a concrete generic List<int> — a shape the
        // cross-process (Newtonsoft) serializer cannot reliably round-trip, which would
        // otherwise silently drop the reply and hang the caller's Ask. The handler must
        // normalize the value (via the same NormalizeRoutedReturnValue projection that
        // RouteInboundApiCall uses) to a plain CLR graph that survives transport while
        // preserving the matched elements.
        var actor = CreateDeploymentManager();
        await Task.Delay(500); // empty startup

        // Static List attribute "Setpoints" = [10,20,30] (decoded to List<int>, Good).
        actor.Tell(new DeployInstanceCommand(
            "dep-wait-list", "WaitPumpList", "sha256:wait-list",
            MakeConfigWithListAttributeJson("WaitPumpList"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000); // let the InstanceActor spin up + decode the List default

        // Encode the target the same canonical way the Instance Actor encodes the
        // current List<int> for its codec-equality match (value-equality across the wire).
        var encodedTarget = AttributeValueCodec.Encode(new List<int> { 10, 20, 30 });
        actor.Tell(new RouteToWaitForAttributeRequest(
            "wait-corr-list", "WaitPumpList", "Setpoints", encodedTarget,
            TimeSpan.FromSeconds(5), DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToWaitForAttributeResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("wait-corr-list", response.CorrelationId);
        Assert.True(response.Success, $"Routed wait failed: {response.ErrorMessage}");
        Assert.True(response.Matched, "Expected fast-path match (List attribute already at target).");
        Assert.False(response.TimedOut);
        Assert.Equal("Good", response.Quality);

        // The matched value must have been projected to a serializer-safe graph:
        // a plain List<object?> of primitives — NOT the Instance Actor's concrete
        // List<int> (which Akka's cross-process serializer drops). The JSON round-trip
        // re-materializes the numbers as boxed primitives; assert numeric equality
        // without pinning the exact CLR numeric type (long vs double is an artifact).
        Assert.NotNull(response.Value);
        var list = Assert.IsType<List<object?>>(response.Value);
        Assert.Equal(
            new[] { 10.0, 20.0, 30.0 },
            list.Select(e => Convert.ToDouble(e, System.Globalization.CultureInfo.InvariantCulture)));
    }

    // ── #162: routed RouteToGetAttributesRequest → InstanceActor (List normalization) ──

    [Fact]
    public async Task RouteInboundApiGetAttributes_ListValuedAttribute_ReturnsSerializerSafeValueAndScalarUnchanged()
    {
        // #162: a routed GetAttributes read of a List-typed static attribute comes back
        // from the Instance Actor as a concrete generic List<int> — the SAME non-primitive
        // shape the cross-process (Newtonsoft) serializer cannot reliably round-trip that
        // WS-4 fixed for WaitForAttribute. Un-normalized, the reply is silently dropped and
        // the caller's Route.To().GetAttributes() Ask hangs to timeout. The handler must
        // project EACH value through NormalizeRoutedReturnValue so the List survives the
        // wire — while leaving scalar/string/null values unchanged (no regression).
        var actor = CreateDeploymentManager();
        await Task.Delay(500); // empty startup

        // Static List attribute "Setpoints" = [10,20,30] (decoded to List<int>, Good)
        // alongside a scalar string attribute "Mode" = "Auto".
        actor.Tell(new DeployInstanceCommand(
            "dep-getattr-list", "GetAttrPumpList", "sha256:getattr-list",
            MakeConfigWithListAndScalarAttributesJson("GetAttrPumpList"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(1000); // let the InstanceActor spin up + decode the List default

        // Request the List attribute, the scalar attribute, and a name that does not
        // exist (must come back null — the Found:false path) in one round-trip.
        actor.Tell(new RouteToGetAttributesRequest(
            "getattr-corr-list", "GetAttrPumpList",
            ["Setpoints", "Mode", "DoesNotExist"], DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToGetAttributesResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal("getattr-corr-list", response.CorrelationId);
        Assert.True(response.Success, $"Routed GetAttributes failed: {response.ErrorMessage}");

        // Keys preserved: all three requested names present.
        Assert.True(response.Values.ContainsKey("Setpoints"));
        Assert.True(response.Values.ContainsKey("Mode"));
        Assert.True(response.Values.ContainsKey("DoesNotExist"));

        // The List value must have been projected to a serializer-safe graph: a plain
        // List<object?> of primitives — NOT the Instance Actor's concrete List<int>
        // (which Akka's cross-process serializer drops). The JSON round-trip re-materializes
        // the numbers as boxed primitives; assert numeric equality without pinning the exact
        // CLR numeric type (long vs double is an artifact of the round-trip).
        var listValue = Assert.IsType<List<object?>>(response.Values["Setpoints"]);
        Assert.Equal(
            new[] { 10.0, 20.0, 30.0 },
            listValue.Select(e => Convert.ToDouble(e, System.Globalization.CultureInfo.InvariantCulture)));

        // Scalar/string value passes through the normalizer UNCHANGED (no regression).
        Assert.Equal("Auto", response.Values["Mode"]);

        // A not-found attribute remains null (the Found:false path is untouched by
        // normalization — NormalizeRoutedReturnValue(null) is a no-op).
        Assert.Null(response.Values["DoesNotExist"]);
    }

    [Fact]
    public async Task RouteInboundApiGetAttributes_UnknownInstance_RepliesNotFound()
    {
        // A routed GetAttributes for an instance never deployed to this site must come
        // back Success:false with a not-found message (routing-level outcome), mirroring
        // the other RouteTo* unknown-instance paths.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new RouteToGetAttributesRequest(
            "getattr-corr-nf", "NeverDeployedGetAttr", ["Setpoints"], DateTimeOffset.UtcNow));

        var response = ExpectMsg<RouteToGetAttributesResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal("getattr-corr-nf", response.CorrelationId);
        Assert.False(response.Success);
        Assert.Empty(response.Values);
        Assert.NotNull(response.ErrorMessage);
        Assert.Contains("not found", response.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
    }

    // ── M2.11: Debug-view routing — unknown-instance not-found signal ──

    [Fact]
    public async Task RouteDebugSnapshot_UnknownInstance_SetsInstanceNotFound()
    {
        // An instance that was never deployed to this site must return a
        // DebugViewSnapshot with InstanceNotFound=true so the caller can
        // distinguish "not deployed here" from a deployed-but-empty instance.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new DebugSnapshotRequest("NeverDeployed", "snap-corr-1"));

        var snapshot = ExpectMsg<DebugViewSnapshot>(TimeSpan.FromSeconds(5));
        Assert.Equal("NeverDeployed", snapshot.InstanceUniqueName);
        Assert.True(snapshot.InstanceNotFound,
            "Expected InstanceNotFound=true for an instance not registered on this site.");
        Assert.Empty(snapshot.AttributeValues);
        Assert.Empty(snapshot.AlarmStates);
    }

    [Fact]
    public async Task RouteDebugViewSubscribe_UnknownInstance_SetsInstanceNotFound()
    {
        // Same contract for the subscribe path: unknown instance → InstanceNotFound=true.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new SubscribeDebugViewRequest("NeverDeployed2", "sub-corr-1"));

        var snapshot = ExpectMsg<DebugViewSnapshot>(TimeSpan.FromSeconds(5));
        Assert.Equal("NeverDeployed2", snapshot.InstanceUniqueName);
        Assert.True(snapshot.InstanceNotFound,
            "Expected InstanceNotFound=true for an instance not registered on this site.");
        Assert.Empty(snapshot.AttributeValues);
        Assert.Empty(snapshot.AlarmStates);
    }

    [Fact]
    public async Task RouteDebugSnapshot_KnownButEmptyInstance_DoesNotSetInstanceNotFound()
    {
        // A deployed instance with no runtime data yet must return an empty
        // snapshot with InstanceNotFound=false — the known-empty path is unchanged.
        var actor = CreateDeploymentManager();
        await Task.Delay(500);

        actor.Tell(new DeployInstanceCommand(
            "dep-dbg", "EmptyPump", "sha256:dbg",
            MakeConfigJson("EmptyPump"), "admin", DateTimeOffset.UtcNow));
        ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        await Task.Delay(800); // let InstanceActor spin up

        actor.Tell(new DebugSnapshotRequest("EmptyPump", "snap-corr-2"));

        var snapshot = ExpectMsg<DebugViewSnapshot>(TimeSpan.FromSeconds(5));
        Assert.Equal("EmptyPump", snapshot.InstanceUniqueName);
        Assert.False(snapshot.InstanceNotFound,
            "A deployed (but empty) instance must NOT set InstanceNotFound.");
    }

    // ── Task 10: notify-and-fetch — RefreshDeploymentCommand → fetch → apply ──

    [Fact]
    public async Task RefreshDeployment_FetchSucceeds_AppliesConfigAndRepliesSuccess()
    {
        // The active singleton receives a small RefreshDeploymentCommand, fetches the
        // flattened config over HTTP via IDeploymentConfigFetcher, then runs the existing
        // apply path: the config is persisted to SQLite and a Success status is replied.
        var fetcher = new FakeConfigFetcher(MakeConfigJson("FetchedPump"));
        var actor = CreateDeploymentManager(configFetcher: fetcher);
        await Task.Delay(500); // empty startup

        actor.Tell(new RefreshDeploymentCommand(
            "dep-fetch-1", "FetchedPump", "sha256:fetch", "admin", DateTimeOffset.UtcNow,
            "http://central:9000", "tok-123"));

        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Success, response.Status);
        Assert.Equal("FetchedPump", response.InstanceUniqueName);
        Assert.Equal("dep-fetch-1", response.DeploymentId);

        // The fetcher was called with the command's fetch coordinates.
        Assert.Equal("http://central:9000", fetcher.LastBaseUrl);
        Assert.Equal("dep-fetch-1", fetcher.LastDeploymentId);
        Assert.Equal("tok-123", fetcher.LastToken);

        // The existing apply path ran end-to-end: the fetched config is persisted.
        var configs = await _storage.GetAllDeployedConfigsAsync();
        Assert.Contains(configs, c => c.InstanceUniqueName == "FetchedPump");
    }

    [Fact]
    public async Task RefreshDeployment_FetchFails_RepliesFailedAndDoesNotApply()
    {
        // A config fetch failure (DeploymentConfigFetchException surfaces as a faulted
        // Task) must reply Failed with a "Communication failure:" message and must NOT
        // apply anything — no config persisted, no Instance Actor created.
        var fetcher = new FakeConfigFetcher(
            new DeploymentConfigFetchException("central unreachable", isSuperseded: false));
        var actor = CreateDeploymentManager(configFetcher: fetcher);
        await Task.Delay(500);

        actor.Tell(new RefreshDeploymentCommand(
            "dep-fetch-fail", "UnfetchedPump", "sha256:x", "admin", DateTimeOffset.UtcNow,
            "http://central:9000", "tok-x"));

        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Failed, response.Status);
        Assert.Equal("UnfetchedPump", response.InstanceUniqueName);
        Assert.NotNull(response.ErrorMessage);
        Assert.StartsWith("Communication failure:", response.ErrorMessage!);

        // No apply occurred: nothing was persisted for the instance.
        await Task.Delay(500);
        var configs = await _storage.GetAllDeployedConfigsAsync();
        Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "UnfetchedPump");
    }

    [Fact]
    public void RefreshDeployment_NullFetcher_RepliesFailed()
    {
        // A node with no configured fetcher (the `_configFetcher is null` guard) must reply
        // Failed rather than NRE-crashing or silently dropping the central Ask.
        var actor = CreateDeploymentManager(); // no configFetcher

        actor.Tell(new RefreshDeploymentCommand(
            "dep-null", "Pump", "sha256:x", "admin", DateTimeOffset.UtcNow,
            "http://central:9000", "tok"));

        var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(2));
        Assert.Equal(DeploymentStatus.Failed, response.Status);
        Assert.Equal("Pump", response.InstanceUniqueName);
    }

    [Fact]
    public async Task RefreshDeployment_ReplyGoesToOriginalSender_AcrossAsyncFetch()
    {
        // The reply must reach the ORIGINAL sender (central's Ask temp actor), proving
        // the captured replyTo survives the async fetch + PipeTo continuation — where
        // Akka's Sender is no longer valid. Send with an explicit probe as the sender
        // and assert the probe (not the default test actor) receives the response.
        var fetcher = new FakeConfigFetcher(MakeConfigJson("SenderPump"));
        var probe = CreateTestProbe();
        var actor = CreateDeploymentManager(configFetcher: fetcher);
        await Task.Delay(500);

        actor.Tell(new RefreshDeploymentCommand(
            "dep-sender", "SenderPump", "sha256:s", "admin", DateTimeOffset.UtcNow,
            "http://central:9000", "tok-s"), probe.Ref);

        var response = probe.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
        Assert.Equal(DeploymentStatus.Success, response.Status);
        Assert.Equal("SenderPump", response.InstanceUniqueName);
    }

    // ── LocalDb Phase 2 / Task 12: the active-node central-only purge ──

    [Fact]
    public async Task ApplyingArtifacts_PurgesCentralOnlyNotificationConfig()
    {
        // Security cleanup. notification_lists and smtp_configurations can hold plaintext
        // SMTP passwords written by a pre-2026-07-10 build, and the ACTIVE node's artifact
        // apply is what clears them (DeploymentManagerActor.HandleDeployArtifacts). The
        // standby used to hold a second copy of this call in SiteReplicationActor; LocalDb
        // Phase 2 deleted that actor, so this is now the ONLY call site that keeps the
        // tables empty.
        //
        // ArtifactStorageTests covers the storage method, not the actor's call to it, so
        // without this test the call could be dropped and every suite would stay green.
        // That is precisely the kind of silent security regression it exists to prevent —
        // verified red-first by commenting out the call.
        await SeedCentralOnlyRowsAsync();
        Assert.Equal(1, await RowCountAsync("notification_lists"));
        Assert.Equal(1, await RowCountAsync("smtp_configurations"));

        var manager = CreateDeploymentManager();
        manager.Tell(new DeployArtifactsCommand(
            DeploymentId: "dep-purge-1",
            SharedScripts: null,
            ExternalSystems: null,
            DatabaseConnections: null,
            NotificationLists: null,
            DataConnections: null,
            SmtpConfigurations: null,
            Timestamp: DateTimeOffset.UtcNow));

        // The apply runs on a Task.Run inside the actor, so poll rather than assert once.
        await AwaitPurgedAsync();
    }

    private async Task SeedCentralOnlyRowsAsync()
    {
        // Seeded through the service's own (already-open) LocalDb connection — a raw
        // SqliteConnection would lack the pragmas and the zb_hlc_next() UDF the site
        // tables' capture triggers call. Raw SQL is the only way these rows can exist at
        // all now that the site-side write paths are gone.
        await using var connection = _storage.CreateConnection();
        await using var command = connection.CreateCommand();
        command.CommandText = """
            INSERT INTO notification_lists (name, recipient_emails, updated_at)
                VALUES ('Ops Team', '["ops@example.com"]', @u);
            INSERT INTO smtp_configurations
                (name, server, port, auth_mode, from_address, username, password, oauth_config, updated_at)
                VALUES ('smtp.example.com:587', 'smtp.example.com', 587, 'BasicAuth',
                        'noreply@example.com', 'smtpuser', 'PLAINTEXT-SECRET', NULL, @u);
            """;
        command.Parameters.AddWithValue("@u", DateTimeOffset.UtcNow.ToString("O"));
        await command.ExecuteNonQueryAsync();
    }

    private async Task<long> RowCountAsync(string table)
    {
        await using var connection = _storage.CreateConnection();
        await using var command = connection.CreateCommand();
        command.CommandText = $"SELECT COUNT(*) FROM {table}";
        return (long)(await command.ExecuteScalarAsync())!;
    }

    private async Task AwaitPurgedAsync()
    {
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
        while (DateTime.UtcNow < deadline)
        {
            if (await RowCountAsync("notification_lists") == 0 &&
                await RowCountAsync("smtp_configurations") == 0)
            {
                return;
            }

            await Task.Delay(50);
        }

        Assert.Fail(
            "Artifact apply did not purge the central-only notification/SMTP rows. " +
            "The plaintext SMTP password is still on disk.");
    }

    /// <summary>
    /// In-test fake <see cref="IDeploymentConfigFetcher"/>: returns a canned config JSON
    /// (notify-and-fetch success) or throws a canned exception (fetch failure), and records
    /// the fetch coordinates it was called with. It is async so a throw surfaces as a faulted
    /// <see cref="Task"/> — mirroring the real HttpDeploymentConfigFetcher; a synchronous
    /// throw would instead crash the actor before the ContinueWith/PipeTo could produce a
    /// RefreshFetchFailed.
    /// </summary>
    private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
    {
        private readonly string? _result;
        private readonly Exception? _error;

        public FakeConfigFetcher(string result) => _result = result;
        public FakeConfigFetcher(Exception error) => _error = error;

        public string? LastBaseUrl { get; private set; }
        public string? LastDeploymentId { get; private set; }
        public string? LastToken { get; private set; }

        public async Task<string> FetchAsync(
            string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
        {
            LastBaseUrl = centralFetchBaseUrl;
            LastDeploymentId = deploymentId;
            LastToken = token;
            await Task.Yield();
            if (_error != null)
                throw _error;
            return _result!;
        }
    }
}