ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/SelfHealCreatesDclConnectionTests.cs

using System.Collections.Concurrent;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Regression tests for deferred register row <b>85</b> — <b>an Instance Actor started by a
/// self-heal got no DCL connection actors</b>, so every tag it subscribed was answered
/// <c>Unknown connection: X</c> forever.
///
/// <para>
/// <b>The defect.</b> A DCL connection actor is never created from the
/// <c>data_connection_definitions</c> table — that table has no production reader at all. It is
/// created only as a side effect of starting an instance, from the <c>Connections</c> map inside
/// that instance's own flattened config, via <c>EnsureDclConnections</c>. That call used to be
/// each CALLER's job, and only two of the four callers made it: the boot-time batch and
/// <c>ApplyDeployment</c>. The two self-heal starts did not:
/// </para>
/// <list type="bullet">
///   <item><c>StartStoredInstance</c> — the path shared by the startup-reconcile
///   ensure-started command and the sweep.</item>
///   <item><c>HandleEnableResult</c> — enabling an instance that was disabled at boot.</item>
/// </list>
///
/// <para>
/// <b>The live sequence (2026-08-16).</b> A fresh node-a was seeded by LocalDb CDC
/// replication; the sweep correctly started the enabled instances, and then every tag
/// subscribe failed for 6+ minutes with <c>Unknown connection: OPC PLC Simulator</c> /
/// <c>No connection actor for OPC PLC Simulator</c>, because nothing on that node had created the
/// connection actor. A second pair restart healed it — the boot load creates connections, so it
/// papered over the gap.
/// </para>
///
/// <para>
/// <b>The enable path proves this is not a replication-only bug.</b> The boot-time load reads
/// only ENABLED configs, so a node that boots with an instance disabled never creates that
/// instance's connections either; enabling it afterwards hit the identical dead end with no
/// replication anywhere in the picture.
/// </para>
///
/// <para>
/// <b>The fix</b> moves <c>EnsureDclConnections</c> into <c>CreateInstanceActor</c> — already
/// documented as the single Instance Actor creation funnel — so all four paths get it and a
/// fifth path cannot forget. Design note:
/// <c>docs/plans/2026-08-15-instance-sweep-self-heal-design.md</c> §7.
/// </para>
/// </summary>
public class SelfHealCreatesDclConnectionTests : TestKit, IDisposable
{
    private const string ConnectionName = "OPC PLC Simulator";

    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    /// <summary>
    /// Every <see cref="CreateConnectionCommand"/> the Deployment Manager sent to the DCL, in
    /// order. This is the exact message whose absence WAS register row 85.
    /// </summary>
    private readonly ConcurrentQueue<CreateConnectionCommand> _createdConnections = new();

    /// <summary>Initializes a new instance of the <c>SelfHealCreatesDclConnectionTests</c> class.</summary>
    public SelfHealCreatesDclConnectionTests()
    {
        _localDb = TestLocalDb.CreateTemp("dcl-selfheal-test");
        _storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    /// <inheritdoc />
    void IDisposable.Dispose()
    {
        // Shut the actors down BEFORE the ILocalDb they write through is disposed.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    /// <summary>Shortest sweep the (whole-second) option allows.</summary>
    private const int SweepSeconds = 1;

    /// <summary>Long enough for at least three sweep ticks to have come and gone.</summary>
    private static readonly TimeSpan SeveralSweeps = TimeSpan.FromSeconds(4);

    /// <summary>
    /// Starts a Deployment Manager wired to a recording stand-in for the DCL manager, and blocks
    /// until its boot-time config load has PROVABLY read storage (its first instance-count
    /// report — see <see cref="DeploymentManagerStartupProbe"/>).
    ///
    /// <para>
    /// That gate is load-bearing for the post-boot tests here for the same reason it is in
    /// <c>InstanceSweepSelfHealTests</c>: a config row written before the boot load reads storage
    /// is picked up by the boot load, which DOES create connections — so an ungated test would
    /// pass while proving the boot load works rather than the self-heal path it names.
    /// </para>
    /// </summary>
    private IActorRef CreateBootedDeploymentManager(int sweepSeconds = SweepSeconds)
    {
        var dclSpy = ActorOf(Props.Create(() => new DclConnectionSpy(_createdConnections)));
        var health = new DeploymentManagerStartupProbe();

        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null,
            new SiteRuntimeOptions { InstanceSweepIntervalSeconds = sweepSeconds },
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager: dclSpy,
            healthCollector: health,
            serviceProvider: null,
            loggerFactory: null,
            configFetcher: null)));

        AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
            TimeSpan.FromMilliseconds(25));
        return dm;
    }

    /// <summary>
    /// A flattened config carrying one data connection — the <c>Connections</c> map is what
    /// <c>EnsureDclConnections</c> reads, and it travels inside every instance's own config.
    /// </summary>
    private static string MakeConfigJson(string instanceName, string endpoint = "opc.tcp://plc:4840") =>
        JsonSerializer.Serialize(new FlattenedConfiguration
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute
                {
                    CanonicalName = "TestAttr", Value = "42", DataType = "Int32"
                }
            ],
            Connections = new Dictionary<string, ConnectionConfig>
            {
                [ConnectionName] = new()
                {
                    Protocol = "OpcUa",
                    ConfigurationJson = $"{{\"endpointUrl\":\"{endpoint}\"}}",
                    FailoverRetryCount = 3
                }
            }
        });

    private IReadOnlyList<CreateConnectionCommand> CreatesFor(string connectionName) =>
        _createdConnections.Where(c => c.ConnectionName == connectionName).ToArray();

    private async Task AwaitConnectionCreatedAsync(string connectionName, string because) =>
        await AwaitAssertAsync(() =>
        {
            Assert.True(CreatesFor(connectionName).Count > 0, because);
            return Task.CompletedTask;
        }, TimeSpan.FromSeconds(20));

    // ── The reproduction: a self-heal start must bring its connections with it ──

    /// <summary>
    /// THE reproduction. The node boots with an empty store, then a config for an ENABLED
    /// instance appears in storage on its own — exactly what LocalDb's CDC sync applier does when
    /// it seeds a fresh node from its peer. The sweep starts the Instance Actor; the DCL
    /// must be told to create the connection that instance's tags subscribe through.
    ///
    /// <para>
    /// <b>Fails before the fix</b> at the connection assert: the instance starts (the sweep works)
    /// but NO <see cref="CreateConnectionCommand"/> is ever sent, which is the live symptom —
    /// the instance is up and every tag subscribe is answered <c>Unknown connection</c>.
    /// </para>
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task SweepStartedInstance_AlsoCreatesItsDclConnection()
    {
        const string instanceName = "ReplicatedPump";
        var dm = CreateBootedDeploymentManager();

        // Baseline: the node genuinely booted with nothing.
        Assert.Empty(CreatesFor(ConnectionName));

        // Replication seeds the row — a bare storage write, the CDC applier's only side effect.
        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-replicated", "rev-replicated", true);

        await AwaitConnectionCreatedAsync(ConnectionName,
            "The sweep started a replication-seeded instance but never asked the DCL to create "
            + "its connection — every tag subscribe would be answered 'Unknown connection' "
            + "until the node was restarted again (register row 85)");
    }

    /// <summary>
    /// The startup-reconcile heal shares <c>StartStoredInstance</c> with the sweep,
    /// so it had the identical gap: the reconcile fetched and stored a config, the Deployment
    /// Manager started the instance, and its connections were never created.
    ///
    /// <para><b>Fails before the fix.</b></para>
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ReconcileEnsureStartedInstance_AlsoCreatesItsDclConnection()
    {
        const string instanceName = "ReconciledPump";
        var dm = CreateBootedDeploymentManager(sweepSeconds: 0); // isolate the reconcile path

        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-r", "rev-r", true);

        dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));

        await AwaitConnectionCreatedAsync(ConnectionName,
            "The startup-reconcile ensure-started heal started the instance without creating "
            + "its DCL connection (register row 85, row-67 path)");
    }

    /// <summary>
    /// <b>Row 85 without any replication at all.</b> The boot-time load reads only ENABLED
    /// configs, so a node that boots with this instance DISABLED never created its connection.
    /// Enabling it afterwards then started the Instance Actor through
    /// <c>HandleEnableResult</c> — the second caller that never ensured connections — leaving a
    /// running instance with no connection actor on a perfectly ordinary single site.
    ///
    /// <para><b>Fails before the fix.</b></para>
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task EnabledInstance_DisabledAtBoot_AlsoCreatesItsDclConnection()
    {
        const string instanceName = "EnabledLaterPump";

        // Stored DISABLED before boot, so the boot load skips it entirely.
        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-e", "rev-e", false);

        var dm = CreateBootedDeploymentManager(sweepSeconds: 0); // isolate the enable path
        Assert.Empty(CreatesFor(ConnectionName));

        dm.Tell(new EnableInstanceCommand("cmd-enable", instanceName, DateTimeOffset.UtcNow));
        ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));

        await AwaitConnectionCreatedAsync(ConnectionName,
            "Enabling an instance that was disabled at boot started it without creating its DCL "
            + "connection — register row 85 reached with no replication involved");
    }

    // ── Guards: the repair must be one-shot and silent in steady state ──

    /// <summary>
    /// Idempotence: many sweeps, ONE create command. The sweep runs forever on a live node, and
    /// re-sending the create every tick would be an unbounded message stream — and, since
    /// <c>HandleCreateConnection</c> warns and returns on a name it already holds, an unbounded
    /// stream of Warning log lines too.
    ///
    /// <para>
    /// <b>Fails before the fix</b> (zero commands, not one), and would fail again if the
    /// <c>_createdConnections</c> hash guard inside <c>EnsureDclConnection</c> were ever lost.
    /// </para>
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task SweepStartedInstance_CreatesTheConnection_ExactlyOnce_AcrossManySweeps()
    {
        const string instanceName = "IdempotentConnectionPump";
        var dm = CreateBootedDeploymentManager();

        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-i", "rev-i", true);

        await AwaitConnectionCreatedAsync(ConnectionName, "the sweep never created the connection");

        // Let several more sweeps run over the now-healthy state.
        await Task.Delay(SeveralSweeps);

        Assert.Single(CreatesFor(ConnectionName));
    }

    /// <summary>
    /// A sweep over a healthy, deployed-the-ordinary-way instance must send the DCL nothing at
    /// all. The hash guard is what makes the connection-ensure safe to sit in the creation
    /// funnel, and this pins it against a config that never changes.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Sweep_SendsNoConnectionCommands_WhenThereIsNothingToHeal()
    {
        const string instanceName = "HealthyConnectionPump";
        var dm = CreateBootedDeploymentManager();

        dm.Tell(new DeployInstanceCommand(
            "dep-1", instanceName, "rev-1", MakeConfigJson(instanceName),
            "admin", DateTimeOffset.UtcNow));
        Assert.Equal(
            DeploymentStatus.Success,
            ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);

        await AwaitConnectionCreatedAsync(ConnectionName, "the deploy never created the connection");
        var afterDeploy = _createdConnections.Count;

        await Task.Delay(SeveralSweeps);

        Assert.Equal(afterDeploy, _createdConnections.Count);
    }

    // ── Regression guards for the two paths that already worked ──

    /// <summary>
    /// The fix moved <c>EnsureDclConnections</c> out of <c>ApplyDeployment</c> and into
    /// <c>CreateInstanceActor</c>. This pins that an ordinary central deploy still creates its
    /// connections — it passes both before and after the fix, which is the point of a
    /// refactor guard.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task OrdinaryDeploy_StillCreatesItsDclConnection()
    {
        const string instanceName = "DeployedPump";
        var dm = CreateBootedDeploymentManager(sweepSeconds: 0);

        dm.Tell(new DeployInstanceCommand(
            "dep-1", instanceName, "rev-1", MakeConfigJson(instanceName),
            "admin", DateTimeOffset.UtcNow));
        Assert.Equal(
            DeploymentStatus.Success,
            ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);

        await AwaitConnectionCreatedAsync(ConnectionName,
            "an ordinary deploy stopped creating its DCL connection");

        var created = CreatesFor(ConnectionName).Single();
        Assert.Equal("OpcUa", created.ProtocolType);
    }

    /// <summary>
    /// The other path the fix moved the call out of: the boot-time staggered batch. The config is
    /// stored BEFORE the manager starts, so the boot load — not the sweep — is what creates the
    /// instance, and it must still bring its connections with it.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task BootLoadedInstance_StillCreatesItsDclConnection()
    {
        const string instanceName = "BootLoadedPump";

        await _storage.StoreDeployedConfigAsync(
            instanceName, MakeConfigJson(instanceName), "dep-b", "rev-b", true);

        CreateBootedDeploymentManager(sweepSeconds: 0);

        await AwaitConnectionCreatedAsync(ConnectionName,
            "the boot-time load stopped creating its DCL connection");
    }

    /// <summary>
    /// Stand-in for the DCL manager actor: records every
    /// <see cref="CreateConnectionCommand"/> and swallows everything else (Instance Actors send
    /// their tag subscribes here too, and those are not what these tests assert on).
    /// </summary>
    private sealed class DclConnectionSpy : ReceiveActor
    {
        /// <summary>Initializes a new instance of the <c>DclConnectionSpy</c> class.</summary>
        /// <param name="creates">The creates.</param>
        public DclConnectionSpy(ConcurrentQueue<CreateConnectionCommand> creates)
        {
            Receive<CreateConnectionCommand>(creates.Enqueue);
            ReceiveAny(_ => { });
        }
    }
}