lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbSourcedAcksTests.cs

using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;

/// <summary>
///     Per-cluster mesh Phase 1: the coordinator's expected-ack set comes from enabled
///     <c>ClusterNode</c> rows, not from <c>Akka.Cluster.State.Members</c> filtered by role.
/// </summary>
/// <remarks>
///     <para>
///         <b>Why every test here seeds ClusterNode rows and asserts a non-empty expected set.</b>
///         The harness's ActorSystem self-joins with role <c>admin</c> and no <c>driver</c> members,
///         so the OLD derivation returned an empty set for every one of these scenarios and sealed
///         immediately. Any test that merely observes "the deployment did not seal" would therefore
///         have passed against the old code too. Each test below distinguishes the two by requiring
///         the coordinator to WAIT for a specific node and then seal when exactly that node acks —
///         behaviour the membership rule cannot produce in this harness at all.
///     </para>
///     <para>
///         Verified by reverting <c>DiscoverDriverNodes</c> to the membership scan: the first three
///         go red. <see cref="No_enabled_rows_seals_empty"/> stays green under both derivations and
///         is <b>not</b> a test of the derivation — the harness has no driver members AND no enabled
///         rows, so both rules produce an empty set. It pins the seal-empty branch and its reworded
///         reason, nothing more.
///     </para>
/// </remarks>
public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneActorTestBase
{
    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('b', 64));
    private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);

    /// <summary>
    ///     An enabled row is expected to ack; the deployment waits for it and seals when it arrives.
    ///     This is the DB-derived set doing something the membership set demonstrably cannot here.
    /// </summary>
    [Fact]
    public void Enabled_ClusterNode_rows_are_the_expected_ack_set()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, ("site-a-1:4053", true, false));

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));

        // The seeded row must have produced a NodeDeploymentState row — proof the coordinator
        // expected it, independent of whether it later sealed.
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }, duration: TimeSpan.FromSeconds(3));

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: TimeSpan.FromSeconds(3));
    }

    /// <summary>
    ///     A disabled row is NOT expected, so a deployment seals without it — for a node genuinely
    ///     decommissioned rather than temporarily down. <b>This is not the maintenance hatch</b>: the
    ///     topology gate rejects a deployment whose enabled-node count differs from the cluster's
    ///     <c>NodeCount</c>, so on any 2-node pair clearing <c>Enabled</c> never reaches this code.
    ///     See <see cref="MaintenanceMode_rows_are_not_expected_to_ack_while_staying_enabled"/>.
    /// </summary>
    [Fact]
    public void Disabled_ClusterNode_rows_are_not_expected_to_ack()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", false, false));

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));

        // Only the enabled node is expected — the disabled one gets no NodeDeploymentState row.
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
        }, duration: TimeSpan.FromSeconds(3));

        // ...and the enabled node's ack alone seals it. If the disabled row were expected this would
        // sit at AwaitingApplyAcks until the deadline.
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: TimeSpan.FromSeconds(3));
    }

    /// <summary>
    ///     An ack from a node with no <c>ClusterNode</c> row is discarded — it is not in the expected
    ///     set, so it must not count toward the seal. Under the membership rule a running-but-
    ///     unregistered node WAS expected, and then threw FK 547 when its state row was written.
    /// </summary>
    [Fact]
    public void Ack_from_a_node_absent_from_ClusterNode_is_discarded()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, ("site-a-1:4053", true, false));

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }, duration: TimeSpan.FromSeconds(3));

        // A node nobody registered acks. One expected node, one ack — if it were counted, the
        // coordinator would seal here.
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("ghost-9:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        ExpectNoMsg(TimeSpan.FromMilliseconds(400));

        using (var db = dbFactory.CreateDbContext())
        {
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }

        // Positive control on the wait itself: the real node's ack DOES seal it, so the assertion
        // above measured the discard rather than a coordinator that had simply stopped responding.
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: TimeSpan.FromSeconds(3));
    }

    /// <summary>
    ///     No enabled rows at all seals empty. Same outcome as the old empty-membership branch, but
    ///     it now means "the fleet has no enabled nodes registered" — a configuration error that
    ///     cannot resolve itself — rather than "no driver node has joined yet".
    /// </summary>
    [Fact]
    public void No_enabled_rows_seals_empty()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, ("site-a-1:4053", false, false), ("site-a-2:4053", false, false));

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
            db.NodeDeploymentStates.ShouldBeEmpty();
        }, duration: TimeSpan.FromSeconds(3));
    }

    /// <summary>
    ///     The maintenance hatch, and the reason it is a separate flag. The Phase 1 live gate set
    ///     <c>Enabled = 0</c> on one node of a 2-node pair and the deployment was rejected outright by
    ///     <c>DraftValidator.ValidateClusterTopology</c> (enabled-node count must equal
    ///     <c>NodeCount</c>) — so the intended escape hatch could not be used on any pair, which is
    ///     every cluster in the target topology. <c>MaintenanceMode</c> excludes a node from the
    ///     expected-ack set while leaving it enabled, so the topology gate still passes.
    /// </summary>
    [Fact]
    public void MaintenanceMode_rows_are_not_expected_to_ack_while_staying_enabled()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", true, true));

        // The topology gate counts Enabled, so both nodes still count — this is what Enabled = 0
        // broke. Assert it here so the fix cannot be "quietly disable it after all".
        using (var check = dbFactory.CreateDbContext())
        {
            check.ClusterNodes.Count(n => n.Enabled).ShouldBe(2);
        }

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
        }, duration: TimeSpan.FromSeconds(3));

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: TimeSpan.FromSeconds(3));
    }

    private static void SeedNodes(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
        params (string NodeId, bool Enabled, bool Maintenance)[] nodes)
    {
        using var db = dbFactory.CreateDbContext();
        db.ServerClusters.Add(new ServerCluster
        {
            ClusterId = "SITE-A",
            Name = "Site A",
            Enterprise = "zb",
            Site = "site-a",
            NodeCount = 2,
            RedundancyMode = RedundancyMode.Warm,
            CreatedBy = "test",
        });
        foreach (var (nodeId, enabled, maintenance) in nodes)
        {
            db.ClusterNodes.Add(new ClusterNode
            {
                NodeId = nodeId,
                ClusterId = "SITE-A",
                Host = nodeId.Split(':')[0],
                ApplicationUri = $"urn:OtOpcUa:{nodeId}",
                Enabled = enabled,
                MaintenanceMode = maintenance,
                CreatedBy = "test",
            });
        }
        db.SaveChanges();
    }

    private static DeploymentId SeedDispatchingDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory)
    {
        var id = DeploymentId.NewId();
        using var db = dbFactory.CreateDbContext();
        db.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = TestRevision.Value,
            Status = DeploymentStatus.Dispatching,
            CreatedBy = "test",
        });
        db.SaveChanges();
        return id;
    }
}