lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs

using Akka.Cluster;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;

namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;

/// <summary>
/// Failover scenarios layered on <see cref="TwoNodeClusterHarness"/> Stop/Restart primitives.
/// Covers graceful node loss, rejoin on the same Akka port, and deployment under reduced membership.
/// </summary>
public sealed class FailoverDuringDeployTests
{
    private static CancellationToken Ct => TestContext.Current.CancellationToken;

    /// <summary>Verifies that stopping node B shrinks the cluster to one up member.</summary>
    [Fact]
    public async Task Stopping_node_b_shrinks_cluster_to_one_up_member()
    {
        await using var harness = await TwoNodeClusterHarness.StartAsync();
        Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members
            .Count(m => m.Status == MemberStatus.Up).ShouldBe(2);

        await harness.StopNodeBAsync();
        await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20));

        Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members
            .Count(m => m.Status == MemberStatus.Up).ShouldBe(1);
    }

    /// <summary>Verifies that a restarted node B rejoins the cluster on the same port.</summary>
    [Fact]
    public async Task Restarted_node_b_rejoins_cluster_on_same_port()
    {
        await using var harness = await TwoNodeClusterHarness.StartAsync();

        await harness.StopNodeBAsync();
        await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20));

        await harness.RestartNodeBAsync();

        Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members
            .Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
        Akka.Cluster.Cluster.Get(harness.NodeBSystem).State.Members
            .Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
    }

    /// <summary>
    ///     A deployment started with node B down no longer seals without it — B's <c>ClusterNode</c>
    ///     row is enabled, so it is expected to ack and the deployment waits.
    /// </summary>
    /// <remarks>
    ///     <b>This test asserted the opposite until per-cluster mesh Phase 1.</b> It was named
    ///     <c>Deployment_started_with_node_b_down_seals_with_one_node_state</c> and documented that
    ///     "<c>DiscoverDriverNodes</c> snapshots membership at dispatch time — when only node A is Up,
    ///     only one ApplyAck is expected and the deployment seals without B ever participating". That
    ///     is exactly the behaviour Phase 1 removed: it told the operator the fleet was deployed while
    ///     a configured node had not received it. The expected-ack set now comes from enabled
    ///     <c>ClusterNode</c> rows, so a node that is merely switched off is still expected.
    /// </remarks>
    [Fact]
    public async Task Deployment_started_with_node_b_down_does_not_seal_without_it()
    {
        await using var harness = await TwoNodeClusterHarness.StartAsync();
        await harness.SeedDefaultClusterAsync();

        await harness.StopNodeBAsync();
        await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20));

        await using var scope = harness.NodeA.Services.CreateAsyncScope();
        var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();

        var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
        result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
        var deploymentId = result.DeploymentId!.Value.Value;

        // Positive evidence that B was EXPECTED, not merely slow: the coordinator seeds a row per
        // expected node at dispatch, so both rows must exist with B still Applying. Asserting only
        // "it didn't seal" would pass just as well against a coordinator that had died.
        await WaitForAsync(async () =>
        {
            await using var pollDb = await CreateDbAsync(harness);
            return await pollDb.NodeDeploymentStates.AsNoTracking()
                .CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2;
        }, TwoNodeClusterHarness.DeploySealTimeout);

        await using var db = await CreateDbAsync(harness);
        var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
            .Where(s => s.DeploymentId == deploymentId)
            .ToListAsync(Ct);
        nodeStates.Count.ShouldBe(2, "both configured nodes are expected to ack");
        nodeStates.Count(s => s.Status == NodeDeploymentStatus.Applied)
            .ShouldBe(1, "only the running node applied");
        nodeStates.ShouldContain(s => s.Status == NodeDeploymentStatus.Applying,
            "the stopped node's ack is still outstanding");

        var deployment = await db.Deployments.AsNoTracking()
            .FirstAsync(d => d.DeploymentId == deploymentId, Ct);
        deployment.Status.ShouldNotBe(DeploymentStatus.Sealed,
            "a deployment must not seal green while a configured node has not received it");
    }

    /// <summary>
    ///     The maintenance hatch, end-to-end: a node down <i>and</i> flagged
    ///     <c>MaintenanceMode</c> is not expected, so the deployment seals with one node state — the
    ///     behaviour the test above used to assert unconditionally, now something an operator has to
    ///     ask for.
    /// </summary>
    [Fact]
    public async Task Deployment_seals_without_a_node_flagged_for_maintenance()
    {
        await using var harness = await TwoNodeClusterHarness.StartAsync();
        await harness.SeedDefaultClusterAsync();

        await harness.StopNodeBAsync();
        await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20));

        await using (var setup = await CreateDbAsync(harness))
        {
            var nodeB = await setup.ClusterNodes.FirstAsync(n => n.NodeId == harness.NodeBNodeId, Ct);
            nodeB.MaintenanceMode = true;
            await setup.SaveChangesAsync(Ct);
            // Still Enabled — DraftValidator.ValidateClusterTopology requires the enabled-node count
            // to equal ServerCluster.NodeCount, which is why MaintenanceMode exists as its own flag.
            nodeB.Enabled.ShouldBeTrue();
        }

        await using var scope = harness.NodeA.Services.CreateAsyncScope();
        var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();

        var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
        result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}");
        var deploymentId = result.DeploymentId!.Value.Value;

        await WaitForAsync(async () =>
        {
            await using var pollDb = await CreateDbAsync(harness);
            var d = await pollDb.Deployments.AsNoTracking()
                .FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
            return d?.Status == DeploymentStatus.Sealed;
        }, TwoNodeClusterHarness.DeploySealTimeout);

        await using var db = await CreateDbAsync(harness);
        var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
            .Where(s => s.DeploymentId == deploymentId)
            .ToListAsync(Ct);
        nodeStates.Count.ShouldBe(1, "the maintenance node is not expected to ack");
        nodeStates[0].NodeId.ShouldBe(harness.NodeANodeId);
        nodeStates[0].Status.ShouldBe(NodeDeploymentStatus.Applied);
    }

    private static async Task<OtOpcUaConfigDbContext> CreateDbAsync(TwoNodeClusterHarness harness)
    {
        var factory = harness.NodeA.Services.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>();
        return await factory.CreateDbContextAsync();
    }

    private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
    {
        var deadline = DateTime.UtcNow + timeout;
        while (DateTime.UtcNow < deadline)
        {
            if (await condition()) return;
            await Task.Delay(200);
        }
        throw new TimeoutException($"Condition not met within {timeout}");
    }
}