lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDegradedAckTests.cs

using System.Text;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;

/// <summary>
///     Gitea #601 — central's side of the honest deploy ack. A node that applied a deployment but whose
///     address-space apply was DEGRADED acks <see cref="ApplyAckOutcome.Applied"/> with
///     <see cref="ApplyAck.Degraded"/>. The deployment still SEALS (the revision is what the node now runs, and
///     "latest sealed" drives drift + the publish actor's fallback) but never looks clean: the node's
///     <c>NodeDeploymentState</c> row keeps status Applied with its degraded reason in <c>FailureReason</c>,
///     and the <c>Deployment</c> row's <c>FailureReason</c> names the degraded nodes. An ack from an old node
///     (no degraded field) is handled exactly as before.
/// </summary>
public sealed class ConfigPublishCoordinatorDegradedAckTests : ControlPlaneActorTestBase
{
    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
    private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
    private const string NodeA = "site-a-1:4053";
    private const string NodeB = "site-a-2:4053";

    [Fact]
    public void A_degraded_ack_seals_but_is_recorded_as_degraded_on_the_node_and_the_deployment()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, NodeA, NodeB);
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
        AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null,
            CorrelationId.NewId()));
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null,
            CorrelationId.NewId(), Degraded: true, DegradedReason: "address-space apply degraded (failedNodes=3)"));

        AwaitStatus(dbFactory, DeploymentStatus.Sealed);
        using var db = dbFactory.CreateDbContext();

        var deployment = db.Deployments.Single();
        deployment.FailureReason.ShouldNotBeNull();
        deployment.FailureReason.ShouldContain("degraded");
        deployment.FailureReason.ShouldContain(NodeB);
        deployment.FailureReason.ShouldNotContain(NodeA);

        var rowB = db.NodeDeploymentStates.Single(s => s.NodeId == NodeB);
        rowB.Status.ShouldBe(NodeDeploymentStatus.Applied);
        rowB.FailureReason.ShouldNotBeNull();
        rowB.FailureReason.ShouldStartWith(ConfigPublishCoordinator.DegradedReasonPrefix);
        rowB.FailureReason.ShouldContain("failedNodes=3");
        ConfigPublishCoordinator.IsDegraded(rowB.Status, rowB.FailureReason).ShouldBeTrue();

        var rowA = db.NodeDeploymentStates.Single(s => s.NodeId == NodeA);
        rowA.Status.ShouldBe(NodeDeploymentStatus.Applied);
        rowA.FailureReason.ShouldBeNull();
        ConfigPublishCoordinator.IsDegraded(rowA.Status, rowA.FailureReason).ShouldBeFalse();
    }

    [Fact]
    public void Old_node_acks_without_the_degraded_field_seal_clean_exactly_as_before()
    {
        // The five-argument constructor IS the pre-#601 ack shape: Degraded defaults to false.
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, NodeA, NodeB);
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
        AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitStatus(dbFactory, DeploymentStatus.Sealed);
        using var db = dbFactory.CreateDbContext();
        db.Deployments.Single().FailureReason.ShouldBeNull();
        db.NodeDeploymentStates.All(s => s.Status == NodeDeploymentStatus.Applied && s.FailureReason == null)
            .ShouldBeTrue();
    }

    [Fact]
    public void A_degraded_ack_alongside_a_failed_one_is_still_PartiallyFailed()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        SeedNodes(dbFactory, NodeA, NodeB);
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
        AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Failed, "boom", CorrelationId.NewId()));
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null,
            CorrelationId.NewId(), Degraded: true, DegradedReason: "x"));

        AwaitStatus(dbFactory, DeploymentStatus.PartiallyFailed);
        using var db = dbFactory.CreateDbContext();
        db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).FailureReason.ShouldBe("boom");
        db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).Status.ShouldBe(NodeDeploymentStatus.Failed);
    }

    [Fact]
    public void A_recovered_coordinator_remembers_a_degraded_ack_that_landed_before_failover()
    {
        // Node B's degraded ack was persisted by a previous singleton instance; this instance recovers the
        // in-flight deployment from the DB and must still seal it as degraded when node A acks clean.
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks);
        SeedNodes(dbFactory, NodeA, NodeB);
        using (var db = dbFactory.CreateDbContext())
        {
            db.NodeDeploymentStates.Add(new NodeDeploymentState
            {
                NodeId = NodeA, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying,
            });
            db.NodeDeploymentStates.Add(new NodeDeploymentState
            {
                NodeId = NodeB, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
                FailureReason = ConfigPublishCoordinator.DegradedReasonPrefix + "failedNodes=1",
                AppliedAtUtc = DateTime.UtcNow,
            });
            db.SaveChanges();
        }

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitStatus(dbFactory, DeploymentStatus.Sealed);
        using var verify = dbFactory.CreateDbContext();
        var reason = verify.Deployments.Single().FailureReason;
        reason.ShouldNotBeNull();
        reason.ShouldContain(NodeB);
    }

    [Fact]
    public void An_old_nodes_DPS_ack_without_the_degraded_fields_deserializes_as_a_clean_ack()
    {
        // The DPS ack path rides Akka's default serializer for the record. Serialize a NEW ack, strip the two
        // #601 members from the payload (what an old node sends), and decode it with this build.
        var ack = new ApplyAck(DeploymentId.NewId(), NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null,
            CorrelationId.NewId(), Degraded: true, DegradedReason: "will be stripped");
        var serializer = Sys.Serialization.FindSerializerFor(ack);
        var json = JObject.Parse(Encoding.UTF8.GetString(serializer.ToBinary(ack)));
        json.Remove("Degraded").ShouldBeTrue();
        json.Remove("DegradedReason").ShouldBeTrue();

        var back = (ApplyAck)serializer.FromBinary(
            Encoding.UTF8.GetBytes(json.ToString(Newtonsoft.Json.Formatting.None)), typeof(ApplyAck));

        back.Outcome.ShouldBe(ApplyAckOutcome.Applied);
        back.Degraded.ShouldBeFalse();
        back.DegradedReason.ShouldBeNull();
        back.DeploymentId.ShouldBe(ack.DeploymentId);

        // ...and the new fields survive the same serializer when present.
        var full = (ApplyAck)serializer.FromBinary(serializer.ToBinary(ack), typeof(ApplyAck));
        full.ShouldBe(ack);
    }

    private void AwaitStatus(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus expected) =>
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(expected);
        }, duration: TimeSpan.FromSeconds(5));

    private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodes)
    {
        using var db = dbFactory.CreateDbContext();
        db.ServerClusters.Add(new ServerCluster
        {
            ClusterId = "SITE-A",
            Name = "Site A",
            Enterprise = "zb",
            Site = "site-a",
            NodeCount = 2,
            RedundancyMode = RedundancyMode.Warm,
            CreatedBy = "test",
        });
        foreach (var nodeId in nodes)
        {
            db.ClusterNodes.Add(new ClusterNode
            {
                NodeId = nodeId,
                ClusterId = "SITE-A",
                Host = nodeId.Split(':')[0],
                ApplicationUri = $"urn:OtOpcUa:{nodeId}",
                Enabled = true,
                CreatedBy = "test",
            });
        }
        db.SaveChanges();
    }

    private static DeploymentId SeedDispatchingDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
        DeploymentStatus status = DeploymentStatus.Dispatching)
    {
        var id = DeploymentId.NewId();
        using var db = dbFactory.CreateDbContext();
        db.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = TestRevision.Value,
            Status = status,
            CreatedBy = "test",
        });
        db.SaveChanges();
        return id;
    }
}