using System.Text; using Akka.Actor; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json.Linq; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; /// /// Gitea #601 — central's side of the honest deploy ack. A node that applied a deployment but whose /// address-space apply was DEGRADED acks with /// . The deployment still SEALS (the revision is what the node now runs, and /// "latest sealed" drives drift + the publish actor's fallback) but never looks clean: the node's /// NodeDeploymentState row keeps status Applied with its degraded reason in FailureReason, /// and the Deployment row's FailureReason names the degraded nodes. An ack from an old node /// (no degraded field) is handled exactly as before. /// public sealed class ConfigPublishCoordinatorDegradedAckTests : ControlPlaneActorTestBase { private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); private const string NodeA = "site-a-1:4053"; private const string NodeB = "site-a-2:4053"; [Fact] public void A_degraded_ack_seals_but_is_recorded_as_degraded_on_the_node_and_the_deployment() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, NodeA, NodeB); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null, CorrelationId.NewId(), Degraded: true, DegradedReason: "address-space apply degraded (failedNodes=3)")); AwaitStatus(dbFactory, DeploymentStatus.Sealed); using var db = dbFactory.CreateDbContext(); var deployment = db.Deployments.Single(); deployment.FailureReason.ShouldNotBeNull(); deployment.FailureReason.ShouldContain("degraded"); deployment.FailureReason.ShouldContain(NodeB); deployment.FailureReason.ShouldNotContain(NodeA); var rowB = db.NodeDeploymentStates.Single(s => s.NodeId == NodeB); rowB.Status.ShouldBe(NodeDeploymentStatus.Applied); rowB.FailureReason.ShouldNotBeNull(); rowB.FailureReason.ShouldStartWith(ConfigPublishCoordinator.DegradedReasonPrefix); rowB.FailureReason.ShouldContain("failedNodes=3"); ConfigPublishCoordinator.IsDegraded(rowB.Status, rowB.FailureReason).ShouldBeTrue(); var rowA = db.NodeDeploymentStates.Single(s => s.NodeId == NodeA); rowA.Status.ShouldBe(NodeDeploymentStatus.Applied); rowA.FailureReason.ShouldBeNull(); ConfigPublishCoordinator.IsDegraded(rowA.Status, rowA.FailureReason).ShouldBeFalse(); } [Fact] public void Old_node_acks_without_the_degraded_field_seal_clean_exactly_as_before() { // The five-argument constructor IS the pre-#601 ack shape: Degraded defaults to false. var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, NodeA, NodeB); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitStatus(dbFactory, DeploymentStatus.Sealed); using var db = dbFactory.CreateDbContext(); db.Deployments.Single().FailureReason.ShouldBeNull(); db.NodeDeploymentStates.All(s => s.Status == NodeDeploymentStatus.Applied && s.FailureReason == null) .ShouldBeTrue(); } [Fact] public void A_degraded_ack_alongside_a_failed_one_is_still_PartiallyFailed() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, NodeA, NodeB); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Failed, "boom", CorrelationId.NewId())); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null, CorrelationId.NewId(), Degraded: true, DegradedReason: "x")); AwaitStatus(dbFactory, DeploymentStatus.PartiallyFailed); using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).FailureReason.ShouldBe("boom"); db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).Status.ShouldBe(NodeDeploymentStatus.Failed); } [Fact] public void A_recovered_coordinator_remembers_a_degraded_ack_that_landed_before_failover() { // Node B's degraded ack was persisted by a previous singleton instance; this instance recovers the // in-flight deployment from the DB and must still seal it as degraded when node A acks clean. var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks); SeedNodes(dbFactory, NodeA, NodeB); using (var db = dbFactory.CreateDbContext()) { db.NodeDeploymentStates.Add(new NodeDeploymentState { NodeId = NodeA, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying, }); db.NodeDeploymentStates.Add(new NodeDeploymentState { NodeId = NodeB, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, FailureReason = ConfigPublishCoordinator.DegradedReasonPrefix + "failedNodes=1", AppliedAtUtc = DateTime.UtcNow, }); db.SaveChanges(); } var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitStatus(dbFactory, DeploymentStatus.Sealed); using var verify = dbFactory.CreateDbContext(); var reason = verify.Deployments.Single().FailureReason; reason.ShouldNotBeNull(); reason.ShouldContain(NodeB); } [Fact] public void An_old_nodes_DPS_ack_without_the_degraded_fields_deserializes_as_a_clean_ack() { // The DPS ack path rides Akka's default serializer for the record. Serialize a NEW ack, strip the two // #601 members from the payload (what an old node sends), and decode it with this build. var ack = new ApplyAck(DeploymentId.NewId(), NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId(), Degraded: true, DegradedReason: "will be stripped"); var serializer = Sys.Serialization.FindSerializerFor(ack); var json = JObject.Parse(Encoding.UTF8.GetString(serializer.ToBinary(ack))); json.Remove("Degraded").ShouldBeTrue(); json.Remove("DegradedReason").ShouldBeTrue(); var back = (ApplyAck)serializer.FromBinary( Encoding.UTF8.GetBytes(json.ToString(Newtonsoft.Json.Formatting.None)), typeof(ApplyAck)); back.Outcome.ShouldBe(ApplyAckOutcome.Applied); back.Degraded.ShouldBeFalse(); back.DegradedReason.ShouldBeNull(); back.DeploymentId.ShouldBe(ack.DeploymentId); // ...and the new fields survive the same serializer when present. var full = (ApplyAck)serializer.FromBinary(serializer.ToBinary(ack), typeof(ApplyAck)); full.ShouldBe(ack); } private void AwaitStatus(IDbContextFactory dbFactory, DeploymentStatus expected) => AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(expected); }, duration: TimeSpan.FromSeconds(5)); private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodes) { using var db = dbFactory.CreateDbContext(); db.ServerClusters.Add(new ServerCluster { ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", NodeCount = 2, RedundancyMode = RedundancyMode.Warm, CreatedBy = "test", }); foreach (var nodeId in nodes) { db.ClusterNodes.Add(new ClusterNode { NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", }); } db.SaveChanges(); } private static DeploymentId SeedDispatchingDeployment( IDbContextFactory dbFactory, DeploymentStatus status = DeploymentStatus.Dispatching) { var id = DeploymentId.NewId(); using var db = dbFactory.CreateDbContext(); db.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = status, CreatedBy = "test", }); db.SaveChanges(); return id; } }