feat(deploy): coordinator records degraded acks; deployment seals but names degraded nodes (#601)
An Applied+Degraded ack keeps NodeDeploymentState.Status = Applied with FailureReason 'Applied degraded: <reason>' (no enum/migration change). The deployment still seals (the revision is running; latest-sealed drives drift) but Deployment.FailureReason names the degraded nodes and the seal logs at Warning. Recovered from the DB on singleton failover. Old-node acks (no field, DPS JSON or gRPC) behave exactly as before. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
+220
@@ -0,0 +1,220 @@
|
||||
using System.Text;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #601 — central's side of the honest deploy ack. A node that applied a deployment but whose
|
||||
/// address-space apply was DEGRADED acks <see cref="ApplyAckOutcome.Applied"/> with
|
||||
/// <see cref="ApplyAck.Degraded"/>. The deployment still SEALS (the revision is what the node now runs, and
|
||||
/// "latest sealed" drives drift + the publish actor's fallback) but never looks clean: the node's
|
||||
/// <c>NodeDeploymentState</c> row keeps status Applied with its degraded reason in <c>FailureReason</c>,
|
||||
/// and the <c>Deployment</c> row's <c>FailureReason</c> names the degraded nodes. An ack from an old node
|
||||
/// (no degraded field) is handled exactly as before.
|
||||
/// </summary>
|
||||
public sealed class ConfigPublishCoordinatorDegradedAckTests : ControlPlaneActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
|
||||
private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
|
||||
private const string NodeA = "site-a-1:4053";
|
||||
private const string NodeB = "site-a-2:4053";
|
||||
|
||||
[Fact]
|
||||
public void A_degraded_ack_seals_but_is_recorded_as_degraded_on_the_node_and_the_deployment()
|
||||
{
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDispatchingDeployment(dbFactory);
|
||||
SeedNodes(dbFactory, NodeA, NodeB);
|
||||
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
|
||||
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);
|
||||
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null,
|
||||
CorrelationId.NewId()));
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null,
|
||||
CorrelationId.NewId(), Degraded: true, DegradedReason: "address-space apply degraded (failedNodes=3)"));
|
||||
|
||||
AwaitStatus(dbFactory, DeploymentStatus.Sealed);
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
|
||||
var deployment = db.Deployments.Single();
|
||||
deployment.FailureReason.ShouldNotBeNull();
|
||||
deployment.FailureReason.ShouldContain("degraded");
|
||||
deployment.FailureReason.ShouldContain(NodeB);
|
||||
deployment.FailureReason.ShouldNotContain(NodeA);
|
||||
|
||||
var rowB = db.NodeDeploymentStates.Single(s => s.NodeId == NodeB);
|
||||
rowB.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
rowB.FailureReason.ShouldNotBeNull();
|
||||
rowB.FailureReason.ShouldStartWith(ConfigPublishCoordinator.DegradedReasonPrefix);
|
||||
rowB.FailureReason.ShouldContain("failedNodes=3");
|
||||
ConfigPublishCoordinator.IsDegraded(rowB.Status, rowB.FailureReason).ShouldBeTrue();
|
||||
|
||||
var rowA = db.NodeDeploymentStates.Single(s => s.NodeId == NodeA);
|
||||
rowA.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
rowA.FailureReason.ShouldBeNull();
|
||||
ConfigPublishCoordinator.IsDegraded(rowA.Status, rowA.FailureReason).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Old_node_acks_without_the_degraded_field_seal_clean_exactly_as_before()
|
||||
{
|
||||
// The five-argument constructor IS the pre-#601 ack shape: Degraded defaults to false.
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDispatchingDeployment(dbFactory);
|
||||
SeedNodes(dbFactory, NodeA, NodeB);
|
||||
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
|
||||
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);
|
||||
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
|
||||
AwaitStatus(dbFactory, DeploymentStatus.Sealed);
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single().FailureReason.ShouldBeNull();
|
||||
db.NodeDeploymentStates.All(s => s.Status == NodeDeploymentStatus.Applied && s.FailureReason == null)
|
||||
.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_degraded_ack_alongside_a_failed_one_is_still_PartiallyFailed()
|
||||
{
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDispatchingDeployment(dbFactory);
|
||||
SeedNodes(dbFactory, NodeA, NodeB);
|
||||
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
|
||||
actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitStatus(dbFactory, DeploymentStatus.AwaitingApplyAcks);
|
||||
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Failed, "boom", CorrelationId.NewId()));
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeB), ApplyAckOutcome.Applied, null,
|
||||
CorrelationId.NewId(), Degraded: true, DegradedReason: "x"));
|
||||
|
||||
AwaitStatus(dbFactory, DeploymentStatus.PartiallyFailed);
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).FailureReason.ShouldBe("boom");
|
||||
db.NodeDeploymentStates.Single(s => s.NodeId == NodeA).Status.ShouldBe(NodeDeploymentStatus.Failed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_recovered_coordinator_remembers_a_degraded_ack_that_landed_before_failover()
|
||||
{
|
||||
// Node B's degraded ack was persisted by a previous singleton instance; this instance recovers the
|
||||
// in-flight deployment from the DB and must still seal it as degraded when node A acks clean.
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDispatchingDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks);
|
||||
SeedNodes(dbFactory, NodeA, NodeB);
|
||||
using (var db = dbFactory.CreateDbContext())
|
||||
{
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = NodeA, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying,
|
||||
});
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = NodeB, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
|
||||
FailureReason = ConfigPublishCoordinator.DegradedReasonPrefix + "failedNodes=1",
|
||||
AppliedAtUtc = DateTime.UtcNow,
|
||||
});
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
|
||||
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
|
||||
AwaitStatus(dbFactory, DeploymentStatus.Sealed);
|
||||
using var verify = dbFactory.CreateDbContext();
|
||||
var reason = verify.Deployments.Single().FailureReason;
|
||||
reason.ShouldNotBeNull();
|
||||
reason.ShouldContain(NodeB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_old_nodes_DPS_ack_without_the_degraded_fields_deserializes_as_a_clean_ack()
|
||||
{
|
||||
// The DPS ack path rides Akka's default serializer for the record. Serialize a NEW ack, strip the two
|
||||
// #601 members from the payload (what an old node sends), and decode it with this build.
|
||||
var ack = new ApplyAck(DeploymentId.NewId(), NodeId.Parse(NodeA), ApplyAckOutcome.Applied, null,
|
||||
CorrelationId.NewId(), Degraded: true, DegradedReason: "will be stripped");
|
||||
var serializer = Sys.Serialization.FindSerializerFor(ack);
|
||||
var json = JObject.Parse(Encoding.UTF8.GetString(serializer.ToBinary(ack)));
|
||||
json.Remove("Degraded").ShouldBeTrue();
|
||||
json.Remove("DegradedReason").ShouldBeTrue();
|
||||
|
||||
var back = (ApplyAck)serializer.FromBinary(
|
||||
Encoding.UTF8.GetBytes(json.ToString(Newtonsoft.Json.Formatting.None)), typeof(ApplyAck));
|
||||
|
||||
back.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
back.Degraded.ShouldBeFalse();
|
||||
back.DegradedReason.ShouldBeNull();
|
||||
back.DeploymentId.ShouldBe(ack.DeploymentId);
|
||||
|
||||
// ...and the new fields survive the same serializer when present.
|
||||
var full = (ApplyAck)serializer.FromBinary(serializer.ToBinary(ack), typeof(ApplyAck));
|
||||
full.ShouldBe(ack);
|
||||
}
|
||||
|
||||
private void AwaitStatus(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus expected) =>
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(expected);
|
||||
}, duration: TimeSpan.FromSeconds(5));
|
||||
|
||||
private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodes)
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.ServerClusters.Add(new ServerCluster
|
||||
{
|
||||
ClusterId = "SITE-A",
|
||||
Name = "Site A",
|
||||
Enterprise = "zb",
|
||||
Site = "site-a",
|
||||
NodeCount = 2,
|
||||
RedundancyMode = RedundancyMode.Warm,
|
||||
CreatedBy = "test",
|
||||
});
|
||||
foreach (var nodeId in nodes)
|
||||
{
|
||||
db.ClusterNodes.Add(new ClusterNode
|
||||
{
|
||||
NodeId = nodeId,
|
||||
ClusterId = "SITE-A",
|
||||
Host = nodeId.Split(':')[0],
|
||||
ApplicationUri = $"urn:OtOpcUa:{nodeId}",
|
||||
Enabled = true,
|
||||
CreatedBy = "test",
|
||||
});
|
||||
}
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
private static DeploymentId SeedDispatchingDeployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
DeploymentStatus status = DeploymentStatus.Dispatching)
|
||||
{
|
||||
var id = DeploymentId.NewId();
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id.Value,
|
||||
RevisionHash = TestRevision.Value,
|
||||
Status = status,
|
||||
CreatedBy = "test",
|
||||
});
|
||||
db.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user