using Akka.Actor; using Microsoft.EntityFrameworkCore; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; /// /// Per-cluster mesh Phase 1: the coordinator's expected-ack set comes from enabled /// ClusterNode rows, not from Akka.Cluster.State.Members filtered by role. /// /// /// /// Why every test here seeds ClusterNode rows and asserts a non-empty expected set. /// The harness's ActorSystem self-joins with role admin and no driver members, /// so the OLD derivation returned an empty set for every one of these scenarios and sealed /// immediately. Any test that merely observes "the deployment did not seal" would therefore /// have passed against the old code too. Each test below distinguishes the two by requiring /// the coordinator to WAIT for a specific node and then seal when exactly that node acks — /// behaviour the membership rule cannot produce in this harness at all. /// /// /// Verified by reverting DiscoverDriverNodes to the membership scan: the first three /// go red. stays green under both derivations and /// is not a test of the derivation — the harness has no driver members AND no enabled /// rows, so both rules produce an empty set. It pins the seal-empty branch and its reworded /// reason, nothing more. /// /// public sealed class ConfigPublishCoordinatorDbSourcedAcksTests : ControlPlaneActorTestBase { private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('b', 64)); private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); /// /// An enabled row is expected to ack; the deployment waits for it and seals when it arrives. /// This is the DB-derived set doing something the membership set demonstrably cannot here. /// [Fact] public void Enabled_ClusterNode_rows_are_the_expected_ack_set() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, ("site-a-1:4053", true, false)); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); // The seeded row must have produced a NodeDeploymentState row — proof the coordinator // expected it, independent of whether it later sealed. AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: TimeSpan.FromSeconds(3)); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: TimeSpan.FromSeconds(3)); } /// /// A disabled row is NOT expected, so a deployment seals without it — for a node genuinely /// decommissioned rather than temporarily down. This is not the maintenance hatch: the /// topology gate rejects a deployment whose enabled-node count differs from the cluster's /// NodeCount, so on any 2-node pair clearing Enabled never reaches this code. /// See . /// [Fact] public void Disabled_ClusterNode_rows_are_not_expected_to_ack() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", false, false)); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); // Only the enabled node is expected — the disabled one gets no NodeDeploymentState row. AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); }, duration: TimeSpan.FromSeconds(3)); // ...and the enabled node's ack alone seals it. If the disabled row were expected this would // sit at AwaitingApplyAcks until the deadline. actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: TimeSpan.FromSeconds(3)); } /// /// An ack from a node with no ClusterNode row is discarded — it is not in the expected /// set, so it must not count toward the seal. Under the membership rule a running-but- /// unregistered node WAS expected, and then threw FK 547 when its state row was written. /// [Fact] public void Ack_from_a_node_absent_from_ClusterNode_is_discarded() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, ("site-a-1:4053", true, false)); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: TimeSpan.FromSeconds(3)); // A node nobody registered acks. One expected node, one ack — if it were counted, the // coordinator would seal here. actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("ghost-9:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); ExpectNoMsg(TimeSpan.FromMilliseconds(400)); using (var db = dbFactory.CreateDbContext()) { db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); } // Positive control on the wait itself: the real node's ack DOES seal it, so the assertion // above measured the discard rather than a coordinator that had simply stopped responding. actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: TimeSpan.FromSeconds(3)); } /// /// No enabled rows at all seals empty. Same outcome as the old empty-membership branch, but /// it now means "the fleet has no enabled nodes registered" — a configuration error that /// cannot resolve itself — rather than "no driver node has joined yet". /// [Fact] public void No_enabled_rows_seals_empty() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, ("site-a-1:4053", false, false), ("site-a-2:4053", false, false)); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); db.NodeDeploymentStates.ShouldBeEmpty(); }, duration: TimeSpan.FromSeconds(3)); } /// /// The maintenance hatch, and the reason it is a separate flag. The Phase 1 live gate set /// Enabled = 0 on one node of a 2-node pair and the deployment was rejected outright by /// DraftValidator.ValidateClusterTopology (enabled-node count must equal /// NodeCount) — so the intended escape hatch could not be used on any pair, which is /// every cluster in the target topology. MaintenanceMode excludes a node from the /// expected-ack set while leaving it enabled, so the topology gate still passes. /// [Fact] public void MaintenanceMode_rows_are_not_expected_to_ack_while_staying_enabled() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDispatchingDeployment(dbFactory); SeedNodes(dbFactory, ("site-a-1:4053", true, false), ("site-a-2:4053", true, true)); // The topology gate counts Enabled, so both nodes still count — this is what Enabled = 0 // broke. Assert it here so the fix cannot be "quietly disable it after all". using (var check = dbFactory.CreateDbContext()) { check.ClusterNodes.Count(n => n.Enabled).ShouldBe(2); } var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); actor.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); }, duration: TimeSpan.FromSeconds(3)); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: TimeSpan.FromSeconds(3)); } private static void SeedNodes( IDbContextFactory dbFactory, params (string NodeId, bool Enabled, bool Maintenance)[] nodes) { using var db = dbFactory.CreateDbContext(); db.ServerClusters.Add(new ServerCluster { ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", NodeCount = 2, RedundancyMode = RedundancyMode.Warm, CreatedBy = "test", }); foreach (var (nodeId, enabled, maintenance) in nodes) { db.ClusterNodes.Add(new ClusterNode { NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = enabled, MaintenanceMode = maintenance, CreatedBy = "test", }); } db.SaveChanges(); } private static DeploymentId SeedDispatchingDeployment( IDbContextFactory dbFactory) { var id = DeploymentId.NewId(); using var db = dbFactory.CreateDbContext(); db.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, CreatedBy = "test", }); db.SaveChanges(); return id; } }