using Akka.Cluster; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// Failover scenarios layered on Stop/Restart primitives. /// Covers graceful node loss, rejoin on the same Akka port, and deployment under reduced membership. /// public sealed class FailoverDuringDeployTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; /// Verifies that stopping node B shrinks the cluster to one up member. [Fact] public async Task Stopping_node_b_shrinks_cluster_to_one_up_member() { await using var harness = await TwoNodeClusterHarness.StartAsync(); Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members .Count(m => m.Status == MemberStatus.Up).ShouldBe(2); await harness.StopNodeBAsync(); await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20)); Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members .Count(m => m.Status == MemberStatus.Up).ShouldBe(1); } /// Verifies that a restarted node B rejoins the cluster on the same port. [Fact] public async Task Restarted_node_b_rejoins_cluster_on_same_port() { await using var harness = await TwoNodeClusterHarness.StartAsync(); await harness.StopNodeBAsync(); await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20)); await harness.RestartNodeBAsync(); Akka.Cluster.Cluster.Get(harness.NodeASystem).State.Members .Count(m => m.Status == MemberStatus.Up).ShouldBe(2); Akka.Cluster.Cluster.Get(harness.NodeBSystem).State.Members .Count(m => m.Status == MemberStatus.Up).ShouldBe(2); } /// /// A deployment started with node B down no longer seals without it — B's ClusterNode /// row is enabled, so it is expected to ack and the deployment waits. /// /// /// This test asserted the opposite until per-cluster mesh Phase 1. It was named /// Deployment_started_with_node_b_down_seals_with_one_node_state and documented that /// "DiscoverDriverNodes snapshots membership at dispatch time — when only node A is Up, /// only one ApplyAck is expected and the deployment seals without B ever participating". That /// is exactly the behaviour Phase 1 removed: it told the operator the fleet was deployed while /// a configured node had not received it. The expected-ack set now comes from enabled /// ClusterNode rows, so a node that is merely switched off is still expected. /// [Fact] public async Task Deployment_started_with_node_b_down_does_not_seal_without_it() { await using var harness = await TwoNodeClusterHarness.StartAsync(); await harness.SeedDefaultClusterAsync(); await harness.StopNodeBAsync(); await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20)); await using var scope = harness.NodeA.Services.CreateAsyncScope(); var client = scope.ServiceProvider.GetRequiredService(); var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct); result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); var deploymentId = result.DeploymentId!.Value.Value; // Positive evidence that B was EXPECTED, not merely slow: the coordinator seeds a row per // expected node at dispatch, so both rows must exist with B still Applying. Asserting only // "it didn't seal" would pass just as well against a coordinator that had died. await WaitForAsync(async () => { await using var pollDb = await CreateDbAsync(harness); return await pollDb.NodeDeploymentStates.AsNoTracking() .CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2; }, TwoNodeClusterHarness.DeploySealTimeout); await using var db = await CreateDbAsync(harness); var nodeStates = await db.NodeDeploymentStates.AsNoTracking() .Where(s => s.DeploymentId == deploymentId) .ToListAsync(Ct); nodeStates.Count.ShouldBe(2, "both configured nodes are expected to ack"); nodeStates.Count(s => s.Status == NodeDeploymentStatus.Applied) .ShouldBe(1, "only the running node applied"); nodeStates.ShouldContain(s => s.Status == NodeDeploymentStatus.Applying, "the stopped node's ack is still outstanding"); var deployment = await db.Deployments.AsNoTracking() .FirstAsync(d => d.DeploymentId == deploymentId, Ct); deployment.Status.ShouldNotBe(DeploymentStatus.Sealed, "a deployment must not seal green while a configured node has not received it"); } /// /// The maintenance hatch, end-to-end: a node down and flagged /// MaintenanceMode is not expected, so the deployment seals with one node state — the /// behaviour the test above used to assert unconditionally, now something an operator has to /// ask for. /// [Fact] public async Task Deployment_seals_without_a_node_flagged_for_maintenance() { await using var harness = await TwoNodeClusterHarness.StartAsync(); await harness.SeedDefaultClusterAsync(); await harness.StopNodeBAsync(); await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20)); await using (var setup = await CreateDbAsync(harness)) { var nodeB = await setup.ClusterNodes.FirstAsync(n => n.NodeId == harness.NodeBNodeId, Ct); nodeB.MaintenanceMode = true; await setup.SaveChangesAsync(Ct); // Still Enabled — DraftValidator.ValidateClusterTopology requires the enabled-node count // to equal ServerCluster.NodeCount, which is why MaintenanceMode exists as its own flag. nodeB.Enabled.ShouldBeTrue(); } await using var scope = harness.NodeA.Services.CreateAsyncScope(); var client = scope.ServiceProvider.GetRequiredService(); var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct); result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}"); var deploymentId = result.DeploymentId!.Value.Value; await WaitForAsync(async () => { await using var pollDb = await CreateDbAsync(harness); var d = await pollDb.Deployments.AsNoTracking() .FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct); return d?.Status == DeploymentStatus.Sealed; }, TwoNodeClusterHarness.DeploySealTimeout); await using var db = await CreateDbAsync(harness); var nodeStates = await db.NodeDeploymentStates.AsNoTracking() .Where(s => s.DeploymentId == deploymentId) .ToListAsync(Ct); nodeStates.Count.ShouldBe(1, "the maintenance node is not expected to ack"); nodeStates[0].NodeId.ShouldBe(harness.NodeANodeId); nodeStates[0].Status.ShouldBe(NodeDeploymentStatus.Applied); } private static async Task CreateDbAsync(TwoNodeClusterHarness harness) { var factory = harness.NodeA.Services.GetRequiredService>(); return await factory.CreateDbContextAsync(); } private static async Task WaitForAsync(Func> condition, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (await condition()) return; await Task.Delay(200); } throw new TimeoutException($"Condition not met within {timeout}"); } }