diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs index 71b008ee..c64480e7 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs @@ -50,13 +50,22 @@ public sealed class FailoverDuringDeployTests .Count(m => m.Status == MemberStatus.Up).ShouldBe(2); } - /// Verifies that a deployment started with node B down seals with one-node state. + /// + /// A deployment started with node B down no longer seals without it — B's ClusterNode + /// row is enabled, so it is expected to ack and the deployment waits. + /// + /// + /// This test asserted the opposite until per-cluster mesh Phase 1. It was named + /// Deployment_started_with_node_b_down_seals_with_one_node_state and documented that + /// "DiscoverDriverNodes snapshots membership at dispatch time — when only node A is Up, + /// only one ApplyAck is expected and the deployment seals without B ever participating". That + /// is exactly the behaviour Phase 1 removed: it told the operator the fleet was deployed while + /// a configured node had not received it. The expected-ack set now comes from enabled + /// ClusterNode rows, so a node that is merely switched off is still expected. + /// [Fact] - public async Task Deployment_started_with_node_b_down_seals_with_one_node_state() + public async Task Deployment_started_with_node_b_down_does_not_seal_without_it() { - // Establishes that ConfigPublishCoordinator.DiscoverDriverNodes snapshots membership at - // dispatch time — when only node A is Up, only one ApplyAck is expected and the - // deployment seals without B ever participating. await using var harness = await TwoNodeClusterHarness.StartAsync(); await harness.SeedDefaultClusterAsync(); @@ -70,10 +79,68 @@ public sealed class FailoverDuringDeployTests result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); var deploymentId = result.DeploymentId!.Value.Value; + // Positive evidence that B was EXPECTED, not merely slow: the coordinator seeds a row per + // expected node at dispatch, so both rows must exist with B still Applying. Asserting only + // "it didn't seal" would pass just as well against a coordinator that had died. await WaitForAsync(async () => { - await using var db = await CreateDbAsync(harness); - var d = await db.Deployments.AsNoTracking() + await using var pollDb = await CreateDbAsync(harness); + return await pollDb.NodeDeploymentStates.AsNoTracking() + .CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2; + }, TimeSpan.FromSeconds(15)); + + await using var db = await CreateDbAsync(harness); + var nodeStates = await db.NodeDeploymentStates.AsNoTracking() + .Where(s => s.DeploymentId == deploymentId) + .ToListAsync(Ct); + nodeStates.Count.ShouldBe(2, "both configured nodes are expected to ack"); + nodeStates.Count(s => s.Status == NodeDeploymentStatus.Applied) + .ShouldBe(1, "only the running node applied"); + nodeStates.ShouldContain(s => s.Status == NodeDeploymentStatus.Applying, + "the stopped node's ack is still outstanding"); + + var deployment = await db.Deployments.AsNoTracking() + .FirstAsync(d => d.DeploymentId == deploymentId, Ct); + deployment.Status.ShouldNotBe(DeploymentStatus.Sealed, + "a deployment must not seal green while a configured node has not received it"); + } + + /// + /// The maintenance hatch, end-to-end: a node down and flagged + /// MaintenanceMode is not expected, so the deployment seals with one node state — the + /// behaviour the test above used to assert unconditionally, now something an operator has to + /// ask for. + /// + [Fact] + public async Task Deployment_seals_without_a_node_flagged_for_maintenance() + { + await using var harness = await TwoNodeClusterHarness.StartAsync(); + await harness.SeedDefaultClusterAsync(); + + await harness.StopNodeBAsync(); + await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20)); + + await using (var setup = await CreateDbAsync(harness)) + { + var nodeB = await setup.ClusterNodes.FirstAsync(n => n.NodeId == harness.NodeBNodeId, Ct); + nodeB.MaintenanceMode = true; + await setup.SaveChangesAsync(Ct); + // Still Enabled — DraftValidator.ValidateClusterTopology requires the enabled-node count + // to equal ServerCluster.NodeCount, which is why MaintenanceMode exists as its own flag. + nodeB.Enabled.ShouldBeTrue(); + } + + await using var scope = harness.NodeA.Services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + + var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}"); + var deploymentId = result.DeploymentId!.Value.Value; + + await WaitForAsync(async () => + { + await using var pollDb = await CreateDbAsync(harness); + var d = await pollDb.Deployments.AsNoTracking() .FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct); return d?.Status == DeploymentStatus.Sealed; }, TimeSpan.FromSeconds(15)); @@ -82,7 +149,8 @@ public sealed class FailoverDuringDeployTests var nodeStates = await db.NodeDeploymentStates.AsNoTracking() .Where(s => s.DeploymentId == deploymentId) .ToListAsync(Ct); - nodeStates.Count.ShouldBe(1); + nodeStates.Count.ShouldBe(1, "the maintenance node is not expected to ack"); + nodeStates[0].NodeId.ShouldBe(harness.NodeANodeId); nodeStates[0].Status.ShouldBe(NodeDeploymentStatus.Applied); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs index cd18796f..c9e73d42 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs @@ -122,16 +122,26 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable /// Seeds a default plus a row for BOTH /// harness nodes ( / ) so the real-SQL FK /// constraint FK_NodeDeploymentState_ClusterNode_NodeId is satisfied when a deployment - /// records per-node state. The EF in-memory provider ignores FK constraints, so deploy E2E - /// tests pass without this; against SQL Server each node's NodeDeploymentState INSERT + /// records per-node state. Against SQL Server each node's NodeDeploymentState INSERT /// fails without its parent row and the deployment never seals. - /// No-op unless OTOPCUA_HARNESS_USE_SQL=1 (in-memory needs no seeding). Call once, before - /// StartDeploymentAsync, in tests that don't already seed their own cluster + both nodes. + /// Call once, before StartDeploymentAsync, in tests that don't already seed their own + /// cluster + both nodes. /// + /// + /// This used to no-op in in-memory mode ("the in-memory provider ignores FK + /// constraints, so deploy E2E tests pass without it"). That stopped being true in per-cluster + /// mesh Phase 1: ConfigPublishCoordinator now derives its expected-ack set from these + /// rows rather than from cluster membership, so with none seeded it seals immediately + /// with an empty set. "Wait for Sealed" then no longer implies "every node has applied", and + /// since DriverHostActor.UpsertNodeDeploymentState writes each node's row on its own + /// schedule, any test counting those rows after a seal became a race — reproducibly green + /// alone and red under suite load. Seeding in both modes restores the invariant the tests + /// were written against, and is closer to production either way: a real fleet always has + /// these rows, because the FK requires them. + /// /// Cluster id for the seeded rows. Defaults to MAIN. public async Task SeedDefaultClusterAsync(string clusterId = "MAIN") { - if (!Mode.UseSqlServer) return; await using var db = await CreateConfigDbContextAsync(); db.ServerClusters.Add(new ServerCluster {