diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs
index 71b008ee..c64480e7 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs
@@ -50,13 +50,22 @@ public sealed class FailoverDuringDeployTests
.Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
}
- /// Verifies that a deployment started with node B down seals with one-node state.
+ ///
+ /// A deployment started with node B down no longer seals without it — B's ClusterNode
+ /// row is enabled, so it is expected to ack and the deployment waits.
+ ///
+ ///
+ /// This test asserted the opposite until per-cluster mesh Phase 1. It was named
+ /// Deployment_started_with_node_b_down_seals_with_one_node_state and documented that
+ /// "DiscoverDriverNodes snapshots membership at dispatch time — when only node A is Up,
+ /// only one ApplyAck is expected and the deployment seals without B ever participating". That
+ /// is exactly the behaviour Phase 1 removed: it told the operator the fleet was deployed while
+ /// a configured node had not received it. The expected-ack set now comes from enabled
+ /// ClusterNode rows, so a node that is merely switched off is still expected.
+ ///
[Fact]
- public async Task Deployment_started_with_node_b_down_seals_with_one_node_state()
+ public async Task Deployment_started_with_node_b_down_does_not_seal_without_it()
{
- // Establishes that ConfigPublishCoordinator.DiscoverDriverNodes snapshots membership at
- // dispatch time — when only node A is Up, only one ApplyAck is expected and the
- // deployment seals without B ever participating.
await using var harness = await TwoNodeClusterHarness.StartAsync();
await harness.SeedDefaultClusterAsync();
@@ -70,10 +79,68 @@ public sealed class FailoverDuringDeployTests
result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
var deploymentId = result.DeploymentId!.Value.Value;
+ // Positive evidence that B was EXPECTED, not merely slow: the coordinator seeds a row per
+ // expected node at dispatch, so both rows must exist with B still Applying. Asserting only
+ // "it didn't seal" would pass just as well against a coordinator that had died.
await WaitForAsync(async () =>
{
- await using var db = await CreateDbAsync(harness);
- var d = await db.Deployments.AsNoTracking()
+ await using var pollDb = await CreateDbAsync(harness);
+ return await pollDb.NodeDeploymentStates.AsNoTracking()
+ .CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2;
+ }, TimeSpan.FromSeconds(15));
+
+ await using var db = await CreateDbAsync(harness);
+ var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
+ .Where(s => s.DeploymentId == deploymentId)
+ .ToListAsync(Ct);
+ nodeStates.Count.ShouldBe(2, "both configured nodes are expected to ack");
+ nodeStates.Count(s => s.Status == NodeDeploymentStatus.Applied)
+ .ShouldBe(1, "only the running node applied");
+ nodeStates.ShouldContain(s => s.Status == NodeDeploymentStatus.Applying,
+ "the stopped node's ack is still outstanding");
+
+ var deployment = await db.Deployments.AsNoTracking()
+ .FirstAsync(d => d.DeploymentId == deploymentId, Ct);
+ deployment.Status.ShouldNotBe(DeploymentStatus.Sealed,
+ "a deployment must not seal green while a configured node has not received it");
+ }
+
+ ///
+ /// The maintenance hatch, end-to-end: a node down and flagged
+ /// MaintenanceMode is not expected, so the deployment seals with one node state — the
+ /// behaviour the test above used to assert unconditionally, now something an operator has to
+ /// ask for.
+ ///
+ [Fact]
+ public async Task Deployment_seals_without_a_node_flagged_for_maintenance()
+ {
+ await using var harness = await TwoNodeClusterHarness.StartAsync();
+ await harness.SeedDefaultClusterAsync();
+
+ await harness.StopNodeBAsync();
+ await harness.WaitForClusterSizeAsync(1, TimeSpan.FromSeconds(20));
+
+ await using (var setup = await CreateDbAsync(harness))
+ {
+ var nodeB = await setup.ClusterNodes.FirstAsync(n => n.NodeId == harness.NodeBNodeId, Ct);
+ nodeB.MaintenanceMode = true;
+ await setup.SaveChangesAsync(Ct);
+ // Still Enabled — DraftValidator.ValidateClusterTopology requires the enabled-node count
+ // to equal ServerCluster.NodeCount, which is why MaintenanceMode exists as its own flag.
+ nodeB.Enabled.ShouldBeTrue();
+ }
+
+ await using var scope = harness.NodeA.Services.CreateAsyncScope();
+ var client = scope.ServiceProvider.GetRequiredService();
+
+ var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
+ result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}");
+ var deploymentId = result.DeploymentId!.Value.Value;
+
+ await WaitForAsync(async () =>
+ {
+ await using var pollDb = await CreateDbAsync(harness);
+ var d = await pollDb.Deployments.AsNoTracking()
.FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
return d?.Status == DeploymentStatus.Sealed;
}, TimeSpan.FromSeconds(15));
@@ -82,7 +149,8 @@ public sealed class FailoverDuringDeployTests
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
.Where(s => s.DeploymentId == deploymentId)
.ToListAsync(Ct);
- nodeStates.Count.ShouldBe(1);
+ nodeStates.Count.ShouldBe(1, "the maintenance node is not expected to ack");
+ nodeStates[0].NodeId.ShouldBe(harness.NodeANodeId);
nodeStates[0].Status.ShouldBe(NodeDeploymentStatus.Applied);
}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs
index cd18796f..c9e73d42 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs
@@ -122,16 +122,26 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable
/// Seeds a default plus a row for BOTH
/// harness nodes ( / ) so the real-SQL FK
/// constraint FK_NodeDeploymentState_ClusterNode_NodeId is satisfied when a deployment
- /// records per-node state. The EF in-memory provider ignores FK constraints, so deploy E2E
- /// tests pass without this; against SQL Server each node's NodeDeploymentState INSERT
+ /// records per-node state. Against SQL Server each node's NodeDeploymentState INSERT
/// fails without its parent row and the deployment never seals.
- /// No-op unless OTOPCUA_HARNESS_USE_SQL=1 (in-memory needs no seeding). Call once, before
- /// StartDeploymentAsync, in tests that don't already seed their own cluster + both nodes.
+ /// Call once, before StartDeploymentAsync, in tests that don't already seed their own
+ /// cluster + both nodes.
///
+ ///
+ /// This used to no-op in in-memory mode ("the in-memory provider ignores FK
+ /// constraints, so deploy E2E tests pass without it"). That stopped being true in per-cluster
+ /// mesh Phase 1: ConfigPublishCoordinator now derives its expected-ack set from these
+ /// rows rather than from cluster membership, so with none seeded it seals immediately
+ /// with an empty set. "Wait for Sealed" then no longer implies "every node has applied", and
+ /// since DriverHostActor.UpsertNodeDeploymentState writes each node's row on its own
+ /// schedule, any test counting those rows after a seal became a race — reproducibly green
+ /// alone and red under suite load. Seeding in both modes restores the invariant the tests
+ /// were written against, and is closer to production either way: a real fleet always has
+ /// these rows, because the FK requires them.
+ ///
/// Cluster id for the seeded rows. Defaults to MAIN.
public async Task SeedDefaultClusterAsync(string clusterId = "MAIN")
{
- if (!Mode.UseSqlServer) return;
await using var db = await CreateConfigDbContextAsync();
db.ServerClusters.Add(new ServerCluster
{