From 0b7c53f64f93ab6754df89d9054fb29d7eb364e6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 22 Jul 2026 09:28:32 -0400 Subject: [PATCH] test(harness): widen the deploy-seal timeout to match a real two-node deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comparing full-suite runs on this branch against master in a clean worktree: both fail 11 tests, 9 identical. Master's two extras are load-flaky unit tests (AbCip Probe_loops, Galaxy EventPumpBoundedChannel); this branch's two extras were both Host.IntegrationTests deploy-path tests — the area this change re-times — so they are attributable here rather than to background flakiness. Cause is margin, not correctness. These waits used to observe a coordinator that sealed instantly on an empty expected-ack set; they now observe a real ApplyAck round-trip from every configured node. 15s was enormous margin against "instant" and thin against the real thing, so they failed only under full-suite CPU contention. Hoisted to TwoNodeClusterHarness.DeploySealTimeout (45s) so the reason is recorded once rather than as four unexplained numbers. DriverReconnectE2eTests is deliberately left alone: it seeds both ClusterNode rows itself and unconditionally, so its expected-ack set is identical before and after this change. Widening its timeout would be papering over a flake this change did not cause. Host.IntegrationTests 195/201; sole failure AbCip_Green_AgainstSim, verified failing on master. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- .../DeployHappyPathTests.cs | 6 +++--- .../EquipmentNamespaceMaterializationTests.cs | 4 ++-- .../FailoverDuringDeployTests.cs | 4 ++-- .../FleetDiagnosticsRoundTripTests.cs | 2 +- .../TwoNodeClusterHarness.cs | 10 ++++++++++ 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DeployHappyPathTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DeployHappyPathTests.cs index e0fb69b7..076c9c4c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DeployHappyPathTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DeployHappyPathTests.cs @@ -40,7 +40,7 @@ public sealed class DeployHappyPathTests await using var db = await CreateDbAsync(harness); var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct); return d?.Status == DeploymentStatus.Sealed; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); await using var db = await CreateDbAsync(harness); var deployment = await db.Deployments.AsNoTracking() @@ -74,7 +74,7 @@ public sealed class DeployHappyPathTests var d = await db.Deployments.AsNoTracking() .FirstOrDefaultAsync(d => d.DeploymentId == first.DeploymentId!.Value.Value, Ct); return d?.Status == DeploymentStatus.Sealed; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); // Same DB state → same revision hash → AdminOperations short-circuits with NoChanges, // OR (if it accepts) the second dispatch is idempotent (DriverHostActor recognises @@ -90,7 +90,7 @@ public sealed class DeployHappyPathTests var d = await db.Deployments.AsNoTracking() .FirstOrDefaultAsync(d => d.DeploymentId == second.DeploymentId!.Value.Value, Ct); return d?.Status == DeploymentStatus.Sealed; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); second.RevisionHash!.Value.Value.ShouldBe(first.RevisionHash!.Value.Value); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs index 3811e137..dd78d608 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs @@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); var composition = DeploymentArtifact.ParseComposition(artifact); @@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TimeSpan.FromSeconds(20)); + }, TwoNodeClusterHarness.DeploySealTimeout); await using (var db = await CreateDbAsync(harness)) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs index c64480e7..ebe35df7 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FailoverDuringDeployTests.cs @@ -87,7 +87,7 @@ public sealed class FailoverDuringDeployTests await using var pollDb = await CreateDbAsync(harness); return await pollDb.NodeDeploymentStates.AsNoTracking() .CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); await using var db = await CreateDbAsync(harness); var nodeStates = await db.NodeDeploymentStates.AsNoTracking() @@ -143,7 +143,7 @@ public sealed class FailoverDuringDeployTests var d = await pollDb.Deployments.AsNoTracking() .FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct); return d?.Status == DeploymentStatus.Sealed; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); await using var db = await CreateDbAsync(harness); var nodeStates = await db.NodeDeploymentStates.AsNoTracking() diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FleetDiagnosticsRoundTripTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FleetDiagnosticsRoundTripTests.cs index 2d8afa3c..d7243abc 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FleetDiagnosticsRoundTripTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/FleetDiagnosticsRoundTripTests.cs @@ -67,7 +67,7 @@ public sealed class FleetDiagnosticsRoundTripTests { var snap = await diagnostics.GetDiagnosticsAsync(nodeId, Ct); return snap.CurrentRevision == expectedRev; - }, TimeSpan.FromSeconds(15)); + }, TwoNodeClusterHarness.DeploySealTimeout); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs index c9e73d42..9e2558b5 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs @@ -96,6 +96,16 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable // host:port into NodeId so the cluster membership stays distinct on different ports. public const string LoopbackHost = "127.0.0.1"; + /// + /// How long a deploy-seal assertion waits. Generous on purpose: since per-cluster mesh + /// Phase 1 the coordinator waits for a real ApplyAck from every configured node before + /// sealing, so this measures an end-to-end two-node deploy rather than — as it did when the + /// expected-ack set came up empty — an immediate seal. The old 15s had enormous margin + /// against "instant" and thin margin against the real thing, which showed up as deploy + /// tests failing only under full-suite CPU contention. + /// + public static readonly TimeSpan DeploySealTimeout = TimeSpan.FromSeconds(45); + /// Gets the Akka ActorSystem for node A. public ActorSystem NodeASystem => NodeA.Services.GetRequiredService(); /// Gets the Akka ActorSystem for node B.