test(harness): widen the deploy-seal timeout to match a real two-node deploy
Comparing full-suite runs on this branch against master in a clean worktree: both fail 11 tests, 9 identical. Master's two extras are load-flaky unit tests (AbCip Probe_loops, Galaxy EventPumpBoundedChannel); this branch's two extras were both Host.IntegrationTests deploy-path tests — the area this change re-times — so they are attributable here rather than to background flakiness. Cause is margin, not correctness. These waits used to observe a coordinator that sealed instantly on an empty expected-ack set; they now observe a real ApplyAck round-trip from every configured node. 15s was enormous margin against "instant" and thin against the real thing, so they failed only under full-suite CPU contention. Hoisted to TwoNodeClusterHarness.DeploySealTimeout (45s) so the reason is recorded once rather than as four unexplained numbers. DriverReconnectE2eTests is deliberately left alone: it seeds both ClusterNode rows itself and unconditionally, so its expected-ack set is identical before and after this change. Widening its timeout would be papering over a flake this change did not cause. Host.IntegrationTests 195/201; sole failure AbCip_Green_AgainstSim, verified failing on master. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
@@ -40,7 +40,7 @@ public sealed class DeployHappyPathTests
|
|||||||
await using var db = await CreateDbAsync(harness);
|
await using var db = await CreateDbAsync(harness);
|
||||||
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
|
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
|
||||||
return d?.Status == DeploymentStatus.Sealed;
|
return d?.Status == DeploymentStatus.Sealed;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
await using var db = await CreateDbAsync(harness);
|
await using var db = await CreateDbAsync(harness);
|
||||||
var deployment = await db.Deployments.AsNoTracking()
|
var deployment = await db.Deployments.AsNoTracking()
|
||||||
@@ -74,7 +74,7 @@ public sealed class DeployHappyPathTests
|
|||||||
var d = await db.Deployments.AsNoTracking()
|
var d = await db.Deployments.AsNoTracking()
|
||||||
.FirstOrDefaultAsync(d => d.DeploymentId == first.DeploymentId!.Value.Value, Ct);
|
.FirstOrDefaultAsync(d => d.DeploymentId == first.DeploymentId!.Value.Value, Ct);
|
||||||
return d?.Status == DeploymentStatus.Sealed;
|
return d?.Status == DeploymentStatus.Sealed;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
// Same DB state → same revision hash → AdminOperations short-circuits with NoChanges,
|
// Same DB state → same revision hash → AdminOperations short-circuits with NoChanges,
|
||||||
// OR (if it accepts) the second dispatch is idempotent (DriverHostActor recognises
|
// OR (if it accepts) the second dispatch is idempotent (DriverHostActor recognises
|
||||||
@@ -90,7 +90,7 @@ public sealed class DeployHappyPathTests
|
|||||||
var d = await db.Deployments.AsNoTracking()
|
var d = await db.Deployments.AsNoTracking()
|
||||||
.FirstOrDefaultAsync(d => d.DeploymentId == second.DeploymentId!.Value.Value, Ct);
|
.FirstOrDefaultAsync(d => d.DeploymentId == second.DeploymentId!.Value.Value, Ct);
|
||||||
return d?.Status == DeploymentStatus.Sealed;
|
return d?.Status == DeploymentStatus.Sealed;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
second.RevisionHash!.Value.Value.ShouldBe(first.RevisionHash!.Value.Value);
|
second.RevisionHash!.Value.Value.ShouldBe(first.RevisionHash!.Value.Value);
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
var composition = DeploymentArtifact.ParseComposition(artifact);
|
var composition = DeploymentArtifact.ParseComposition(artifact);
|
||||||
|
|
||||||
@@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}, TimeSpan.FromSeconds(20));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
await using (var db = await CreateDbAsync(harness))
|
await using (var db = await CreateDbAsync(harness))
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ public sealed class FailoverDuringDeployTests
|
|||||||
await using var pollDb = await CreateDbAsync(harness);
|
await using var pollDb = await CreateDbAsync(harness);
|
||||||
return await pollDb.NodeDeploymentStates.AsNoTracking()
|
return await pollDb.NodeDeploymentStates.AsNoTracking()
|
||||||
.CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2;
|
.CountAsync(s => s.DeploymentId == deploymentId, Ct) == 2;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
await using var db = await CreateDbAsync(harness);
|
await using var db = await CreateDbAsync(harness);
|
||||||
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
||||||
@@ -143,7 +143,7 @@ public sealed class FailoverDuringDeployTests
|
|||||||
var d = await pollDb.Deployments.AsNoTracking()
|
var d = await pollDb.Deployments.AsNoTracking()
|
||||||
.FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
|
.FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
|
||||||
return d?.Status == DeploymentStatus.Sealed;
|
return d?.Status == DeploymentStatus.Sealed;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
|
|
||||||
await using var db = await CreateDbAsync(harness);
|
await using var db = await CreateDbAsync(harness);
|
||||||
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
||||||
|
|||||||
+1
-1
@@ -67,7 +67,7 @@ public sealed class FleetDiagnosticsRoundTripTests
|
|||||||
{
|
{
|
||||||
var snap = await diagnostics.GetDiagnosticsAsync(nodeId, Ct);
|
var snap = await diagnostics.GetDiagnosticsAsync(nodeId, Ct);
|
||||||
return snap.CurrentRevision == expectedRev;
|
return snap.CurrentRevision == expectedRev;
|
||||||
}, TimeSpan.FromSeconds(15));
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -96,6 +96,16 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable
|
|||||||
// host:port into NodeId so the cluster membership stays distinct on different ports.
|
// host:port into NodeId so the cluster membership stays distinct on different ports.
|
||||||
public const string LoopbackHost = "127.0.0.1";
|
public const string LoopbackHost = "127.0.0.1";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How long a deploy-seal assertion waits. Generous on purpose: since per-cluster mesh
|
||||||
|
/// Phase 1 the coordinator waits for a real ApplyAck from every configured node before
|
||||||
|
/// sealing, so this measures an end-to-end two-node deploy rather than — as it did when the
|
||||||
|
/// expected-ack set came up empty — an immediate seal. The old 15s had enormous margin
|
||||||
|
/// against "instant" and thin margin against the real thing, which showed up as deploy
|
||||||
|
/// tests failing only under full-suite CPU contention.
|
||||||
|
/// </summary>
|
||||||
|
public static readonly TimeSpan DeploySealTimeout = TimeSpan.FromSeconds(45);
|
||||||
|
|
||||||
/// <summary>Gets the Akka ActorSystem for node A.</summary>
|
/// <summary>Gets the Akka ActorSystem for node A.</summary>
|
||||||
public ActorSystem NodeASystem => NodeA.Services.GetRequiredService<ActorSystem>();
|
public ActorSystem NodeASystem => NodeA.Services.GetRequiredService<ActorSystem>();
|
||||||
/// <summary>Gets the Akka ActorSystem for node B.</summary>
|
/// <summary>Gets the Akka ActorSystem for node B.</summary>
|
||||||
|
|||||||
Reference in New Issue
Block a user