From 997c78df8a4a008a26639a1d9dcf0afd1ba1e633 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 05:09:36 -0500 Subject: [PATCH] test(host): deploy-seal waits in EquipmentNamespaceMaterializationTests report where the deploy stood on timeout (#843) The one failure on record (48 s vs 3 s, load 140-280) was a bare 'Condition not met within 00:00:45'. It did not reproduce here (0/10 alone and 0/6 full Host integration runs under 28 CPU burners at load ~300), so no timing was changed; the timeout now carries the deployment row (status, artifact size, failure reason), every per-node state (status, started/applied, failure) vs the expected node ids, and both nodes' cluster views - enough to tell a slow apply from a node that never got the dispatch from a failed apply. Co-Authored-By: Claude --- .../EquipmentNamespaceMaterializationTests.cs | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs index dd78d6087..824336829 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs @@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TwoNodeClusterHarness.DeploySealTimeout); + }, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId)); var composition = DeploymentArtifact.ParseComposition(artifact); @@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TwoNodeClusterHarness.DeploySealTimeout); + }, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId)); await using (var db = await CreateDbAsync(harness)) { @@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests return await factory.CreateDbContextAsync(); } - private static async Task WaitForAsync(Func> condition, TimeSpan timeout) + private static async Task WaitForAsync(Func> condition, TimeSpan timeout, + Func>? diagnose = null) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) @@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests if (await condition()) return; await Task.Delay(200); } - throw new TimeoutException($"Condition not met within {timeout}"); + throw new TimeoutException($"Condition not met within {timeout}" + + (diagnose is null ? "" : $"; state at timeout: {await diagnose()}")); + } + + /// + /// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load + /// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's + /// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node + /// that never received the dispatch (no ack, still Applying), or a failed apply. + /// + private static async Task DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId) + { + await using var db = await CreateDbAsync(harness); + var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId); + var nodes = await db.NodeDeploymentStates.AsNoTracking() + .Where(s => s.DeploymentId == deploymentId).ToListAsync(); + static string View(Akka.Actor.ActorSystem s) => string.Join(",", + Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}")) + + $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}"; + return $"deployment={d?.Status.ToString() ?? ""} artifact={d?.ArtifactBlob.Length ?? 0}B " + + $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; " + + $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] " + + $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); " + + $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]"; } }