diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs index dd78d6087..824336829 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/EquipmentNamespaceMaterializationTests.cs @@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TwoNodeClusterHarness.DeploySealTimeout); + }, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId)); var composition = DeploymentArtifact.ParseComposition(artifact); @@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests return true; } return false; - }, TwoNodeClusterHarness.DeploySealTimeout); + }, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId)); await using (var db = await CreateDbAsync(harness)) { @@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests return await factory.CreateDbContextAsync(); } - private static async Task WaitForAsync(Func> condition, TimeSpan timeout) + private static async Task WaitForAsync(Func> condition, TimeSpan timeout, + Func>? diagnose = null) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) @@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests if (await condition()) return; await Task.Delay(200); } - throw new TimeoutException($"Condition not met within {timeout}"); + throw new TimeoutException($"Condition not met within {timeout}" + + (diagnose is null ? "" : $"; state at timeout: {await diagnose()}")); + } + + /// + /// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load + /// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's + /// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node + /// that never received the dispatch (no ack, still Applying), or a failed apply. + /// + private static async Task DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId) + { + await using var db = await CreateDbAsync(harness); + var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId); + var nodes = await db.NodeDeploymentStates.AsNoTracking() + .Where(s => s.DeploymentId == deploymentId).ToListAsync(); + static string View(Akka.Actor.ActorSystem s) => string.Join(",", + Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}")) + + $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}"; + return $"deployment={d?.Status.ToString() ?? ""} artifact={d?.ArtifactBlob.Length ?? 0}B " + + $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; " + + $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] " + + $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); " + + $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]"; } }