test(host): deploy-seal waits in EquipmentNamespaceMaterializationTests report where the deploy stood on timeout (#843)

The one failure on record (48 s vs 3 s, load 140-280) was a bare 'Condition
not met within 00:00:45'. It did not reproduce here (0/10 alone and 0/6 full
Host integration runs under 28 CPU burners at load ~300), so no timing was
changed; the timeout now carries the deployment row (status, artifact size,
failure reason), every per-node state (status, started/applied, failure) vs
the expected node ids, and both nodes' cluster views - enough to tell a slow
apply from a node that never got the dispatch from a failed apply.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-11-11 05:09:36 -05:00
parent e89ac2df33
commit 997c78df8a
@@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests
return true;
}
return false;
}, TwoNodeClusterHarness.DeploySealTimeout);
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
var composition = DeploymentArtifact.ParseComposition(artifact);
@@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests
return true;
}
return false;
}, TwoNodeClusterHarness.DeploySealTimeout);
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
await using (var db = await CreateDbAsync(harness))
{
@@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests
return await factory.CreateDbContextAsync();
}
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout,
Func<Task<string>>? diagnose = null)
{
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
@@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests
if (await condition()) return;
await Task.Delay(200);
}
throw new TimeoutException($"Condition not met within {timeout}");
throw new TimeoutException($"Condition not met within {timeout}"
+ (diagnose is null ? "" : $"; state at timeout: {await diagnose()}"));
}
/// <summary>
/// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load
/// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's
/// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node
/// that never received the dispatch (no ack, still Applying), or a failed apply.
/// </summary>
private static async Task<string> DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId)
{
await using var db = await CreateDbAsync(harness);
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId);
var nodes = await db.NodeDeploymentStates.AsNoTracking()
.Where(s => s.DeploymentId == deploymentId).ToListAsync();
static string View(Akka.Actor.ActorSystem s) => string.Join(",",
Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}"))
+ $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}";
return $"deployment={d?.Status.ToString() ?? "<missing>"} artifact={d?.ArtifactBlob.Length ?? 0}B "
+ $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; "
+ $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] "
+ $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); "
+ $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]";
}
}