test(host): deploy-seal waits in EquipmentNamespaceMaterializationTests report where the deploy stood on timeout (#843)
The one failure on record (48 s vs 3 s, load 140-280) was a bare 'Condition not met within 00:00:45'. It did not reproduce here (0/10 alone and 0/6 full Host integration runs under 28 CPU burners at load ~300), so no timing was changed; the timeout now carries the deployment row (status, artifact size, failure reason), every per-node state (status, started/applied, failure) vs the expected node ids, and both nodes' cluster views - enough to tell a slow apply from a node that never got the dispatch from a failed apply. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+28
-4
@@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||||
|
||||
var composition = DeploymentArtifact.ParseComposition(artifact);
|
||||
|
||||
@@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||||
|
||||
await using (var db = await CreateDbAsync(harness))
|
||||
{
|
||||
@@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return await factory.CreateDbContextAsync();
|
||||
}
|
||||
|
||||
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
|
||||
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout,
|
||||
Func<Task<string>>? diagnose = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
@@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
if (await condition()) return;
|
||||
await Task.Delay(200);
|
||||
}
|
||||
throw new TimeoutException($"Condition not met within {timeout}");
|
||||
throw new TimeoutException($"Condition not met within {timeout}"
|
||||
+ (diagnose is null ? "" : $"; state at timeout: {await diagnose()}"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load
|
||||
/// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's
|
||||
/// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node
|
||||
/// that never received the dispatch (no ack, still Applying), or a failed apply.
|
||||
/// </summary>
|
||||
private static async Task<string> DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId)
|
||||
{
|
||||
await using var db = await CreateDbAsync(harness);
|
||||
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId);
|
||||
var nodes = await db.NodeDeploymentStates.AsNoTracking()
|
||||
.Where(s => s.DeploymentId == deploymentId).ToListAsync();
|
||||
static string View(Akka.Actor.ActorSystem s) => string.Join(",",
|
||||
Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}"))
|
||||
+ $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}";
|
||||
return $"deployment={d?.Status.ToString() ?? "<missing>"} artifact={d?.ArtifactBlob.Length ?? 0}B "
|
||||
+ $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; "
|
||||
+ $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] "
|
||||
+ $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); "
|
||||
+ $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]";
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user