From 80118a01c501b171eeb4b4014a65e0967774604a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 21:16:53 -0500 Subject: [PATCH] fix(deploy): persist which nodes never acked on a deploy timeout (lmxopcua#784) In the normal timeout branch (some acks landed) the deployment row was set to TimedOut with a null FailureReason, so GET /api/deployments/{id} could not say which node to fix; only the central log named it. The branch now persists a reason naming the silent nodes ("Timed out waiting for acks (n/m landed). No ack from: ..."), bounded to 20 named nodes plus a count of the rest, and still backstopped by the 2048-char Truncate. Tests: the reason names the silent node and not the acked one; on a 25-node silence it names 20, counts 5 and fits the column. Mutation-checked: dropping the SetFailureReason fails both, dropping the bound fails the second. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- docs/reviews/2026-10-21/gates.md | 2 + .../Coordinators/ConfigPublishCoordinator.cs | 28 +++++++- .../ConfigPublishCoordinatorTimeoutTests.cs | 68 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/docs/reviews/2026-10-21/gates.md b/docs/reviews/2026-10-21/gates.md index ec7b178a2..3276bba2e 100644 --- a/docs/reviews/2026-10-21/gates.md +++ b/docs/reviews/2026-10-21/gates.md @@ -186,6 +186,8 @@ hold. `GET /api/deployments/{id}` returns `status: TimedOut` with `failureReason: null`. Only the central log names the silent node (`No ack from: site-b-1:4053`). The no-expected-rows branch does set a reason. An automation client polling the API cannot see which node to fix. + **Resolved (lmxopcua#784):** the timeout branch now persists `failureReason` = `Timed out waiting for acks + (n/m landed). No ack from: . …` (at most 20 nodes named, the rest counted). 2. **Rig: mesh PSK warnings on the centrals.** central logs `Mesh PSK secret ot-mesh-psk-{main,site-a,site-b} … is absent (or blank) in the store; the key is REVOKED — commands to that cluster and acks from it are refused`. Yet every gRPC dispatch and ack in these runs succeeded. Either the rig's PSK secrets are not seeded and enforcement is diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index d408343cb..01143dd4d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -470,11 +470,14 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers ResetForNext(); return; } - db.SaveChanges(); // Name the silent nodes, not just the counts. Since the expected set became DB-derived this // is the failure an operator meets after deploying while a configured node is switched off, // so it has to say which node — "4/5 acks landed" leaves them reading logs on five machines. var missing = _expectedAcks.Where(n => !_acks.ContainsKey(n)).Select(n => n.Value).Order().ToList(); + // lmxopcua#784: persist the same answer on the deployment row, so GET /api/deployments/{id} says which + // node to fix instead of TimedOut + a null reason (only the no-expected-rows branch above set one). + SetFailureReason(db, TimeoutFailureReason(_acks.Count, _expectedAcks.Count, missing)); + db.SaveChanges(); _log.Warning( "Deployment {Id} timed out after {Deadline} ({Acked}/{Total} acks landed). " + "No ack from: {MissingNodes}. Each is an enabled, non-maintenance ClusterNode row — start " + @@ -483,6 +486,29 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers ResetForNext(); } + /// + /// The most silent nodes a timeout reason names before summarising the rest as a count. Bounds the + /// reason on a large fleet well inside ; + /// still backstops pathological node ids. + /// + private const int TimeoutReasonMaxNamedNodes = 20; + + /// The persisted reason for a deployment that timed out with some acks outstanding. + /// Acks that landed. + /// Acks that were expected. + /// The nodes that never acked, ordered. + /// A reason naming the silent nodes, bounded in length. + private static string TimeoutFailureReason(int acked, int expected, IReadOnlyList missing) + { + var named = string.Join(", ", missing.Take(TimeoutReasonMaxNamedNodes)); + var rest = missing.Count > TimeoutReasonMaxNamedNodes + ? $" (and {missing.Count - TimeoutReasonMaxNamedNodes} more)" + : string.Empty; + return Truncate( + $"Timed out waiting for acks ({acked}/{expected} landed). No ack from: {named}{rest}. Start the node, " + + "or set ClusterNode.MaintenanceMode = 1 if it is down for maintenance, then redeploy."); + } + private void ResetForNext() { Timers.Cancel(DeadlineTimerKey); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTimeoutTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTimeoutTests.cs index 0490eecee..5639996f4 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTimeoutTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTimeoutTests.cs @@ -111,6 +111,74 @@ public sealed class ConfigPublishCoordinatorTimeoutTests : ControlPlaneActorTest }, duration: TimeSpan.FromSeconds(3)); } + /// + /// lmxopcua#784 — a deployment that timed out with SOME acks landed must persist WHY: the status API + /// returned TimedOut with a null reason, and only the central log named the silent node. The reason + /// names the node(s) that never acked and not the one that did. + /// + [Fact] + public void Timeout_with_some_acks_landed_persists_a_reason_naming_the_silent_nodes() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks); + SeedNodeRows(dbFactory, deploymentId, applied: new[] { "site-a-1:4053" }, applying: new[] { "site-b-1:4053" }); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5))); + actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("No ack from: site-b-1:4053."); + row.FailureReason.ShouldContain("(1/2 landed)"); + row.FailureReason.ShouldNotContain("site-a-1"); + }, duration: TimeSpan.FromSeconds(3)); + } + + /// lmxopcua#784 — the timeout reason stays bounded on a large fleet: it names 20 nodes and counts the rest. + [Fact] + public void Timeout_reason_is_bounded_when_many_nodes_are_silent() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks); + var silent = Enumerable.Range(0, 25).Select(i => $"node-{i:D2}-with-a-fairly-long-host-name:4053").ToArray(); + SeedNodeRows(dbFactory, deploymentId, applied: new[] { "acked-node:4053" }, applying: silent); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5))); + actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var reason = db.Deployments.Single().FailureReason; + reason.ShouldNotBeNull(); + reason.ShouldContain("node-19-with-a-fairly-long-host-name:4053"); + reason.ShouldNotContain("node-20-"); + reason.ShouldContain("(and 5 more)"); + reason.Length.ShouldBeLessThanOrEqualTo(2048); + }, duration: TimeSpan.FromSeconds(3)); + } + + private static void SeedNodeRows( + IDbContextFactory dbFactory, DeploymentId deploymentId, string[] applied, string[] applying) + { + using var db = dbFactory.CreateDbContext(); + foreach (var n in applied) + db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState + { + NodeId = n, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, + }); + foreach (var n in applying) + db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState + { + NodeId = n, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying, + }); + db.SaveChanges(); + } + private static DeploymentId SeedDispatchingDeployment( IDbContextFactory dbFactory, DeploymentStatus status = DeploymentStatus.Dispatching)