dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(controlplane): recover a lost dispatch instead of wedging the deploy pipeline

Browse Source 
A Deployment row left in Dispatching with no NodeDeploymentState rows (HandleDispatch threw
before its SaveChanges, or the process died between the AdminOps row write and the
coordinator's Tell) was recovered with an empty expected set; the deadline then hit the
0 == 0 "race" branch and returned without touching the row, so every later StartDeployment was
refused AnotherDeploymentInFlight until someone hand-edited SQL.

- PreStart: a recovered row with no expected-ack rows is a lost dispatch. Inside the apply
  deadline it is re-driven through HandleDispatch (bounded: CreatedAtUtc never moves); past it
  the row is closed TimedOut with a reason naming the condition.
- HandleDispatch is idempotent on a re-dispatch (rows already seeded ARE the expected set).
- HandleDeadline takes the race branch only when _expectedAcks.Count > 0.
- The PreStart recovery read projects ArtifactBlob away (R03:3.5).

Refs #733

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:22:20 -04:00
parent ad2b5c4044
commit b437c34f92
2 changed files with 316 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs
+82 -17
View File
@@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self));
using var db = _dbFactory.CreateDbContext();
// Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never
// reads — and this query runs on every singleton start and hand-over.
var inflight = db.Deployments
.Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
.OrderByDescending(d => d.CreatedAtUtc)
.Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc })
.FirstOrDefault();
if (inflight is null) return;
@@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
.AsNoTracking()
.ToList();
// Resume the deadline timer using the remaining time. The deadline runs from when the
// deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
// proxy — we don't track a separate "dispatched at" column).
var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
var remaining = _applyDeadline - elapsed;
if (nodeStates.Count == 0)
{
// The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by
// HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed:
// HandleDispatch threw before its SaveChanges (this actor restarted and the message was
// dropped), the process died between AdminOps' row write and the coordinator's Tell, or the
// Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to
// let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight
// forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply
// re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that
// keeps throwing ends up on the other branch once the deadline passes); past it the row is
// closed as TimedOut with a reason that names the condition.
if (remaining > TimeSpan.Zero)
{
_log.Warning(
"Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
"before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
"re-dispatching with {Remaining} of the apply deadline left",
_current, remaining);
Self.Tell(new DispatchDeployment(
_current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
}
else
{
Self.Tell(new DeadlineElapsed(_current.Value));
}
return;
}
_expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer);
foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying))
{
@@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
}
// Resume the deadline timer using the remaining time. The deadline runs from when the
// deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
// proxy — we don't track a separate "dispatched at" column).
var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
var remaining = _applyDeadline - elapsed;
if (remaining <= TimeSpan.Zero)
{
Self.Tell(new DeadlineElapsed(_current.Value));
@@ -208,19 +241,34 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
// Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack.
using (var db = _dbFactory.CreateDbContext())
{
// Discovery reuses this context rather than opening a second one — it reads the same
// ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
// context keeps the expected set and the seeded rows consistent by construction.
_expectedAcks = DiscoverDriverNodes(db);
foreach (var node in _expectedAcks)
// Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows
// already seeded for this deployment ARE the expected set — re-discovering would both duplicate
// the primary key and let a node toggled since the first attempt change who is expected.
var seeded = db.NodeDeploymentStates
.AsNoTracking()
.Where(x => x.DeploymentId == msg.DeploymentId.Value)
.Select(x => x.NodeId)
.ToList();
if (seeded.Count > 0)
{
db.NodeDeploymentStates.Add(new NodeDeploymentState
_expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer);
}
else
{
// Discovery reuses this context rather than opening a second one — it reads the same
// ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
// context keeps the expected set and the seeded rows consistent by construction.
_expectedAcks = DiscoverDriverNodes(db);
foreach (var node in _expectedAcks)
{
NodeId = node.Value,
DeploymentId = msg.DeploymentId.Value,
Status = NodeDeploymentStatus.Applying,
});
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = node.Value,
DeploymentId = msg.DeploymentId.Value,
Status = NodeDeploymentStatus.Applying,
});
}
}
UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks);
db.SaveChanges();
@@ -360,7 +408,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
msg.DeploymentId, _current);
return;
}
if (_acks.Count == _expectedAcks.Count)
// Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the
// everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped.
if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count)
{
// Race: every node acked just as the deadline fired. Already sealed/failed elsewhere.
return;
@@ -368,6 +418,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
using var db = _dbFactory.CreateDbContext();
UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut);
if (_expectedAcks.Count == 0)
{
var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value);
if (d is not null)
d.FailureReason = Truncate(
"Dispatch was lost before any node was expected to ack (coordinator restart or singleton " +
"hand-over mid-dispatch) and the apply deadline has passed. Redeploy.");
db.SaveChanges();
_log.Warning(
"Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " +
"expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy",
_current.Value, _applyDeadline);
ResetForNext();
return;
}
db.SaveChanges();
// Name the silent nodes, not just the counts. Since the expected set became DB-derived this
// is the failure an operator meets after deploying while a configured node is switched off,