fix(controlplane): #868 review — complete fully-acked recoveries, replay queued acks before a deadline, retry failed ack persists
Review findings on02503a4a(ConfigPublishCoordinator): 1 (HIGH) A recovered deployment whose every expected ack was already in NodeDeploymentState (a seal whose SaveChanges failed, a fused node's own row) was never completed: inside the deadline it waited for acks that had landed, past it HandleDeadline's "race" guard returned without sealing — stuck in AwaitingApplyAcks, refusing every later deploy. ApplyRecovery now completes it (Seal / PartiallyFailed), and HandleDeadline completes a full ack set instead of returning. 2 (MED-HIGH) The past-deadline recovery of a populated expected set ran HandleDeadline inline before the stash replay, discarding on-time acks queued during the outage. It is now self-sent, behind the replay. The zero-rows past-deadline branch stays inline on purpose: no ack can count there, and a self-sent deadline would time out a queued dispatch of that same row right after publishing it. 3 (MED) A failed PersistNodeAck threw, restarted the actor and dropped the ack (driver-only nodes never write their own row and do not resend). Acks are now persisted before being counted, and a failed persist is re-queued on a timer. The class remarks document that the Loading stash is node-local (lost on a hand-over; queued dispatches are recovered from their rows). Minor: MarkReady/Become(Ready) only after the recovery applied; a throw while applying is a failed load (backoff) rather than a tight restart loop; a queued dispatch matching the recovered Dispatching row logs Info, not the "lost dispatch" Warning. Tests (ConfigPublishCoordinatorDbOutageTests): fully-acked recovery seals / partially fails inside and past the deadline (4 cases); queued ack applied before a past-deadline recovery; failed ack persist retried. All 6 fail on02503a4a. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
+93
-18
@@ -64,6 +64,18 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
/// during the outage is queued, never dropped. <see cref="ConfigPublishCoordinatorReadiness"/> reports the
|
||||
/// loading state (and the last error) to the node's health endpoint as Degraded.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The stash is node-local.</b> A singleton hand-over while the coordinator is still Loading discards what it
|
||||
/// queued (the old incarnation stops with its mailbox). That loses nothing durable: a queued dispatch's row is
|
||||
/// <c>Dispatching</c> with no expected-ack rows, which the new incarnation's recovery (or AdminOps' lost-dispatch
|
||||
/// closure) re-drives or closes; a queued ack is the one exception — its node does not resend, so that node is
|
||||
/// named in the deadline's timeout reason.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>An ack is held in memory only once it is persisted.</b> <c>NodeDeploymentState</c> is the system of record
|
||||
/// (driver-only nodes never write their own row), so a failed <c>PersistNodeAck</c> re-queues the ack on a timer
|
||||
/// instead of throwing — a throw would restart the actor and drop it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithUnboundedStash
|
||||
{
|
||||
@@ -86,6 +98,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
private readonly ConfigPublishCoordinatorReadiness? _readiness;
|
||||
private int _failedLoads;
|
||||
|
||||
/// <summary>Deployments whose dispatch was queued while Loading (only to word the recovery log correctly).</summary>
|
||||
private readonly HashSet<DeploymentId> _queuedDispatches = new();
|
||||
|
||||
/// <summary>
|
||||
/// Where central→node commands are handed off (per-cluster mesh Phase 2), or
|
||||
/// <see langword="null"/> to publish on the mediator directly.
|
||||
@@ -204,6 +219,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
"DispatchDeployment {Id} queued: the coordinator has not yet loaded its recovery state from the " +
|
||||
"ConfigDb ({Failed} failed attempt(s) so far); it is dispatched as soon as the load succeeds",
|
||||
msg.DeploymentId, _failedLoads);
|
||||
_queuedDispatches.Add(msg.DeploymentId);
|
||||
Stash.Stash();
|
||||
});
|
||||
ReceiveAny(_ => Stash.Stash());
|
||||
@@ -214,6 +230,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
{
|
||||
Receive<DispatchDeployment>(HandleDispatch);
|
||||
Receive<ApplyAck>(HandleAck);
|
||||
Receive<RetryAckPersist>(m => HandleAck(m.Ack));
|
||||
Receive<DeadlineElapsed>(HandleDeadline);
|
||||
Receive<SubscribeAck>(_ => { /* DPS subscribe confirmation */ });
|
||||
// A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed.
|
||||
@@ -296,26 +313,38 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies the recovered state, switches to <see cref="Ready"/> and replays every stashed message. The
|
||||
/// recovery DECISIONS (re-dispatch, time out) run inline, before the replay, so a queued message for the
|
||||
/// same deployment is judged against the recovered state rather than racing a self-sent one.
|
||||
/// Applies the recovered state, then switches to <see cref="Ready"/> and replays every stashed message. A
|
||||
/// decision that must see the queued messages first (the deadline of a recovered deployment that is already
|
||||
/// past it) is SELF-SENT, so it lands behind the replay; one that must precede them (re-driving a lost
|
||||
/// dispatch, so a queued duplicate of it is recognised) runs inline. A throw while applying is treated as a
|
||||
/// failed load — the actor stays Loading and retries on the backoff, never a tight restart loop.
|
||||
/// </summary>
|
||||
private void HandleRecoveryLoaded(RecoveryLoaded msg)
|
||||
{
|
||||
Timers.Cancel(LoadRetryTimerKey);
|
||||
if (_failedLoads > 0)
|
||||
_log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads);
|
||||
_failedLoads = 0;
|
||||
Become(Ready);
|
||||
_readiness?.MarkReady();
|
||||
try
|
||||
{
|
||||
ApplyRecovery(msg.Inflight);
|
||||
}
|
||||
finally
|
||||
catch (Exception ex)
|
||||
{
|
||||
Stash.UnstashAll();
|
||||
_log.Error(ex, "ConfigPublishCoordinator failed to apply its recovered state; retrying the load");
|
||||
Timers.Cancel(DeadlineTimerKey);
|
||||
_current = null;
|
||||
_expectedAcks.Clear();
|
||||
_acks.Clear();
|
||||
_degraded.Clear();
|
||||
HandleRecoveryLoadFailed(new RecoveryLoadFailed(ex));
|
||||
return;
|
||||
}
|
||||
|
||||
if (_failedLoads > 0)
|
||||
_log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads);
|
||||
_failedLoads = 0;
|
||||
_queuedDispatches.Clear();
|
||||
Become(Ready);
|
||||
_readiness?.MarkReady();
|
||||
Stash.UnstashAll();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -350,16 +379,25 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
// closed as TimedOut with a reason that names the condition.
|
||||
if (remaining > TimeSpan.Zero)
|
||||
{
|
||||
_log.Warning(
|
||||
"Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
|
||||
"before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
|
||||
"re-dispatching with {Remaining} of the apply deadline left",
|
||||
_current, remaining);
|
||||
if (_queuedDispatches.Contains(_current.Value))
|
||||
_log.Info(
|
||||
"Coordinator dispatching deployment {Id}, queued while its recovery state was loading",
|
||||
_current);
|
||||
else
|
||||
_log.Warning(
|
||||
"Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
|
||||
"before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
|
||||
"re-dispatching with {Remaining} of the apply deadline left",
|
||||
_current, remaining);
|
||||
HandleDispatch(new DispatchDeployment(
|
||||
_current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Deliberately INLINE, unlike the populated-set deadline below: with no expected set no queued ack
|
||||
// can count, so there is nothing to let the replay apply first — and a queued dispatch of this very
|
||||
// row must meet the TimedOut row (terminal-state guard ignores it), not publish to every node and
|
||||
// be timed out the moment a self-sent deadline caught up with it.
|
||||
HandleDeadline(new DeadlineElapsed(_current.Value));
|
||||
}
|
||||
return;
|
||||
@@ -374,9 +412,20 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
|
||||
}
|
||||
|
||||
// Every expected ack is already in the DB (a seal whose SaveChanges failed, or a fused node's own
|
||||
// DriverHostActor row): finish it now. Waiting for the deadline would leave it in flight — the deadline
|
||||
// can only time out an INCOMPLETE set.
|
||||
if (_acks.Count == _expectedAcks.Count)
|
||||
{
|
||||
_log.Info("Coordinator recovered deployment {Id} with every ack already landed; completing it", _current);
|
||||
CompleteFromAcks();
|
||||
return;
|
||||
}
|
||||
|
||||
if (remaining <= TimeSpan.Zero)
|
||||
{
|
||||
HandleDeadline(new DeadlineElapsed(_current.Value));
|
||||
// Self-sent, so acks queued during the load (which arrived in time) are applied first.
|
||||
Self.Tell(new DeadlineElapsed(_current.Value));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -516,20 +565,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
return;
|
||||
}
|
||||
|
||||
// Persist FIRST: the row is the system of record (driver-only nodes never write their own, and nodes do not
|
||||
// resend). A throw here used to restart the actor and drop the ack; it is re-queued instead.
|
||||
try
|
||||
{
|
||||
PersistNodeAck(msg);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex,
|
||||
"Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}",
|
||||
msg.NodeId, msg.DeploymentId, _loadRetryMin);
|
||||
Timers.StartSingleTimer(AckRetryTimerKey(msg), new RetryAckPersist(msg), _loadRetryMin);
|
||||
return;
|
||||
}
|
||||
|
||||
_acks[msg.NodeId] = msg.Outcome;
|
||||
// An old node never sets Degraded, so its ack takes the clean branch exactly as before.
|
||||
if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId);
|
||||
else _degraded.Remove(msg.NodeId);
|
||||
PersistNodeAck(msg);
|
||||
|
||||
if (_acks.Count < _expectedAcks.Count) return;
|
||||
CompleteFromAcks();
|
||||
}
|
||||
|
||||
/// <summary>Seals (all Applied) or marks partially failed — called once every expected ack is in.</summary>
|
||||
private void CompleteFromAcks()
|
||||
{
|
||||
if (_acks.Values.All(o => o == ApplyAckOutcome.Applied))
|
||||
SealDeployment();
|
||||
else
|
||||
MarkPartiallyFailed();
|
||||
}
|
||||
|
||||
private static string AckRetryTimerKey(ApplyAck msg) => $"ack-retry:{msg.DeploymentId.Value:N}:{msg.NodeId.Value}";
|
||||
|
||||
private void PersistNodeAck(ApplyAck msg)
|
||||
{
|
||||
using var db = _dbFactory.CreateDbContext();
|
||||
@@ -612,7 +682,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
// everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped.
|
||||
if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count)
|
||||
{
|
||||
// Race: every node acked just as the deadline fired. Already sealed/failed elsewhere.
|
||||
// Every ack is in but nothing completed it (a completion normally resets _current, so the stale-id
|
||||
// check above catches the ack-vs-deadline race). Complete it rather than leave it in flight forever.
|
||||
CompleteFromAcks();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -682,6 +754,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
|
||||
private const string DeadlineTimerKey = "apply-deadline";
|
||||
private const string LoadRetryTimerKey = "recovery-load-retry";
|
||||
|
||||
/// <summary>An ack whose persist failed, re-delivered by its retry timer.</summary>
|
||||
private sealed record RetryAckPersist(ApplyAck Ack);
|
||||
|
||||
public sealed record DeadlineElapsed(DeploymentId DeploymentId);
|
||||
|
||||
/// <summary>The recovery read's result: the newest in-flight deployment, or null when none.</summary>
|
||||
|
||||
Reference in New Issue
Block a user