fix(controlplane,runtime): #868 third review — bounded failure reasons, poison ack writes, cache-booted revisions unconfirmed

1 PersistNodeAck wrote a Failed ack's FailureReason unbounded; the column is
  nvarchar(2048), so on SQL Server the write failed forever, the ack retried
  forever, the deadline deferred forever and every later deploy was refused.
  - One shared bound, Configuration.FailureReasons.Truncate (MaxLength 2048),
    used by the coordinator and DriverHostActor.UpsertNodeDeploymentState
    (exception messages and degraded reasons).
  - A non-transient DbUpdateException (not concurrency, no transient
    DbException / timeout / socket / IO underneath) is POISON: the ack is
    counted from memory with an Error, never retried.
  - Deadline deferral on unpersisted acks is capped at 10 x the retry ceiling;
    past it the outcome is decided from the acks held in memory.
2 A node restarting in the outcome window boots the cached revision (cache is
  written before the outcome) and never acks. Cache-booted revisions
  (BootstrapFromCache / TryBootFromCache) are now marked unconfirmed, so a
  re-dispatch re-applies and acks the real outcome instead of a clean
  short-circuit. docs/Redundancy.md corrected (replay only on the ConfigDb boot
  path).
3 A lost dispatch re-driven near its deadline now gets a FULL deadline (a node
  needs its whole outcome timeout + margin = ApplyDeadline, so any shorter
  remainder timed out nodes that applied). No dispatched-at column: a second
  restart recomputes from CreatedAtUtc and closes it instead (documented).
4 docs/Redundancy.md upgrade note: do not deploy while the central pair is mid
  rolling-upgrade across this change.

Tests (all fail on the previous code): oversized Failed reason truncated (on a
SaveChanges interceptor that enforces the column like SQL Server); rejected ack
write counted from memory; bounded deadline deferral; full deadline on a
near-deadline re-drive; cache-booted revision re-applies on re-dispatch;
oversized degraded reason truncated on the node's own row.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:13:36 -05:00
parent 74496b4fec
commit 780c87eba3
8 changed files with 378 additions and 24 deletions
@@ -109,6 +109,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
private readonly Dictionary<NodeId, PendingPersist> _unpersisted = new(NodeIdComparer);
private string? _lastPersistError;
/// <summary>When the current deadline was first deferred on an unpersisted ack (bounds the deferral).</summary>
private DateTime? _deadlineDeferredSinceUtc;
/// <summary>How long a deadline may wait on unpersisted acks before deciding from memory (10 × the retry ceiling).</summary>
private TimeSpan MaxDeadlineDeferral => TimeSpan.FromTicks(_loadRetryMax.Ticks * 10);
/// <summary>
/// Where central→node commands are handed off (per-cluster mesh Phase 2), or
/// <see langword="null"/> to publish on the mediator directly.
@@ -419,11 +425,15 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
_log.Warning(
"Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
"before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
"re-dispatching with {Remaining} of the apply deadline left",
"re-dispatching with a fresh apply deadline ({Remaining} of the original was left)",
_current, remaining);
// The REMAINING time (review L4): CreatedAtUtc never moves, so a re-drive must not restart the clock.
// A FULL deadline (third review #3, superseding the second review's "remaining time"): a node needs its
// whole outcome timeout (DeployTimings.ApplyDeadline - ApplyOutcomeSafetyMargin) plus that margin to
// report — i.e. the full ApplyDeadline — so ANY remaining time shorter than that let the nodes apply
// and the deployment time out moments later. Bounded anyway: there is no dispatched-at column, so a
// SECOND restart recomputes from CreatedAtUtc and, past the deadline by then, closes the row instead.
HandleDispatch(new DispatchDeployment(
_current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()), remaining);
_current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
}
else
{
@@ -468,11 +478,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
_current, _acks.Count, _expectedAcks.Count);
}
private void HandleDispatch(DispatchDeployment msg) => HandleDispatch(msg, deadline: null);
/// <param name="msg">The dispatch.</param>
/// <param name="deadline">The apply deadline to arm; null = the full <see cref="_applyDeadline"/>.</param>
private void HandleDispatch(DispatchDeployment msg, TimeSpan? deadline)
private void HandleDispatch(DispatchDeployment msg)
{
// A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the
// recovery load ran, for the very row that load re-drove) must not reset the acks already gathered.
@@ -567,7 +573,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
// directly, so this actor no longer knows which transport is in force.
if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg));
else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg));
Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), deadline ?? _applyDeadline);
Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline);
if (_expectedAcks.Count == 0)
{
@@ -621,6 +627,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
{
PersistNodeAck(msg);
}
catch (Exception ex) when (IsPoison(ex))
{
// Third review #1: a write the DB REJECTS (constraint, truncation, …) fails identically on every retry;
// retrying it forever wedged the deployment (the deadline deferred on it) and every later deploy. The
// node DID ack, so count it from memory and say loudly that its row is not recorded.
_log.Error(ex,
"The ApplyAck from {Node} for {Id} ({Outcome}) was REJECTED by the ConfigDb and will not be retried; " +
"counting it from memory. Its NodeDeploymentState row stays unrecorded",
msg.NodeId, msg.DeploymentId, msg.Outcome);
DropUnpersisted(msg.NodeId);
CountAck(msg);
return true;
}
catch (Exception ex)
{
var attempts = _unpersisted.TryGetValue(msg.NodeId, out var prior) ? prior.Attempts + 1 : 1;
@@ -639,17 +658,42 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
return false;
}
if (_unpersisted.Remove(msg.NodeId))
{
Timers.Cancel(AckRetryTimerKey(msg.NodeId));
if (_unpersisted.Count == 0) _lastPersistError = null;
ReportPersistBacklog();
}
DropUnpersisted(msg.NodeId);
CountAck(msg);
return true;
}
private void CountAck(ApplyAck msg)
{
_acks[msg.NodeId] = msg.Outcome;
// An old node never sets Degraded, so its ack takes the clean branch exactly as before.
if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId);
else _degraded.Remove(msg.NodeId);
}
private void DropUnpersisted(NodeId node)
{
if (!_unpersisted.Remove(node)) return;
Timers.Cancel(AckRetryTimerKey(node));
if (_unpersisted.Count == 0) _lastPersistError = null;
ReportPersistBacklog();
}
/// <summary>
/// A non-transient rejection of the write itself: an EF <see cref="DbUpdateException"/> (not a concurrency
/// conflict) with no transient database error, timeout or socket failure underneath. Connectivity failures
/// surface as other exception types (or as transient <see cref="System.Data.Common.DbException"/>s) and stay
/// retryable.
/// </summary>
internal static bool IsPoison(Exception ex)
{
if (ex is not DbUpdateException || ex is DbUpdateConcurrencyException) return false;
for (var e = ex.InnerException; e is not null; e = e.InnerException)
{
if (e is System.Data.Common.DbException { IsTransient: true }
or TimeoutException or System.Net.Sockets.SocketException or System.IO.IOException)
return false;
}
return true;
}
@@ -697,18 +741,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
? Truncate(DegradedReasonPrefix + (string.IsNullOrWhiteSpace(msg.DegradedReason)
? "address-space apply degraded (no reason given)"
: msg.DegradedReason))
: msg.FailureReason;
// Third review #1: a Failed ack's reason is the node's free text (often an exception message) and was
// written UNBOUNDED — on SQL Server an overflow refuses the SaveChanges forever.
: FailureReasons.Truncate(msg.FailureReason);
db.SaveChanges();
}
private static bool IsDegradedAck(ApplyAck msg) => msg.Outcome == ApplyAckOutcome.Applied && msg.Degraded;
// FailureReason is nvarchar(2048) on both tables (OtOpcUaConfigDbContext); a reason built from a node's
// free text must never fail the SaveChanges that records the ack.
private const int FailureReasonMaxLength = 2048;
private static string Truncate(string s) =>
s.Length <= FailureReasonMaxLength ? s : s[..(FailureReasonMaxLength - 1)] + "…";
// free text must never fail the SaveChanges that records the ack. One shared bound (FailureReasons).
private static string Truncate(string s) => FailureReasons.Truncate(s)!;
private void SealDeployment()
{
@@ -763,6 +806,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
if (_unpersisted.Count > 0)
{
foreach (var pending in _unpersisted.Values.ToList()) TryPersistAck(pending.Ack);
_deadlineDeferredSinceUtc ??= DateTime.UtcNow;
if (_unpersisted.Count > 0 && DateTime.UtcNow - _deadlineDeferredSinceUtc.Value >= MaxDeadlineDeferral)
{
// Third review #1: bounded. Past the cap the outcome is decided from what was RECEIVED — the nodes did
// ack — rather than waiting on rows that may never be writable.
_log.Error(
"Deployment {Id}: {Count} received ack(s) still could not be persisted after deferring the deadline " +
"{Cap}; deciding the outcome from the acks held in memory (last error: {Error})",
_current.Value, _unpersisted.Count, MaxDeadlineDeferral, _lastPersistError);
foreach (var pending in _unpersisted.Values.ToList())
{
DropUnpersisted(pending.Ack.NodeId);
CountAck(pending.Ack);
}
}
if (_unpersisted.Count > 0)
{
var delay = Backoff(_unpersisted.Values.Min(p => p.Attempts));
@@ -841,6 +899,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
private void ResetForNext()
{
Timers.Cancel(DeadlineTimerKey);
_deadlineDeferredSinceUtc = null;
foreach (var node in _unpersisted.Keys) Timers.Cancel(AckRetryTimerKey(node));
if (_unpersisted.Count > 0)
{
@@ -295,7 +295,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied
/// for an address space known to be partial, and the re-apply is what repairs it (a degraded
/// non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing
/// nodes). Reset by every clean outcome and by every boot path that restores a revision.
/// nodes). Reset by every clean outcome and by the ConfigDb boot path (an Applied row is a final outcome);
/// SET by the cache boot paths, whose cached revision may never have reported an outcome (lmxopcua#868).
/// </summary>
private bool _currentRevisionDegraded;
@@ -1434,6 +1435,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
}
_currentRevision = revision;
// Unconfirmed (lmxopcua#868 third review #2): the cache is written BEFORE the address-space outcome, so a
// restart in that window boots a revision central never got an ack for. Marking it unconfirmed makes a
// re-dispatch of it RE-APPLY (and ack the real outcome) instead of short-circuiting a clean Applied.
_currentRevisionDegraded = true;
BecomeSteady();
_log.Info(
@@ -1514,6 +1519,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// Stale would have started does not run here, so recovery rides on the next dispatch —
// matching how a normally-booted node behaves.
_currentRevision = revision;
// Unconfirmed, as in BootstrapFromCache (third review #2): a re-dispatch of it re-applies.
_currentRevisionDegraded = true;
BecomeSteady();
_isRunningFromCache = true;
@@ -4336,6 +4343,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason)
{
// nvarchar(2048): an exception message or a degraded reason can exceed it, and SQL Server then refuses the
// write — the row would stay Applying forever (lmxopcua#868 third review #1).
failureReason = FailureReasons.Truncate(failureReason);
// Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of
// record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every
// ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is