fix(controlplane,runtime): #868 second review — final-outcome node rows, unpersisted acks survive deadline and restart

M1 DriverHostActor wrote its own NodeDeploymentState row Applied/null as soon as the revision advanced, up to ~105 s before the address-space outcome. The coordinator's recovery counts any non-Applying row as an ack, so a coordinator restart in that window sealed CLEAN while the node was about to report degraded (#601). The row now stays Applying until ReleasePendingAck (or the no-publish-actor path) writes the final Applied, clean or with the degraded reason. A node restart in the window replays the orphan Applying row; /fleet shows Applying until the outcome (docs/Redundancy.md). M2 Acks whose persist failed are tracked in _unpersisted (newest per node). HandleDeadline persists that backlog first and defers its decision while any remain, so a node that acked is never timed out as silent; PreRestart re-sends them to Self so a restart (timers lost) does not drop them. L3 A successful persist cancels the node's retry timer and drops its older pending ack; a retry only fires for the node's current pending ack. L4 A dispatch queued during Loading is sent with a fresh deadline however long the outage lasted (never closed "Dispatch was lost"); the inline re-drive of a genuinely lost dispatch arms the REMAINING deadline. L5 A recovered row with a blank RevisionHash (re-dispatch) or blank NodeId is closed PartiallyFailed with a reason instead of failing every load. L6 Readiness carries PendingAckPersists + LastPersistError; the config-publish health check is Degraded while Ready with a backlog. L7 Ack-persist retries back off (shared Backoff); stack trace on the first failure only. L8 Terminal writes (Sealed / PartiallyFailed / TimedOut / failure reason) only apply to a row still Dispatching/AwaitingApplyAcks. Tests (all fail on the previous code): ControlPlane DbOutage — deadline does not time out an unpersisted ack; unpersisted ack survives a restart; stale retry does not overwrite a newer ack; queued dispatch through an outage longer than the deadline is sent; malformed row closed and coordinator carries on; readiness reports the persist backlog. Runtime DegradedAck — own row stays Applying until the outcome. Host — Ready + backlog is Degraded. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 12:54:30 -05:00
parent ff85b8511a
commit 494dcb5b87
8 changed files with 475 additions and 53 deletions
@@ -98,9 +98,17 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    private readonly ConfigPublishCoordinatorReadiness? _readiness;
    private int _failedLoads;

-    /// <summary>Deployments whose dispatch was queued while Loading (only to word the recovery log correctly).</summary>
+    /// <summary>Deployments whose dispatch was queued while Loading (never sent: recovery re-dispatches them).</summary>
    private readonly HashSet<DeploymentId> _queuedDispatches = new();

+    /// <summary>
+    ///     Acks of the current deployment that arrived but could not be persisted yet (review M2), one per node — the
+    ///     newest wins. They are retried on a backoff, block the deadline from deciding without them, and are
+    ///     re-sent to Self across a restart (<see cref="PreRestart"/>).
+    /// </summary>
+    private readonly Dictionary<NodeId, PendingPersist> _unpersisted = new(NodeIdComparer);
+    private string? _lastPersistError;
+
    /// <summary>
    ///     Where central→node commands are handed off (per-cluster mesh Phase 2), or
    ///     <see langword="null"/> to publish on the mediator directly.
@@ -230,7 +238,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    {
        Receive<DispatchDeployment>(HandleDispatch);
        Receive<ApplyAck>(HandleAck);
-        Receive<RetryAckPersist>(m => HandleAck(m.Ack));
+        Receive<RetryAckPersist>(HandleRetryAckPersist);
        Receive<DeadlineElapsed>(HandleDeadline);
        Receive<SubscribeAck>(_ => { /* DPS subscribe confirmation */ });
        // A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed.
@@ -252,6 +260,15 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        StartRecoveryLoad();
    }

+    /// <inheritdoc />
+    protected override void PreRestart(Exception reason, object message)
+    {
+        // Timers die with this incarnation; the mailbox survives. Re-send every ack that is still unpersisted so the
+        // next incarnation (which stashes while Loading) persists and counts it — its node will not resend (M2).
+        foreach (var pending in _unpersisted.Values) Self.Tell(pending.Ack);
+        base.PreRestart(reason, message);
+    }
+
    /// <inheritdoc />
    protected override void PostStop()
    {
@@ -302,8 +319,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    {
        _failedLoads++;
        // Exponential backoff, capped: min * 2^(n-1), with the shift bounded so it never overflows.
-        var factor = 1L << Math.Min(_failedLoads - 1, 20);
-        var delay = TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor));
+        var delay = Backoff(_failedLoads);
        _readiness?.MarkLoading(_failedLoads, msg.Error.Message);
        _log.Warning(msg.Error,
            "ConfigPublishCoordinator could not load its recovery state from the ConfigDb (attempt {Attempt}); " +
@@ -329,11 +345,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        catch (Exception ex)
        {
            _log.Error(ex, "ConfigPublishCoordinator failed to apply its recovered state; retrying the load");
-            Timers.Cancel(DeadlineTimerKey);
-            _current = null;
-            _expectedAcks.Clear();
-            _acks.Clear();
-            _degraded.Clear();
+            ResetForNext();
            HandleRecoveryLoadFailed(new RecoveryLoadFailed(ex));
            return;
        }
@@ -359,6 +371,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        _current = new DeploymentId(inflight.DeploymentId);
        var nodeStates = inflight.NodeStates;

+        // Review L5: a row recovery cannot interpret (blank revision on a row it must re-dispatch, blank node id) would
+        // throw on every load and keep the coordinator Loading forever. Close it with a reason and carry on.
+        var malformed = DescribeMalformed(inflight);
+        if (malformed is not null)
+        {
+            _log.Error("Coordinator cannot recover deployment {Id}: {Problem}. Closing it as PartiallyFailed", _current, malformed);
+            using var db = _dbFactory.CreateDbContext();
+            UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed, onlyIfInFlight: true);
+            SetFailureReason(db, Truncate(
+                $"The deploy coordinator could not recover this deployment: {malformed}. Redeploy."));
+            db.SaveChanges();
+            ResetForNext();
+            return;
+        }
+
        // Resume the deadline timer using the remaining time. The deadline runs from when the
        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
        // proxy — we don't track a separate "dispatched at" column).
@@ -377,21 +404,27 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
            // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that
            // keeps throwing ends up on the other branch once the deadline passes); past it the row is
            // closed as TimedOut with a reason that names the condition.
-            if (remaining > TimeSpan.Zero)
+            if (_queuedDispatches.Contains(_current.Value))
            {
-                if (_queuedDispatches.Contains(_current.Value))
-                    _log.Info(
-                        "Coordinator dispatching deployment {Id}, queued while its recovery state was loading",
-                        _current);
-                else
-                    _log.Warning(
-                        "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
-                        "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
-                        "re-dispatching with {Remaining} of the apply deadline left",
-                        _current, remaining);
+                // Review L4: its dispatch was QUEUED here while the load was failing — it was never sent, so however
+                // long the outage lasted it is not a lost dispatch. Send it now with a full, fresh deadline.
+                _log.Info(
+                    "Coordinator dispatching deployment {Id}, queued while its recovery state was loading",
+                    _current);
                HandleDispatch(new DispatchDeployment(
                    _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
            }
+            else if (remaining > TimeSpan.Zero)
+            {
+                _log.Warning(
+                    "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
+                    "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
+                    "re-dispatching with {Remaining} of the apply deadline left",
+                    _current, remaining);
+                // The REMAINING time (review L4): CreatedAtUtc never moves, so a re-drive must not restart the clock.
+                HandleDispatch(new DispatchDeployment(
+                    _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()), remaining);
+            }
            else
            {
                // Deliberately INLINE, unlike the populated-set deadline below: with no expected set no queued ack
@@ -413,7 +446,8 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        }

        // Every expected ack is already in the DB (a seal whose SaveChanges failed, or a fused node's own
-        // DriverHostActor row): finish it now. Waiting for the deadline would leave it in flight — the deadline
+        // DriverHostActor row — which that actor writes Applied only once the address-space outcome is known, so a
+        // non-Applying row is always a FINAL outcome; review M1): finish it now. Waiting for the deadline would leave it in flight — the deadline
        // can only time out an INCOMPLETE set.
        if (_acks.Count == _expectedAcks.Count)
        {
@@ -434,7 +468,11 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
            _current, _acks.Count, _expectedAcks.Count);
    }

-    private void HandleDispatch(DispatchDeployment msg)
+    private void HandleDispatch(DispatchDeployment msg) => HandleDispatch(msg, deadline: null);
+
+    /// <param name="msg">The dispatch.</param>
+    /// <param name="deadline">The apply deadline to arm; null = the full <see cref="_applyDeadline"/>.</param>
+    private void HandleDispatch(DispatchDeployment msg, TimeSpan? deadline)
    {
        // A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the
        // recovery load ran, for the very row that load re-drove) must not reset the acks already gathered.
@@ -529,7 +567,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        // directly, so this actor no longer knows which transport is in force.
        if (_meshRouter is not null) _meshRouter.Tell(new MeshCommand(DeploymentsTopic, msg));
        else DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentsTopic, msg));
-        Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), _applyDeadline);
+        Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(msg.DeploymentId), deadline ?? _applyDeadline);

        if (_expectedAcks.Count == 0)
        {
@@ -567,26 +605,68 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU

        // Persist FIRST: the row is the system of record (driver-only nodes never write their own, and nodes do not
        // resend). A throw here used to restart the actor and drop the ack; it is re-queued instead.
+        if (!TryPersistAck(msg)) return;
+        if (_acks.Count < _expectedAcks.Count) return;
+        CompleteFromAcks();
+    }
+
+    /// <summary>
+    ///     Persists an ack and, only on success, counts it (and drops any older unpersisted ack of that node, with its
+    ///     retry timer — review L3). On failure the ack is held in <see cref="_unpersisted"/> and retried on a backoff.
+    /// </summary>
+    /// <returns><see langword="true"/> when the ack is persisted and counted.</returns>
+    private bool TryPersistAck(ApplyAck msg)
+    {
        try
        {
            PersistNodeAck(msg);
        }
        catch (Exception ex)
        {
-            _log.Warning(ex,
-                "Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}",
-                msg.NodeId, msg.DeploymentId, _loadRetryMin);
-            Timers.StartSingleTimer(AckRetryTimerKey(msg), new RetryAckPersist(msg), _loadRetryMin);
-            return;
+            var attempts = _unpersisted.TryGetValue(msg.NodeId, out var prior) ? prior.Attempts + 1 : 1;
+            _unpersisted[msg.NodeId] = new PendingPersist(msg, attempts);
+            _lastPersistError = ex.Message;
+            var delay = Backoff(attempts);
+            // Review L7: the stack trace once, then one line per retry.
+            if (attempts == 1)
+                _log.Warning(ex, "Could not persist the ApplyAck from {Node} for {Id}; retrying in {Delay}",
+                    msg.NodeId, msg.DeploymentId, delay);
+            else
+                _log.Warning("Still cannot persist the ApplyAck from {Node} for {Id} (attempt {Attempt}: {Error}); retrying in {Delay}",
+                    msg.NodeId, msg.DeploymentId, attempts, ex.Message, delay);
+            Timers.StartSingleTimer(AckRetryTimerKey(msg.NodeId), new RetryAckPersist(msg), delay);
+            ReportPersistBacklog();
+            return false;
+        }
+
+        if (_unpersisted.Remove(msg.NodeId))
+        {
+            Timers.Cancel(AckRetryTimerKey(msg.NodeId));
+            if (_unpersisted.Count == 0) _lastPersistError = null;
+            ReportPersistBacklog();
        }

        _acks[msg.NodeId] = msg.Outcome;
        // An old node never sets Degraded, so its ack takes the clean branch exactly as before.
        if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId);
        else _degraded.Remove(msg.NodeId);
+        return true;
+    }

-        if (_acks.Count < _expectedAcks.Count) return;
-        CompleteFromAcks();
+    /// <summary>A retry tick: only the node's CURRENT unpersisted ack is retried; a superseded one is dropped (L3).</summary>
+    private void HandleRetryAckPersist(RetryAckPersist msg)
+    {
+        if (!_unpersisted.TryGetValue(msg.Ack.NodeId, out var pending) || !ReferenceEquals(pending.Ack, msg.Ack)) return;
+        HandleAck(msg.Ack);
+    }
+
+    private void ReportPersistBacklog() => _readiness?.MarkAckPersistBacklog(_unpersisted.Count, _lastPersistError);
+
+    /// <summary>min * 2^(attempt-1), capped at the max — shared by the load and ack-persist retries.</summary>
+    private TimeSpan Backoff(int attempt)
+    {
+        var factor = 1L << Math.Min(Math.Max(attempt, 1) - 1, 20);
+        return TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor));
    }

    /// <summary>Seals (all Applied) or marks partially failed — called once every expected ack is in.</summary>
@@ -598,7 +678,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
            MarkPartiallyFailed();
    }

-    private static string AckRetryTimerKey(ApplyAck msg) => $"ack-retry:{msg.DeploymentId.Value:N}:{msg.NodeId.Value}";
+    private static string AckRetryTimerKey(NodeId node) => $"ack-retry:{node.Value.ToUpperInvariant()}";

    private void PersistNodeAck(ApplyAck msg)
    {
@@ -634,7 +714,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    {
        if (_current is null) return;
        using var db = _dbFactory.CreateDbContext();
-        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true);
+        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true, onlyIfInFlight: true);
        if (_degraded.Count > 0)
        {
            // Still Sealed: every node DID apply this revision and is running it, and "latest sealed" is
@@ -663,7 +743,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    {
        if (_current is null) return;
        using var db = _dbFactory.CreateDbContext();
-        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed);
+        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.PartiallyFailed, onlyIfInFlight: true);
        db.SaveChanges();
        _log.Warning("Deployment {Id} partially failed; acks={Acks}", _current.Value,
            string.Join(",", _acks.Select(kv => $"{kv.Key.Value}={kv.Value}")));
@@ -678,6 +758,22 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
                msg.DeploymentId, _current);
            return;
        }
+        // Review M2: an ack that ARRIVED but is not yet persisted must never be timed out as "no ack from" its node.
+        // Persist the backlog now; if the DB still refuses, defer the decision (the terminal write needs it too).
+        if (_unpersisted.Count > 0)
+        {
+            foreach (var pending in _unpersisted.Values.ToList()) TryPersistAck(pending.Ack);
+            if (_unpersisted.Count > 0)
+            {
+                var delay = Backoff(_unpersisted.Values.Min(p => p.Attempts));
+                _log.Warning(
+                    "Deployment {Id} reached its apply deadline with {Count} received ack(s) not yet persisted; deferring " +
+                    "the outcome {Delay} until they are", _current.Value, _unpersisted.Count, delay);
+                Timers.StartSingleTimer(DeadlineTimerKey, msg, delay);
+                return;
+            }
+        }
+
        // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the
        // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped.
        if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count)
@@ -689,7 +785,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
        }

        using var db = _dbFactory.CreateDbContext();
-        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut);
+        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut, onlyIfInFlight: true);
        if (_expectedAcks.Count == 0)
        {
            SetFailureReason(db, Truncate(
@@ -745,6 +841,13 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    private void ResetForNext()
    {
        Timers.Cancel(DeadlineTimerKey);
+        foreach (var node in _unpersisted.Keys) Timers.Cancel(AckRetryTimerKey(node));
+        if (_unpersisted.Count > 0)
+        {
+            _unpersisted.Clear();
+            _lastPersistError = null;
+            ReportPersistBacklog();
+        }
        _current = null;
        _expectedAcks.Clear();
        _acks.Clear();
@@ -757,6 +860,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    /// <summary>An ack whose persist failed, re-delivered by its retry timer.</summary>
    private sealed record RetryAckPersist(ApplyAck Ack);

+    /// <summary>A received ack not yet persisted, and how many persists of it have failed.</summary>
+    private sealed record PendingPersist(ApplyAck Ack, int Attempts);
+
+    /// <summary>Why recovery cannot interpret a row (review L5), or null when it can.</summary>
+    private static string? DescribeMalformed(RecoveredDeployment d)
+    {
+        if (d.NodeStates.Count == 0 && string.IsNullOrWhiteSpace(d.RevisionHash))
+            return "its RevisionHash is blank, so it cannot be re-dispatched";
+        if (d.NodeStates.Any(n => string.IsNullOrWhiteSpace(n.NodeId)))
+            return "one of its NodeDeploymentState rows has a blank NodeId";
+        return null;
+    }
+
    public sealed record DeadlineElapsed(DeploymentId DeploymentId);

    /// <summary>The recovery read's result: the newest in-flight deployment, or null when none.</summary>
@@ -784,9 +900,10 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    ///     stub stays tracked so <see cref="SetFailureReason"/> can add to the same UPDATE.
    /// </summary>
    private static void UpdateDeploymentStatus(
-        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null)
+        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null,
+        bool onlyIfInFlight = false)
    {
-        var entry = TrackStub(db, id, rowVersion);
+        var entry = TrackStub(db, id, rowVersion, onlyIfInFlight);
        if (entry is null) return;
        entry.Entity.Status = status;
        entry.Property(x => x.Status).IsModified = true;
@@ -801,7 +918,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    private void SetFailureReason(OtOpcUaConfigDbContext db, string reason)
    {
        if (_current is null) return;
-        var entry = TrackStub(db, _current.Value);
+        var entry = TrackStub(db, _current.Value, onlyIfInFlight: true);
        if (entry is null) return;
        entry.Entity.FailureReason = reason;
        entry.Property(x => x.FailureReason).IsModified = true;
@@ -815,7 +932,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
    ///     guard); <see langword="null"/> reads the current one.
    /// </param>
    private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry<Deployment>? TrackStub(
-        OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null)
+        OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null, bool onlyIfInFlight = false)
    {
        var tracked = db.ChangeTracker.Entries<Deployment>().FirstOrDefault(e => e.Entity.DeploymentId == id.Value);
        if (tracked is not null) return tracked;
@@ -824,9 +941,13 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithU
            var stamp = db.Deployments
                .AsNoTracking()
                .Where(x => x.DeploymentId == id.Value)
-                .Select(x => new { x.RowVersion })
+                .Select(x => new { x.RowVersion, x.Status })
                .FirstOrDefault();
            if (stamp is null) return null;
+            // Review L8: a terminal outcome is written once. A row something else already closed (AdminOps' lost-
+            // dispatch closure, a previous incarnation) is left as it is; the RowVersion check covers the race.
+            if (onlyIfInFlight && stamp.Status is not (DeploymentStatus.Dispatching or DeploymentStatus.AwaitingApplyAcks))
+                return null;
            rowVersion = stamp.RowVersion;
        }
        return db.Deployments.Attach(new Deployment
@@ -18,8 +18,11 @@ public enum ConfigPublishCoordinatorState
 /// <param name="FailedLoadAttempts">Consecutive failed recovery loads (0 unless <see cref="ConfigPublishCoordinatorState.Loading"/>).</param>
 /// <param name="LastError">The last load failure's message, or null.</param>
 /// <param name="SinceUtc">When the state was entered.</param>
+/// <param name="PendingAckPersists">Received acks the coordinator could not persist yet and is retrying.</param>
+/// <param name="LastPersistError">The last ack-persist failure's message, or null.</param>
 public sealed record ConfigPublishCoordinatorSnapshot(
-    ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc);
+    ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc,
+    int PendingAckPersists = 0, string? LastPersistError = null);

 /// <summary>
 ///     Thread-safe state bridge from the <see cref="ConfigPublishCoordinator"/> actor to the node's health endpoint
@@ -39,11 +42,23 @@ public sealed class ConfigPublishCoordinatorReadiness
    public void MarkLoading(int failedAttempts, string? lastError)
    {
        var since = _current.State == ConfigPublishCoordinatorState.Loading ? _current.SinceUtc : DateTime.UtcNow;
-        _current = new(ConfigPublishCoordinatorState.Loading, failedAttempts, lastError, since);
+        _current = _current with
+        {
+            State = ConfigPublishCoordinatorState.Loading, FailedLoadAttempts = failedAttempts, LastError = lastError, SinceUtc = since,
+        };
    }

+    /// <summary>The ack-persist backlog changed (review L6): acks received but not yet written, and why.</summary>
+    /// <param name="pending">Acks awaiting a persist retry.</param>
+    /// <param name="lastError">The last persist failure's message, or null.</param>
+    public void MarkAckPersistBacklog(int pending, string? lastError) =>
+        _current = _current with { PendingAckPersists = pending, LastPersistError = pending > 0 ? lastError : null };
+
    /// <summary>The recovery state is loaded.</summary>
-    public void MarkReady() => _current = new(ConfigPublishCoordinatorState.Ready, 0, null, DateTime.UtcNow);
+    public void MarkReady() => _current = _current with
+    {
+        State = ConfigPublishCoordinatorState.Ready, FailedLoadAttempts = 0, LastError = null, SinceUtc = DateTime.UtcNow,
+    };

    /// <summary>The singleton stopped on this node.</summary>
    public void MarkNotHosted() => _current = new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow);
@@ -29,9 +29,15 @@ public sealed class ConfigPublishCoordinatorHealthCheck(ConfigPublishCoordinator
            ["failedLoadAttempts"] = s.FailedLoadAttempts,
        };
        if (s.LastError is not null) data["lastError"] = s.LastError;
+        data["pendingAckPersists"] = s.PendingAckPersists;
+        if (s.LastPersistError is not null) data["lastPersistError"] = s.LastPersistError;

        return Task.FromResult(s.State switch
        {
+            ConfigPublishCoordinatorState.Ready when s.PendingAckPersists > 0 => HealthCheckResult.Degraded(
+                $"Deploy coordinator cannot persist {s.PendingAckPersists} received deploy ack(s) to the ConfigDb " +
+                $"(last: {s.LastPersistError}); retrying with backoff. The deployment's outcome waits for them.",
+                data: data),
            ConfigPublishCoordinatorState.Loading => HealthCheckResult.Degraded(
                s.FailedLoadAttempts == 0
                    ? "Deploy coordinator is loading its recovery state from the ConfigDb; deployments are queued until it completes."
@@ -2984,7 +2984,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
            }

            _currentRevision = msg.Revision;
-            UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
+            // lmxopcua#868 review M1: the own row stays APPLYING (written by BeginDirectApply) until the
+            // address-space outcome is known — AwaitAddressSpaceOutcomeThenAck / ReleasePendingAck write the
+            // final Applied (clean, or with the degraded reason). Writing Applied/null here, up to ~105 s before
+            // that outcome, made the row read as a CLEAN ack: a coordinator restart in that window recovered it
+            // as complete and sealed the deployment clean while this node was about to report degraded (#601).
+            // A restart of THIS node in the window now replays the orphan Applying row (re-applies), which is
+            // the honest answer to "outcome unknown".
            // The Applied ack is NOT sent here any more. It used to go out right now, before the
            // address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults,
            // _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held
@@ -3069,6 +3075,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        if (_opcUaPublishActor is null)
        {
            _currentRevisionDegraded = false;
+            UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null);
            SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
            return;
        }
@@ -3126,13 +3133,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        if (_currentRevision is { } cur && cur == pending.Revision)
            _currentRevisionDegraded = degraded;

+        // The own row has been Applying since the apply began (lmxopcua#868 review M1); the FINAL outcome is
+        // written here, clean or degraded. Central adds the reason from the ack only when it ACCEPTS the ack (a
+        // timed-out or superseded deployment's ack is discarded), so a node with a ConfigDb records it itself; a
+        // driver-only node no-ops here.
+        UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
+            degraded ? ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded") : null);
        if (degraded)
        {
-            // The apply wrote this node's own row Applied/null before the outcome was known. Central
-            // adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
-            // is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
-            UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
-                ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
            OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
            _log.Warning(
                "DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",