feat(deploy): coordinator records degraded acks; deployment seals but names degraded nodes (#601)

An Applied+Degraded ack keeps NodeDeploymentState.Status = Applied with
FailureReason 'Applied degraded: <reason>' (no enum/migration change). The
deployment still seals (the revision is running; latest-sealed drives drift)
but Deployment.FailureReason names the degraded nodes and the seal logs at
Warning. Recovered from the DB on singleton failover. Old-node acks (no field,
DPS JSON or gRPC) behave exactly as before.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs

@@ -83,6 +83,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
     private static readonly IEqualityComparer<NodeId> NodeIdComparer = new CaseInsensitiveNodeIdComparer();
     private readonly Dictionary<NodeId, ApplyAckOutcome> _acks = new(NodeIdComparer);
 
+    /// <summary>Gitea #601 — the expected nodes whose Applied ack was DEGRADED (address-space apply partial).
+    /// Recovered from the DB on failover via <see cref="IsDegraded"/>.</summary>
+    private readonly HashSet<NodeId> _degraded = new(NodeIdComparer);
+
+    /// <summary>
+    ///     Gitea #601 — prefix of <c>NodeDeploymentState.FailureReason</c> on an <b>Applied</b> row whose apply
+    ///     was degraded. The status model is deliberately unchanged (no new enum value, no migration): a
+    ///     degraded node DID apply the revision, so its status stays <see cref="NodeDeploymentStatus.Applied"/>
+    ///     and the degradation rides the existing reason column, which <c>/fleet</c> already renders.
+    /// </summary>
+    public const string DegradedReasonPrefix = "Applied degraded: ";
+
+    /// <summary>
+    ///     Gitea #601 — whether a <c>NodeDeploymentState</c> row records a degraded apply: status
+    ///     <see cref="NodeDeploymentStatus.Applied"/> with a non-empty <c>FailureReason</c>. A clean Applied row
+    ///     never carries a reason (neither this coordinator nor <c>DriverHostActor</c> writes one), so an
+    ///     old node's row reads as clean exactly as before.
+    /// </summary>
+    /// <param name="status">The row's status.</param>
+    /// <param name="failureReason">The row's failure reason.</param>
+    /// <returns><see langword="true"/> for an Applied-but-degraded row.</returns>
+    public static bool IsDegraded(NodeDeploymentStatus status, string? failureReason) =>
+        status == NodeDeploymentStatus.Applied && !string.IsNullOrEmpty(failureReason);
+
     private DeploymentId? _current;
     private HashSet<NodeId> _expectedAcks = new(NodeIdComparer);
 
@@ -149,9 +173,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
         _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer);
         foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying))
+        {
             _acks[NodeId.Parse(s.NodeId)] = s.Status == NodeDeploymentStatus.Applied
                 ? ApplyAckOutcome.Applied
                 : ApplyAckOutcome.Failed;
+            if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
+        }
 
         // Resume the deadline timer using the remaining time. The deadline runs from when the
         // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
@@ -174,6 +201,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
     {
         _current = msg.DeploymentId;
         _acks.Clear();
+        _degraded.Clear();
 
         // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack.
         using (var db = _dbFactory.CreateDbContext())
@@ -237,6 +265,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         }
 
         _acks[msg.NodeId] = msg.Outcome;
+        // #601 — an old node never sets Degraded, so its ack takes the clean branch exactly as before.
+        if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId);
+        else _degraded.Remove(msg.NodeId);
         PersistNodeAck(msg);
 
         if (_acks.Count < _expectedAcks.Count) return;
@@ -258,15 +289,51 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             ? NodeDeploymentStatus.Applied
             : NodeDeploymentStatus.Failed;
         row.AppliedAtUtc = DateTime.UtcNow;
-        row.FailureReason = msg.FailureReason;
+        // #601 — an Applied+degraded ack keeps status Applied and records WHY in the reason column, so the row
+        // reads as degraded (IsDegraded) rather than clean. A clean or Failed ack writes what it always did.
+        row.FailureReason = IsDegradedAck(msg)
+            ? Truncate(DegradedReasonPrefix + (string.IsNullOrWhiteSpace(msg.DegradedReason)
+                ? "address-space apply degraded (no reason given)"
+                : msg.DegradedReason))
+            : msg.FailureReason;
         db.SaveChanges();
     }
 
+    private static bool IsDegradedAck(ApplyAck msg) => msg.Outcome == ApplyAckOutcome.Applied && msg.Degraded;
+
+    // FailureReason is nvarchar(2048) on both tables (OtOpcUaConfigDbContext); a reason built from a node's
+    // free text must never fail the SaveChanges that records the ack.
+    private const int FailureReasonMaxLength = 2048;
+
+    private static string Truncate(string s) =>
+        s.Length <= FailureReasonMaxLength ? s : s[..(FailureReasonMaxLength - 1)] + "…";
+
     private void SealDeployment()
     {
         if (_current is null) return;
         using var db = _dbFactory.CreateDbContext();
         UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true);
+        if (_degraded.Count > 0)
+        {
+            // #601 — still Sealed: every node DID apply this revision and is running it, and "latest sealed" is
+            // what drift detection and the publish actor's fallback key on, so a degraded deploy that was not
+            // sealed would make the fleet look one revision behind what it actually runs. Sealed is not the
+            // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on
+            // each NodeDeploymentState row) and the seal logs at Warning.
+            var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList();
+            var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value);
+            if (d is not null)
+                d.FailureReason = Truncate(
+                    $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
+                    "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.");
+            db.SaveChanges();
+            _log.Warning(
+                "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}",
+                _current.Value, _acks.Count, string.Join(", ", nodes));
+            ResetForNext();
+            return;
+        }
+
         db.SaveChanges();
         _log.Info("Deployment {Id} sealed (acks={Count})", _current.Value, _acks.Count);
         ResetForNext();
@@ -318,6 +385,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         _current = null;
         _expectedAcks.Clear();
         _acks.Clear();
+        _degraded.Clear();
     }
 
     private const string DeadlineTimerKey = "apply-deadline";