Merge b640-601: honest deploy acks — Applied only after the address-space outcome, degraded flag + reason (#601)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe # Conflicts: # src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs
2026-10-08 18:00:23 -04:00
parent fdee129fd0 0a821417ea
commit 8801631739
28 changed files with 1259 additions and 55 deletions
@@ -57,7 +57,8 @@
            <tr>
                <td><code>@Short(d.DeploymentId)</code></td>
                <td><code>@DisplayText.Abbreviate(d.RevisionHash, 12)</code></td>
-                <td>@d.Status</td>
+                @* #601: Sealed + a reason = sealed with a degraded address-space apply on some node(s). *@
+                <td title="@(d.FailureReason ?? "")">@(d.Status == DeploymentStatus.Sealed && !string.IsNullOrEmpty(d.FailureReason) ? "Sealed (degraded)" : d.Status.ToString())</td>
                <td>@d.CreatedBy</td>
                <td>@d.CreatedAtUtc.ToString("u")</td>
                <td>@(d.SealedAtUtc?.ToString("u") ?? "—")</td>
@@ -77,7 +78,8 @@
        DeploymentStatus Status,
        string CreatedBy,
        DateTime CreatedAtUtc,
-        DateTime? SealedAtUtc);
+        DateTime? SealedAtUtc,
+        string? FailureReason);

    private IReadOnlyList<DeploymentRow> _deployments = Array.Empty<DeploymentRow>();
    private bool _busy;
@@ -102,7 +104,8 @@
                d.Status,
                d.CreatedBy,
                d.CreatedAtUtc,
-                d.SealedAtUtc))
+                d.SealedAtUtc,
+                d.FailureReason))
            .Take(50)
            .ToListAsync();

@@ -88,7 +88,7 @@ else
                                    <span class="chip chip-idle me-1">@role</span>
                                }
                            </td>
-                            <td><span class="chip @StatusChipClass(r.Status)">@StatusLabel(r.Status)</span></td>
+                            <td><span class="chip @StatusChipClass(r.Status, r.FailureReason)">@StatusLabel(r.Status, r.FailureReason)</span></td>
                            <td>@(r.AppliedAtUtc?.ToString("u") ?? "—")</td>
                            <td>@(r.StartedAtUtc?.ToString("u") ?? "—")</td>
                            <td title="@(r.FailureReason ?? "")">@(string.IsNullOrEmpty(r.FailureReason) ? "" : DisplayText.Abbreviate(r.FailureReason, 80))</td>
@@ -184,15 +184,22 @@ else
        }
    }

-    private static string StatusChipClass(NodeDeploymentStatus? status) => status switch
-    {
-        NodeDeploymentStatus.Applied => "chip-ok",
-        NodeDeploymentStatus.Applying => "chip-caution",
-        NodeDeploymentStatus.Failed => "chip-alert",
-        _ => "chip-idle",
-    };
+    // #601 — an Applied row carrying a reason is a DEGRADED apply (the node runs the revision but its address
+    // space may be partial); it must not render as the same green chip as a clean apply.
+    private static bool IsDegraded(NodeDeploymentStatus? status, string? failureReason) =>
+        status is { } s && ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators.ConfigPublishCoordinator.IsDegraded(s, failureReason);

-    private static string StatusLabel(NodeDeploymentStatus? status) => status?.ToString() ?? "waiting";
+    private static string StatusChipClass(NodeDeploymentStatus? status, string? failureReason) =>
+        IsDegraded(status, failureReason) ? "chip-caution" : status switch
+        {
+            NodeDeploymentStatus.Applied => "chip-ok",
+            NodeDeploymentStatus.Applying => "chip-caution",
+            NodeDeploymentStatus.Failed => "chip-alert",
+            _ => "chip-idle",
+        };
+
+    private static string StatusLabel(NodeDeploymentStatus? status, string? failureReason) =>
+        IsDegraded(status, failureReason) ? "Applied (degraded)" : status?.ToString() ?? "waiting";

    public async ValueTask DisposeAsync()
    {
@@ -57,7 +57,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 {
    public const string DeploymentsTopic = "deployments";
    public const string DeploymentAcksTopic = "deployment-acks";
-    public static readonly TimeSpan DefaultApplyDeadline = TimeSpan.FromMinutes(2);
+    /// <summary>Central's apply deadline — the shared <see cref="DeployTimings.ApplyDeadline"/>, which the node's
+    /// apply-outcome timeout is derived from (Gitea #601).</summary>
+    public static readonly TimeSpan DefaultApplyDeadline = DeployTimings.ApplyDeadline;

    private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
    private readonly TimeSpan _applyDeadline;
@@ -83,6 +85,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
    private static readonly IEqualityComparer<NodeId> NodeIdComparer = new CaseInsensitiveNodeIdComparer();
    private readonly Dictionary<NodeId, ApplyAckOutcome> _acks = new(NodeIdComparer);

+    /// <summary>Gitea #601 — the expected nodes whose Applied ack was DEGRADED (address-space apply partial).
+    /// Recovered from the DB on failover via <see cref="IsDegraded"/>.</summary>
+    private readonly HashSet<NodeId> _degraded = new(NodeIdComparer);
+
+    /// <summary>
+    ///     Gitea #601 — prefix of <c>NodeDeploymentState.FailureReason</c> on an <b>Applied</b> row whose apply
+    ///     was degraded. The status model is deliberately unchanged (no new enum value, no migration): a
+    ///     degraded node DID apply the revision, so its status stays <see cref="NodeDeploymentStatus.Applied"/>
+    ///     and the degradation rides the existing reason column, which <c>/fleet</c> already renders.
+    /// </summary>
+    public const string DegradedReasonPrefix = ApplyAck.DegradedReasonPrefix;
+
+    /// <summary>
+    ///     Gitea #601 — whether a <c>NodeDeploymentState</c> row records a degraded apply: status
+    ///     <see cref="NodeDeploymentStatus.Applied"/> with a non-empty <c>FailureReason</c>. A clean Applied row
+    ///     never carries a reason (neither this coordinator nor <c>DriverHostActor</c> writes one), so an
+    ///     old node's row reads as clean exactly as before.
+    /// </summary>
+    /// <param name="status">The row's status.</param>
+    /// <param name="failureReason">The row's failure reason.</param>
+    /// <returns><see langword="true"/> for an Applied-but-degraded row.</returns>
+    public static bool IsDegraded(NodeDeploymentStatus status, string? failureReason) =>
+        status == NodeDeploymentStatus.Applied && !string.IsNullOrEmpty(failureReason);
+
    private DeploymentId? _current;
    private HashSet<NodeId> _expectedAcks = new(NodeIdComparer);

@@ -149,9 +175,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers

        _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer);
        foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying))
+        {
            _acks[NodeId.Parse(s.NodeId)] = s.Status == NodeDeploymentStatus.Applied
                ? ApplyAckOutcome.Applied
                : ApplyAckOutcome.Failed;
+            if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
+        }

        // Resume the deadline timer using the remaining time. The deadline runs from when the
        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
@@ -174,6 +203,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
    {
        _current = msg.DeploymentId;
        _acks.Clear();
+        _degraded.Clear();

        // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack.
        using (var db = _dbFactory.CreateDbContext())
@@ -237,6 +267,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
        }

        _acks[msg.NodeId] = msg.Outcome;
+        // #601 — an old node never sets Degraded, so its ack takes the clean branch exactly as before.
+        if (IsDegradedAck(msg)) _degraded.Add(msg.NodeId);
+        else _degraded.Remove(msg.NodeId);
        PersistNodeAck(msg);

        if (_acks.Count < _expectedAcks.Count) return;
@@ -258,15 +291,51 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
            ? NodeDeploymentStatus.Applied
            : NodeDeploymentStatus.Failed;
        row.AppliedAtUtc = DateTime.UtcNow;
-        row.FailureReason = msg.FailureReason;
+        // #601 — an Applied+degraded ack keeps status Applied and records WHY in the reason column, so the row
+        // reads as degraded (IsDegraded) rather than clean. A clean or Failed ack writes what it always did.
+        row.FailureReason = IsDegradedAck(msg)
+            ? Truncate(DegradedReasonPrefix + (string.IsNullOrWhiteSpace(msg.DegradedReason)
+                ? "address-space apply degraded (no reason given)"
+                : msg.DegradedReason))
+            : msg.FailureReason;
        db.SaveChanges();
    }

+    private static bool IsDegradedAck(ApplyAck msg) => msg.Outcome == ApplyAckOutcome.Applied && msg.Degraded;
+
+    // FailureReason is nvarchar(2048) on both tables (OtOpcUaConfigDbContext); a reason built from a node's
+    // free text must never fail the SaveChanges that records the ack.
+    private const int FailureReasonMaxLength = 2048;
+
+    private static string Truncate(string s) =>
+        s.Length <= FailureReasonMaxLength ? s : s[..(FailureReasonMaxLength - 1)] + "…";
+
    private void SealDeployment()
    {
        if (_current is null) return;
        using var db = _dbFactory.CreateDbContext();
        UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.Sealed, sealNow: true);
+        if (_degraded.Count > 0)
+        {
+            // #601 — still Sealed: every node DID apply this revision and is running it, and "latest sealed" is
+            // what drift detection and the publish actor's fallback key on, so a degraded deploy that was not
+            // sealed would make the fleet look one revision behind what it actually runs. Sealed is not the
+            // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on
+            // each NodeDeploymentState row) and the seal logs at Warning.
+            var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList();
+            var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value);
+            if (d is not null)
+                d.FailureReason = Truncate(
+                    $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
+                    "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.");
+            db.SaveChanges();
+            _log.Warning(
+                "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}",
+                _current.Value, _acks.Count, string.Join(", ", nodes));
+            ResetForNext();
+            return;
+        }
+
        db.SaveChanges();
        _log.Info("Deployment {Id} sealed (acks={Count})", _current.Value, _acks.Count);
        ResetForNext();
@@ -318,6 +387,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
        _current = null;
        _expectedAcks.Clear();
        _acks.Clear();
+        _degraded.Clear();
    }

    private const string DeadlineTimerKey = "apply-deadline";
@@ -84,6 +84,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
    /// </remarks>
    public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5);

+    /// <summary>
+    ///     Gitea #601 — how long an Applied ack waits for <c>OpcUaPublishActor</c>'s address-space apply
+    ///     outcome (<see cref="OpcUa.OpcUaPublishActor.RebuildCompleted"/>) before it is sent anyway.
+    /// </summary>
+    /// <remarks>
+    ///     Derived from central's apply deadline (<see cref="DeployTimings.ApplyDeadline"/>, 2 min — the same
+    ///     constant <c>ConfigPublishCoordinator.DefaultApplyDeadline</c> reads) minus
+    ///     <see cref="DeployTimings.ApplyOutcomeSafetyMargin"/> (15 s) = 105 s: long enough that a
+    ///     slow-but-clean rebuild of a large address space is not labelled degraded, short enough that the ack
+    ///     still lands before central times the deployment out. An ack sent on this timeout is marked
+    ///     <b>degraded</b> with a reason starting <see cref="OutcomeNotReportedReasonPrefix"/>, never clean —
+    ///     the revision did advance, but nobody has confirmed the address space matches it.
+    /// </remarks>
+    public static readonly TimeSpan DefaultApplyOutcomeTimeout = DeployTimings.ApplyOutcomeTimeout;
+
+    /// <summary>Gitea #601 — the reason prefix of a degraded ack sent because the outcome TIMED OUT, so an
+    /// operator can tell "not reported in time" from a real sink failure.</summary>
+    public const string OutcomeNotReportedReasonPrefix = "address-space apply outcome not reported in time";
+
    /// <summary>Minimum interval between two supervision reprovisions of the SAME driver child (archreview
    /// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the
    /// reprovision hook makes every restart do ShutdownAsync + InitializeAsync + SubscribeAsync — so an
@@ -244,6 +263,21 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
    private IActorRef? _scriptedAlarmHost;

    private RevisionHash? _currentRevision;
+
+    /// <summary>
+    ///     Gitea #601 — the last address-space apply of <see cref="_currentRevision"/> reported DEGRADED (or no
+    ///     outcome in time). A re-dispatch of that same revision then RE-APPLIES instead of taking the
+    ///     idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied
+    ///     for an address space known to be partial, and the re-apply is what repairs it (a degraded
+    ///     non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing
+    ///     nodes). Reset by every clean outcome and by every boot path that restores a revision.
+    /// </summary>
+    private bool _currentRevisionDegraded;
+
+    /// <summary>Gitea #601 — Applied acks held until the address-space apply reports its outcome, keyed by
+    /// the apply's correlation id (echoed back in <see cref="OpcUa.OpcUaPublishActor.RebuildCompleted"/>).</summary>
+    private readonly Dictionary<CorrelationId, PendingAck> _pendingAcks = new();
+
    private DeploymentId? _applyingDeploymentId;

    private readonly Dictionary<string, ChildEntry> _children = new(StringComparer.Ordinal);
@@ -431,6 +465,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
    /// </summary>
    private readonly Func<CommonsNodeId, bool>? _sharesHistorizationScopeWithLocal;

+    /// <summary>Gitea #601 — how long an Applied ack waits for the address-space outcome (see
+    /// <see cref="DefaultApplyOutcomeTimeout"/>).</summary>
+    private readonly TimeSpan _applyOutcomeTimeout;
+
    /// <summary>This host's <see cref="Applying"/>-wedge watchdog delay (see
    /// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
    private readonly TimeSpan _applyWedgeTimeout;
@@ -595,6 +633,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
    /// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
    /// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
    /// <see cref="DefaultCommsRecoveryAuditDelay"/>; tests shorten it.</param>
+    /// <param name="applyOutcomeTimeout">Gitea #601: how long an Applied ack waits for the address-space
+    /// apply outcome before it is sent anyway, marked degraded ("outcome unknown"). Defaults to
+    /// <see cref="DefaultApplyOutcomeTimeout"/>; tests shorten it.</param>
    /// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
    public static Props Props(
        IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
@@ -626,7 +667,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        TimeSpan? applyWedgeTimeout = null,
        TimeSpan? reprovisionBackoff = null,
        TimeSpan? commsRecoveryAuditDelay = null,
-        Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null) =>
+        Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
+        TimeSpan? applyOutcomeTimeout = null) =>
        // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
        // expression tree. Six IActorRef? parameters and several interface-typed ones mean a
        // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -639,7 +681,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
            deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
            alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
            sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
-            sharesAlarmQueueWithLocal));
+            sharesAlarmQueueWithLocal, applyOutcomeTimeout));

    /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
    /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -737,8 +779,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        TimeSpan? applyWedgeTimeout = null,
        TimeSpan? reprovisionBackoff = null,
        TimeSpan? commsRecoveryAuditDelay = null,
-        Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null)
+        Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
+        TimeSpan? applyOutcomeTimeout = null)
    {
+        _applyOutcomeTimeout = applyOutcomeTimeout ?? DefaultApplyOutcomeTimeout;
        _sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
        _applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
        _reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
@@ -1288,6 +1332,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
    private void Steady()
    {
        Receive<DispatchDeployment>(HandleDispatchFromSteady);
+        // #601 — the held Applied ack is released by the address-space outcome (or its timeout); both can
+        // land in any behaviour, so all three register them.
+        Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
+        Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
        Receive<ArtifactCached>(HandleArtifactCached);
        Receive<ArtifactCacheFailed>(HandleArtifactCacheFailed);
        // A wedge tick already in the mailbox when the apply finished — dropped at Debug, see the handler.
@@ -1348,6 +1396,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        // Direct ConfigDb read (BeginFetchAndCacheApply / BeginDirectApply ran in Steady, then
        // Become(Applying)); completing the apply transitions back to Steady.
        Receive<ArtifactForApply>(HandleArtifactForApply);
+        Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
+        Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
        // The cache write is piped back too, and it outlives the apply that started it by design — so
        // its completion can land in EITHER state and both must handle it or it dead-letters.
        Receive<ArtifactCached>(HandleArtifactCached);
@@ -2180,6 +2230,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        // The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
        // We don't consume it here — drop it so it doesn't dead-letter (matches PeerProbeSupervisor).
        Receive<ZB.MOM.WW.OtOpcUa.Runtime.Health.PeerOpcUaProbeActor.OpcUaProbeResult>(_ => { });
+        // #601 — an apply that completed just before the host went Stale still owes its ack; the
+        // coordinator is not behind the ConfigDb, so release it exactly as in Steady.
+        Receive<OpcUa.OpcUaPublishActor.RebuildCompleted>(HandleRebuildCompleted);
+        Receive<ApplyOutcomeTimedOut>(HandleApplyOutcomeTimedOut);
        // An inbound operator write can't be serviced while the config DB is unreachable — fast-fail so the
        // node-manager's bounded Ask gets an immediate clear status instead of dead-lettering into a timeout.
        Receive<RouteNodeWrite>(_ =>
@@ -2226,7 +2280,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS

    private void HandleDispatchFromSteady(DispatchDeployment msg)
    {
-        if (_currentRevision is { } cur && cur == msg.RevisionHash)
+        // #601: the short-circuit claims a CLEAN Applied, so it is only honest while the current revision's
+        // address space is known-good — not after a degraded apply, and not while its outcome is still
+        // pending. Either way the dispatch re-applies instead, which is also what repairs a degraded apply.
+        var currentRevisionUnconfirmed = _currentRevisionDegraded
+            || _pendingAcks.Values.Any(p => p.Revision == msg.RevisionHash);
+        if (_currentRevision is { } cur && cur == msg.RevisionHash && !currentRevisionUnconfirmed)
        {
            // Idempotent — already at this rev. Ack and stay Steady.
            _log.Debug("DriverHost {Node}: dispatch {Id} matches current rev {Rev}; immediate ACK",
@@ -2235,6 +2294,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
            return;
        }

+        if (_currentRevision is { } same && same == msg.RevisionHash)
+            _log.Info(
+                "DriverHost {Node}: dispatch {Id} matches current rev {Rev}, but its address-space apply was degraded or is unconfirmed — re-applying rather than acking clean",
+                _localNode, msg.DeploymentId, msg.RevisionHash);
+
        if (_fetchAndCacheMode)
        {
            BeginFetchAndCacheApply(msg.DeploymentId, msg.RevisionHash, msg.CorrelationId);
@@ -2394,7 +2458,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS

            _currentRevision = msg.Revision;
            UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
-            SendAck(msg.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, msg.Correlation);
+            // Gitea #601: the Applied ack is NOT sent here any more. It used to go out right now, before the
+            // address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults,
+            // _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held
+            // until OpcUaPublishActor replies RebuildCompleted (HandleRebuildCompleted) or the outcome
+            // timeout fires (sent degraded, "outcome unknown"). With no publish actor there is no address
+            // space to wait for, so the ack goes out clean immediately — exactly the old behaviour.
+            AwaitAddressSpaceOutcomeThenAck(msg.DeploymentId, msg.Revision, msg.Correlation);
            // Rebuild the address space + push subscriptions FROM the bytes this apply already has, so
            // nothing downstream re-reads the artifact: under FetchAndCache OpcUaPublishActor has no
            // central SQL to read at all, and under Direct a bytes-less RebuildAddressSpace would make
@@ -2406,8 +2476,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
            // different actor with its own mailbox, and ParsedDeploymentArtifact is a disposable,
            // non-thread-safe object owned by this handler (disposed in the finally below, likely before
            // the other actor even dequeues). It opens its own single parse on the far side.
+            // ReportOutcome: the reply (to Self — this Tell runs inside a receive, so the implicit sender is
+            // this actor) releases the held ack. Plain Tell + reply message, no Ask/await, so nothing here
+            // resumes off the actor thread.
            _opcUaPublishActor?.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RebuildAddressSpace(
-                msg.Correlation, msg.DeploymentId, appliedBlob));
+                msg.Correlation, msg.DeploymentId, appliedBlob, ReportOutcome: true));
            // SubscribeBulk pass: hand each driver its desired tag references so live values flow into
            // the just-rebuilt address space instead of staying BadWaitingForInitialData.
            PushDesiredSubscriptionsFromParsed(msg.DeploymentId, parsed);
@@ -2427,6 +2500,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        }
        catch (Exception ex)
        {
+            // #601: a throw after the Applied ack was put on hold — the Failed ack below is this apply's one
+            // answer, so the held Applied must never follow it when the address-space outcome lands.
+            DropPendingAck(msg.Correlation);
            UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Failed, ex.Message);
            SendAck(msg.DeploymentId, ApplyAckOutcome.Failed, ex.Message, msg.Correlation);
            OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "reject"));
@@ -2447,6 +2523,99 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        }
    }

+    /// <summary>Gitea #601 — an Applied ack held until the address-space apply reports its outcome.</summary>
+    private sealed record PendingAck(DeploymentId DeploymentId, RevisionHash Revision, CorrelationId Correlation);
+
+    /// <summary>Gitea #601 — self-timer: the address-space outcome for this correlation did not arrive in time.</summary>
+    private sealed record ApplyOutcomeTimedOut(CorrelationId Correlation);
+
+    private static string ApplyOutcomeTimerKey(CorrelationId correlation) => $"apply-outcome-{correlation}";
+
+    /// <summary>
+    ///     Gitea #601 — holds the Applied ack for an apply whose revision has just advanced until the
+    ///     address-space apply outcome is known, arming <see cref="_applyOutcomeTimeout"/> as the backstop.
+    ///     With no publish actor wired (tests, a node with no OPC UA data plane) there is no outcome to wait
+    ///     for, so the ack is sent clean at once — byte-for-byte the pre-#601 behaviour.
+    /// </summary>
+    private void AwaitAddressSpaceOutcomeThenAck(DeploymentId deploymentId, RevisionHash revision, CorrelationId correlation)
+    {
+        if (_opcUaPublishActor is null)
+        {
+            _currentRevisionDegraded = false;
+            SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
+            return;
+        }
+
+        if (_pendingAcks.Remove(correlation, out var superseded))
+            _log.Warning(
+                "DriverHost {Node}: a second apply reused correlation {Correlation} while deployment {Old}'s ack was still waiting for its address-space outcome — that ack is superseded",
+                _localNode, correlation, superseded.DeploymentId);
+        _pendingAcks[correlation] = new PendingAck(deploymentId, revision, correlation);
+        Timers.StartSingleTimer(ApplyOutcomeTimerKey(correlation), new ApplyOutcomeTimedOut(correlation), _applyOutcomeTimeout);
+    }
+
+    /// <summary>Gitea #601 — forgets a held ack (its apply failed after all, and acks Failed instead).</summary>
+    private void DropPendingAck(CorrelationId correlation)
+    {
+        if (_pendingAcks.Remove(correlation))
+            Timers.Cancel(ApplyOutcomeTimerKey(correlation));
+    }
+
+    /// <summary>
+    ///     Gitea #601 — the address-space apply reported its outcome: release the held ack, clean or
+    ///     degraded. Registered in every behaviour (Steady, Applying, Stale) because the reply can land in
+    ///     any of them — the host returns to Steady before the publish actor has even dequeued the rebuild.
+    /// </summary>
+    private void HandleRebuildCompleted(OpcUa.OpcUaPublishActor.RebuildCompleted msg)
+    {
+        if (!_pendingAcks.Remove(msg.Correlation, out var pending))
+        {
+            _log.Debug("DriverHost {Node}: address-space outcome for {Correlation} has no held ack (already timed out or superseded); dropping",
+                _localNode, msg.Correlation);
+            return;
+        }
+
+        Timers.Cancel(ApplyOutcomeTimerKey(msg.Correlation));
+        ReleasePendingAck(pending, msg.Degraded, msg.DegradedReason);
+    }
+
+    /// <summary>
+    ///     Gitea #601 — the address-space outcome did not arrive within <see cref="_applyOutcomeTimeout"/>.
+    ///     The ack is sent anyway so the coordinator does not time the deployment out on a node that DID
+    ///     advance its revision — but degraded, never clean: nobody has confirmed the address space.
+    /// </summary>
+    private void HandleApplyOutcomeTimedOut(ApplyOutcomeTimedOut msg)
+    {
+        if (!_pendingAcks.Remove(msg.Correlation, out var pending)) return;
+
+        ReleasePendingAck(pending, degraded: true,
+            $"{OutcomeNotReportedReasonPrefix} ({_applyOutcomeTimeout.TotalSeconds:0}s) — no sink failure was observed; the apply may still be running (slow) or its outcome was lost");
+    }
+
+    private void ReleasePendingAck(PendingAck pending, bool degraded, string? reason)
+    {
+        // Only the CURRENT revision's flag is ours to set — an older apply's late outcome must not mark a
+        // newer revision degraded (or clean).
+        if (_currentRevision is { } cur && cur == pending.Revision)
+            _currentRevisionDegraded = degraded;
+
+        if (degraded)
+        {
+            // #601 review — the apply wrote this node's own row Applied/null before the outcome was known. Central
+            // adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
+            // is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
+            UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
+                ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
+            OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
+            _log.Warning(
+                "DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
+                _localNode, pending.DeploymentId, pending.Revision, reason);
+        }
+
+        SendAck(pending.DeploymentId, ApplyAckOutcome.Applied, failureReason: null, pending.Correlation,
+            degraded, degraded ? reason ?? "address-space apply degraded" : null);
+    }
+
    /// <summary>
    ///     Self-timer telling this host it has been in <see cref="Applying"/> for
    ///     <see cref="_applyWedgeTimeout"/> without finishing.
@@ -3677,9 +3846,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
        }
    }

-    private void SendAck(DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation)
+    private void SendAck(
+        DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation,
+        bool degraded = false, string? degradedReason = null)
    {
-        var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation);
+        var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation, degraded, degradedReason);
        if (_ackRouter is not null)
        {
            // Either a direct coordinator handle (tests) or, under MeshTransport:Mode = Grpc, the
@@ -100,8 +100,31 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
    ///     that cannot succeed, leaving the rebuild a no-op and clients browsing an empty server. Null
    ///     on the normal path, where loading from the ConfigDb by id is correct.
    /// </param>
+    /// <param name="ReportOutcome">
+    ///     Gitea #601 — when <see langword="true"/>, the actor replies to the sender with a
+    ///     <see cref="RebuildCompleted"/> once the apply outcome is known (clean or degraded), on EVERY path
+    ///     through the handler. <c>DriverHostActor</c> sets it on the deploy path and sends the deployment's
+    ///     ack only after that reply, so a degraded apply can no longer seal a deployment green. Off for the
+    ///     boot/restore paths, which re-ack nothing.
+    /// </param>
    public sealed record RebuildAddressSpace(
-        CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null);
+        CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null,
+        bool ReportOutcome = false);
+
+    /// <summary>
+    ///     Gitea #601 — the reply to a <see cref="RebuildAddressSpace"/> carrying
+    ///     <see cref="RebuildAddressSpace.ReportOutcome"/>: whether the address-space apply was clean.
+    /// </summary>
+    /// <param name="Correlation">The rebuild's correlation id, echoed.</param>
+    /// <param name="DeploymentId">The rebuild's deployment id, echoed (null for a legacy caller).</param>
+    /// <param name="Degraded">
+    ///     <see langword="true"/> when the apply swallowed per-node sink faults, its rebuild failed, the
+    ///     pipeline threw, or there was no usable artifact — i.e. the served address space may be partial or
+    ///     stale. An empty plan (nothing changed) is clean.
+    /// </param>
+    /// <param name="DegradedReason">A short operator-readable reason; null when clean.</param>
+    public sealed record RebuildCompleted(
+        CorrelationId Correlation, DeploymentId? DeploymentId, bool Degraded, string? DegradedReason);

    public sealed record ServiceLevelChanged(byte ServiceLevel);

@@ -134,6 +157,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
    /// protecting. Distinguishes "the artifact went missing under a running server" (issue #485 — warn, and
    /// hold what is materialised) from "nothing has been deployed here yet" (a quiet no-op at boot).</summary>
    private bool _hasAppliedComposition;
+    /// <summary>
+    ///     Gitea #601 review — <see cref="_lastApplied"/> was advanced by a DEGRADED rebuild (a rebuild advances
+    ///     even with swallowed failures, see <see cref="HandleRebuildCore"/>). A re-deploy of the same revision
+    ///     then diffs to an EMPTY plan; without this flag that empty plan reported clean, so the "re-deploy
+    ///     re-applies" repair path sealed green over an address space still known to be partial. While set, an
+    ///     empty plan re-materialises the whole composition (idempotent ensures) and reports clean only when
+    ///     that pass is itself clean.
+    /// </summary>
+    private bool _lastAppliedDegraded;
    private DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
    private RedundancyStateChanged? _lastSnapshot;
    private (bool Ok, DateTime At)? _probeAboutMe;
@@ -438,9 +470,13 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers

    private void HandleRebuild(RebuildAddressSpace msg)
    {
+        // Captured up front: the pipeline below is synchronous on this actor's thread today, but the reply
+        // must reach whoever asked even if a future edit introduces an await into it.
+        var requester = Sender;
+        (bool Degraded, string? Reason) outcome;
        try
        {
-            HandleRebuildCore(msg);
+            outcome = HandleRebuildCore(msg);
        }
        finally
        {
@@ -455,9 +491,14 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                _log.Error(ex, "OpcUaPublish: sink.CompleteAddressSpaceApply threw (correlation={Correlation})", msg.Correlation);
            }
        }
+
+        // Gitea #601 — the outcome goes out only after the apply (including #641's completion step) is done.
+        if (msg.ReportOutcome)
+            requester.Tell(new RebuildCompleted(msg.Correlation, msg.DeploymentId, outcome.Degraded, outcome.Reason));
    }

-    private void HandleRebuildCore(RebuildAddressSpace msg)
+    /// <summary>The rebuild pipeline. Returns whether the apply was degraded, and why (Gitea #601).</summary>
+    private (bool Degraded, string? Reason) HandleRebuildCore(RebuildAddressSpace msg)
    {
        using var span = OtOpcUaTelemetry.StartAddressSpaceRebuildSpan();
        span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
@@ -481,8 +522,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
            {
                _log.Error(ex, "OpcUaPublish: sink.RebuildAddressSpace threw (correlation={Correlation})",
                    msg.Correlation);
+                return (true, $"address-space rebuild threw: {ex.Message}");
            }
-            return;
+            return (false, null);
        }

        try
@@ -520,7 +562,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                    _log.Debug(
                        "OpcUaPublish: no artifact to materialise yet (deployment={Id}, correlation={Correlation})",
                        msg.DeploymentId, msg.Correlation);
-                return;
+                return (true, "no usable artifact reached the address-space apply; the previously served address space was kept");
            }

            // Parse the artifact ONCE for this rebuild. Every consumer below (data-plane authority, then the
@@ -551,9 +593,31 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers

            if (plan.IsEmpty)
            {
+                if (_lastAppliedDegraded)
+                {
+                    // #601 review — the composition is unchanged but the last apply of it was a degraded
+                    // rebuild: re-ensure every node (each Ensure* no-ops on a node that stands) so the
+                    // re-deploy actually repairs what that rebuild could not place.
+                    var repairFailed = _applier.MaterialiseHierarchy(composition)
+                        + MaterialiseNodes(plan, composition, fullComposition: true);
+                    if (repairFailed > 0)
+                    {
+                        OtOpcUaTelemetry.OpcUaApplyFailed.Add(1, new KeyValuePair<string, object?>("kind", "nodes"));
+                        _log.Error(
+                            "OpcUaPublish: re-apply of a degraded composition is STILL degraded (correlation={Correlation}, failedNodes={FailedNodes})",
+                            msg.Correlation, repairFailed);
+                        return (true, $"re-apply of a previously degraded address space is still degraded (failedNodes={repairFailed})");
+                    }
+
+                    _lastAppliedDegraded = false;
+                    _log.Info("OpcUaPublish: re-apply repaired the previously degraded address space (correlation={Correlation})",
+                        msg.Correlation);
+                    return (false, null);
+                }
+
                _log.Debug("OpcUaPublish: rebuild requested but plan is empty (correlation={Correlation})",
                    msg.Correlation);
-                return;
+                return (false, null);
            }

            var outcome = _applier.Apply(plan);
@@ -596,6 +660,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
            {
                _lastApplied = composition;
                _hasAppliedComposition = true;
+                // #601 review — advanced despite failures only on a rebuild; remember it so an identical
+                // re-deploy repairs instead of short-circuiting clean on the empty plan.
+                _lastAppliedDegraded = outcome.RebuildFailed || failedNodes > 0;
            }
            else
            {
@@ -629,16 +696,19 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                    "OpcUaPublish: address-space apply DEGRADED (correlation={Correlation}, rebuildFailed={RebuildFailed}, failedNodes={FailedNodes}, added={Added}, removed={Removed}, changed={Changed}) — the running address space may be stale or partial",
                    msg.Correlation, outcome.RebuildFailed, failedNodes,
                    outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes);
+                return (true, outcome.RebuildFailed
+                    ? $"address-space rebuild failed (failedNodes={failedNodes})"
+                    : $"address-space apply degraded (failedNodes={failedNodes}); not recorded as applied, the next deploy of this revision re-diffs and repairs it");
            }
-            else
-            {
-                _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
-                    msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
-            }
+
+            _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
+                msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
+            return (false, null);
        }
        catch (Exception ex)
        {
            _log.Error(ex, "OpcUaPublish: rebuild pipeline threw (correlation={Correlation})", msg.Correlation);
+            return (true, $"address-space apply threw: {ex.Message}");
        }
    }