Merge b640-601: honest deploy acks — Applied only after the address-space outcome, degraded flag + reason (#601)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe

# Conflicts:
#	src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs

@@ -100,8 +100,31 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     ///     that cannot succeed, leaving the rebuild a no-op and clients browsing an empty server. Null
     ///     on the normal path, where loading from the ConfigDb by id is correct.
     /// </param>
+    /// <param name="ReportOutcome">
+    ///     Gitea #601 — when <see langword="true"/>, the actor replies to the sender with a
+    ///     <see cref="RebuildCompleted"/> once the apply outcome is known (clean or degraded), on EVERY path
+    ///     through the handler. <c>DriverHostActor</c> sets it on the deploy path and sends the deployment's
+    ///     ack only after that reply, so a degraded apply can no longer seal a deployment green. Off for the
+    ///     boot/restore paths, which re-ack nothing.
+    /// </param>
     public sealed record RebuildAddressSpace(
-        CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null);
+        CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null,
+        bool ReportOutcome = false);
+
+    /// <summary>
+    ///     Gitea #601 — the reply to a <see cref="RebuildAddressSpace"/> carrying
+    ///     <see cref="RebuildAddressSpace.ReportOutcome"/>: whether the address-space apply was clean.
+    /// </summary>
+    /// <param name="Correlation">The rebuild's correlation id, echoed.</param>
+    /// <param name="DeploymentId">The rebuild's deployment id, echoed (null for a legacy caller).</param>
+    /// <param name="Degraded">
+    ///     <see langword="true"/> when the apply swallowed per-node sink faults, its rebuild failed, the
+    ///     pipeline threw, or there was no usable artifact — i.e. the served address space may be partial or
+    ///     stale. An empty plan (nothing changed) is clean.
+    /// </param>
+    /// <param name="DegradedReason">A short operator-readable reason; null when clean.</param>
+    public sealed record RebuildCompleted(
+        CorrelationId Correlation, DeploymentId? DeploymentId, bool Degraded, string? DegradedReason);
 
     public sealed record ServiceLevelChanged(byte ServiceLevel);
 
@@ -134,6 +157,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// protecting. Distinguishes "the artifact went missing under a running server" (issue #485 — warn, and
     /// hold what is materialised) from "nothing has been deployed here yet" (a quiet no-op at boot).</summary>
     private bool _hasAppliedComposition;
+    /// <summary>
+    ///     Gitea #601 review — <see cref="_lastApplied"/> was advanced by a DEGRADED rebuild (a rebuild advances
+    ///     even with swallowed failures, see <see cref="HandleRebuildCore"/>). A re-deploy of the same revision
+    ///     then diffs to an EMPTY plan; without this flag that empty plan reported clean, so the "re-deploy
+    ///     re-applies" repair path sealed green over an address space still known to be partial. While set, an
+    ///     empty plan re-materialises the whole composition (idempotent ensures) and reports clean only when
+    ///     that pass is itself clean.
+    /// </summary>
+    private bool _lastAppliedDegraded;
     private DbHealthProbeActor.DbHealthStatus? _lastDbHealth;
     private RedundancyStateChanged? _lastSnapshot;
     private (bool Ok, DateTime At)? _probeAboutMe;
@@ -438,9 +470,13 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
 
     private void HandleRebuild(RebuildAddressSpace msg)
     {
+        // Captured up front: the pipeline below is synchronous on this actor's thread today, but the reply
+        // must reach whoever asked even if a future edit introduces an await into it.
+        var requester = Sender;
+        (bool Degraded, string? Reason) outcome;
         try
         {
-            HandleRebuildCore(msg);
+            outcome = HandleRebuildCore(msg);
         }
         finally
         {
@@ -455,9 +491,14 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                 _log.Error(ex, "OpcUaPublish: sink.CompleteAddressSpaceApply threw (correlation={Correlation})", msg.Correlation);
             }
         }
+
+        // Gitea #601 — the outcome goes out only after the apply (including #641's completion step) is done.
+        if (msg.ReportOutcome)
+            requester.Tell(new RebuildCompleted(msg.Correlation, msg.DeploymentId, outcome.Degraded, outcome.Reason));
     }
 
-    private void HandleRebuildCore(RebuildAddressSpace msg)
+    /// <summary>The rebuild pipeline. Returns whether the apply was degraded, and why (Gitea #601).</summary>
+    private (bool Degraded, string? Reason) HandleRebuildCore(RebuildAddressSpace msg)
     {
         using var span = OtOpcUaTelemetry.StartAddressSpaceRebuildSpan();
         span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
@@ -481,8 +522,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             {
                 _log.Error(ex, "OpcUaPublish: sink.RebuildAddressSpace threw (correlation={Correlation})",
                     msg.Correlation);
+                return (true, $"address-space rebuild threw: {ex.Message}");
             }
-            return;
+            return (false, null);
         }
 
         try
@@ -520,7 +562,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                     _log.Debug(
                         "OpcUaPublish: no artifact to materialise yet (deployment={Id}, correlation={Correlation})",
                         msg.DeploymentId, msg.Correlation);
-                return;
+                return (true, "no usable artifact reached the address-space apply; the previously served address space was kept");
             }
 
             // Parse the artifact ONCE for this rebuild. Every consumer below (data-plane authority, then the
@@ -551,9 +593,31 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
 
             if (plan.IsEmpty)
             {
+                if (_lastAppliedDegraded)
+                {
+                    // #601 review — the composition is unchanged but the last apply of it was a degraded
+                    // rebuild: re-ensure every node (each Ensure* no-ops on a node that stands) so the
+                    // re-deploy actually repairs what that rebuild could not place.
+                    var repairFailed = _applier.MaterialiseHierarchy(composition)
+                        + MaterialiseNodes(plan, composition, fullComposition: true);
+                    if (repairFailed > 0)
+                    {
+                        OtOpcUaTelemetry.OpcUaApplyFailed.Add(1, new KeyValuePair<string, object?>("kind", "nodes"));
+                        _log.Error(
+                            "OpcUaPublish: re-apply of a degraded composition is STILL degraded (correlation={Correlation}, failedNodes={FailedNodes})",
+                            msg.Correlation, repairFailed);
+                        return (true, $"re-apply of a previously degraded address space is still degraded (failedNodes={repairFailed})");
+                    }
+
+                    _lastAppliedDegraded = false;
+                    _log.Info("OpcUaPublish: re-apply repaired the previously degraded address space (correlation={Correlation})",
+                        msg.Correlation);
+                    return (false, null);
+                }
+
                 _log.Debug("OpcUaPublish: rebuild requested but plan is empty (correlation={Correlation})",
                     msg.Correlation);
-                return;
+                return (false, null);
             }
 
             var outcome = _applier.Apply(plan);
@@ -596,6 +660,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             {
                 _lastApplied = composition;
                 _hasAppliedComposition = true;
+                // #601 review — advanced despite failures only on a rebuild; remember it so an identical
+                // re-deploy repairs instead of short-circuiting clean on the empty plan.
+                _lastAppliedDegraded = outcome.RebuildFailed || failedNodes > 0;
             }
             else
             {
@@ -629,16 +696,19 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
                     "OpcUaPublish: address-space apply DEGRADED (correlation={Correlation}, rebuildFailed={RebuildFailed}, failedNodes={FailedNodes}, added={Added}, removed={Removed}, changed={Changed}) — the running address space may be stale or partial",
                     msg.Correlation, outcome.RebuildFailed, failedNodes,
                     outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes);
+                return (true, outcome.RebuildFailed
+                    ? $"address-space rebuild failed (failedNodes={failedNodes})"
+                    : $"address-space apply degraded (failedNodes={failedNodes}); not recorded as applied, the next deploy of this revision re-diffs and repairs it");
             }
-            else
-            {
-                _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
-                    msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
-            }
+
+            _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
+                msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
+            return (false, null);
         }
         catch (Exception ex)
         {
             _log.Error(ex, "OpcUaPublish: rebuild pipeline threw (correlation={Correlation})", msg.Correlation);
+            return (true, $"address-space apply threw: {ex.Message}");
         }
     }