fix(deploy): derive the apply-outcome timeout from central's apply deadline (105 s) and name a timeout reason distinctly (#601 review)

60 s labelled a slow-but-clean large rebuild degraded. The deadline moves to a
shared Commons DeployTimings constant that ConfigPublishCoordinator reads, and
the node's timeout is deadline - 15 s margin, so the held ack still lands in
time. A timed-out ack's reason now says the outcome was not reported in time,
distinct from a real sink failure.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe

CLAUDE.md

@@ -185,7 +185,7 @@ is still default-closed; what changed is which deltas it whitelists.
   apply's removes (a re-remove returns false and ratchets to a rebuild, which is the correct,
   bounded resync). A **rebuild** still advances even with failures.
 - **…and the deploy ack now SAYS so (#601).** The Applied ack waits for `OpcUaPublishActor`'s
-  `RebuildCompleted` (60 s timeout ⇒ degraded, never clean) and carries an additive `Degraded`/`DegradedReason`
+  `RebuildCompleted` (timeout = shared `DeployTimings.ApplyDeadline` − 15 s = 105 s ⇒ degraded "outcome not reported in time", never clean) and carries an additive `Degraded`/`DegradedReason`
   on both ack transports; central seals but records `NodeDeploymentState.FailureReason = "Applied degraded: …"`
   (`/fleet` "Applied (degraded)"), and a re-dispatch of a degraded revision re-applies instead of short-circuiting.
   See `docs/Redundancy.md` §"What an `Applied` ack means".

docs/Redundancy.md

@@ -598,9 +598,16 @@ deployment green fleet-wide. Now:
   is clean), and only then is the ack sent. It is a plain `Tell` + reply message — no `Ask`, no `await`, so
   nothing resumes off the actor thread.
 - **A missing outcome is degraded, never clean.** If no reply arrives within
-  `DriverHostActor.DefaultApplyOutcomeTimeout` (60 s — inside the coordinator's 2-minute apply deadline,
-  which also has to cover the artifact read/fetch), the ack goes out `Applied` + degraded
-  (*"did not report an outcome within 60s"*). A node with no publish actor wired acks clean at once, as before.
+  `DriverHostActor.DefaultApplyOutcomeTimeout` the ack goes out `Applied` + degraded, with a reason starting
+  *"address-space apply outcome not reported in time"* — distinct from a real sink failure (which carries
+  `failedNodes=` / "rebuild failed" / "threw"), so an operator can tell a slow or lost apply from a broken one.
+  The timeout is **derived**, not a free constant: `DeployTimings.ApplyDeadline` (Commons, 2 min — the same
+  value `ConfigPublishCoordinator.DefaultApplyDeadline` reads) minus `DeployTimings.ApplyOutcomeSafetyMargin`
+  (15 s) = **105 s**, so a slow-but-clean rebuild of a large address space is not labelled degraded and the
+  ack still reaches central before its deadline. Change the deadline there and both sides move together.
+  Accepted edge: the timer starts after the artifact read + driver reconcile, so if those alone exceed the
+  15 s margin a never-answered apply can still time the deployment out. A node with no publish actor wired
+  acks clean at once, as before.
 - **The ack carries an ADDITIVE degraded flag on both transports.** `ApplyAck` gained trailing
   `Degraded = false, DegradedReason = null` (an old node's DPS JSON decodes clean); `ApplyAckMessage` gained
   `bool degraded = 6; optional string degraded_reason = 7;` (an old node's gRPC ack decodes clean). The

src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Deploy/DeployTimings.cs

@@ -0,0 +1,25 @@
+namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+
+/// <summary>
+///     The deploy-path timing contract shared by central (<c>ConfigPublishCoordinator</c>) and the node
+///     (<c>DriverHostActor</c>). One place, so the node's ack-hold timeout can never drift past the deadline
+///     central times the deployment out on.
+/// </summary>
+public static class DeployTimings
+{
+    /// <summary>How long central waits for every expected node's ack before sealing the deployment TimedOut
+    /// (<c>ConfigPublishCoordinator.DefaultApplyDeadline</c>).</summary>
+    public static readonly TimeSpan ApplyDeadline = TimeSpan.FromMinutes(2);
+
+    /// <summary>Gitea #601 — how far inside <see cref="ApplyDeadline"/> a node's held Applied ack is released
+    /// when the address-space outcome has not arrived, leaving room for transport latency.</summary>
+    public static readonly TimeSpan ApplyOutcomeSafetyMargin = TimeSpan.FromSeconds(15);
+
+    /// <summary>Gitea #601 — the node's default apply-outcome timeout: <see cref="ApplyDeadline"/> −
+    /// <see cref="ApplyOutcomeSafetyMargin"/> (105 s). Long enough that a slow-but-clean rebuild of a large
+    /// address space is not labelled degraded, short enough to ack before central gives up.</summary>
+    /// <remarks>The timer is armed after the artifact read and driver reconcile, so if those alone took
+    /// longer than the margin the ack can still land after the deadline — accepted: they are seconds in
+    /// practice, and lengthening the margin would label more slow-but-clean rebuilds degraded.</remarks>
+    public static TimeSpan ApplyOutcomeTimeout => ApplyDeadline - ApplyOutcomeSafetyMargin;
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs

@@ -57,7 +57,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 {
     public const string DeploymentsTopic = "deployments";
     public const string DeploymentAcksTopic = "deployment-acks";
-    public static readonly TimeSpan DefaultApplyDeadline = TimeSpan.FromMinutes(2);
+    /// <summary>Central's apply deadline — the shared <see cref="DeployTimings.ApplyDeadline"/>, which the node's
+    /// apply-outcome timeout is derived from (Gitea #601).</summary>
+    public static readonly TimeSpan DefaultApplyDeadline = DeployTimings.ApplyDeadline;
 
     private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
     private readonly TimeSpan _applyDeadline;

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -89,13 +89,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     ///     outcome (<see cref="OpcUa.OpcUaPublishActor.RebuildCompleted"/>) before it is sent anyway.
     /// </summary>
     /// <remarks>
-    ///     Deliberately well inside the coordinator's apply deadline
-    ///     (<c>ConfigPublishCoordinator.DefaultApplyDeadline</c>, 2 min, which also has to cover the artifact
-    ///     read/fetch): a hung or very slow apply must still produce an ack before central times the
-    ///     deployment out. An ack sent on this timeout is marked <b>degraded</b> ("outcome unknown"), never
-    ///     clean — the revision did advance, but nobody has confirmed the address space matches it.
+    ///     Derived from central's apply deadline (<see cref="DeployTimings.ApplyDeadline"/>, 2 min — the same
+    ///     constant <c>ConfigPublishCoordinator.DefaultApplyDeadline</c> reads) minus
+    ///     <see cref="DeployTimings.ApplyOutcomeSafetyMargin"/> (15 s) = 105 s: long enough that a
+    ///     slow-but-clean rebuild of a large address space is not labelled degraded, short enough that the ack
+    ///     still lands before central times the deployment out. An ack sent on this timeout is marked
+    ///     <b>degraded</b> with a reason starting <see cref="OutcomeNotReportedReasonPrefix"/>, never clean —
+    ///     the revision did advance, but nobody has confirmed the address space matches it.
     /// </remarks>
-    public static readonly TimeSpan DefaultApplyOutcomeTimeout = TimeSpan.FromSeconds(60);
+    public static readonly TimeSpan DefaultApplyOutcomeTimeout = DeployTimings.ApplyOutcomeTimeout;
+
+    /// <summary>Gitea #601 — the reason prefix of a degraded ack sent because the outcome TIMED OUT, so an
+    /// operator can tell "not reported in time" from a real sink failure.</summary>
+    public const string OutcomeNotReportedReasonPrefix = "address-space apply outcome not reported in time";
 
     /// <summary>Minimum interval between two supervision reprovisions of the SAME driver child (archreview
     /// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the
@@ -2576,7 +2582,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         if (!_pendingAcks.Remove(msg.Correlation, out var pending)) return;
 
         ReleasePendingAck(pending, degraded: true,
-            $"the address-space apply did not report an outcome within {_applyOutcomeTimeout.TotalSeconds:0}s; it may still be running or may have been lost");
+            $"{OutcomeNotReportedReasonPrefix} ({_applyOutcomeTimeout.TotalSeconds:0}s) — no sink failure was observed; the apply may still be running (slow) or its outcome was lost");
     }
 
     private void ReleasePendingAck(PendingAck pending, bool degraded, string? reason)

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs

@@ -134,7 +134,9 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
         ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
         ack.Degraded.ShouldBeTrue();
         ack.DegradedReason.ShouldNotBeNull();
-        ack.DegradedReason.ShouldContain("did not report an outcome");
+        // Distinct from a real sink failure: it says the outcome was not reported in time.
+        ack.DegradedReason.ShouldStartWith(DriverHostActor.OutcomeNotReportedReasonPrefix);
+        ack.DegradedReason.ShouldNotContain("failedNodes=");
 
         // A late outcome after the timeout must not produce a second ack.
         var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
@@ -199,6 +201,18 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
         }, PresenceBudget);
     }
 
+    [Fact]
+    public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin()
+    {
+        // A slow-but-clean rebuild must not be labelled degraded at 60 s, yet the held ack must still
+        // reach central before its apply deadline. Both sides read ONE shared constant.
+        DeployTimings.ApplyDeadline.ShouldBe(TimeSpan.FromMinutes(2));
+        DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBe(DeployTimings.ApplyDeadline - DeployTimings.ApplyOutcomeSafetyMargin);
+        DeployTimings.ApplyOutcomeSafetyMargin.ShouldBe(TimeSpan.FromSeconds(15));
+        DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeGreaterThan(TimeSpan.FromSeconds(60));
+        DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeLessThan(DeployTimings.ApplyDeadline);
+    }
+
     private (IDbContextFactory<OtOpcUaConfigDbContext> Db, DeploymentId Id) SeedOneTagDeployment()
     {
         var db = NewInMemoryDbFactory();