diff --git a/CLAUDE.md b/CLAUDE.md index d1a2535d..7c5eff6d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -185,7 +185,7 @@ is still default-closed; what changed is which deltas it whitelists. apply's removes (a re-remove returns false and ratchets to a rebuild, which is the correct, bounded resync). A **rebuild** still advances even with failures. - **…and the deploy ack now SAYS so (#601).** The Applied ack waits for `OpcUaPublishActor`'s - `RebuildCompleted` (60 s timeout ⇒ degraded, never clean) and carries an additive `Degraded`/`DegradedReason` + `RebuildCompleted` (timeout = shared `DeployTimings.ApplyDeadline` − 15 s = 105 s ⇒ degraded "outcome not reported in time", never clean) and carries an additive `Degraded`/`DegradedReason` on both ack transports; central seals but records `NodeDeploymentState.FailureReason = "Applied degraded: …"` (`/fleet` "Applied (degraded)"), and a re-dispatch of a degraded revision re-applies instead of short-circuiting. See `docs/Redundancy.md` §"What an `Applied` ack means". diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 94ad1a4c..e1f5d67c 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -598,9 +598,16 @@ deployment green fleet-wide. Now: is clean), and only then is the ack sent. It is a plain `Tell` + reply message — no `Ask`, no `await`, so nothing resumes off the actor thread. - **A missing outcome is degraded, never clean.** If no reply arrives within - `DriverHostActor.DefaultApplyOutcomeTimeout` (60 s — inside the coordinator's 2-minute apply deadline, - which also has to cover the artifact read/fetch), the ack goes out `Applied` + degraded - (*"did not report an outcome within 60s"*). A node with no publish actor wired acks clean at once, as before. + `DriverHostActor.DefaultApplyOutcomeTimeout` the ack goes out `Applied` + degraded, with a reason starting + *"address-space apply outcome not reported in time"* — distinct from a real sink failure (which carries + `failedNodes=` / "rebuild failed" / "threw"), so an operator can tell a slow or lost apply from a broken one. + The timeout is **derived**, not a free constant: `DeployTimings.ApplyDeadline` (Commons, 2 min — the same + value `ConfigPublishCoordinator.DefaultApplyDeadline` reads) minus `DeployTimings.ApplyOutcomeSafetyMargin` + (15 s) = **105 s**, so a slow-but-clean rebuild of a large address space is not labelled degraded and the + ack still reaches central before its deadline. Change the deadline there and both sides move together. + Accepted edge: the timer starts after the artifact read + driver reconcile, so if those alone exceed the + 15 s margin a never-answered apply can still time the deployment out. A node with no publish actor wired + acks clean at once, as before. - **The ack carries an ADDITIVE degraded flag on both transports.** `ApplyAck` gained trailing `Degraded = false, DegradedReason = null` (an old node's DPS JSON decodes clean); `ApplyAckMessage` gained `bool degraded = 6; optional string degraded_reason = 7;` (an old node's gRPC ack decodes clean). The diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Deploy/DeployTimings.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Deploy/DeployTimings.cs new file mode 100644 index 00000000..1f0866de --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Deploy/DeployTimings.cs @@ -0,0 +1,25 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; + +/// +/// The deploy-path timing contract shared by central (ConfigPublishCoordinator) and the node +/// (DriverHostActor). One place, so the node's ack-hold timeout can never drift past the deadline +/// central times the deployment out on. +/// +public static class DeployTimings +{ + /// How long central waits for every expected node's ack before sealing the deployment TimedOut + /// (ConfigPublishCoordinator.DefaultApplyDeadline). + public static readonly TimeSpan ApplyDeadline = TimeSpan.FromMinutes(2); + + /// Gitea #601 — how far inside a node's held Applied ack is released + /// when the address-space outcome has not arrived, leaving room for transport latency. + public static readonly TimeSpan ApplyOutcomeSafetyMargin = TimeSpan.FromSeconds(15); + + /// Gitea #601 — the node's default apply-outcome timeout: − + /// (105 s). Long enough that a slow-but-clean rebuild of a large + /// address space is not labelled degraded, short enough to ack before central gives up. + /// The timer is armed after the artifact read and driver reconcile, so if those alone took + /// longer than the margin the ack can still land after the deadline — accepted: they are seconds in + /// practice, and lengthening the margin would label more slow-but-clean rebuilds degraded. + public static TimeSpan ApplyOutcomeTimeout => ApplyDeadline - ApplyOutcomeSafetyMargin; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index d45a20e4..32b31116 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -57,7 +57,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers { public const string DeploymentsTopic = "deployments"; public const string DeploymentAcksTopic = "deployment-acks"; - public static readonly TimeSpan DefaultApplyDeadline = TimeSpan.FromMinutes(2); + /// Central's apply deadline — the shared , which the node's + /// apply-outcome timeout is derived from (Gitea #601). + public static readonly TimeSpan DefaultApplyDeadline = DeployTimings.ApplyDeadline; private readonly IDbContextFactory _dbFactory; private readonly TimeSpan _applyDeadline; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 6bb1f3f4..b5ca4204 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -89,13 +89,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// outcome () before it is sent anyway. /// /// - /// Deliberately well inside the coordinator's apply deadline - /// (ConfigPublishCoordinator.DefaultApplyDeadline, 2 min, which also has to cover the artifact - /// read/fetch): a hung or very slow apply must still produce an ack before central times the - /// deployment out. An ack sent on this timeout is marked degraded ("outcome unknown"), never - /// clean — the revision did advance, but nobody has confirmed the address space matches it. + /// Derived from central's apply deadline (, 2 min — the same + /// constant ConfigPublishCoordinator.DefaultApplyDeadline reads) minus + /// (15 s) = 105 s: long enough that a + /// slow-but-clean rebuild of a large address space is not labelled degraded, short enough that the ack + /// still lands before central times the deployment out. An ack sent on this timeout is marked + /// degraded with a reason starting , never clean — + /// the revision did advance, but nobody has confirmed the address space matches it. /// - public static readonly TimeSpan DefaultApplyOutcomeTimeout = TimeSpan.FromSeconds(60); + public static readonly TimeSpan DefaultApplyOutcomeTimeout = DeployTimings.ApplyOutcomeTimeout; + + /// Gitea #601 — the reason prefix of a degraded ack sent because the outcome TIMED OUT, so an + /// operator can tell "not reported in time" from a real sink failure. + public const string OutcomeNotReportedReasonPrefix = "address-space apply outcome not reported in time"; /// Minimum interval between two supervision reprovisions of the SAME driver child (archreview /// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the @@ -2576,7 +2582,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS if (!_pendingAcks.Remove(msg.Correlation, out var pending)) return; ReleasePendingAck(pending, degraded: true, - $"the address-space apply did not report an outcome within {_applyOutcomeTimeout.TotalSeconds:0}s; it may still be running or may have been lost"); + $"{OutcomeNotReportedReasonPrefix} ({_applyOutcomeTimeout.TotalSeconds:0}s) — no sink failure was observed; the apply may still be running (slow) or its outcome was lost"); } private void ReleasePendingAck(PendingAck pending, bool degraded, string? reason) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs index 5da97d25..e79f0ec2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs @@ -134,7 +134,9 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeTrue(); ack.DegradedReason.ShouldNotBeNull(); - ack.DegradedReason.ShouldContain("did not report an outcome"); + // Distinct from a real sink failure: it says the outcome was not reported in time. + ack.DegradedReason.ShouldStartWith(DriverHostActor.OutcomeNotReportedReasonPrefix); + ack.DegradedReason.ShouldNotContain("failedNodes="); // A late outcome after the timeout must not produce a second ack. var rebuild = publish.FishForMessage(_ => true, PresenceBudget); @@ -199,6 +201,18 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase }, PresenceBudget); } + [Fact] + public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin() + { + // A slow-but-clean rebuild must not be labelled degraded at 60 s, yet the held ack must still + // reach central before its apply deadline. Both sides read ONE shared constant. + DeployTimings.ApplyDeadline.ShouldBe(TimeSpan.FromMinutes(2)); + DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBe(DeployTimings.ApplyDeadline - DeployTimings.ApplyOutcomeSafetyMargin); + DeployTimings.ApplyOutcomeSafetyMargin.ShouldBe(TimeSpan.FromSeconds(15)); + DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeGreaterThan(TimeSpan.FromSeconds(60)); + DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeLessThan(DeployTimings.ApplyDeadline); + } + private (IDbContextFactory Db, DeploymentId Id) SeedOneTagDeployment() { var db = NewInMemoryDbFactory();