feat(deploy): additive degraded flag on ApplyAck + publish-actor apply outcome reply (#601)

ApplyAck gains trailing Degraded/DegradedReason (defaulted, so an old node's
DPS JSON or gRPC ack decodes exactly as before). mesh_transport.proto adds
ApplyAckMessage.degraded = 6 / optional degraded_reason = 7. OpcUaPublishActor
replies RebuildCompleted(Degraded, Reason) on every handler path when the
RebuildAddressSpace carries ReportOutcome.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 17:19:57 -04:00
parent 9b74a65b25
commit 521e1eb89b
6 changed files with 286 additions and 11 deletions
@@ -100,8 +100,31 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// that cannot succeed, leaving the rebuild a no-op and clients browsing an empty server. Null
/// on the normal path, where loading from the ConfigDb by id is correct.
/// </param>
/// <param name="ReportOutcome">
/// Gitea #601 — when <see langword="true"/>, the actor replies to the sender with a
/// <see cref="RebuildCompleted"/> once the apply outcome is known (clean or degraded), on EVERY path
/// through the handler. <c>DriverHostActor</c> sets it on the deploy path and sends the deployment's
/// ack only after that reply, so a degraded apply can no longer seal a deployment green. Off for the
/// boot/restore paths, which re-ack nothing.
/// </param>
public sealed record RebuildAddressSpace(
CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null);
CorrelationId Correlation, DeploymentId? DeploymentId = null, byte[]? Artifact = null,
bool ReportOutcome = false);
/// <summary>
/// Gitea #601 — the reply to a <see cref="RebuildAddressSpace"/> carrying
/// <see cref="RebuildAddressSpace.ReportOutcome"/>: whether the address-space apply was clean.
/// </summary>
/// <param name="Correlation">The rebuild's correlation id, echoed.</param>
/// <param name="DeploymentId">The rebuild's deployment id, echoed (null for a legacy caller).</param>
/// <param name="Degraded">
/// <see langword="true"/> when the apply swallowed per-node sink faults, its rebuild failed, the
/// pipeline threw, or there was no usable artifact — i.e. the served address space may be partial or
/// stale. An empty plan (nothing changed) is clean.
/// </param>
/// <param name="DegradedReason">A short operator-readable reason; null when clean.</param>
public sealed record RebuildCompleted(
CorrelationId Correlation, DeploymentId? DeploymentId, bool Degraded, string? DegradedReason);
public sealed record ServiceLevelChanged(byte ServiceLevel);
@@ -437,6 +460,17 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
}
private void HandleRebuild(RebuildAddressSpace msg)
{
// Captured up front: the pipeline below is synchronous on this actor's thread today, but the reply
// must reach whoever asked even if a future edit introduces an await into it.
var requester = Sender;
var (degraded, reason) = HandleRebuildCore(msg);
if (msg.ReportOutcome)
requester.Tell(new RebuildCompleted(msg.Correlation, msg.DeploymentId, degraded, reason));
}
/// <summary>The rebuild pipeline. Returns whether the apply was degraded, and why (Gitea #601).</summary>
private (bool Degraded, string? Reason) HandleRebuildCore(RebuildAddressSpace msg)
{
using var span = OtOpcUaTelemetry.StartAddressSpaceRebuildSpan();
span?.SetTag("otopcua.correlation_id", msg.Correlation.ToString());
@@ -460,8 +494,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
{
_log.Error(ex, "OpcUaPublish: sink.RebuildAddressSpace threw (correlation={Correlation})",
msg.Correlation);
return (true, $"address-space rebuild threw: {ex.Message}");
}
return;
return (false, null);
}
try
@@ -499,7 +534,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
_log.Debug(
"OpcUaPublish: no artifact to materialise yet (deployment={Id}, correlation={Correlation})",
msg.DeploymentId, msg.Correlation);
return;
return (true, "no usable artifact reached the address-space apply; the previously served address space was kept");
}
// Parse the artifact ONCE for this rebuild. Every consumer below (data-plane authority, then the
@@ -532,7 +567,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
{
_log.Debug("OpcUaPublish: rebuild requested but plan is empty (correlation={Correlation})",
msg.Correlation);
return;
return (false, null);
}
var outcome = _applier.Apply(plan);
@@ -608,16 +643,19 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
"OpcUaPublish: address-space apply DEGRADED (correlation={Correlation}, rebuildFailed={RebuildFailed}, failedNodes={FailedNodes}, added={Added}, removed={Removed}, changed={Changed}) — the running address space may be stale or partial",
msg.Correlation, outcome.RebuildFailed, failedNodes,
outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes);
return (true, outcome.RebuildFailed
? $"address-space rebuild failed (failedNodes={failedNodes})"
: $"address-space apply degraded (failedNodes={failedNodes}); not recorded as applied, the next deploy of this revision re-diffs and repairs it");
}
else
{
_log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
}
_log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
return (false, null);
}
catch (Exception ex)
{
_log.Error(ex, "OpcUaPublish: rebuild pipeline threw (correlation={Correlation})", msg.Correlation);
return (true, $"address-space apply threw: {ex.Message}");
}
}