fix(deploy): a node with a ConfigDb records its degraded apply on its own NodeDeploymentState row (#601 review)
The apply writes the node's row Applied/null before the address-space outcome is known; central adds the reason only when it ACCEPTS the ack, so a discarded (stale/timed-out) ack left a fused node's row reading clean. The prefix moves to ApplyAck.DegradedReasonPrefix so both writers spell the row identically. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
@@ -27,4 +27,12 @@ public sealed record ApplyAck(
|
|||||||
string? FailureReason,
|
string? FailureReason,
|
||||||
CorrelationId CorrelationId,
|
CorrelationId CorrelationId,
|
||||||
bool Degraded = false,
|
bool Degraded = false,
|
||||||
string? DegradedReason = null);
|
string? DegradedReason = null)
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gitea #601 — prefix of <c>NodeDeploymentState.FailureReason</c> on an <b>Applied</b> row whose
|
||||||
|
/// address-space apply was degraded. Shared by central (which writes it from the ack) and a node with
|
||||||
|
/// a ConfigDb (which writes its own row), so both spell the row identically.
|
||||||
|
/// </summary>
|
||||||
|
public const string DegradedReasonPrefix = "Applied degraded: ";
|
||||||
|
}
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
|
|||||||
/// degraded node DID apply the revision, so its status stays <see cref="NodeDeploymentStatus.Applied"/>
|
/// degraded node DID apply the revision, so its status stays <see cref="NodeDeploymentStatus.Applied"/>
|
||||||
/// and the degradation rides the existing reason column, which <c>/fleet</c> already renders.
|
/// and the degradation rides the existing reason column, which <c>/fleet</c> already renders.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public const string DegradedReasonPrefix = "Applied degraded: ";
|
public const string DegradedReasonPrefix = ApplyAck.DegradedReasonPrefix;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gitea #601 — whether a <c>NodeDeploymentState</c> row records a degraded apply: status
|
/// Gitea #601 — whether a <c>NodeDeploymentState</c> row records a degraded apply: status
|
||||||
|
|||||||
@@ -2588,6 +2588,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
|||||||
|
|
||||||
if (degraded)
|
if (degraded)
|
||||||
{
|
{
|
||||||
|
// #601 review — the apply wrote this node's own row Applied/null before the outcome was known. Central
|
||||||
|
// adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
|
||||||
|
// is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
|
||||||
|
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
|
||||||
|
ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
|
||||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
|
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
|
||||||
_log.Warning(
|
_log.Warning(
|
||||||
"DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
|
"DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
|
||||||
|
|||||||
+27
@@ -172,6 +172,33 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
|
|||||||
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row()
|
||||||
|
{
|
||||||
|
// #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known.
|
||||||
|
// Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out
|
||||||
|
// deployment's ack is discarded), so the node must not leave its own row reading clean.
|
||||||
|
var (db, deploymentId) = SeedOneTagDeployment();
|
||||||
|
var coordinator = CreateTestProbe();
|
||||||
|
var publish = CreateTestProbe();
|
||||||
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||||
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
||||||
|
opcUaPublishActor: publish.Ref));
|
||||||
|
|
||||||
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||||
|
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
||||||
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, "failedNodes=3"));
|
||||||
|
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeTrue();
|
||||||
|
|
||||||
|
AwaitAssert(() =>
|
||||||
|
{
|
||||||
|
using var ctx = db.CreateDbContext();
|
||||||
|
var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
|
||||||
|
row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
|
||||||
|
row.FailureReason.ShouldBe(ApplyAck.DegradedReasonPrefix + "failedNodes=3");
|
||||||
|
}, PresenceBudget);
|
||||||
|
}
|
||||||
|
|
||||||
private (IDbContextFactory<OtOpcUaConfigDbContext> Db, DeploymentId Id) SeedOneTagDeployment()
|
private (IDbContextFactory<OtOpcUaConfigDbContext> Db, DeploymentId Id) SeedOneTagDeployment()
|
||||||
{
|
{
|
||||||
var db = NewInMemoryDbFactory();
|
var db = NewInMemoryDbFactory();
|
||||||
|
|||||||
Reference in New Issue
Block a user