fix(controlplane,runtime): #868 third review — bounded failure reasons, poison ack writes, cache-booted revisions unconfirmed

1 PersistNodeAck wrote a Failed ack's FailureReason unbounded; the column is
  nvarchar(2048), so on SQL Server the write failed forever, the ack retried
  forever, the deadline deferred forever and every later deploy was refused.
  - One shared bound, Configuration.FailureReasons.Truncate (MaxLength 2048),
    used by the coordinator and DriverHostActor.UpsertNodeDeploymentState
    (exception messages and degraded reasons).
  - A non-transient DbUpdateException (not concurrency, no transient
    DbException / timeout / socket / IO underneath) is POISON: the ack is
    counted from memory with an Error, never retried.
  - Deadline deferral on unpersisted acks is capped at 10 x the retry ceiling;
    past it the outcome is decided from the acks held in memory.
2 A node restarting in the outcome window boots the cached revision (cache is
  written before the outcome) and never acks. Cache-booted revisions
  (BootstrapFromCache / TryBootFromCache) are now marked unconfirmed, so a
  re-dispatch re-applies and acks the real outcome instead of a clean
  short-circuit. docs/Redundancy.md corrected (replay only on the ConfigDb boot
  path).
3 A lost dispatch re-driven near its deadline now gets a FULL deadline (a node
  needs its whole outcome timeout + margin = ApplyDeadline, so any shorter
  remainder timed out nodes that applied). No dispatched-at column: a second
  restart recomputes from CreatedAtUtc and closes it instead (documented).
4 docs/Redundancy.md upgrade note: do not deploy while the central pair is mid
  rolling-upgrade across this change.

Tests (all fail on the previous code): oversized Failed reason truncated (on a
SaveChanges interceptor that enforces the column like SQL Server); rejected ack
write counted from memory; bounded deadline deferral; full deadline on a
near-deadline re-drive; cache-booted revision re-applies on re-dispatch;
oversized degraded reason truncated on the node's own row.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:13:36 -05:00
parent 74496b4fec
commit 780c87eba3
8 changed files with 378 additions and 24 deletions
@@ -402,6 +402,115 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Third review #1: a Failed ack's reason (node free text) was written unbounded; on SQL Server the overflow
/// refuses the SaveChanges forever and the deployment wedged. It is truncated to the column.
/// </summary>
[Fact]
public void An_oversized_failure_reason_is_truncated_not_retried_forever()
{
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed,
new string('x', 5000), CorrelationId.NewId()));
AwaitAssert(() =>
{
using var ctx = db.CreateDbContext();
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.PartiallyFailed);
var row = ctx.NodeDeploymentStates.Single();
row.Status.ShouldBe(NodeDeploymentStatus.Failed);
row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Third review #1: a write the DB REJECTS (non-transient DbUpdateException) is not retried forever — the ack is
/// counted from memory and the deployment completes.
/// </summary>
[Fact]
public void A_rejected_ack_write_is_counted_from_memory_not_retried_forever()
{
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
db.PoisonNodeStateUpdates = true;
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var ctx = db.CreateDbContext();
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Third review #1: a write that keeps failing transiently defers the deadline only up to a cap
/// (10 × the retry ceiling); past it the outcome is decided from the acks held in memory.
/// </summary>
[Fact]
public void Deadline_deferral_on_unpersisted_acks_is_bounded()
{
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(TimeSpan.FromSeconds(1));
db.FailNodeStateUpdatesTransiently = true;
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
// Deadline 1 s + cap 10 × 200 ms: decided within a few seconds, never left in flight.
AwaitAssert(() =>
{
using var ctx = db.CreateDbContext();
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Third review #3: a lost dispatch recovered with only a sliver of its deadline left used to be re-driven with
/// that sliver — the nodes applied and the deployment timed out moments later. It gets a full deadline.
/// </summary>
[Fact]
public void A_lost_dispatch_redriven_near_its_deadline_gets_a_full_deadline()
{
var dbFactory = NewInMemoryDbFactory();
SeedNodes(dbFactory, Node);
var deadline = TimeSpan.FromSeconds(3);
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow - TimeSpan.FromSeconds(2.5));
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
// Well past the 0.5 s that was left, still inside a fresh 3 s deadline.
Thread.Sleep(TimeSpan.FromSeconds(1.5));
using (var db = dbFactory.CreateDbContext())
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
private (ConstraintEnforcingDbFactory Db, DeploymentId Id, IActorRef Coordinator) DispatchedOnConstraintDb(TimeSpan applyDeadline)
{
var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N"));
SeedNodes(db, Node);
var deploymentId = SeedDeployment(db, DeploymentStatus.Dispatching, DateTime.UtcNow);
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
db, applyDeadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var ctx = db.CreateDbContext();
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
return (db, deploymentId, coordinator);
}
/// <summary>A single-node deployment dispatched and AwaitingApplyAcks through an outage-capable factory.</summary>
private (IDbContextFactory<OtOpcUaConfigDbContext> Direct, OutageDbFactory Outage, DeploymentId Id, IActorRef Coordinator,
ConfigPublishCoordinatorReadiness Readiness) DispatchedSingleNode(TimeSpan applyDeadline)
@@ -0,0 +1,69 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
/// <summary>
/// An in-memory ConfigDb that behaves like SQL Server where the InMemory provider does not (lmxopcua#868 third
/// review): a <c>FailureReason</c> longer than the column refuses the whole <c>SaveChanges</c> with a
/// <see cref="DbUpdateException"/>. It can also reject (<see cref="PoisonNodeStateUpdates"/>, a non-transient
/// <see cref="DbUpdateException"/>) or fail transiently (<see cref="FailNodeStateUpdatesTransiently"/>) every
/// update of an existing <c>NodeDeploymentState</c> row.
/// </summary>
public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
{
private readonly Interceptor _interceptor = new();
/// <summary>Every update of an existing node row is rejected as a constraint violation.</summary>
public bool PoisonNodeStateUpdates { get => _interceptor.Poison; set => _interceptor.Poison = value; }
/// <summary>Every update of an existing node row fails as if the connection dropped.</summary>
public bool FailNodeStateUpdatesTransiently { get => _interceptor.Transient; set => _interceptor.Transient = value; }
/// <inheritdoc />
public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseInMemoryDatabase(dbName)
.AddInterceptors(_interceptor)
.Options);
private sealed class Interceptor : SaveChangesInterceptor
{
public volatile bool Poison;
public volatile bool Transient;
public override InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
{
Check(eventData.Context!);
return result;
}
public override ValueTask<InterceptionResult<int>> SavingChangesAsync(
DbContextEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default)
{
Check(eventData.Context!);
return ValueTask.FromResult(result);
}
private void Check(DbContext ctx)
{
foreach (var e in ctx.ChangeTracker.Entries())
{
var reason = e.Entity switch
{
NodeDeploymentState n => n.FailureReason,
Deployment d => d.FailureReason,
_ => null,
};
if (reason is { Length: > FailureReasons.MaxLength })
throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
if (e.Entity is NodeDeploymentState && e.State == EntityState.Modified)
{
if (Poison) throw new DbUpdateException("The UPDATE statement conflicted with a CHECK constraint.");
if (Transient) throw new InvalidOperationException("A network-related error occurred (simulated).");
}
}
}
}
}
@@ -56,6 +56,31 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
cache.UnkeyedReads.ShouldBe(1);
}
/// <summary>
/// lmxopcua#868 third review #2: the cache is written before the address-space outcome, so a cache-booted
/// revision may never have been acked. A re-dispatch of it must RE-APPLY (here: fail honestly, since the
/// ConfigDb is unreachable) — never short-circuit a clean Applied for an outcome nobody confirmed.
/// </summary>
[Fact]
public void A_cache_booted_revision_is_unconfirmed_so_a_redispatch_reapplies_instead_of_acking_clean()
{
var cached = new CachedDeploymentArtifact(
DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow.AddMinutes(-1));
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, ackRouter: coordinator.Ref,
localRoles: new HashSet<string> { "driver" },
deploymentArtifactCache: new StubArtifactCache(cached)));
AskDiagnostics(actor).CurrentRevision.ShouldBe(CachedRev);
actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.DispatchDeployment(
DeploymentId.NewId(), CachedRev, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg<ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAck>(
PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ack.Outcome.ShouldNotBe(ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAckOutcome.Applied);
}
[Fact]
public void BootFromCache_HandsTheCachedArtifactToTheAddressSpaceRebuild()
{
@@ -238,6 +238,58 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
[Fact]
public void An_oversized_degraded_reason_is_truncated_so_the_local_row_still_records_the_outcome()
{
// lmxopcua#868 third review #1: the row's FailureReason is nvarchar(2048). An unbounded reason made SQL Server
// refuse the write (logged and swallowed), so the row stayed Applying forever. A DB that enforces the column.
var db = new LengthEnforcingDbFactory(Guid.NewGuid().ToString("N"));
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish.Ref));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, new string('f', 5000)));
coordinator.ExpectMsg<ApplyAck>(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeTrue();
AwaitAssert(() =>
{
using var ctx = db.CreateDbContext();
var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
row.FailureReason.ShouldStartWith(ApplyAck.DegradedReasonPrefix);
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>In-memory ConfigDb that refuses an over-length FailureReason, as SQL Server does.</summary>
private sealed class LengthEnforcingDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseInMemoryDatabase(dbName)
.AddInterceptors(new Enforcer())
.Options);
private sealed class Enforcer : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor
{
public override Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult<int> SavingChanges(
Microsoft.EntityFrameworkCore.Diagnostics.DbContextEventData eventData,
Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult<int> result)
{
foreach (var e in eventData.Context!.ChangeTracker.Entries<ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState>())
if (e.Entity.FailureReason is { Length: > FailureReasons.MaxLength })
throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
return result;
}
}
}
[Fact]
public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin()
{