dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge #868 review fixes: recovered fully-acked deployments seal, stashed acks beat a past deadline, failed ack writes retry

Browse Source 
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 12:38:57 -05:00
parent 2bdff975fa 19c7b5d7e4
commit ff85b8511a
2 changed files with 204 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs
+111
View File
@@ -115,6 +115,105 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Review finding 1: a recovered deployment whose every expected ack is already in the DB (a seal whose
/// SaveChanges failed, or a fused node's own row) must be completed — inside the deadline it used to wait for
/// acks that had already landed, past it the deadline's "race" guard returned without sealing; either way it
/// sat in AwaitingApplyAcks forever and AdminOps refused every later deploy.
/// </summary>
[Theory]
[InlineData(false, false, DeploymentStatus.Sealed)]
[InlineData(true, false, DeploymentStatus.Sealed)]
[InlineData(false, true, DeploymentStatus.PartiallyFailed)]
[InlineData(true, true, DeploymentStatus.PartiallyFailed)]
public void Recovered_deployment_with_every_ack_already_persisted_is_completed(bool pastDeadline, bool oneFailed, DeploymentStatus expected)
{
var dbFactory = NewInMemoryDbFactory();
SeedNodes(dbFactory, Node, "site-a-2:4053");
var deadline = TimeSpan.FromMinutes(1);
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks,
pastDeadline ? DateTime.UtcNow.AddMinutes(-10) : DateTime.UtcNow);
SeedNodeState(dbFactory, deploymentId, Node, NodeDeploymentStatus.Applied);
SeedNodeState(dbFactory, deploymentId, "site-a-2:4053", oneFailed ? NodeDeploymentStatus.Failed : NodeDeploymentStatus.Applied);
Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(expected);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Review finding 2: a recovered deployment that is already past its deadline must first apply the acks that
/// were queued while the load was failing (they arrived in time); the deadline used to run inline before the
/// replay, so they were discarded as stale and the deployment was timed out.
/// </summary>
[Fact]
public void Ack_queued_during_the_outage_is_applied_before_a_past_deadline_recovery_times_out()
{
var dbName = Guid.NewGuid().ToString("N");
var direct = NewInMemoryDbFactory(dbName);
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
SeedNodes(direct, Node);
var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow.AddMinutes(-10));
SeedNodeState(direct, deploymentId, Node, NodeDeploymentStatus.Applying);
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
outage, TimeSpan.FromMinutes(1), loadRetryMin: RetryMin, loadRetryMax: RetryMax));
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
outage.Down = false;
AwaitAssert(() =>
{
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
/// <summary>
/// Review finding 3: an ack whose persist fails (ConfigDb blip while Ready) is retried, not dropped. It used to
/// throw, restart the actor and lose the ack — the node never resends and never writes its own row.
/// </summary>
[Fact]
public void Ack_whose_persist_fails_is_retried_not_dropped()
{
var dbName = Guid.NewGuid().ToString("N");
var direct = NewInMemoryDbFactory(dbName);
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
SeedNodes(direct, Node);
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
var readiness = new ConfigPublishCoordinatorReadiness();
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
outage.Down = true;
var before = outage.Attempts;
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(before + 2),
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
outage.Down = false;
AwaitAssert(() =>
{
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
[Fact]
public void Readiness_is_NotHosted_after_the_coordinator_stops()
{
@@ -146,6 +245,18 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes
return id;
}
private static void SeedNodeState(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentId deploymentId, string nodeId, NodeDeploymentStatus status)
{
using var db = dbFactory.CreateDbContext();
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = nodeId, DeploymentId = deploymentId.Value, Status = status,
AppliedAtUtc = status == NodeDeploymentStatus.Applying ? null : DateTime.UtcNow,
});
db.SaveChanges();
}
private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
{
using var db = dbFactory.CreateDbContext();