722a222db5
DeadlineElapsed_for_current_deployment_marks_TimedOut carried two races, both test-side. ConfigPublishCoordinator itself is untouched. (a) The NodeDeploymentState row was inserted AFTER Sys.ActorOf. The coordinator's PreStart reads NodeDeploymentStates to recover the expected-ack set, and it runs as soon as the actor is scheduled — so if PreStart won, the expected-ack set was empty and the coordinator could short-circuit to Sealed instead of ever reaching TimedOut. The row is now seeded before the spawn. (b) The applyDeadline was 150 ms of real wall clock, and it arms a real Akka timer. Under load that timer could fire on its own and race the test's explicit DeadlineElapsed tell — so the test was intermittently exercising the timer rather than the message handler it names. The deadline is now TimeSpan.FromMinutes(5), matching the other two tests in this class, which makes it un-hittable inside the test's own 3 s AwaitAssert bound and leaves the explicit tell as the only deadline signal the coordinator ever sees. That tell, and the AwaitAssert bound, are unchanged. Verified: the single test 5/5 green in a row, then the whole class 3/3. Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
131 lines
5.9 KiB
C#
131 lines
5.9 KiB
C#
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
|
|
|
public sealed class ConfigPublishCoordinatorTimeoutTests : ControlPlaneActorTestBase
|
|
{
|
|
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('b', 64));
|
|
|
|
/// <summary>Verifies that DeadlineElapsed marks the current deployment as timed out.</summary>
|
|
[Fact]
|
|
public void DeadlineElapsed_for_current_deployment_marks_TimedOut()
|
|
{
|
|
var dbFactory = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDispatchingDeployment(dbFactory);
|
|
|
|
// Seed the NodeDeploymentState row BEFORE the actor is spawned. PreStart reads
|
|
// NodeDeploymentStates to recover the expected-ack set, and it runs as soon as Sys.ActorOf
|
|
// schedules the actor — so seeding afterwards is a race: if PreStart wins, the coordinator
|
|
// sees zero expected acks and can short-circuit straight to Sealed, and the assertion below
|
|
// never sees TimedOut. We pretend a driver node exists in the cluster.
|
|
using (var db = dbFactory.CreateDbContext())
|
|
{
|
|
db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = "phantom-driver",
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applying,
|
|
});
|
|
db.SaveChanges();
|
|
}
|
|
|
|
// Deliberately un-hittable wall-clock deadline (matching the other tests in this class).
|
|
// The apply deadline is a real Akka timer: a short one can self-fire under load and race
|
|
// the explicit DeadlineElapsed below, so the test would sometimes be measuring the timer
|
|
// rather than the message handler. Five minutes guarantees the ONLY deadline signal the
|
|
// coordinator ever receives is the one this test sends.
|
|
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));
|
|
|
|
// Drive the deadline ourselves rather than waiting for the cluster's empty driver-set
|
|
// bypass. Tell the actor the deadline elapsed for this id.
|
|
actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId));
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
using var db = dbFactory.CreateDbContext();
|
|
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.TimedOut);
|
|
}, duration: TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
/// <summary>Verifies that stale DeadlineElapsed for other deployments is ignored.</summary>
|
|
[Fact]
|
|
public void Stale_DeadlineElapsed_for_other_deployment_is_ignored()
|
|
{
|
|
var dbFactory = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDispatchingDeployment(dbFactory);
|
|
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(1)));
|
|
|
|
// Tell the actor a deadline elapsed for a completely different deployment id.
|
|
actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(DeploymentId.NewId()));
|
|
|
|
// The seeded one should remain in its starting state (no transition triggered).
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(250));
|
|
using var db = dbFactory.CreateDbContext();
|
|
var status = db.Deployments.Single().Status;
|
|
status.ShouldBeOneOf(DeploymentStatus.Dispatching, DeploymentStatus.AwaitingApplyAcks, DeploymentStatus.Sealed);
|
|
status.ShouldNotBe(DeploymentStatus.TimedOut);
|
|
}
|
|
|
|
/// <summary>Verifies that PreStart recovers in-flight deployment state.</summary>
|
|
[Fact]
|
|
public void PreStart_recovers_inflight_deployment_state()
|
|
{
|
|
var dbFactory = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks);
|
|
|
|
// Seed two NodeDeploymentState rows — one already Applied, one still Applying.
|
|
using (var db = dbFactory.CreateDbContext())
|
|
{
|
|
db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = "driver-a", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
|
|
});
|
|
db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = "driver-b", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying,
|
|
});
|
|
db.SaveChanges();
|
|
}
|
|
|
|
// Start a fresh coordinator — simulates singleton failover to this node.
|
|
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));
|
|
|
|
// Send the missing ACK; the recovered state should expect exactly that node, and the
|
|
// deployment should now seal (both nodes acked Applied).
|
|
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("driver-b"),
|
|
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
using var db = dbFactory.CreateDbContext();
|
|
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
|
}, duration: TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
private static DeploymentId SeedDispatchingDeployment(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
|
DeploymentStatus status = DeploymentStatus.Dispatching)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var db = dbFactory.CreateDbContext();
|
|
db.Deployments.Add(new Configuration.Entities.Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = TestRevision.Value,
|
|
Status = status,
|
|
CreatedBy = "test",
|
|
});
|
|
db.SaveChanges();
|
|
return id;
|
|
}
|
|
}
|