using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
///
/// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState
/// rows — HandleDispatch threw before its SaveChanges, or the process died between the
/// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected
/// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching
/// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever.
///
public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase
{
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
///
/// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it,
/// the expected set is seeded from the enabled rows, and the node's ack seals it.
///
[Fact]
public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals()
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
SeedNodes(dbFactory, "site-a-1:4053");
// No DispatchDeployment is ever sent — PreStart alone has to recover it.
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
}
///
/// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason
/// naming the condition — never left in Dispatching — and the next deployment proceeds
/// (positive control: it is dispatched, acked and sealed through the same coordinator).
///
[Fact]
public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged()
{
var dbFactory = NewInMemoryDbFactory();
var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
SeedNodes(dbFactory, "site-a-1:4053");
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value);
row.Status.ShouldBe(DeploymentStatus.TimedOut);
row.FailureReason.ShouldNotBeNull();
row.FailureReason.ShouldContain("lost");
}, duration: PresenceBudget);
// Positive control: the coordinator is not wedged — a fresh deployment runs end to end.
var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
}
///
/// An AwaitingApplyAcks row with no rows (the process died between the status flip and the
/// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh
/// dispatch would — the recovery keys on the missing expected set, not on the status value.
///
[Fact]
public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty()
{
var dbFactory = NewInMemoryDbFactory();
SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
db.NodeDeploymentStates.ShouldBeEmpty();
}, duration: PresenceBudget);
}
///
/// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the
/// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and
/// the deployment completes; a second StartDeployment is then Accepted (positive control: a
/// normal deploy seals).
///
[Fact]
public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds()
{
var dbName = Guid.NewGuid().ToString("N");
var adminFactory = NewInMemoryDbFactory(dbName);
var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName));
SeedNodes(adminFactory, "site-a-1:4053");
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty()));
// Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
// the actor would be stopped, not restarted). Identify is processed after creation, so a
// resolved selection proves PreStart completed.
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);
faulting.FailNext();
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var first = ExpectMsg(PresenceBudget);
first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
first.DeploymentId.ShouldNotBeNull();
// The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands.
AwaitAssert(() =>
{
faulting.Faulted.ShouldBeTrue();
using var db = adminFactory.CreateDbContext();
db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1);
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = adminFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
// Positive control: nothing is wedged — the next deploy is Accepted and seals too.
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var second = ExpectMsg(PresenceBudget);
second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = adminFactory.CreateDbContext();
db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2);
}, duration: PresenceBudget);
}
private static DeploymentId SeedDeployment(
IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc)
{
var id = DeploymentId.NewId();
using var db = dbFactory.CreateDbContext();
db.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = TestRevision.Value,
Status = status,
CreatedBy = "test",
CreatedAtUtc = createdAtUtc,
});
db.SaveChanges();
return id;
}
private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds)
{
using var db = dbFactory.CreateDbContext();
db.ServerClusters.Add(new ServerCluster
{
ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
});
foreach (var nodeId in nodeIds)
{
db.ClusterNodes.Add(new ClusterNode
{
NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
});
}
db.SaveChanges();
}
/// Throws from exactly one CreateDbContext call after .
private sealed class OneShotFaultingDbFactory(IDbContextFactory inner)
: IDbContextFactory
{
private int _armed;
public bool Faulted { get; private set; }
public void FailNext() => Interlocked.Exchange(ref _armed, 1);
public OtOpcUaConfigDbContext CreateDbContext()
{
if (Interlocked.Exchange(ref _armed, 0) == 1)
{
Faulted = true;
throw new InvalidOperationException("injected transient fault seeding the expected set");
}
return inner.CreateDbContext();
}
}
}