using Akka.Actor; using Microsoft.EntityFrameworkCore; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; /// /// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState /// rows — HandleDispatch threw before its SaveChanges, or the process died between the /// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected /// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching /// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. /// public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase { private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); /// /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, /// the expected set is seeded from the enabled rows, and the node's ack seals it. /// [Fact] public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); SeedNodes(dbFactory, "site-a-1:4053"); // No DispatchDeployment is ever sent — PreStart alone has to recover it. var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); } /// /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason /// naming the condition — never left in Dispatching — and the next deployment proceeds /// (positive control: it is dispatched, acked and sealed through the same coordinator). /// [Fact] public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() { var dbFactory = NewInMemoryDbFactory(); var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); SeedNodes(dbFactory, "site-a-1:4053"); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); row.Status.ShouldBe(DeploymentStatus.TimedOut); row.FailureReason.ShouldNotBeNull(); row.FailureReason.ShouldContain("lost"); }, duration: PresenceBudget); // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); } /// /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh /// dispatch would — the recovery keys on the missing expected set, not on the status value. /// [Fact] public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() { var dbFactory = NewInMemoryDbFactory(); SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); db.NodeDeploymentStates.ShouldBeEmpty(); }, duration: PresenceBudget); } /// /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and /// the deployment completes; a second StartDeployment is then Accepted (positive control: a /// normal deploy seals). /// [Fact] public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() { var dbName = Guid.NewGuid().ToString("N"); var adminFactory = NewInMemoryDbFactory(dbName); var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); SeedNodes(adminFactory, "site-a-1:4053"); var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and // the actor would be stopped, not restarted). Identify is processed after creation, so a // resolved selection proves PreStart completed. await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); faulting.FailNext(); admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var first = ExpectMsg(PresenceBudget); first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); first.DeploymentId.ShouldNotBeNull(); // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. AwaitAssert(() => { faulting.Faulted.ShouldBeTrue(); using var db = adminFactory.CreateDbContext(); db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = adminFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); // Positive control: nothing is wedged — the next deploy is Accepted and seals too. admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var second = ExpectMsg(PresenceBudget); second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = adminFactory.CreateDbContext(); db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); }, duration: PresenceBudget); } private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { var id = DeploymentId.NewId(); using var db = dbFactory.CreateDbContext(); db.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = status, CreatedBy = "test", CreatedAtUtc = createdAtUtc, }); db.SaveChanges(); return id; } private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) { using var db = dbFactory.CreateDbContext(); db.ServerClusters.Add(new ServerCluster { ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", }); foreach (var nodeId in nodeIds) { db.ClusterNodes.Add(new ClusterNode { NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", }); } db.SaveChanges(); } /// Throws from exactly one CreateDbContext call after . private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) : IDbContextFactory { private int _armed; public bool Faulted { get; private set; } public void FailNext() => Interlocked.Exchange(ref _armed, 1); public OtOpcUaConfigDbContext CreateDbContext() { if (Interlocked.Exchange(ref _armed, 0) == 1) { Faulted = true; throw new InvalidOperationException("injected transient fault seeding the expected set"); } return inner.CreateDbContext(); } } }