using Akka.Actor; using Microsoft.EntityFrameworkCore; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; /// /// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState /// rows — HandleDispatch threw before its SaveChanges, or the process died between the /// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected /// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching /// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. /// public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase { private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); /// /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, /// the expected set is seeded from the enabled rows, and the node's ack seals it. /// [Fact] public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() { var dbFactory = NewInMemoryDbFactory(); var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); SeedNodes(dbFactory, "site-a-1:4053"); // No DispatchDeployment is ever sent — PreStart alone has to recover it. var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); } /// /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason /// naming the condition — never left in Dispatching — and the next deployment proceeds /// (positive control: it is dispatched, acked and sealed through the same coordinator). /// [Fact] public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() { var dbFactory = NewInMemoryDbFactory(); var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); SeedNodes(dbFactory, "site-a-1:4053"); var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); row.Status.ShouldBe(DeploymentStatus.TimedOut); row.FailureReason.ShouldNotBeNull(); row.FailureReason.ShouldContain("lost"); }, duration: PresenceBudget); // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); } /// /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh /// dispatch would — the recovery keys on the missing expected set, not on the status value. /// [Fact] public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() { var dbFactory = NewInMemoryDbFactory(); SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); db.NodeDeploymentStates.ShouldBeEmpty(); }, duration: PresenceBudget); } /// /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and /// the deployment completes; a second StartDeployment is then Accepted (positive control: a /// normal deploy seals). /// [Fact] public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() { var dbName = Guid.NewGuid().ToString("N"); var adminFactory = NewInMemoryDbFactory(dbName); var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); SeedNodes(adminFactory, "site-a-1:4053"); var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and // the actor would be stopped, not restarted). Identify is processed after creation, so a // resolved selection proves PreStart completed. await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); faulting.FailNext(); admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var first = ExpectMsg(PresenceBudget); first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); first.DeploymentId.ShouldNotBeNull(); // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. AwaitAssert(() => { faulting.Faulted.ShouldBeTrue(); using var db = adminFactory.CreateDbContext(); db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = adminFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); // Positive control: nothing is wedged — the next deploy is Accepted and seals too. admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var second = ExpectMsg(PresenceBudget); second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = adminFactory.CreateDbContext(); db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); }, duration: PresenceBudget); } /// /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next /// StartDeployment is Accepted (and seals — positive control on the pipeline). /// [Fact] public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() { var dbFactory = NewInMemoryDbFactory(); SeedNodes(dbFactory, "site-a-1:4053"); var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); var admin = Sys.ActorOf(AdminOperationsActor.Props( dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var result = ExpectMsg(PresenceBudget); result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); using (var db = dbFactory.CreateDbContext()) { var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); row.Status.ShouldBe(DeploymentStatus.TimedOut); row.FailureReason.ShouldNotBeNull(); row.FailureReason.ShouldContain("lost"); } coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status .ShouldBe(DeploymentStatus.Sealed); }, duration: PresenceBudget); } /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. [Fact] public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() { var dbFactory = NewInMemoryDbFactory(); var coordinator = CreateTestProbe(); var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); var admin = Sys.ActorOf(AdminOperationsActor.Props( dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); var result = ExpectMsg(PresenceBudget); result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); result.DeploymentId.ShouldBe(pending); using var db = dbFactory.CreateDbContext(); db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); } /// /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right /// after it IS seeded (so the first message was processed, not merely not-yet-processed). /// [Theory] [InlineData(DeploymentStatus.Sealed)] [InlineData(DeploymentStatus.TimedOut)] [InlineData(DeploymentStatus.PartiallyFailed)] public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) { var dbFactory = NewInMemoryDbFactory(); SeedNodes(dbFactory, "site-a-1:4053"); var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); AwaitAssert(() => { using var db = dbFactory.CreateDbContext(); db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); using var verify = dbFactory.CreateDbContext(); verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); } /// /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are /// untouched by the partial UPDATEs. /// [Fact] [Trait("Category", "SchemaCompliance")] public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() { using var sql = new SqlServerTestDatabase(); var factory = sql.Factory(); SeedNodes(factory, "site-a-1:4053"); var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); var id = DeploymentId.NewId(); using (var db = sql.NewContext()) { db.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, CreatedBy = "test", ArtifactBlob = blob, }); db.SaveChanges(); } var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); AwaitAssert(() => { using var db = sql.NewContext(); db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); AwaitAssert(() => { using var db = sql.NewContext(); var row = db.Deployments.AsNoTracking().Single(); row.Status.ShouldBe(DeploymentStatus.Sealed); row.SealedAtUtc.ShouldNotBeNull(); row.ArtifactBlob.ShouldBe(blob); }, duration: PresenceBudget); } private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { var id = DeploymentId.NewId(); using var db = dbFactory.CreateDbContext(); db.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = status, CreatedBy = "test", CreatedAtUtc = createdAtUtc, }); db.SaveChanges(); return id; } private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) { using var db = dbFactory.CreateDbContext(); db.ServerClusters.Add(new ServerCluster { ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", }); foreach (var nodeId in nodeIds) { db.ClusterNodes.Add(new ClusterNode { NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", }); } db.SaveChanges(); } /// Throws from exactly one CreateDbContext call after . private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) : IDbContextFactory { private int _armed; public bool Faulted { get; private set; } public void FailNext() => Interlocked.Exchange(ref _armed, 1); public OtOpcUaConfigDbContext CreateDbContext() { if (Interlocked.Exchange(ref _armed, 0) == 1) { Faulted = true; throw new InvalidOperationException("injected transient fault seeding the expected set"); } return inner.CreateDbContext(); } } }