fix(controlplane): close a lost dispatch from the in-flight check; guard terminal rows; blob-free status writes

I7: a Dispatching row committed after the coordinator's PreStart whose Tell never arrived had no owner and refused every later deploy. The in-flight check (now fully off the admin singleton's thread, serialised by the actor-local compose flag alone) closes a Dispatching row with zero NodeDeploymentState rows that is past the apply deadline as TimedOut (optimistic on RowVersion) and proceeds. HandleDispatch ignores a dispatch for a Sealed/PartiallyFailed/TimedOut row, so a late or duplicated dispatch cannot flip it back. Coordinator status/reason writes attach a key+RowVersion stub instead of loading the multi-MB ArtifactBlob; proven on the real SQL Server (blob untouched, seal lands). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 13:46:54 -04:00
parent f7e7ba0263
commit 9466b8b242
3 changed files with 332 additions and 59 deletions
@@ -176,6 +176,141 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes
        }, duration: PresenceBudget);
    }

+    /// <summary>
+    ///     Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the
+    ///     coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between
+    ///     its commit and the Tell). Nothing would ever close it. The in-flight check now closes a
+    ///     Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next
+    ///     StartDeployment is Accepted (and seals — positive control on the pipeline).
+    /// </summary>
+    [Fact]
+    public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        SeedNodes(dbFactory, "site-a-1:4053");
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);   // PreStart saw nothing
+
+        var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
+        var admin = Sys.ActorOf(AdminOperationsActor.Props(
+            dbFactory, coordinator, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
+
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message);
+
+        using (var db = dbFactory.CreateDbContext())
+        {
+            var row = db.Deployments.Single(d => d.DeploymentId == stale.Value);
+            row.Status.ShouldBe(DeploymentStatus.TimedOut);
+            row.FailureReason.ShouldNotBeNull();
+            row.FailureReason.ShouldContain("lost");
+        }
+
+        coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status
+                .ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>Control for the closure: the same row INSIDE the deadline is still in flight and still refuses.</summary>
+    [Fact]
+    public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var coordinator = CreateTestProbe();
+        var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5));
+        var admin = Sys.ActorOf(AdminOperationsActor.Props(
+            dbFactory, coordinator.Ref, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
+
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight);
+        result.DeploymentId.ShouldBe(pending);
+
+        using var db = dbFactory.CreateDbContext();
+        db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
+    }
+
+    /// <summary>
+    ///     Fix round 1 minor — a late or duplicated <c>DispatchDeployment</c> for a row that is already
+    ///     terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip
+    ///     it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right
+    ///     after it IS seeded (so the first message was processed, not merely not-yet-processed).
+    /// </summary>
+    [Theory]
+    [InlineData(DeploymentStatus.Sealed)]
+    [InlineData(DeploymentStatus.TimedOut)]
+    [InlineData(DeploymentStatus.PartiallyFailed)]
+    public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal)
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        SeedNodes(dbFactory, "site-a-1:4053");
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+
+        var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1));
+        coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId()));
+        var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId()));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        using var verify = dbFactory.CreateDbContext();
+        verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal);
+        verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of
+    ///     materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the
+    ///     optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are
+    ///     untouched by the partial UPDATEs.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "SchemaCompliance")]
+    public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob()
+    {
+        using var sql = new SqlServerTestDatabase();
+        var factory = sql.Factory();
+        SeedNodes(factory, "site-a-1:4053");
+        var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray();
+        var id = DeploymentId.NewId();
+        using (var db = sql.NewContext())
+        {
+            db.Deployments.Add(new Deployment
+            {
+                DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching,
+                CreatedBy = "test", ArtifactBlob = blob,
+            });
+            db.SaveChanges();
+        }
+
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout));
+        AwaitAssert(() =>
+        {
+            using var db = sql.NewContext();
+            db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);   // PreStart re-drives the un-dispatched row
+        coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+
+        AwaitAssert(() =>
+        {
+            using var db = sql.NewContext();
+            var row = db.Deployments.AsNoTracking().Single();
+            row.Status.ShouldBe(DeploymentStatus.Sealed);
+            row.SealedAtUtc.ShouldNotBeNull();
+            row.ArtifactBlob.ShouldBe(blob);
+        }, duration: PresenceBudget);
+    }
+
    private static DeploymentId SeedDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
    {