From 9466b8b242f80d36f3ef6c481a6058e9eee46722 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:46:54 -0400 Subject: [PATCH] fix(controlplane): close a lost dispatch from the in-flight check; guard terminal rows; blob-free status writes I7: a Dispatching row committed after the coordinator's PreStart whose Tell never arrived had no owner and refused every later deploy. The in-flight check (now fully off the admin singleton's thread, serialised by the actor-local compose flag alone) closes a Dispatching row with zero NodeDeploymentState rows that is past the apply deadline as TimedOut (optimistic on RowVersion) and proceeds. HandleDispatch ignores a dispatch for a Sealed/PartiallyFailed/TimedOut row, so a late or duplicated dispatch cannot flip it back. Coordinator status/reason writes attach a key+RowVersion stub instead of loading the multi-MB ArtifactBlob; proven on the real SQL Server (blob untouched, seal lands). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 172 +++++++++++++----- .../Coordinators/ConfigPublishCoordinator.cs | 84 +++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 135 ++++++++++++++ 3 files changed, 332 insertions(+), 59 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index e78c926e..81f09dae 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -28,6 +29,12 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + /// + /// The coordinator's apply deadline. A Dispatching row with no expected-ack rows that is older + /// than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7). + /// + private readonly TimeSpan _applyDeadline; + /// /// Where central→node commands are handed off (per-cluster mesh), or /// to publish on the mediator directly. @@ -48,15 +55,17 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// + /// The coordinator's apply deadline; defaults to . /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) => + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) => Akka.Actor.Props.Create(() => - new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. @@ -64,13 +73,16 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. + /// The coordinator's apply deadline; defaults to . public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) { + _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); @@ -201,16 +213,17 @@ public sealed class AdminOperationsActor : ReceiveActor } /// - /// Serialised part of StartDeployment: the in-flight check. Everything after it — the - /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a - /// / message, so a large compose no - /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// StartDeployment. The only serialised part is the actor-local + /// flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the + /// thread pool and come back as a / + /// message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the + /// in-flight query off the actor thread too) and alarm acks keep flowing while it runs. /// /// - /// is what keeps the check correct once the compose is off-actor: the - /// Deployment row is written at the END of the compose, so a second request arriving mid-compose - /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared - /// when the outcome message lands, on both branches. + /// is what keeps the check correct once it is off-actor: it is set + /// BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a + /// second request is refused in memory for the whole check-plus-compose and two artifacts can never + /// race for one coordinator. /// private void HandleStartDeployment(StartDeployment msg) { @@ -226,49 +239,118 @@ public sealed class AdminOperationsActor : ReceiveActor return; } - try - { - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed - // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. - using var db = _dbFactory.CreateDbContext(); - var inflight = db.Deployments - .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) - .Select(d => d.DeploymentId) - .FirstOrDefault(); - if (inflight != Guid.Empty) - { - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.AnotherDeploymentInFlight, - DeploymentId: new DeploymentId(inflight), - RevisionHash: null, - Message: $"Deployment {inflight:N} is still in flight.", - msg.CorrelationId)); - return; - } - } - catch (Exception ex) - { - _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: ServerFault("StartDeployment", msg.CorrelationId), - msg.CorrelationId)); - return; - } - _composeInFlight = msg; // Captured here: Context/Sender are not available on the thread-pool thread. var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; var self = Self; - Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + Task.Run(async () => + { + var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false); + return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false); + }) .PipeTo(self, success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } + /// + /// The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight — + /// keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first. + /// + /// The request. + /// The refusal, or null to proceed. + /// + /// + /// Lost dispatch (fix round 1 / I7). A Dispatching row with ZERO + /// NodeDeploymentState rows was never taken by the coordinator (its HandleDispatch + /// seeds the expected set and flips the status in one SaveChanges). The coordinator's PreStart + /// recovers such a row — but only one that already existed when it started. A row committed AFTER + /// PreStart whose Tell never arrived (this actor died between the compose's commit and + /// ) has no owner at all, and refused every later deploy forever. + /// Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is + /// closed TimedOut with a reason and the new deploy proceeds. A late DispatchDeployment + /// for it is ignored by the coordinator's terminal-state guard. + /// + /// + /// One indexed projection (IX_Deployment_Status) plus, only for a Dispatching row past the + /// deadline, a count and a tracked update. + /// + /// + private async Task CheckInFlightAsync(StartDeployment msg) + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + var inflight = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc }) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var row in inflight) + { + if (row.Status == DeploymentStatus.Dispatching + && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline + && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false) + && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false)) + { + continue; + } + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: new DeploymentId(row.DeploymentId), + RevisionHash: null, + Message: $"Deployment {row.DeploymentId:N} is still in flight.", + msg.CorrelationId), + Dispatch: null); + } + return null; + } + + /// Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion). + /// The context. + /// The row. + /// when the row is now closed. + private async Task CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId) + { + var stamp = await db.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching) + .Select(d => new { d.RowVersion }) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (stamp is null) return true; // no longer Dispatching — not in flight as far as this check goes + + // Stub update: the row's ArtifactBlob is never read just to change its status. + var stub = new Deployment + { + DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty, + RowVersion = stamp.RowVersion, + Status = DeploymentStatus.TimedOut, + FailureReason = + "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " + + "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.", + }; + var entry = db.Deployments.Attach(stub); + entry.Property(d => d.Status).IsModified = true; + entry.Property(d => d.FailureReason).IsModified = true; + try + { + await db.SaveChangesAsync().ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException) + { + // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight. + return false; + } + _log.Warning( + "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " + + "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline); + return true; + } + private void HandleDeployComposed(DeployComposed msg) { _composeInFlight = null; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 3c99e998..49b54df4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -234,6 +234,26 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private void HandleDispatch(DispatchDeployment msg) { + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already + // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message + // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + using (var check = _dbFactory.CreateDbContext()) + { + var status = check.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == msg.DeploymentId.Value) + .Select(d => (DeploymentStatus?)d.Status) + .FirstOrDefault(); + if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + { + _log.Warning( + "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", + msg.DeploymentId, status); + if (_current == msg.DeploymentId) ResetForNext(); + return; + } + } + _current = msg.DeploymentId; _acks.Clear(); _degraded.Clear(); @@ -371,11 +391,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on // each NodeDeploymentState row) and the seal logs at Warning. var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList(); - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + - "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."); + SetFailureReason(db, Truncate( + $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + + "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.")); db.SaveChanges(); _log.Warning( "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}", @@ -420,11 +438,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); if (_expectedAcks.Count == 0) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + - "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + SetFailureReason(db, Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy.")); db.SaveChanges(); _log.Warning( "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + @@ -459,13 +475,53 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// + /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob + /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub + /// carrying only the key and the row's current RowVersion is attached and the changed columns marked + /// modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged + /// stub stays tracked so can add to the same UPDATE. + /// private static void UpdateDeploymentStatus( OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value); - if (d is null) return; - d.Status = status; - if (sealNow) d.SealedAtUtc = DateTime.UtcNow; + var entry = TrackStub(db, id); + if (entry is null) return; + entry.Entity.Status = status; + entry.Property(x => x.Status).IsModified = true; + if (sealNow) + { + entry.Entity.SealedAtUtc = DateTime.UtcNow; + entry.Property(x => x.SealedAtUtc).IsModified = true; + } + } + + /// Stages the current deployment's failure reason onto the stub staged. + private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) + { + if (_current is null) return; + var entry = TrackStub(db, _current.Value); + if (entry is null) return; + entry.Entity.FailureReason = reason; + entry.Property(x => x.FailureReason).IsModified = true; + } + + /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( + OtOpcUaConfigDbContext db, DeploymentId id) + { + var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); + if (tracked is not null) return tracked; + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + return db.Deployments.Attach(new Deployment + { + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + }); } /// diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 86f6efa7..0added3d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -176,6 +176,141 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the + /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between + /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a + /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next + /// StartDeployment is Accepted (and seals — positive control on the pipeline). + /// + [Fact] + public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing + + var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); + + using (var db = dbFactory.CreateDbContext()) + { + var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + } + + coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status + .ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. + [Fact] + public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe(); + var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + result.DeploymentId.ShouldBe(pending); + + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + } + + /// + /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already + /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip + /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right + /// after it IS seeded (so the first message was processed, not merely not-yet-processed). + /// + [Theory] + [InlineData(DeploymentStatus.Sealed)] + [InlineData(DeploymentStatus.TimedOut)] + [InlineData(DeploymentStatus.PartiallyFailed)] + public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); + coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); + var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = dbFactory.CreateDbContext(); + verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); + } + + /// + /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of + /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the + /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are + /// untouched by the partial UPDATEs. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() + { + using var sql = new SqlServerTestDatabase(); + var factory = sql.Factory(); + SeedNodes(factory, "site-a-1:4053"); + var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + var id = DeploymentId.NewId(); + using (var db = sql.NewContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, + CreatedBy = "test", ArtifactBlob = blob, + }); + db.SaveChanges(); + } + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row + coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = sql.NewContext(); + var row = db.Deployments.AsNoTracking().Single(); + row.Status.ShouldBe(DeploymentStatus.Sealed); + row.SealedAtUtc.ShouldNotBeNull(); + row.ArtifactBlob.ShouldBe(blob); + }, duration: PresenceBudget); + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) {