fix(deploymentmanager): bulk site deploy skips up-to-date instances and un-poisons the shared DbContext (arch-review Low P7-5 + P7-6)
P7-6 (behaviour fix, plan decision 1): DeploySiteAsync redeployed EVERY deployable instance unconditionally. A redeploy is not idempotent at the site — it resets static attribute overrides and clears native alarm state by design — so one site-wide push destroyed live operational state on every unchanged instance in the batch, which is normally most of them. Bulk deploy now pre-filters with IStaleInstanceProbe (the same seam the bundle importer and Deployments page use; its memo timing was fixed by the Medium wave's P7-4): an instance whose freshly computed revision hash still matches its DeployedConfigSnapshot is reported in the existing additive SkippedInstances list with reason "up-to-date" and never flattened, staged or sent. The filter is deliberately conservative — every uncertainty deploys: only an Enabled instance qualifies, a missing snapshot disqualifies, a prior record that ShouldQuerySiteBeforeRedeploy flags goes through prepare so the reconcile that HEALS it still runs, and a null/faulting probe means "indeterminate". New additive MgmtDeploySiteCommand.Force (default false = the new filtered behaviour) plus CLI `deploy site --force` restores the unconditional push for the case central cannot see (restored volume, rebuilt node). Composes with the Medium wave's P7-2: Disabled wins — force suppresses the up-to-date filter only, never the Disabled skip. P7-5 (verified, then fixed): the cascade REPRODUCES. All three bulk phases share one scoped DbContext, and a failed SaveChanges does not untrack the offending entity — EF re-attempts it on the next save, including MarkDeploymentFailedAsync's cleanup write. Measured on a real SQLite-backed ScadaBridgeDbContext: a permanent save failure on instance 1's post-success side effects failed instances 2, 3 and 4 and left all three deployment records stranded InProgress. MarkDeploymentFailedAsync now detaches the pending changes (new IDeploymentManagerRepository.DetachPendingChanges) when the cleanup save fails and retries the Failed write once on the clean tracker, so later instances save cleanly and the record reaches a terminal status. The DeployFailed audit row moved outside the status-write try (best-effort, must not be lost because the status write failed). Tests: 5 new DeploySiteAsyncTests (skip + reason + no site round-trip; --force deploys; Disabled skipped under both force values; the three not-enough-to-skip guards), 2 new IntegrationTests over a real EF context (the poisoned-context cascade, and the narrow EF tracking fact the fix rests on), CLI --force flag parse test, and a MgmtDeploySite wire-default test. Proof of failure: both P7-6 behaviour tests and both P7-5 assertions fail against the unfixed code.
This commit is contained in:
Joseph Doherty
@@ -10,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
@@ -34,6 +35,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
private readonly IFlatteningPipeline _pipeline = Substitute.For<IFlatteningPipeline>();
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
private readonly IAuditService _audit = Substitute.For<IAuditService>();
|
||||
private readonly IStaleInstanceProbe _staleProbe = Substitute.For<IStaleInstanceProbe>();
|
||||
private readonly OperationLockManager _lockManager = new();
|
||||
|
||||
public DeploySiteAsyncTests()
|
||||
@@ -99,6 +101,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
_staleProbe,
|
||||
options,
|
||||
Options.Create(new CommunicationOptions
|
||||
{
|
||||
@@ -302,7 +305,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
var service = CreateService(commActor, maxParallelism: 2);
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => service.DeploySiteAsync(SiteId, "admin", cts.Token));
|
||||
() => service.DeploySiteAsync(SiteId, "admin", cancellationToken: cts.Token));
|
||||
|
||||
Assert.Equal(0, _lockManager.TrackedLockCount);
|
||||
|
||||
@@ -425,6 +428,203 @@ public class DeploySiteAsyncTests : TestKit
|
||||
Assert.Equal(0, result.Value.FailureCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks an instance as one a bulk deploy must treat as up to date: it is
|
||||
/// <c>Enabled</c>, central holds a deployed snapshot for it, and the staleness
|
||||
/// probe recomputes the very hash that snapshot recorded.
|
||||
/// </summary>
|
||||
/// <param name="instance">The instance to make up-to-date.</param>
|
||||
private void ArrangeUpToDate(Instance instance)
|
||||
{
|
||||
var hash = $"sha256:{instance.Id}";
|
||||
instance.State = InstanceState.Enabled;
|
||||
|
||||
_repo.GetDeployedSnapshotByInstanceIdAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeployedConfigSnapshot($"prior-{instance.Id}", hash, "{}") { InstanceId = instance.Id });
|
||||
_repo.GetCurrentDeploymentStatusAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeploymentRecord($"prior-{instance.Id}", "admin")
|
||||
{
|
||||
InstanceId = instance.Id,
|
||||
Status = DeploymentStatus.Success,
|
||||
RevisionHash = hash,
|
||||
});
|
||||
_staleProbe.GetCurrentRevisionHashAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(hash);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6: a bulk site deploy must not redeploy instances whose configuration has
|
||||
/// not changed.
|
||||
///
|
||||
/// <para>
|
||||
/// A redeploy is NOT a no-op at the site: applying a config RESETS that
|
||||
/// instance's static attribute overrides and CLEARS its native alarm state,
|
||||
/// both by design. Pushing one site-wide therefore destroyed live operational
|
||||
/// state on every instance in the batch, including the majority whose config
|
||||
/// the operator had not touched at all.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The central-side proof that the site state survives is that the site is
|
||||
/// never told to apply anything: no <c>RefreshDeploymentCommand</c> is sent
|
||||
/// for a skipped instance, so nothing at the site runs the reset. The
|
||||
/// assertions below therefore pin the whole chain — no flatten, no deployment
|
||||
/// record, no staged config, no command — for the unchanged pair.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_SkipsUpToDateInstances_SoTheirSiteStateIsNeverReset()
|
||||
{
|
||||
var instances = ArrangeInstances(4);
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 3));
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 4).DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
var summary = result.Value;
|
||||
|
||||
// Only the two genuinely-stale instances were deployed.
|
||||
Assert.Equal(2, summary.InstanceResults.Count);
|
||||
Assert.Equal(2, summary.SuccessCount);
|
||||
Assert.Equal(0, summary.FailureCount);
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-04" }, summary.InstanceResults.Select(r => r.UniqueName).Order().ToArray());
|
||||
|
||||
// The unchanged pair is reported as skipped, with the reason the operator needs.
|
||||
Assert.NotNull(summary.SkippedInstances);
|
||||
var skipped = summary.SkippedInstances;
|
||||
Assert.Equal(new[] { "Inst-02", "Inst-03" }, skipped.Select(s => s.UniqueName).Order().ToArray());
|
||||
Assert.All(skipped, s => Assert.Contains("up-to-date", s.Reason));
|
||||
|
||||
// ...and nothing reached the site for them, which is what leaves their
|
||||
// static overrides and native alarm state alone.
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-04" }, applied.Order().ToArray());
|
||||
|
||||
foreach (var id in new[] { 2, 3 })
|
||||
{
|
||||
await _pipeline.DidNotReceive().FlattenAndValidateAsync(
|
||||
id, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
|
||||
await _repo.DidNotReceive().AddDeploymentRecordAsync(
|
||||
Arg.Is<DeploymentRecord>(r => r.InstanceId == id), Arg.Any<CancellationToken>());
|
||||
await _repo.DidNotReceive().AddPendingDeploymentAsync(
|
||||
Arg.Is<PendingDeployment>(p => p.InstanceId == id), Arg.Any<CancellationToken>());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6: <c>force</c> restores the unconditional behaviour for operators whose
|
||||
/// site state central cannot see — a restored data volume or a rebuilt node
|
||||
/// looks perfectly up to date from central and still needs the push.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_Force_RedeploysUpToDateInstances()
|
||||
{
|
||||
var instances = ArrangeInstances(3);
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3)
|
||||
.DeploySiteAsync(SiteId, "admin", force: true);
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.Equal(3, result.Value.SuccessCount);
|
||||
Assert.Empty(result.Value.SkippedInstances!);
|
||||
Assert.Contains("Inst-02", applied);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6 composed with P7-2: <c>force</c> suppresses the up-to-date filter and
|
||||
/// ONLY that filter. A Disabled instance is never deployed at bulk scope,
|
||||
/// because forcing a bulk push is an operator saying "the site's config may be
|
||||
/// wrong", not "restart the equipment I stopped" — the single-instance deploy
|
||||
/// remains the explicit re-enable action.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task DeploySiteAsync_DisabledInstancesAreSkipped_RegardlessOfForce(bool force)
|
||||
{
|
||||
var instances = ArrangeInstances(3, disabledIds: [2]);
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3)
|
||||
.DeploySiteAsync(SiteId, "admin", force: force);
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.DoesNotContain("Inst-02", applied);
|
||||
Assert.Equal(InstanceState.Disabled, instances.Single(i => i.Id == 2).State);
|
||||
|
||||
var skipped = Assert.Single(result.Value.SkippedInstances!);
|
||||
Assert.Equal("Inst-02", skipped.UniqueName);
|
||||
Assert.Contains("deploy individually to re-enable", skipped.Reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The up-to-date filter is deliberately conservative: every uncertainty
|
||||
/// resolves to "deploy it". These are the three cases where a matching hash is
|
||||
/// NOT sufficient — each one would be a silently skipped instance that actually
|
||||
/// needed the deploy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_MatchingHashIsNotEnough_WhenTheInstanceStillNeedsDeploying()
|
||||
{
|
||||
var instances = ArrangeInstances(3);
|
||||
|
||||
// 1: NotDeployed. A leftover snapshot says nothing about an instance that
|
||||
// is not running — it must be deployed to become Enabled.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 1));
|
||||
instances.Single(i => i.Id == 1).State = InstanceState.NotDeployed;
|
||||
|
||||
// 2: prior record stuck InProgress. Prepare is where query-before-redeploy
|
||||
// reconciliation HEALS such a record; skipping here strands it forever.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
_repo.GetCurrentDeploymentStatusAsync(2, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeploymentRecord("prior-2", "admin")
|
||||
{
|
||||
InstanceId = 2,
|
||||
Status = DeploymentStatus.InProgress,
|
||||
RevisionHash = "sha256:2",
|
||||
});
|
||||
|
||||
// 3: staleness indeterminate — the probe cannot compute a hash.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 3));
|
||||
_staleProbe.GetCurrentRevisionHashAsync(3, Arg.Any<CancellationToken>()).Returns((string?)null);
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3).DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.Empty(result.Value.SkippedInstances!);
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-02", "Inst-03" }, applied.Order().ToArray());
|
||||
}
|
||||
|
||||
/// <summary>Answers every deploy with Success, recording which instances the site was asked to apply.</summary>
|
||||
private sealed class RecordingSiteActor : ReceiveActor
|
||||
{
|
||||
public RecordingSiteActor(ConcurrentBag<string> applied)
|
||||
{
|
||||
Receive<SiteEnvelope>(env =>
|
||||
{
|
||||
if (env.Message is not RefreshDeploymentCommand cmd)
|
||||
return;
|
||||
|
||||
applied.Add(cmd.InstanceUniqueName);
|
||||
Sender.Tell(new DeploymentStatusResponse(
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName,
|
||||
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Answers <c>RefreshDeploymentCommand</c> normally but NEVER answers a
|
||||
/// <c>DeploymentStateQueryRequest</c>, counting how many arrived — a site that
|
||||
|
||||
@@ -9,6 +9,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
@@ -54,6 +55,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
options,
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -165,6 +167,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -385,6 +388,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -1153,6 +1157,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(centralFetchBaseUrl),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -1743,6 +1748,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1801,6 +1807,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1851,6 +1858,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1898,6 +1906,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
|
||||
+3
-1
@@ -7,6 +7,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
@@ -60,7 +61,8 @@ public class DeploymentStatusNotifierTests : TestKit
|
||||
|
||||
_service = new DeploymentService(
|
||||
_repo, siteRepo, _pipeline, _comms, _lockManager, _audit,
|
||||
new DiffService(), new RevisionHashService(), _notifier, options,
|
||||
new DiffService(), new RevisionHashService(), _notifier,
|
||||
Substitute.For<IStaleInstanceProbe>(), options,
|
||||
// Notify-and-fetch (Task 6): a non-empty CentralFetchBaseUrl lets the
|
||||
// deploy path reach the send (and, here, the catch block) rather than
|
||||
// tripping the fail-fast configuration guard.
|
||||
|
||||
Reference in New Issue
Block a user