fix(deploymentmanager): bulk site deploy skips up-to-date instances and un-poisons the shared DbContext (arch-review Low P7-5 + P7-6)
P7-6 (behaviour fix, plan decision 1): DeploySiteAsync redeployed EVERY deployable instance unconditionally. A redeploy is not idempotent at the site — it resets static attribute overrides and clears native alarm state by design — so one site-wide push destroyed live operational state on every unchanged instance in the batch, which is normally most of them. Bulk deploy now pre-filters with IStaleInstanceProbe (the same seam the bundle importer and Deployments page use; its memo timing was fixed by the Medium wave's P7-4): an instance whose freshly computed revision hash still matches its DeployedConfigSnapshot is reported in the existing additive SkippedInstances list with reason "up-to-date" and never flattened, staged or sent. The filter is deliberately conservative — every uncertainty deploys: only an Enabled instance qualifies, a missing snapshot disqualifies, a prior record that ShouldQuerySiteBeforeRedeploy flags goes through prepare so the reconcile that HEALS it still runs, and a null/faulting probe means "indeterminate". New additive MgmtDeploySiteCommand.Force (default false = the new filtered behaviour) plus CLI `deploy site --force` restores the unconditional push for the case central cannot see (restored volume, rebuilt node). Composes with the Medium wave's P7-2: Disabled wins — force suppresses the up-to-date filter only, never the Disabled skip. P7-5 (verified, then fixed): the cascade REPRODUCES. All three bulk phases share one scoped DbContext, and a failed SaveChanges does not untrack the offending entity — EF re-attempts it on the next save, including MarkDeploymentFailedAsync's cleanup write. Measured on a real SQLite-backed ScadaBridgeDbContext: a permanent save failure on instance 1's post-success side effects failed instances 2, 3 and 4 and left all three deployment records stranded InProgress. MarkDeploymentFailedAsync now detaches the pending changes (new IDeploymentManagerRepository.DetachPendingChanges) when the cleanup save fails and retries the Failed write once on the clean tracker, so later instances save cleanly and the record reaches a terminal status. The DeployFailed audit row moved outside the status-write try (best-effort, must not be lost because the status write failed). Tests: 5 new DeploySiteAsyncTests (skip + reason + no site round-trip; --force deploys; Disabled skipped under both force values; the three not-enough-to-skip guards), 2 new IntegrationTests over a real EF context (the poisoned-context cascade, and the narrow EF tracking fact the fix rests on), CLI --force flag parse test, and a MgmtDeploySite wire-default test. Proof of failure: both P7-6 behaviour tests and both P7-5 assertions fail against the unfixed code.
This commit is contained in:
Joseph Doherty
@@ -157,6 +157,28 @@ public class CommandTreeTests
|
||||
Assert.Empty(site.Parse(["--site-id", "3"]).Errors);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeploySite_HasForceFlag_DefaultingToTheNonDestructiveBehaviour()
|
||||
{
|
||||
// P7-6: bulk deploy skips instances whose config is unchanged, because a
|
||||
// redeploy resets their static overrides and native alarm state at the
|
||||
// site. --force is the operator escape hatch (restored volume, rebuilt
|
||||
// node) and must be a FLAG — absent means skip-unchanged, which is the
|
||||
// safe default a scripted `deploy site` keeps getting.
|
||||
var deploy = DeployCommands.Build(Url, Format, Username, Password);
|
||||
var site = deploy.Subcommands.Single(c => c.Name == "site");
|
||||
var forceOption = (Option<bool>)site.Options.Single(o => o.Name == "--force");
|
||||
|
||||
var withoutForce = site.Parse(["--site-id", "3"]);
|
||||
Assert.Empty(withoutForce.Errors);
|
||||
Assert.False(withoutForce.GetValue(forceOption));
|
||||
|
||||
// A bare --force (no value) must parse: it is a flag, not a value option.
|
||||
var withForce = site.Parse(["--site-id", "3", "--force"]);
|
||||
Assert.Empty(withForce.Errors);
|
||||
Assert.True(withForce.GetValue(forceOption));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InstanceNativeAlarmSource_HasSetAndClear()
|
||||
{
|
||||
|
||||
@@ -13,6 +13,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
@@ -74,6 +75,11 @@ public class TopologyPageTests : BunitContext
|
||||
Services.AddSingleton<ZB.MOM.WW.ScadaBridge.DeploymentManager.IDeploymentStatusNotifier>(
|
||||
new ZB.MOM.WW.ScadaBridge.DeploymentManager.DeploymentStatusNotifier(
|
||||
NullLogger<ZB.MOM.WW.ScadaBridge.DeploymentManager.DeploymentStatusNotifier>.Instance));
|
||||
// P7-6: the bulk site deploy's up-to-date pre-filter asks the staleness
|
||||
// probe for each instance's current revision hash, so DeploymentService now
|
||||
// depends on it too. The page under test never bulk-deploys; a substitute
|
||||
// keeps the DI graph resolvable without dragging in the flattening stack.
|
||||
Services.AddSingleton(Substitute.For<IStaleInstanceProbe>());
|
||||
Services.AddScoped<DeploymentService>();
|
||||
Services.AddScoped<AreaService>();
|
||||
Services.AddScoped<InstanceService>();
|
||||
|
||||
@@ -10,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
@@ -34,6 +35,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
private readonly IFlatteningPipeline _pipeline = Substitute.For<IFlatteningPipeline>();
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
private readonly IAuditService _audit = Substitute.For<IAuditService>();
|
||||
private readonly IStaleInstanceProbe _staleProbe = Substitute.For<IStaleInstanceProbe>();
|
||||
private readonly OperationLockManager _lockManager = new();
|
||||
|
||||
public DeploySiteAsyncTests()
|
||||
@@ -99,6 +101,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
_staleProbe,
|
||||
options,
|
||||
Options.Create(new CommunicationOptions
|
||||
{
|
||||
@@ -302,7 +305,7 @@ public class DeploySiteAsyncTests : TestKit
|
||||
var service = CreateService(commActor, maxParallelism: 2);
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => service.DeploySiteAsync(SiteId, "admin", cts.Token));
|
||||
() => service.DeploySiteAsync(SiteId, "admin", cancellationToken: cts.Token));
|
||||
|
||||
Assert.Equal(0, _lockManager.TrackedLockCount);
|
||||
|
||||
@@ -425,6 +428,203 @@ public class DeploySiteAsyncTests : TestKit
|
||||
Assert.Equal(0, result.Value.FailureCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks an instance as one a bulk deploy must treat as up to date: it is
|
||||
/// <c>Enabled</c>, central holds a deployed snapshot for it, and the staleness
|
||||
/// probe recomputes the very hash that snapshot recorded.
|
||||
/// </summary>
|
||||
/// <param name="instance">The instance to make up-to-date.</param>
|
||||
private void ArrangeUpToDate(Instance instance)
|
||||
{
|
||||
var hash = $"sha256:{instance.Id}";
|
||||
instance.State = InstanceState.Enabled;
|
||||
|
||||
_repo.GetDeployedSnapshotByInstanceIdAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeployedConfigSnapshot($"prior-{instance.Id}", hash, "{}") { InstanceId = instance.Id });
|
||||
_repo.GetCurrentDeploymentStatusAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeploymentRecord($"prior-{instance.Id}", "admin")
|
||||
{
|
||||
InstanceId = instance.Id,
|
||||
Status = DeploymentStatus.Success,
|
||||
RevisionHash = hash,
|
||||
});
|
||||
_staleProbe.GetCurrentRevisionHashAsync(instance.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(hash);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6: a bulk site deploy must not redeploy instances whose configuration has
|
||||
/// not changed.
|
||||
///
|
||||
/// <para>
|
||||
/// A redeploy is NOT a no-op at the site: applying a config RESETS that
|
||||
/// instance's static attribute overrides and CLEARS its native alarm state,
|
||||
/// both by design. Pushing one site-wide therefore destroyed live operational
|
||||
/// state on every instance in the batch, including the majority whose config
|
||||
/// the operator had not touched at all.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The central-side proof that the site state survives is that the site is
|
||||
/// never told to apply anything: no <c>RefreshDeploymentCommand</c> is sent
|
||||
/// for a skipped instance, so nothing at the site runs the reset. The
|
||||
/// assertions below therefore pin the whole chain — no flatten, no deployment
|
||||
/// record, no staged config, no command — for the unchanged pair.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_SkipsUpToDateInstances_SoTheirSiteStateIsNeverReset()
|
||||
{
|
||||
var instances = ArrangeInstances(4);
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 3));
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 4).DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
var summary = result.Value;
|
||||
|
||||
// Only the two genuinely-stale instances were deployed.
|
||||
Assert.Equal(2, summary.InstanceResults.Count);
|
||||
Assert.Equal(2, summary.SuccessCount);
|
||||
Assert.Equal(0, summary.FailureCount);
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-04" }, summary.InstanceResults.Select(r => r.UniqueName).Order().ToArray());
|
||||
|
||||
// The unchanged pair is reported as skipped, with the reason the operator needs.
|
||||
Assert.NotNull(summary.SkippedInstances);
|
||||
var skipped = summary.SkippedInstances;
|
||||
Assert.Equal(new[] { "Inst-02", "Inst-03" }, skipped.Select(s => s.UniqueName).Order().ToArray());
|
||||
Assert.All(skipped, s => Assert.Contains("up-to-date", s.Reason));
|
||||
|
||||
// ...and nothing reached the site for them, which is what leaves their
|
||||
// static overrides and native alarm state alone.
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-04" }, applied.Order().ToArray());
|
||||
|
||||
foreach (var id in new[] { 2, 3 })
|
||||
{
|
||||
await _pipeline.DidNotReceive().FlattenAndValidateAsync(
|
||||
id, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
|
||||
await _repo.DidNotReceive().AddDeploymentRecordAsync(
|
||||
Arg.Is<DeploymentRecord>(r => r.InstanceId == id), Arg.Any<CancellationToken>());
|
||||
await _repo.DidNotReceive().AddPendingDeploymentAsync(
|
||||
Arg.Is<PendingDeployment>(p => p.InstanceId == id), Arg.Any<CancellationToken>());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6: <c>force</c> restores the unconditional behaviour for operators whose
|
||||
/// site state central cannot see — a restored data volume or a rebuilt node
|
||||
/// looks perfectly up to date from central and still needs the push.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_Force_RedeploysUpToDateInstances()
|
||||
{
|
||||
var instances = ArrangeInstances(3);
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3)
|
||||
.DeploySiteAsync(SiteId, "admin", force: true);
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.Equal(3, result.Value.SuccessCount);
|
||||
Assert.Empty(result.Value.SkippedInstances!);
|
||||
Assert.Contains("Inst-02", applied);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6 composed with P7-2: <c>force</c> suppresses the up-to-date filter and
|
||||
/// ONLY that filter. A Disabled instance is never deployed at bulk scope,
|
||||
/// because forcing a bulk push is an operator saying "the site's config may be
|
||||
/// wrong", not "restart the equipment I stopped" — the single-instance deploy
|
||||
/// remains the explicit re-enable action.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task DeploySiteAsync_DisabledInstancesAreSkipped_RegardlessOfForce(bool force)
|
||||
{
|
||||
var instances = ArrangeInstances(3, disabledIds: [2]);
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3)
|
||||
.DeploySiteAsync(SiteId, "admin", force: force);
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.DoesNotContain("Inst-02", applied);
|
||||
Assert.Equal(InstanceState.Disabled, instances.Single(i => i.Id == 2).State);
|
||||
|
||||
var skipped = Assert.Single(result.Value.SkippedInstances!);
|
||||
Assert.Equal("Inst-02", skipped.UniqueName);
|
||||
Assert.Contains("deploy individually to re-enable", skipped.Reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The up-to-date filter is deliberately conservative: every uncertainty
|
||||
/// resolves to "deploy it". These are the three cases where a matching hash is
|
||||
/// NOT sufficient — each one would be a silently skipped instance that actually
|
||||
/// needed the deploy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_MatchingHashIsNotEnough_WhenTheInstanceStillNeedsDeploying()
|
||||
{
|
||||
var instances = ArrangeInstances(3);
|
||||
|
||||
// 1: NotDeployed. A leftover snapshot says nothing about an instance that
|
||||
// is not running — it must be deployed to become Enabled.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 1));
|
||||
instances.Single(i => i.Id == 1).State = InstanceState.NotDeployed;
|
||||
|
||||
// 2: prior record stuck InProgress. Prepare is where query-before-redeploy
|
||||
// reconciliation HEALS such a record; skipping here strands it forever.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 2));
|
||||
_repo.GetCurrentDeploymentStatusAsync(2, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeploymentRecord("prior-2", "admin")
|
||||
{
|
||||
InstanceId = 2,
|
||||
Status = DeploymentStatus.InProgress,
|
||||
RevisionHash = "sha256:2",
|
||||
});
|
||||
|
||||
// 3: staleness indeterminate — the probe cannot compute a hash.
|
||||
ArrangeUpToDate(instances.Single(i => i.Id == 3));
|
||||
_staleProbe.GetCurrentRevisionHashAsync(3, Arg.Any<CancellationToken>()).Returns((string?)null);
|
||||
|
||||
var applied = new ConcurrentBag<string>();
|
||||
var commActor = Sys.ActorOf(Props.Create(() => new RecordingSiteActor(applied)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 3).DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.Empty(result.Value.SkippedInstances!);
|
||||
Assert.Equal(new[] { "Inst-01", "Inst-02", "Inst-03" }, applied.Order().ToArray());
|
||||
}
|
||||
|
||||
/// <summary>Answers every deploy with Success, recording which instances the site was asked to apply.</summary>
|
||||
private sealed class RecordingSiteActor : ReceiveActor
|
||||
{
|
||||
public RecordingSiteActor(ConcurrentBag<string> applied)
|
||||
{
|
||||
Receive<SiteEnvelope>(env =>
|
||||
{
|
||||
if (env.Message is not RefreshDeploymentCommand cmd)
|
||||
return;
|
||||
|
||||
applied.Add(cmd.InstanceUniqueName);
|
||||
Sender.Tell(new DeploymentStatusResponse(
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName,
|
||||
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Answers <c>RefreshDeploymentCommand</c> normally but NEVER answers a
|
||||
/// <c>DeploymentStateQueryRequest</c>, counting how many arrived — a site that
|
||||
|
||||
@@ -9,6 +9,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
@@ -54,6 +55,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
options,
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -165,6 +167,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -385,6 +388,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -1153,6 +1157,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(centralFetchBaseUrl),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
@@ -1743,6 +1748,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1801,6 +1807,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1851,6 +1858,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
@@ -1898,6 +1906,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
|
||||
+3
-1
@@ -7,6 +7,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
@@ -60,7 +61,8 @@ public class DeploymentStatusNotifierTests : TestKit
|
||||
|
||||
_service = new DeploymentService(
|
||||
_repo, siteRepo, _pipeline, _comms, _lockManager, _audit,
|
||||
new DiffService(), new RevisionHashService(), _notifier, options,
|
||||
new DiffService(), new RevisionHashService(), _notifier,
|
||||
Substitute.For<IStaleInstanceProbe>(), options,
|
||||
// Notify-and-fetch (Task 6): a non-empty CentralFetchBaseUrl lets the
|
||||
// deploy path reach the send (and, here, the catch block) rather than
|
||||
// tripping the fail-fast configuration guard.
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.DeploymentManager;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// P7-5: a bulk site deploy runs all three phases over ONE scoped
|
||||
/// <c>DbContext</c>, so a persistent <c>SaveChanges</c> failure on a single
|
||||
/// instance used to fail every instance that came after it.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Why this needs a real EF context.</b> The mechanism is not in the service's
|
||||
/// control flow, it is in EF's: a failed <c>SaveChanges</c> does NOT untrack the
|
||||
/// entity that caused it. The entity stays in the change set and is re-attempted
|
||||
/// by the very next save made through the same context — including the cleanup
|
||||
/// save inside <c>MarkDeploymentFailedAsync</c>, whose whole job is to stop a
|
||||
/// record being stranded <c>InProgress</c>. A mocked repository could not prove
|
||||
/// that; a real <see cref="ScadaBridgeDbContext"/> over SQLite does.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Honesty note on the injected failure.</b> The persistent failure itself is
|
||||
/// injected by a save interceptor rather than provoked by a real constraint
|
||||
/// violation, because the entity that must fail (the <see cref="Instance"/> row
|
||||
/// the deploy path mutates) is created and mutated inside the service, where a
|
||||
/// test cannot reach in and corrupt it mid-batch. The interceptor throws
|
||||
/// <see cref="DbUpdateException"/> at exactly the point a constraint violation,
|
||||
/// a concurrency conflict or a dropped connection would — everything after that,
|
||||
/// including the cascade and the fix, is real EF behaviour.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class BulkDeployPoisonedContextTests : TestKit, IDisposable
|
||||
{
|
||||
private const int InstanceCount = 4;
|
||||
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly ScadaBridgeDbContext _context;
|
||||
private readonly PoisonedEntityInterceptor _poison = new();
|
||||
private int _siteId;
|
||||
|
||||
public BulkDeployPoisonedContextTests()
|
||||
{
|
||||
_connection = new SqliteConnection("DataSource=:memory:");
|
||||
_connection.Open();
|
||||
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(_connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.AddInterceptors(_poison)
|
||||
.Options;
|
||||
|
||||
_context = new SqliteContext(options);
|
||||
_context.Database.EnsureCreated();
|
||||
Seed();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The whole finding in one run: instance 1's post-success persistence fails
|
||||
/// permanently (best-effort, so the deploy still reports Success), which leaves
|
||||
/// its <see cref="Instance"/> row poisoning the shared change tracker. Every
|
||||
/// later instance's status write then rides on that same change set.
|
||||
///
|
||||
/// <para>
|
||||
/// Pre-fix this cascaded: instances 2, 3 and 4 all failed, and their records
|
||||
/// stayed <c>InProgress</c> because the cleanup save re-attempted — and
|
||||
/// re-failed on — the same poisoned entity. Post-fix the cleanup detaches the
|
||||
/// pending work, so instance 2 is the only casualty (correctly recorded
|
||||
/// <c>Failed</c>, not stranded), and 3 and 4 save cleanly.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task BulkDeploy_PersistentSaveFailureOnOneInstance_DoesNotCascadeToTheRest()
|
||||
{
|
||||
_poison.PoisonInstanceNamed("Inst-01");
|
||||
|
||||
var service = CreateService();
|
||||
|
||||
var result = await service.DeploySiteAsync(_siteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
var summary = result.Value;
|
||||
Assert.Equal(InstanceCount, summary.InstanceResults.Count);
|
||||
|
||||
// Instances 3 and 4 are downstream of the poison and must be unaffected:
|
||||
// this is the assertion that fails outright without the detach.
|
||||
foreach (var name in new[] { "Inst-03", "Inst-04" })
|
||||
{
|
||||
var row = Assert.Single(summary.InstanceResults, r => r.UniqueName == name);
|
||||
Assert.True(row.Success, $"{name} failed with: {row.ErrorMessage}");
|
||||
}
|
||||
|
||||
// Instance 2 is the one collateral casualty — its own status write shared
|
||||
// the poisoned change set — but it must be recorded Failed, never left
|
||||
// InProgress, which is what the retry on the cleaned tracker buys.
|
||||
var stranded = await NoLongerInProgressAsync();
|
||||
Assert.Empty(stranded);
|
||||
|
||||
// ...and the fix is not "swallow everything": the poisoned instance's own
|
||||
// side effects genuinely did not persist, so its Instance row is still
|
||||
// NotDeployed even though the site confirmed the apply. That divergence is
|
||||
// the pre-existing best-effort contract, logged for operator reconciliation.
|
||||
await using var verify = NewContext();
|
||||
Assert.Equal(
|
||||
InstanceState.NotDeployed,
|
||||
verify.Instances.Single(i => i.UniqueName == "Inst-01").State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The narrow EF fact the fix rests on, pinned on its own so a future reader
|
||||
/// does not have to re-derive it: a failed save leaves its entity tracked and
|
||||
/// the NEXT save re-attempts it, and
|
||||
/// <c>IDeploymentManagerRepository.DetachPendingChanges</c> is what clears it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task FailedSave_KeepsTheEntityTracked_UntilPendingChangesAreDetached()
|
||||
{
|
||||
var repository = new DeploymentManagerRepository(_context);
|
||||
var instance = _context.Instances.Single(i => i.UniqueName == "Inst-01");
|
||||
|
||||
_poison.PoisonInstanceNamed("Inst-01");
|
||||
instance.State = InstanceState.Enabled;
|
||||
await repository.UpdateInstanceAsync(instance);
|
||||
await Assert.ThrowsAsync<DbUpdateException>(() => repository.SaveChangesAsync());
|
||||
|
||||
// An unrelated, perfectly valid write now fails too — the poisoned entity
|
||||
// is still in the change set.
|
||||
var record = new DeploymentRecord("dep-unrelated", "admin")
|
||||
{
|
||||
InstanceId = _context.Instances.Single(i => i.UniqueName == "Inst-02").Id,
|
||||
Status = DeploymentStatus.Failed,
|
||||
RevisionHash = "sha256:x",
|
||||
DeployedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await repository.AddDeploymentRecordAsync(record);
|
||||
await Assert.ThrowsAsync<DbUpdateException>(() => repository.SaveChangesAsync());
|
||||
|
||||
// Detaching the pending work un-poisons the unit of work; the same write
|
||||
// then succeeds.
|
||||
Assert.True(repository.DetachPendingChanges() > 0);
|
||||
await repository.AddDeploymentRecordAsync(record);
|
||||
Assert.Equal(1, await repository.SaveChangesAsync());
|
||||
}
|
||||
|
||||
/// <summary>Deployment records left non-terminal after the batch — the stranded-InProgress check.</summary>
|
||||
private async Task<List<string>> NoLongerInProgressAsync()
|
||||
{
|
||||
await using var verify = NewContext();
|
||||
return await verify.DeploymentRecords
|
||||
.Where(r => r.Status == DeploymentStatus.InProgress || r.Status == DeploymentStatus.Pending)
|
||||
.Select(r => r.DeploymentId)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private DeploymentService CreateService()
|
||||
{
|
||||
var repository = new DeploymentManagerRepository(_context);
|
||||
var siteRepository = new SiteRepository(_context);
|
||||
|
||||
var pipeline = Substitute.For<IFlatteningPipeline>();
|
||||
pipeline.CreateSession().Returns(_ => new FlattenSession(new TemplateGraphWatermark()));
|
||||
pipeline
|
||||
.FlattenAndValidateAsync(
|
||||
Arg.Any<int>(), Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
|
||||
.Returns(ci => Result<FlatteningPipelineResult>.Success(
|
||||
new FlatteningPipelineResult(
|
||||
new FlattenedConfiguration { InstanceUniqueName = $"Inst-{ci.ArgAt<int>(0):00}" },
|
||||
$"sha256:{ci.ArgAt<int>(0)}",
|
||||
ValidationResult.Success())));
|
||||
|
||||
var comms = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(10) }),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
comms.SetCommunicationActor(Sys.ActorOf(Props.Create(() => new AcceptingSiteActor())));
|
||||
|
||||
// Null probe result = "staleness indeterminate", so the up-to-date
|
||||
// pre-filter never skips: this test is about the persistence cascade and
|
||||
// needs every instance to actually run.
|
||||
var probe = Substitute.For<IStaleInstanceProbe>();
|
||||
probe.GetCurrentRevisionHashAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns((string?)null);
|
||||
|
||||
return new DeploymentService(
|
||||
repository, siteRepository, pipeline, comms, new OperationLockManager(),
|
||||
Substitute.For<IAuditService>(),
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
probe,
|
||||
Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
SiteDeploymentMaxParallelism = 1,
|
||||
SiteDeploymentTimeoutPerInstance = TimeSpan.FromSeconds(10),
|
||||
SiteDeploymentReconcileTimeout = TimeSpan.FromSeconds(2),
|
||||
}),
|
||||
Options.Create(new CommunicationOptions
|
||||
{
|
||||
CentralFetchBaseUrl = "https://central.test:9000",
|
||||
PendingDeploymentTtl = TimeSpan.FromMinutes(5),
|
||||
}),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
}
|
||||
|
||||
private void Seed()
|
||||
{
|
||||
using var ctx = NewContext();
|
||||
var site = new Site("Poison Site", "site-poison");
|
||||
var template = new Template("T-Poison");
|
||||
ctx.Sites.Add(site);
|
||||
ctx.Templates.Add(template);
|
||||
ctx.SaveChanges();
|
||||
_siteId = site.Id;
|
||||
|
||||
for (var i = 1; i <= InstanceCount; i++)
|
||||
{
|
||||
ctx.Instances.Add(new Instance($"Inst-{i:00}")
|
||||
{
|
||||
SiteId = site.Id,
|
||||
TemplateId = template.Id,
|
||||
State = InstanceState.NotDeployed,
|
||||
});
|
||||
}
|
||||
|
||||
ctx.SaveChanges();
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext NewContext() =>
|
||||
new SqliteContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(_connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.Options);
|
||||
|
||||
public new void Dispose()
|
||||
{
|
||||
_context.Dispose();
|
||||
_connection.Dispose();
|
||||
base.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fails every <c>SaveChanges</c> whose change set contains the named
|
||||
/// <see cref="Instance"/> — a stand-in for a permanent constraint violation on
|
||||
/// one tracked entity, raised at the point EF raises a real one.
|
||||
/// </summary>
|
||||
private sealed class PoisonedEntityInterceptor : SaveChangesInterceptor
|
||||
{
|
||||
private string? _poisonedUniqueName;
|
||||
|
||||
public void PoisonInstanceNamed(string uniqueName) => _poisonedUniqueName = uniqueName;
|
||||
|
||||
public override InterceptionResult<int> SavingChanges(
|
||||
DbContextEventData eventData, InterceptionResult<int> result)
|
||||
{
|
||||
ThrowIfPoisoned(eventData);
|
||||
return base.SavingChanges(eventData, result);
|
||||
}
|
||||
|
||||
public override ValueTask<InterceptionResult<int>> SavingChangesAsync(
|
||||
DbContextEventData eventData,
|
||||
InterceptionResult<int> result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ThrowIfPoisoned(eventData);
|
||||
return base.SavingChangesAsync(eventData, result, cancellationToken);
|
||||
}
|
||||
|
||||
private void ThrowIfPoisoned(DbContextEventData eventData)
|
||||
{
|
||||
if (_poisonedUniqueName is null || eventData.Context is null)
|
||||
return;
|
||||
|
||||
var poisoned = eventData.Context.ChangeTracker.Entries<Instance>()
|
||||
.Any(e => e.State != EntityState.Unchanged
|
||||
&& e.State != EntityState.Detached
|
||||
&& e.Entity.UniqueName == _poisonedUniqueName);
|
||||
|
||||
if (poisoned)
|
||||
throw new DbUpdateException($"simulated persistent failure on {_poisonedUniqueName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Answers every deployment with Success, so only persistence can fail here.</summary>
|
||||
private sealed class AcceptingSiteActor : ReceiveActor
|
||||
{
|
||||
public AcceptingSiteActor()
|
||||
{
|
||||
Receive<SiteEnvelope>(env =>
|
||||
{
|
||||
if (env.Message is not RefreshDeploymentCommand cmd)
|
||||
return;
|
||||
|
||||
Sender.Tell(new DeploymentStatusResponse(
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName,
|
||||
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQLite-adapted context: SQLite has no rowversion and no native
|
||||
/// DateTimeOffset. Mirrors the adaptation <c>DeployNotifyFetchTests</c> uses.
|
||||
/// </summary>
|
||||
private sealed class SqliteContext : ScadaBridgeDbContext
|
||||
{
|
||||
public SqliteContext(DbContextOptions<ScadaBridgeDbContext> options)
|
||||
: base(options, new EphemeralDataProtectionProvider())
|
||||
{
|
||||
}
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
base.OnModelCreating(modelBuilder);
|
||||
|
||||
modelBuilder.Entity<DeploymentRecord>(builder =>
|
||||
{
|
||||
builder.Property(d => d.RowVersion)
|
||||
.IsRequired(false)
|
||||
.IsConcurrencyToken(false)
|
||||
.ValueGeneratedNever();
|
||||
});
|
||||
|
||||
var converter = new ValueConverter<DateTimeOffset, string>(
|
||||
v => v.UtcDateTime.ToString("o"),
|
||||
v => DateTimeOffset.Parse(v));
|
||||
var nullableConverter = new ValueConverter<DateTimeOffset?, string?>(
|
||||
v => v.HasValue ? v.Value.UtcDateTime.ToString("o") : null,
|
||||
v => v != null ? DateTimeOffset.Parse(v) : null);
|
||||
|
||||
foreach (var entityType in modelBuilder.Model.GetEntityTypes())
|
||||
{
|
||||
foreach (var property in entityType.GetProperties())
|
||||
{
|
||||
if (property.ClrType == typeof(DateTimeOffset))
|
||||
{
|
||||
property.SetValueConverter(converter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
else if (property.ClrType == typeof(DateTimeOffset?))
|
||||
{
|
||||
property.SetValueConverter(nullableConverter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,28 @@ public class ManagementEndpointsTests
|
||||
Assert.Equal("Desc", command.Description);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P7-6: <c>MgmtDeploySiteCommand.Force</c> is an ADDITIVE field, so a client
|
||||
/// that predates it — every CLI older than this change, and every scripted
|
||||
/// HTTP caller — must keep working and must land on the SAFE default. Absent
|
||||
/// means <c>false</c> means "skip instances whose config is unchanged", which
|
||||
/// is the non-destructive behaviour; only an explicit <c>true</c> restores the
|
||||
/// unconditional redeploy.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("""{ "command": "MgmtDeploySite", "payload": { "siteId": 7 } }""", false)]
|
||||
[InlineData("""{ "command": "MgmtDeploySite", "payload": { "siteId": 7, "force": false } }""", false)]
|
||||
[InlineData("""{ "command": "MgmtDeploySite", "payload": { "siteId": 7, "force": true } }""", true)]
|
||||
public void ParseCommand_DeploySite_ForceDefaultsToTheNonDestructiveBehaviour(string json, bool expectedForce)
|
||||
{
|
||||
var result = ManagementEndpoints.ParseCommand(json);
|
||||
|
||||
Assert.True(result.Success);
|
||||
var command = Assert.IsType<MgmtDeploySiteCommand>(result.Command);
|
||||
Assert.Equal(7, command.SiteId);
|
||||
Assert.Equal(expectedForce, command.Force);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseCommand_WithMissingPayload_DeserializesParameterlessCommand()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user