perf(deploy): flatten-session caching, bulk DeploySiteAsync, paged management queries
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,279 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP2.5: bulk site deployment. Pins the properties that make the fan-out safe —
|
||||
/// concurrency is BOUNDED by the configured degree, one slow instance does not stop
|
||||
/// the others completing, a per-instance timeout contains a wedged instance, and
|
||||
/// every instance still gets its own deployment id and its own operation lock.
|
||||
/// </summary>
|
||||
public class DeploySiteAsyncTests : TestKit
|
||||
{
|
||||
private const int SiteId = 1;
|
||||
|
||||
private readonly IDeploymentManagerRepository _repo = Substitute.For<IDeploymentManagerRepository>();
|
||||
private readonly IFlatteningPipeline _pipeline = Substitute.For<IFlatteningPipeline>();
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
private readonly IAuditService _audit = Substitute.For<IAuditService>();
|
||||
private readonly OperationLockManager _lockManager = new();
|
||||
|
||||
public DeploySiteAsyncTests()
|
||||
{
|
||||
_siteRepo.GetSiteByIdAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(ci => new Site($"Site {ci.ArgAt<int>(0)}", $"site-{ci.ArgAt<int>(0)}") { Id = ci.ArgAt<int>(0) });
|
||||
|
||||
_pipeline.CreateSession().Returns(_ => new FlattenSession(new TemplateGraphWatermark()));
|
||||
}
|
||||
|
||||
/// <summary>Seeds <paramref name="count"/> deployable instances at the test site.</summary>
|
||||
private List<Instance> ArrangeInstances(int count)
|
||||
{
|
||||
var instances = new List<Instance>();
|
||||
for (var i = 1; i <= count; i++)
|
||||
{
|
||||
var instance = new Instance($"Inst-{i:00}")
|
||||
{
|
||||
Id = i,
|
||||
SiteId = SiteId,
|
||||
TemplateId = 10,
|
||||
State = InstanceState.NotDeployed
|
||||
};
|
||||
instances.Add(instance);
|
||||
|
||||
_repo.GetInstanceByIdAsync(i, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
_repo.GetCurrentDeploymentStatusAsync(i, Arg.Any<CancellationToken>()).Returns((DeploymentRecord?)null);
|
||||
|
||||
var config = new FlattenedConfiguration { InstanceUniqueName = instance.UniqueName };
|
||||
_pipeline.FlattenAndValidateAsync(
|
||||
i, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
|
||||
.Returns(Result<FlatteningPipelineResult>.Success(
|
||||
new FlatteningPipelineResult(config, $"sha256:{i}", ValidationResult.Success())));
|
||||
}
|
||||
|
||||
_siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(instances);
|
||||
return instances;
|
||||
}
|
||||
|
||||
private DeploymentService CreateService(IActorRef commActor, int maxParallelism, TimeSpan? perInstanceTimeout = null)
|
||||
{
|
||||
var comms = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(30) }),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
comms.SetCommunicationActor(commActor);
|
||||
|
||||
var options = Options.Create(new DeploymentManagerOptions
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
SiteDeploymentMaxParallelism = maxParallelism,
|
||||
SiteDeploymentTimeoutPerInstance = perInstanceTimeout ?? TimeSpan.FromSeconds(30)
|
||||
});
|
||||
|
||||
return new DeploymentService(
|
||||
_repo, _siteRepo, _pipeline, comms, _lockManager, _audit,
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
options,
|
||||
Options.Create(new CommunicationOptions
|
||||
{
|
||||
CentralFetchBaseUrl = "https://central.test:9000",
|
||||
PendingDeploymentTtl = TimeSpan.FromMinutes(5)
|
||||
}),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_SlowInstance_OthersStillComplete_AndConcurrencyIsBounded()
|
||||
{
|
||||
const int instanceCount = 8;
|
||||
const int maxParallelism = 3;
|
||||
ArrangeInstances(instanceCount);
|
||||
|
||||
var tracker = new ConcurrencyTracker();
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ThrottledSiteActor(tracker, slowInstanceName: "Inst-01", slowDelay: TimeSpan.FromMilliseconds(400))));
|
||||
|
||||
var service = CreateService(commActor, maxParallelism);
|
||||
|
||||
var result = await service.DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
var summary = result.Value;
|
||||
|
||||
// Every instance produced a row and all eight completed, including the
|
||||
// seven that were NOT waiting on the slow site round-trip.
|
||||
Assert.Equal(instanceCount, summary.InstanceResults.Count);
|
||||
Assert.Equal(instanceCount, summary.SuccessCount);
|
||||
Assert.Equal(0, summary.FailureCount);
|
||||
|
||||
// The fan-out never exceeded the configured bound.
|
||||
Assert.True(tracker.MaxObserved <= maxParallelism,
|
||||
$"observed {tracker.MaxObserved} concurrent site round-trips, bound was {maxParallelism}");
|
||||
|
||||
// ...and it genuinely WAS concurrent, so the assertion above is not
|
||||
// vacuously satisfied by a serial implementation.
|
||||
Assert.True(tracker.MaxObserved > 1, "site round-trips did not run concurrently at all");
|
||||
|
||||
// Deployment identity: one distinct deployment id per instance.
|
||||
var deploymentIds = summary.InstanceResults.Select(r => r.DeploymentId).ToList();
|
||||
Assert.Equal(instanceCount, deploymentIds.Distinct().Count());
|
||||
Assert.DoesNotContain(deploymentIds, id => string.IsNullOrEmpty(id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_WedgedInstance_TimesOutAlone_RestSucceed()
|
||||
{
|
||||
ArrangeInstances(4);
|
||||
|
||||
var tracker = new ConcurrencyTracker();
|
||||
// Instance 2's round-trip is never answered — its per-instance deadline
|
||||
// must contain it rather than stalling the batch.
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ThrottledSiteActor(tracker, slowInstanceName: "Inst-02", slowDelay: Timeout.InfiniteTimeSpan)));
|
||||
|
||||
var service = CreateService(commActor, maxParallelism: 4, perInstanceTimeout: TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var result = await service.DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
var summary = result.Value;
|
||||
|
||||
Assert.Equal(4, summary.InstanceResults.Count);
|
||||
Assert.Equal(3, summary.SuccessCount);
|
||||
Assert.Equal(1, summary.FailureCount);
|
||||
|
||||
var wedged = Assert.Single(summary.InstanceResults, r => !r.Success);
|
||||
Assert.Equal("Inst-02", wedged.UniqueName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_ReleasesEveryOperationLock()
|
||||
{
|
||||
ArrangeInstances(5);
|
||||
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
|
||||
|
||||
var service = CreateService(commActor, maxParallelism: 2);
|
||||
|
||||
await service.DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
// The per-instance operation lock is held from prepare through finalize;
|
||||
// once the batch is done every entry must be reclaimed, or a second bulk
|
||||
// deploy of the same site would deadlock on its own leftovers.
|
||||
Assert.Equal(0, _lockManager.TrackedLockCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_SiteWithNoInstances_SucceedsWithEmptySummary()
|
||||
{
|
||||
_siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns([]);
|
||||
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(SiteId, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
Assert.Empty(result.Value.InstanceResults);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploySiteAsync_UnknownSite_ReturnsFailure()
|
||||
{
|
||||
_siteRepo.GetSiteByIdAsync(99, Arg.Any<CancellationToken>()).Returns((Site?)null);
|
||||
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
|
||||
|
||||
var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(99, "admin");
|
||||
|
||||
Assert.True(result.IsFailure);
|
||||
Assert.Contains("not found", result.Error);
|
||||
}
|
||||
|
||||
/// <summary>Records the peak number of simultaneously in-flight site round-trips.</summary>
|
||||
private sealed class ConcurrencyTracker
|
||||
{
|
||||
private int _current;
|
||||
private int _max;
|
||||
|
||||
/// <summary>Highest simultaneous in-flight count observed.</summary>
|
||||
public int MaxObserved => Volatile.Read(ref _max);
|
||||
|
||||
/// <summary>Marks one round-trip as started and updates the peak.</summary>
|
||||
public void Enter()
|
||||
{
|
||||
var now = Interlocked.Increment(ref _current);
|
||||
int observed;
|
||||
while (now > (observed = Volatile.Read(ref _max)))
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _max, now, observed) == observed)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Marks one round-trip as finished.</summary>
|
||||
public void Exit() => Interlocked.Decrement(ref _current);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stand-in site that answers <c>RefreshDeploymentCommand</c> with Success,
|
||||
/// optionally delaying (or never answering) one named instance so the test can
|
||||
/// observe the fan-out's bound and its per-instance deadline.
|
||||
///
|
||||
/// <para>
|
||||
/// The reply is scheduled rather than sent inline, so the actor's mailbox is
|
||||
/// not the thing serialising the batch — otherwise the concurrency the test is
|
||||
/// measuring would be an artefact of the harness.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private sealed class ThrottledSiteActor : ReceiveActor
|
||||
{
|
||||
public ThrottledSiteActor(ConcurrencyTracker tracker, string? slowInstanceName, TimeSpan slowDelay)
|
||||
{
|
||||
Receive<SiteEnvelope>(env =>
|
||||
{
|
||||
if (env.Message is not RefreshDeploymentCommand cmd)
|
||||
return;
|
||||
|
||||
var replyTo = Sender;
|
||||
var isSlow = slowInstanceName != null && cmd.InstanceUniqueName == slowInstanceName;
|
||||
|
||||
if (isSlow && slowDelay == Timeout.InfiniteTimeSpan)
|
||||
{
|
||||
// Never answer: the caller's per-instance deadline must fire.
|
||||
tracker.Enter();
|
||||
return;
|
||||
}
|
||||
|
||||
var delay = isSlow ? slowDelay : TimeSpan.FromMilliseconds(60);
|
||||
tracker.Enter();
|
||||
Context.System.Scheduler.Advanced.ScheduleOnce(delay, () =>
|
||||
{
|
||||
tracker.Exit();
|
||||
replyTo.Tell(new DeploymentStatusResponse(
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName,
|
||||
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user