using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening; namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests; /// /// WP2.5: bulk site deployment. Pins the properties that make the fan-out safe — /// concurrency is BOUNDED by the configured degree, one slow instance does not stop /// the others completing, a per-instance timeout contains a wedged instance, and /// every instance still gets its own deployment id and its own operation lock. /// public class DeploySiteAsyncTests : TestKit { private const int SiteId = 1; private readonly IDeploymentManagerRepository _repo = Substitute.For(); private readonly IFlatteningPipeline _pipeline = Substitute.For(); private readonly ISiteRepository _siteRepo = Substitute.For(); private readonly IAuditService _audit = Substitute.For(); private readonly OperationLockManager _lockManager = new(); public DeploySiteAsyncTests() { _siteRepo.GetSiteByIdAsync(Arg.Any(), Arg.Any()) .Returns(ci => new Site($"Site {ci.ArgAt(0)}", $"site-{ci.ArgAt(0)}") { Id = ci.ArgAt(0) }); _pipeline.CreateSession().Returns(_ => new FlattenSession(new TemplateGraphWatermark())); } /// Seeds deployable instances at the test site. private List ArrangeInstances(int count) { var instances = new List(); for (var i = 1; i <= count; i++) { var instance = new Instance($"Inst-{i:00}") { Id = i, SiteId = SiteId, TemplateId = 10, State = InstanceState.NotDeployed }; instances.Add(instance); _repo.GetInstanceByIdAsync(i, Arg.Any()).Returns(instance); _repo.GetCurrentDeploymentStatusAsync(i, Arg.Any()).Returns((DeploymentRecord?)null); var config = new FlattenedConfiguration { InstanceUniqueName = instance.UniqueName }; _pipeline.FlattenAndValidateAsync( i, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, $"sha256:{i}", ValidationResult.Success()))); } _siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()).Returns(instances); return instances; } private DeploymentService CreateService(IActorRef commActor, int maxParallelism, TimeSpan? perInstanceTimeout = null) { var comms = new CommunicationService( Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(30) }), NullLogger.Instance); comms.SetCommunicationActor(commActor); var options = Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5), SiteDeploymentMaxParallelism = maxParallelism, SiteDeploymentTimeoutPerInstance = perInstanceTimeout ?? TimeSpan.FromSeconds(30) }); return new DeploymentService( _repo, _siteRepo, _pipeline, comms, _lockManager, _audit, new DiffService(), new RevisionHashService(), new DeploymentStatusNotifier(NullLogger.Instance), options, Options.Create(new CommunicationOptions { CentralFetchBaseUrl = "https://central.test:9000", PendingDeploymentTtl = TimeSpan.FromMinutes(5) }), NullLogger.Instance); } [Fact] public async Task DeploySiteAsync_SlowInstance_OthersStillComplete_AndConcurrencyIsBounded() { const int instanceCount = 8; const int maxParallelism = 3; ArrangeInstances(instanceCount); var tracker = new ConcurrencyTracker(); var commActor = Sys.ActorOf(Props.Create(() => new ThrottledSiteActor(tracker, slowInstanceName: "Inst-01", slowDelay: TimeSpan.FromMilliseconds(400)))); var service = CreateService(commActor, maxParallelism); var result = await service.DeploySiteAsync(SiteId, "admin"); Assert.True(result.IsSuccess); var summary = result.Value; // Every instance produced a row and all eight completed, including the // seven that were NOT waiting on the slow site round-trip. Assert.Equal(instanceCount, summary.InstanceResults.Count); Assert.Equal(instanceCount, summary.SuccessCount); Assert.Equal(0, summary.FailureCount); // The fan-out never exceeded the configured bound. Assert.True(tracker.MaxObserved <= maxParallelism, $"observed {tracker.MaxObserved} concurrent site round-trips, bound was {maxParallelism}"); // ...and it genuinely WAS concurrent, so the assertion above is not // vacuously satisfied by a serial implementation. Assert.True(tracker.MaxObserved > 1, "site round-trips did not run concurrently at all"); // Deployment identity: one distinct deployment id per instance. var deploymentIds = summary.InstanceResults.Select(r => r.DeploymentId).ToList(); Assert.Equal(instanceCount, deploymentIds.Distinct().Count()); Assert.DoesNotContain(deploymentIds, id => string.IsNullOrEmpty(id)); } [Fact] public async Task DeploySiteAsync_WedgedInstance_TimesOutAlone_RestSucceed() { ArrangeInstances(4); var tracker = new ConcurrencyTracker(); // Instance 2's round-trip is never answered — its per-instance deadline // must contain it rather than stalling the batch. var commActor = Sys.ActorOf(Props.Create(() => new ThrottledSiteActor(tracker, slowInstanceName: "Inst-02", slowDelay: Timeout.InfiniteTimeSpan))); var service = CreateService(commActor, maxParallelism: 4, perInstanceTimeout: TimeSpan.FromMilliseconds(300)); var result = await service.DeploySiteAsync(SiteId, "admin"); Assert.True(result.IsSuccess); var summary = result.Value; Assert.Equal(4, summary.InstanceResults.Count); Assert.Equal(3, summary.SuccessCount); Assert.Equal(1, summary.FailureCount); var wedged = Assert.Single(summary.InstanceResults, r => !r.Success); Assert.Equal("Inst-02", wedged.UniqueName); } [Fact] public async Task DeploySiteAsync_ReleasesEveryOperationLock() { ArrangeInstances(5); var commActor = Sys.ActorOf(Props.Create(() => new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero))); var service = CreateService(commActor, maxParallelism: 2); await service.DeploySiteAsync(SiteId, "admin"); // The per-instance operation lock is held from prepare through finalize; // once the batch is done every entry must be reclaimed, or a second bulk // deploy of the same site would deadlock on its own leftovers. Assert.Equal(0, _lockManager.TrackedLockCount); } [Fact] public async Task DeploySiteAsync_SiteWithNoInstances_SucceedsWithEmptySummary() { _siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()).Returns([]); var commActor = Sys.ActorOf(Props.Create(() => new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero))); var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(SiteId, "admin"); Assert.True(result.IsSuccess); Assert.Empty(result.Value.InstanceResults); } [Fact] public async Task DeploySiteAsync_UnknownSite_ReturnsFailure() { _siteRepo.GetSiteByIdAsync(99, Arg.Any()).Returns((Site?)null); var commActor = Sys.ActorOf(Props.Create(() => new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero))); var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(99, "admin"); Assert.True(result.IsFailure); Assert.Contains("not found", result.Error); } /// Records the peak number of simultaneously in-flight site round-trips. private sealed class ConcurrencyTracker { private int _current; private int _max; /// Highest simultaneous in-flight count observed. public int MaxObserved => Volatile.Read(ref _max); /// Marks one round-trip as started and updates the peak. public void Enter() { var now = Interlocked.Increment(ref _current); int observed; while (now > (observed = Volatile.Read(ref _max))) { if (Interlocked.CompareExchange(ref _max, now, observed) == observed) break; } } /// Marks one round-trip as finished. public void Exit() => Interlocked.Decrement(ref _current); } /// /// Stand-in site that answers RefreshDeploymentCommand with Success, /// optionally delaying (or never answering) one named instance so the test can /// observe the fan-out's bound and its per-instance deadline. /// /// /// The reply is scheduled rather than sent inline, so the actor's mailbox is /// not the thing serialising the batch — otherwise the concurrency the test is /// measuring would be an artefact of the harness. /// /// private sealed class ThrottledSiteActor : ReceiveActor { public ThrottledSiteActor(ConcurrencyTracker tracker, string? slowInstanceName, TimeSpan slowDelay) { Receive(env => { if (env.Message is not RefreshDeploymentCommand cmd) return; var replyTo = Sender; var isSlow = slowInstanceName != null && cmd.InstanceUniqueName == slowInstanceName; if (isSlow && slowDelay == Timeout.InfiniteTimeSpan) { // Never answer: the caller's per-instance deadline must fire. tracker.Enter(); return; } var delay = isSlow ? slowDelay : TimeSpan.FromMilliseconds(60); tracker.Enter(); Context.System.Scheduler.Advanced.ScheduleOnce(delay, () => { tracker.Exit(); replyTo.Tell(new DeploymentStatusResponse( cmd.DeploymentId, cmd.InstanceUniqueName, DeploymentStatus.Success, null, DateTimeOffset.UtcNow)); }); }); } } }