using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests;
///
/// WP2.5: bulk site deployment. Pins the properties that make the fan-out safe —
/// concurrency is BOUNDED by the configured degree, one slow instance does not stop
/// the others completing, a per-instance timeout contains a wedged instance, and
/// every instance still gets its own deployment id and its own operation lock.
///
public class DeploySiteAsyncTests : TestKit
{
private const int SiteId = 1;
private readonly IDeploymentManagerRepository _repo = Substitute.For();
private readonly IFlatteningPipeline _pipeline = Substitute.For();
private readonly ISiteRepository _siteRepo = Substitute.For();
private readonly IAuditService _audit = Substitute.For();
private readonly OperationLockManager _lockManager = new();
public DeploySiteAsyncTests()
{
_siteRepo.GetSiteByIdAsync(Arg.Any(), Arg.Any())
.Returns(ci => new Site($"Site {ci.ArgAt(0)}", $"site-{ci.ArgAt(0)}") { Id = ci.ArgAt(0) });
_pipeline.CreateSession().Returns(_ => new FlattenSession(new TemplateGraphWatermark()));
}
/// Seeds deployable instances at the test site.
private List ArrangeInstances(int count)
{
var instances = new List();
for (var i = 1; i <= count; i++)
{
var instance = new Instance($"Inst-{i:00}")
{
Id = i,
SiteId = SiteId,
TemplateId = 10,
State = InstanceState.NotDeployed
};
instances.Add(instance);
_repo.GetInstanceByIdAsync(i, Arg.Any()).Returns(instance);
_repo.GetCurrentDeploymentStatusAsync(i, Arg.Any()).Returns((DeploymentRecord?)null);
var config = new FlattenedConfiguration { InstanceUniqueName = instance.UniqueName };
_pipeline.FlattenAndValidateAsync(
i, Arg.Any(), Arg.Any(), Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, $"sha256:{i}", ValidationResult.Success())));
}
_siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()).Returns(instances);
return instances;
}
private DeploymentService CreateService(IActorRef commActor, int maxParallelism, TimeSpan? perInstanceTimeout = null)
{
var comms = new CommunicationService(
Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(30) }),
NullLogger.Instance);
comms.SetCommunicationActor(commActor);
var options = Options.Create(new DeploymentManagerOptions
{
OperationLockTimeout = TimeSpan.FromSeconds(5),
SiteDeploymentMaxParallelism = maxParallelism,
SiteDeploymentTimeoutPerInstance = perInstanceTimeout ?? TimeSpan.FromSeconds(30)
});
return new DeploymentService(
_repo, _siteRepo, _pipeline, comms, _lockManager, _audit,
new DiffService(),
new RevisionHashService(),
new DeploymentStatusNotifier(NullLogger.Instance),
options,
Options.Create(new CommunicationOptions
{
CentralFetchBaseUrl = "https://central.test:9000",
PendingDeploymentTtl = TimeSpan.FromMinutes(5)
}),
NullLogger.Instance);
}
[Fact]
public async Task DeploySiteAsync_SlowInstance_OthersStillComplete_AndConcurrencyIsBounded()
{
const int instanceCount = 8;
const int maxParallelism = 3;
ArrangeInstances(instanceCount);
var tracker = new ConcurrencyTracker();
var commActor = Sys.ActorOf(Props.Create(() =>
new ThrottledSiteActor(tracker, slowInstanceName: "Inst-01", slowDelay: TimeSpan.FromMilliseconds(400))));
var service = CreateService(commActor, maxParallelism);
var result = await service.DeploySiteAsync(SiteId, "admin");
Assert.True(result.IsSuccess);
var summary = result.Value;
// Every instance produced a row and all eight completed, including the
// seven that were NOT waiting on the slow site round-trip.
Assert.Equal(instanceCount, summary.InstanceResults.Count);
Assert.Equal(instanceCount, summary.SuccessCount);
Assert.Equal(0, summary.FailureCount);
// The fan-out never exceeded the configured bound.
Assert.True(tracker.MaxObserved <= maxParallelism,
$"observed {tracker.MaxObserved} concurrent site round-trips, bound was {maxParallelism}");
// ...and it genuinely WAS concurrent, so the assertion above is not
// vacuously satisfied by a serial implementation.
Assert.True(tracker.MaxObserved > 1, "site round-trips did not run concurrently at all");
// Deployment identity: one distinct deployment id per instance.
var deploymentIds = summary.InstanceResults.Select(r => r.DeploymentId).ToList();
Assert.Equal(instanceCount, deploymentIds.Distinct().Count());
Assert.DoesNotContain(deploymentIds, id => string.IsNullOrEmpty(id));
}
[Fact]
public async Task DeploySiteAsync_WedgedInstance_TimesOutAlone_RestSucceed()
{
ArrangeInstances(4);
var tracker = new ConcurrencyTracker();
// Instance 2's round-trip is never answered — its per-instance deadline
// must contain it rather than stalling the batch.
var commActor = Sys.ActorOf(Props.Create(() =>
new ThrottledSiteActor(tracker, slowInstanceName: "Inst-02", slowDelay: Timeout.InfiniteTimeSpan)));
var service = CreateService(commActor, maxParallelism: 4, perInstanceTimeout: TimeSpan.FromMilliseconds(300));
var result = await service.DeploySiteAsync(SiteId, "admin");
Assert.True(result.IsSuccess);
var summary = result.Value;
Assert.Equal(4, summary.InstanceResults.Count);
Assert.Equal(3, summary.SuccessCount);
Assert.Equal(1, summary.FailureCount);
var wedged = Assert.Single(summary.InstanceResults, r => !r.Success);
Assert.Equal("Inst-02", wedged.UniqueName);
}
[Fact]
public async Task DeploySiteAsync_ReleasesEveryOperationLock()
{
ArrangeInstances(5);
var commActor = Sys.ActorOf(Props.Create(() =>
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
var service = CreateService(commActor, maxParallelism: 2);
await service.DeploySiteAsync(SiteId, "admin");
// The per-instance operation lock is held from prepare through finalize;
// once the batch is done every entry must be reclaimed, or a second bulk
// deploy of the same site would deadlock on its own leftovers.
Assert.Equal(0, _lockManager.TrackedLockCount);
}
[Fact]
public async Task DeploySiteAsync_SiteWithNoInstances_SucceedsWithEmptySummary()
{
_siteRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()).Returns([]);
var commActor = Sys.ActorOf(Props.Create(() =>
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(SiteId, "admin");
Assert.True(result.IsSuccess);
Assert.Empty(result.Value.InstanceResults);
}
[Fact]
public async Task DeploySiteAsync_UnknownSite_ReturnsFailure()
{
_siteRepo.GetSiteByIdAsync(99, Arg.Any()).Returns((Site?)null);
var commActor = Sys.ActorOf(Props.Create(() =>
new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
var result = await CreateService(commActor, maxParallelism: 2).DeploySiteAsync(99, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
/// Records the peak number of simultaneously in-flight site round-trips.
private sealed class ConcurrencyTracker
{
private int _current;
private int _max;
/// Highest simultaneous in-flight count observed.
public int MaxObserved => Volatile.Read(ref _max);
/// Marks one round-trip as started and updates the peak.
public void Enter()
{
var now = Interlocked.Increment(ref _current);
int observed;
while (now > (observed = Volatile.Read(ref _max)))
{
if (Interlocked.CompareExchange(ref _max, now, observed) == observed)
break;
}
}
/// Marks one round-trip as finished.
public void Exit() => Interlocked.Decrement(ref _current);
}
///
/// Stand-in site that answers RefreshDeploymentCommand with Success,
/// optionally delaying (or never answering) one named instance so the test can
/// observe the fan-out's bound and its per-instance deadline.
///
///
/// The reply is scheduled rather than sent inline, so the actor's mailbox is
/// not the thing serialising the batch — otherwise the concurrency the test is
/// measuring would be an artefact of the harness.
///
///
private sealed class ThrottledSiteActor : ReceiveActor
{
public ThrottledSiteActor(ConcurrencyTracker tracker, string? slowInstanceName, TimeSpan slowDelay)
{
Receive(env =>
{
if (env.Message is not RefreshDeploymentCommand cmd)
return;
var replyTo = Sender;
var isSlow = slowInstanceName != null && cmd.InstanceUniqueName == slowInstanceName;
if (isSlow && slowDelay == Timeout.InfiniteTimeSpan)
{
// Never answer: the caller's per-instance deadline must fire.
tracker.Enter();
return;
}
var delay = isSlow ? slowDelay : TimeSpan.FromMilliseconds(60);
tracker.Enter();
Context.System.Scheduler.Advanced.ScheduleOnce(delay, () =>
{
tracker.Exit();
replyTo.Tell(new DeploymentStatusResponse(
cmd.DeploymentId, cmd.InstanceUniqueName,
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
});
});
}
}
}