using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening; namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests; /// /// Gitea #73: branches the existing suites leave untested — /// a site that answers but REFUSES (as opposed to timing out or throwing), the query-before- /// redeploy trigger for a non-timeout failure, and the detach-and-retry-once recovery of the /// Failed-status write. Each one decides an instance's or a deployment record's state. /// public class DeploymentServiceFailurePathTests : TestKit { private readonly IDeploymentManagerRepository _repo = Substitute.For(); private readonly IFlatteningPipeline _pipeline = Substitute.For(); private readonly IAuditService _audit = Substitute.For(); /// /// Stand-in central communication actor: answers each site-bound message with whatever /// reply returns for it (null drops it), and counts every message it saw. /// private sealed class ScriptedSiteActor : ReceiveActor { /// Initializes the actor. /// Maps a site-bound message to the site's answer. /// Sink for every message received. public ScriptedSiteActor(Func reply, List seen) { Receive(env => { lock (seen) seen.Add(env.Message); var answer = reply(env.Message); if (answer is not null) Sender.Tell(answer); }); } } private DeploymentService CreateService(IActorRef? commActor) { var comms = new CommunicationService( Options.Create(new CommunicationOptions { QueryTimeout = TimeSpan.FromSeconds(5), DeploymentTimeout = TimeSpan.FromSeconds(5), }), NullLogger.Instance); if (commActor is not null) comms.SetCommunicationActor(commActor); var siteRepo = Substitute.For(); siteRepo.GetSiteByIdAsync(Arg.Any(), Arg.Any()) .Returns(ci => new Site($"Site {ci.ArgAt(0)}", $"site-{ci.ArgAt(0)}") { Id = ci.ArgAt(0) }); return new DeploymentService( _repo, siteRepo, Substitute.For(), _pipeline, comms, new OperationLockManager(), _audit, new DiffService(), new RevisionHashService(), new DeploymentStatusNotifier(NullLogger.Instance), Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5), LifecycleCommandTimeout = TimeSpan.FromSeconds(5), }), Options.Create(new CommunicationOptions { CentralFetchBaseUrl = "https://central.test:9000", PendingDeploymentTtl = TimeSpan.FromMinutes(5), }), NullLogger.Instance); } private (IActorRef Actor, List Seen) SiteActor(Func reply) { var seen = new List(); return (Sys.ActorOf(Props.Create(() => new ScriptedSiteActor(reply, seen))), seen); } private Instance ArrangeDeployable(int id, InstanceState state, string revisionHash = "sha256:new") { var instance = new Instance($"Inst{id}") { Id = id, SiteId = 1, State = state }; _repo.GetInstanceByIdAsync(id, Arg.Any()).Returns(instance); _pipeline.FlattenAndValidateAsync(id, Arg.Any()) .Returns(Result.Success(new FlatteningPipelineResult( new FlattenedConfiguration { InstanceUniqueName = instance.UniqueName }, revisionHash, ValidationResult.Success()))); return instance; } // ── Deploy: the site answers, but with Failed ── /// /// A site that receives the deploy but reports Failed (e.g. its own compile step rejected the /// config) commits a Failed record carrying the SITE's message — not a timeout marker, so the /// next redeploy does not waste a site query — and leaves the instance's state and deployed /// snapshot exactly as they were: the site is not running the new config. /// /// The instance state before the deploy. /// A task that represents the asynchronous operation. [Theory] [InlineData(InstanceState.NotDeployed)] [InlineData(InstanceState.Disabled)] public async Task Deploy_SiteReportsFailed_RecordFailedWithSiteError_InstanceAndSnapshotUntouched(InstanceState initialState) { var instance = ArrangeDeployable(10, initialState); DeploymentRecord? record = null; await _repo.AddDeploymentRecordAsync(Arg.Do(r => record = r), Arg.Any()); var (actor, _) = SiteActor(msg => msg is RefreshDeploymentCommand d ? new DeploymentStatusResponse(d.DeploymentId, d.InstanceUniqueName, DeploymentStatus.Failed, "Script 'OnStart' failed to compile at site", DateTimeOffset.UtcNow) : null); var result = await CreateService(actor).DeployInstanceAsync(10, "admin"); Assert.True(result.IsFailure); Assert.Equal("Deployment failed: Script 'OnStart' failed to compile at site", result.Error); Assert.NotNull(record); Assert.Equal(DeploymentStatus.Failed, record!.Status); Assert.Equal("Script 'OnStart' failed to compile at site", record.ErrorMessage); Assert.NotNull(record.CompletedAt); Assert.Equal(initialState, instance.State); await _repo.DidNotReceive().UpdateInstanceAsync(Arg.Any(), Arg.Any()); await _repo.DidNotReceive().AddDeployedSnapshotAsync(Arg.Any(), Arg.Any()); await _repo.DidNotReceive().UpdateDeployedSnapshotAsync(Arg.Any(), Arg.Any()); // The outcome is audited as a Deploy with Failed status — not as a DeployFailed fault. await _audit.Received(1).LogAsync("admin", "Deploy", "Instance", "10", "Inst10", Arg.Any(), Arg.Any()); await _audit.DidNotReceive().LogAsync(Arg.Any(), "DeployFailed", Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } /// /// A prior deployment that failed for a NON-timeout reason (the site refused it) cannot have /// been applied, so the redeploy goes straight to the site without a state query. Only an /// InProgress or timeout-marked Failed record may hide an apply central never heard about. /// /// A task that represents the asynchronous operation. [Fact] public async Task Redeploy_PriorFailedForNonTimeoutReason_SkipsSiteQuery() { ArrangeDeployable(11, InstanceState.Enabled); _repo.GetCurrentDeploymentStatusAsync(11, Arg.Any()) .Returns(new DeploymentRecord("dep-prior", "admin") { InstanceId = 11, Status = DeploymentStatus.Failed, ErrorMessage = "Script 'OnStart' failed to compile at site", }); var (actor, seen) = SiteActor(msg => msg switch { RefreshDeploymentCommand d => new DeploymentStatusResponse( d.DeploymentId, d.InstanceUniqueName, DeploymentStatus.Success, null, DateTimeOffset.UtcNow), DeploymentStateQueryRequest q => new DeploymentStateQueryResponse( q.CorrelationId, q.InstanceUniqueName, true, "dep-x", "sha256:new", DateTimeOffset.UtcNow), _ => null, }); var result = await CreateService(actor).DeployInstanceAsync(11, "admin"); Assert.True(result.IsSuccess, result.IsFailure ? result.Error : null); lock (seen) { Assert.DoesNotContain(seen, m => m is DeploymentStateQueryRequest); Assert.Single(seen, m => m is RefreshDeploymentCommand); } } // ── Failed-status write: detach and retry once ── /// /// When the cleanup write of the Failed status itself fails, the pending changes are detached /// and the write is retried ONCE on the clean unit of work, so the record still reaches its /// terminal state; with nothing to detach there is nothing to retry. Either way the caller /// gets the ORIGINAL deployment error, never the cleanup error. /// /// What DetachPendingChanges reports. /// Failed-status write attempts expected. /// A task that represents the asynchronous operation. [Theory] [InlineData(2, 2)] [InlineData(0, 1)] public async Task Deploy_FailedStatusWriteFails_DetachesAndRetriesOnce(int detached, int expectedFailedWrites) { ArrangeDeployable(12, InstanceState.NotDeployed); DeploymentRecord? record = null; await _repo.AddDeploymentRecordAsync(Arg.Do(r => record = r), Arg.Any()); _repo.DetachPendingChanges().Returns(detached); var failedWrites = 0; _repo.SaveChangesAsync(Arg.Any()).Returns(_ => { if (record?.Status == DeploymentStatus.Failed && ++failedWrites == 1) throw new InvalidOperationException("poisoned unit of work"); return Task.FromResult(1); }); // No communication actor: the send faults with a non-timeout error. var result = await CreateService(commActor: null).DeployInstanceAsync(12, "admin"); Assert.True(result.IsFailure); Assert.DoesNotContain("poisoned", result.Error); Assert.Equal(DeploymentStatus.Failed, record!.Status); Assert.StartsWith("Deployment error:", record.ErrorMessage); Assert.Equal(expectedFailedWrites, failedWrites); _repo.Received(1).DetachPendingChanges(); // The failure audit row is written even when the status write could not be. await _audit.Received(1).LogAsync("admin", "DeployFailed", "Instance", "12", "Inst12", Arg.Any(), Arg.Any()); } // ── Lifecycle commands: the site answers, but refuses ── /// Lifecycle actions the theory below drives. public enum LifecycleAction { /// DisableInstanceAsync. Disable, /// EnableInstanceAsync. Enable, /// DeleteInstanceAsync. Delete, } /// /// A lifecycle command the site answers with Success=false leaves the central instance /// exactly as it was — state unchanged, nothing deleted — still audits the attempt, and /// surfaces the site's message (or a default when the site gave none). /// /// The lifecycle action. /// The instance state before the command. /// The site's error message, or null. /// The error the caller must see. /// A task that represents the asynchronous operation. [Theory] [InlineData(LifecycleAction.Disable, InstanceState.Enabled, "Instance actor busy", "Instance actor busy")] [InlineData(LifecycleAction.Disable, InstanceState.Enabled, null, "Disable failed.")] [InlineData(LifecycleAction.Enable, InstanceState.Disabled, "No stored config", "No stored config")] [InlineData(LifecycleAction.Enable, InstanceState.Disabled, null, "Enable failed.")] [InlineData(LifecycleAction.Delete, InstanceState.Enabled, "Delete refused", "Delete refused")] [InlineData(LifecycleAction.Delete, InstanceState.Disabled, null, "Delete failed. Site may be unreachable.")] public async Task Lifecycle_SiteRefuses_InstanceUnchanged_AttemptAudited( LifecycleAction action, InstanceState initialState, string? siteError, string expectedError) { var instance = new Instance("LcInst") { Id = 20, SiteId = 1, State = initialState }; _repo.GetInstanceByIdAsync(20, Arg.Any()).Returns(instance); var (actor, _) = SiteActor(msg => msg switch { DisableInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow), EnableInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow), DeleteInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow), _ => null, }); var service = CreateService(actor); var result = action switch { LifecycleAction.Disable => await service.DisableInstanceAsync(20, "admin"), LifecycleAction.Enable => await service.EnableInstanceAsync(20, "admin"), _ => await service.DeleteInstanceAsync(20, "admin"), }; Assert.True(result.IsFailure); Assert.Equal(expectedError, result.Error); Assert.Equal(initialState, instance.State); await _repo.DidNotReceive().UpdateInstanceAsync(Arg.Any(), Arg.Any()); await _repo.DidNotReceive().DeleteInstanceAsync(Arg.Any(), Arg.Any()); await _repo.DidNotReceive().SaveChangesAsync(Arg.Any()); await _audit.Received(1).LogAsync("admin", action.ToString(), "Instance", "20", "LcInst", Arg.Any(), Arg.Any()); } }