using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests;
///
/// Gitea #73: branches the existing suites leave untested —
/// a site that answers but REFUSES (as opposed to timing out or throwing), the query-before-
/// redeploy trigger for a non-timeout failure, and the detach-and-retry-once recovery of the
/// Failed-status write. Each one decides an instance's or a deployment record's state.
///
public class DeploymentServiceFailurePathTests : TestKit
{
private readonly IDeploymentManagerRepository _repo = Substitute.For();
private readonly IFlatteningPipeline _pipeline = Substitute.For();
private readonly IAuditService _audit = Substitute.For();
///
/// Stand-in central communication actor: answers each site-bound message with whatever
/// reply returns for it (null drops it), and counts every message it saw.
///
private sealed class ScriptedSiteActor : ReceiveActor
{
/// Initializes the actor.
/// Maps a site-bound message to the site's answer.
/// Sink for every message received.
public ScriptedSiteActor(Func reply, List seen)
{
Receive(env =>
{
lock (seen) seen.Add(env.Message);
var answer = reply(env.Message);
if (answer is not null)
Sender.Tell(answer);
});
}
}
private DeploymentService CreateService(IActorRef? commActor)
{
var comms = new CommunicationService(
Options.Create(new CommunicationOptions
{
QueryTimeout = TimeSpan.FromSeconds(5),
DeploymentTimeout = TimeSpan.FromSeconds(5),
}),
NullLogger.Instance);
if (commActor is not null)
comms.SetCommunicationActor(commActor);
var siteRepo = Substitute.For();
siteRepo.GetSiteByIdAsync(Arg.Any(), Arg.Any())
.Returns(ci => new Site($"Site {ci.ArgAt(0)}", $"site-{ci.ArgAt(0)}") { Id = ci.ArgAt(0) });
return new DeploymentService(
_repo, siteRepo, Substitute.For(), _pipeline, comms,
new OperationLockManager(), _audit,
new DiffService(),
new RevisionHashService(),
new DeploymentStatusNotifier(NullLogger.Instance),
Options.Create(new DeploymentManagerOptions
{
OperationLockTimeout = TimeSpan.FromSeconds(5),
LifecycleCommandTimeout = TimeSpan.FromSeconds(5),
}),
Options.Create(new CommunicationOptions
{
CentralFetchBaseUrl = "https://central.test:9000",
PendingDeploymentTtl = TimeSpan.FromMinutes(5),
}),
NullLogger.Instance);
}
private (IActorRef Actor, List Seen) SiteActor(Func reply)
{
var seen = new List();
return (Sys.ActorOf(Props.Create(() => new ScriptedSiteActor(reply, seen))), seen);
}
private Instance ArrangeDeployable(int id, InstanceState state, string revisionHash = "sha256:new")
{
var instance = new Instance($"Inst{id}") { Id = id, SiteId = 1, State = state };
_repo.GetInstanceByIdAsync(id, Arg.Any()).Returns(instance);
_pipeline.FlattenAndValidateAsync(id, Arg.Any())
.Returns(Result.Success(new FlatteningPipelineResult(
new FlattenedConfiguration { InstanceUniqueName = instance.UniqueName },
revisionHash,
ValidationResult.Success())));
return instance;
}
// ── Deploy: the site answers, but with Failed ──
///
/// A site that receives the deploy but reports Failed (e.g. its own compile step rejected the
/// config) commits a Failed record carrying the SITE's message — not a timeout marker, so the
/// next redeploy does not waste a site query — and leaves the instance's state and deployed
/// snapshot exactly as they were: the site is not running the new config.
///
/// The instance state before the deploy.
/// A task that represents the asynchronous operation.
[Theory]
[InlineData(InstanceState.NotDeployed)]
[InlineData(InstanceState.Disabled)]
public async Task Deploy_SiteReportsFailed_RecordFailedWithSiteError_InstanceAndSnapshotUntouched(InstanceState initialState)
{
var instance = ArrangeDeployable(10, initialState);
DeploymentRecord? record = null;
await _repo.AddDeploymentRecordAsync(Arg.Do(r => record = r), Arg.Any());
var (actor, _) = SiteActor(msg => msg is RefreshDeploymentCommand d
? new DeploymentStatusResponse(d.DeploymentId, d.InstanceUniqueName,
DeploymentStatus.Failed, "Script 'OnStart' failed to compile at site", DateTimeOffset.UtcNow)
: null);
var result = await CreateService(actor).DeployInstanceAsync(10, "admin");
Assert.True(result.IsFailure);
Assert.Equal("Deployment failed: Script 'OnStart' failed to compile at site", result.Error);
Assert.NotNull(record);
Assert.Equal(DeploymentStatus.Failed, record!.Status);
Assert.Equal("Script 'OnStart' failed to compile at site", record.ErrorMessage);
Assert.NotNull(record.CompletedAt);
Assert.Equal(initialState, instance.State);
await _repo.DidNotReceive().UpdateInstanceAsync(Arg.Any(), Arg.Any());
await _repo.DidNotReceive().AddDeployedSnapshotAsync(Arg.Any(), Arg.Any());
await _repo.DidNotReceive().UpdateDeployedSnapshotAsync(Arg.Any(), Arg.Any());
// The outcome is audited as a Deploy with Failed status — not as a DeployFailed fault.
await _audit.Received(1).LogAsync("admin", "Deploy", "Instance", "10", "Inst10",
Arg.Any(), Arg.Any());
await _audit.DidNotReceive().LogAsync(Arg.Any(), "DeployFailed", Arg.Any(),
Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any());
}
///
/// A prior deployment that failed for a NON-timeout reason (the site refused it) cannot have
/// been applied, so the redeploy goes straight to the site without a state query. Only an
/// InProgress or timeout-marked Failed record may hide an apply central never heard about.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task Redeploy_PriorFailedForNonTimeoutReason_SkipsSiteQuery()
{
ArrangeDeployable(11, InstanceState.Enabled);
_repo.GetCurrentDeploymentStatusAsync(11, Arg.Any())
.Returns(new DeploymentRecord("dep-prior", "admin")
{
InstanceId = 11,
Status = DeploymentStatus.Failed,
ErrorMessage = "Script 'OnStart' failed to compile at site",
});
var (actor, seen) = SiteActor(msg => msg switch
{
RefreshDeploymentCommand d => new DeploymentStatusResponse(
d.DeploymentId, d.InstanceUniqueName, DeploymentStatus.Success, null, DateTimeOffset.UtcNow),
DeploymentStateQueryRequest q => new DeploymentStateQueryResponse(
q.CorrelationId, q.InstanceUniqueName, true, "dep-x", "sha256:new", DateTimeOffset.UtcNow),
_ => null,
});
var result = await CreateService(actor).DeployInstanceAsync(11, "admin");
Assert.True(result.IsSuccess, result.IsFailure ? result.Error : null);
lock (seen)
{
Assert.DoesNotContain(seen, m => m is DeploymentStateQueryRequest);
Assert.Single(seen, m => m is RefreshDeploymentCommand);
}
}
// ── Failed-status write: detach and retry once ──
///
/// When the cleanup write of the Failed status itself fails, the pending changes are detached
/// and the write is retried ONCE on the clean unit of work, so the record still reaches its
/// terminal state; with nothing to detach there is nothing to retry. Either way the caller
/// gets the ORIGINAL deployment error, never the cleanup error.
///
/// What DetachPendingChanges reports.
/// Failed-status write attempts expected.
/// A task that represents the asynchronous operation.
[Theory]
[InlineData(2, 2)]
[InlineData(0, 1)]
public async Task Deploy_FailedStatusWriteFails_DetachesAndRetriesOnce(int detached, int expectedFailedWrites)
{
ArrangeDeployable(12, InstanceState.NotDeployed);
DeploymentRecord? record = null;
await _repo.AddDeploymentRecordAsync(Arg.Do(r => record = r), Arg.Any());
_repo.DetachPendingChanges().Returns(detached);
var failedWrites = 0;
_repo.SaveChangesAsync(Arg.Any()).Returns(_ =>
{
if (record?.Status == DeploymentStatus.Failed && ++failedWrites == 1)
throw new InvalidOperationException("poisoned unit of work");
return Task.FromResult(1);
});
// No communication actor: the send faults with a non-timeout error.
var result = await CreateService(commActor: null).DeployInstanceAsync(12, "admin");
Assert.True(result.IsFailure);
Assert.DoesNotContain("poisoned", result.Error);
Assert.Equal(DeploymentStatus.Failed, record!.Status);
Assert.StartsWith("Deployment error:", record.ErrorMessage);
Assert.Equal(expectedFailedWrites, failedWrites);
_repo.Received(1).DetachPendingChanges();
// The failure audit row is written even when the status write could not be.
await _audit.Received(1).LogAsync("admin", "DeployFailed", "Instance", "12", "Inst12",
Arg.Any(), Arg.Any());
}
// ── Lifecycle commands: the site answers, but refuses ──
/// Lifecycle actions the theory below drives.
public enum LifecycleAction
{
/// DisableInstanceAsync.
Disable,
/// EnableInstanceAsync.
Enable,
/// DeleteInstanceAsync.
Delete,
}
///
/// A lifecycle command the site answers with Success=false leaves the central instance
/// exactly as it was — state unchanged, nothing deleted — still audits the attempt, and
/// surfaces the site's message (or a default when the site gave none).
///
/// The lifecycle action.
/// The instance state before the command.
/// The site's error message, or null.
/// The error the caller must see.
/// A task that represents the asynchronous operation.
[Theory]
[InlineData(LifecycleAction.Disable, InstanceState.Enabled, "Instance actor busy", "Instance actor busy")]
[InlineData(LifecycleAction.Disable, InstanceState.Enabled, null, "Disable failed.")]
[InlineData(LifecycleAction.Enable, InstanceState.Disabled, "No stored config", "No stored config")]
[InlineData(LifecycleAction.Enable, InstanceState.Disabled, null, "Enable failed.")]
[InlineData(LifecycleAction.Delete, InstanceState.Enabled, "Delete refused", "Delete refused")]
[InlineData(LifecycleAction.Delete, InstanceState.Disabled, null, "Delete failed. Site may be unreachable.")]
public async Task Lifecycle_SiteRefuses_InstanceUnchanged_AttemptAudited(
LifecycleAction action, InstanceState initialState, string? siteError, string expectedError)
{
var instance = new Instance("LcInst") { Id = 20, SiteId = 1, State = initialState };
_repo.GetInstanceByIdAsync(20, Arg.Any()).Returns(instance);
var (actor, _) = SiteActor(msg => msg switch
{
DisableInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow),
EnableInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow),
DeleteInstanceCommand c => new InstanceLifecycleResponse(c.CommandId, c.InstanceUniqueName, false, siteError, DateTimeOffset.UtcNow),
_ => null,
});
var service = CreateService(actor);
var result = action switch
{
LifecycleAction.Disable => await service.DisableInstanceAsync(20, "admin"),
LifecycleAction.Enable => await service.EnableInstanceAsync(20, "admin"),
_ => await service.DeleteInstanceAsync(20, "admin"),
};
Assert.True(result.IsFailure);
Assert.Equal(expectedError, result.Error);
Assert.Equal(initialState, instance.State);
await _repo.DidNotReceive().UpdateInstanceAsync(Arg.Any(), Arg.Any());
await _repo.DidNotReceive().DeleteInstanceAsync(Arg.Any(), Arg.Any());
await _repo.DidNotReceive().SaveChangesAsync(Arg.Any());
await _audit.Received(1).LogAsync("admin", action.ToString(), "Instance", "20", "LcInst",
Arg.Any(), Arg.Any());
}
}