using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ScadaLink.Commons.Entities.Deployment;
using ScadaLink.Commons.Entities.Instances;
using ScadaLink.Commons.Interfaces.Repositories;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.Deployment;
using ScadaLink.Commons.Messages.Lifecycle;
using ScadaLink.Commons.Types;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.Commons.Types.Flattening;
using ScadaLink.Communication;
namespace ScadaLink.DeploymentManager.Tests;
///
/// WP-1/2/4/5/6/8/16: Tests for central-side DeploymentService.
///
public class DeploymentServiceTests : TestKit
{
private readonly IDeploymentManagerRepository _repo;
private readonly IFlatteningPipeline _pipeline;
private readonly CommunicationService _comms;
private readonly OperationLockManager _lockManager;
private readonly IAuditService _audit;
private readonly DeploymentService _service;
public DeploymentServiceTests()
{
_repo = Substitute.For();
_pipeline = Substitute.For();
_comms = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger.Instance);
_lockManager = new OperationLockManager();
_audit = Substitute.For();
var options = Options.Create(new DeploymentManagerOptions
{
OperationLockTimeout = TimeSpan.FromSeconds(5)
});
var siteRepo = Substitute.For();
_service = new DeploymentService(
_repo, siteRepo, _pipeline, _comms, _lockManager, _audit, options,
NullLogger.Instance);
}
// ── WP-1: Deployment flow ──
[Fact]
public async Task DeployInstanceAsync_InstanceNotFound_ReturnsFailure()
{
_repo.GetInstanceByIdAsync(1).Returns((Instance?)null);
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
[Fact]
public async Task DeployInstanceAsync_ValidationFails_ReturnsFailure()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var validationResult = new ValidationResult
{
Errors = [ValidationEntry.Error(ValidationCategory.ScriptCompilation, "Compile error")]
};
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(new FlattenedConfiguration(), "hash1", validationResult)));
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("validation failed", result.Error);
}
[Fact]
public async Task DeployInstanceAsync_FlatteningFails_ReturnsFailure()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Failure("Template chain empty"));
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("Validation failed", result.Error);
}
// ── WP-2: Deployment identity ──
[Fact]
public async Task DeployInstanceAsync_CreatesUniqueDeploymentId()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
// Pipeline succeeds
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
var validResult = ValidationResult.Success();
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "sha256:abc", validResult)));
// Capture the deployment record
DeploymentRecord? captured = null;
await _repo.AddDeploymentRecordAsync(Arg.Do(r => captured = r), Arg.Any());
// CommunicationService will throw because actor not set -- this tests the flow up to that point
try
{
await _service.DeployInstanceAsync(1, "admin");
}
catch (InvalidOperationException)
{
// Expected -- CommunicationService not initialized
}
Assert.NotNull(captured);
Assert.False(string.IsNullOrEmpty(captured!.DeploymentId));
Assert.Equal(32, captured.DeploymentId.Length); // GUID without hyphens
Assert.Equal("sha256:abc", captured.RevisionHash);
}
// ── WP-4: State transition validation ──
[Fact]
public async Task DeployInstanceAsync_EnabledInstance_AllowsDeploy()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "hash", ValidationResult.Success())));
// Will fail at communication layer, but passes state validation
try { await _service.DeployInstanceAsync(1, "admin"); } catch (InvalidOperationException) { }
// If we got past state validation, the deployment record was created
await _repo.Received().AddDeploymentRecordAsync(Arg.Any(), Arg.Any());
}
// ── DeploymentManager-001: unexpected exception must not leave record InProgress ──
[Fact]
public async Task DeployInstanceAsync_CommunicationThrowsUnexpectedException_RecordMarkedFailed()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success())));
// Capture the deployment record so we can inspect its final state.
DeploymentRecord? captured = null;
await _repo.AddDeploymentRecordAsync(
Arg.Do(r => captured = r), Arg.Any());
// _comms has no actor set, so DeployInstanceAsync throws
// InvalidOperationException -- a non-timeout, non-cancellation exception.
var result = await _service.DeployInstanceAsync(1, "admin");
// The exception must be handled, not escape.
Assert.True(result.IsFailure);
Assert.Contains("Deployment failed", result.Error);
// The record must not be left stuck in InProgress.
Assert.NotNull(captured);
Assert.Equal(DeploymentStatus.Failed, captured!.Status);
Assert.NotNull(captured.ErrorMessage);
Assert.NotNull(captured.CompletedAt);
}
// ── DeploymentManager-002: failure write must not use a cancelled token ──
[Fact]
public async Task DeployInstanceAsync_FailureWrite_UsesNonCancellableToken()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(Arg.Any(), Arg.Any()).Returns(instance);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(Arg.Any(), Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success())));
DeploymentRecord? captured = null;
await _repo.AddDeploymentRecordAsync(
Arg.Do(r => captured = r), Arg.Any());
// Simulate a repository that rejects already-cancelled tokens (the
// real EF Core behaviour when the operation token is cancelled). If the
// catch block passes the operation's cancelled token, the Failed-status
// write throws and the record stays InProgress -- the exact bug.
_repo.UpdateDeploymentRecordAsync(
Arg.Is(r => r.Status == DeploymentStatus.Failed),
Arg.Is(ct => ct.IsCancellationRequested))
.Returns(_ => throw new OperationCanceledException());
_repo.SaveChangesAsync(Arg.Is(ct => ct.IsCancellationRequested))
.Returns>(_ => throw new OperationCanceledException());
// The communication call fails (no actor set). The catch block must
// persist the Failed status with a non-cancellable token, so cleanup
// succeeds even when the caller's token is cancelled.
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.NotNull(captured);
Assert.Equal(DeploymentStatus.Failed, captured!.Status);
// The Failed-status write happened with a non-cancelled token.
await _repo.Received().UpdateDeploymentRecordAsync(
Arg.Is(r => r.Status == DeploymentStatus.Failed),
Arg.Is(ct => !ct.IsCancellationRequested));
}
// ── WP-6: Lifecycle commands ──
[Fact]
public async Task DisableInstanceAsync_InstanceNotFound_ReturnsFailure()
{
_repo.GetInstanceByIdAsync(1).Returns((Instance?)null);
var result = await _service.DisableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
[Fact]
public async Task DisableInstanceAsync_WhenDisabled_ReturnsTransitionError()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Disabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var result = await _service.DisableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not allowed", result.Error);
}
[Fact]
public async Task EnableInstanceAsync_WhenEnabled_ReturnsTransitionError()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var result = await _service.EnableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not allowed", result.Error);
}
[Fact]
public async Task DeleteInstanceAsync_InstanceNotFound_ReturnsFailure()
{
_repo.GetInstanceByIdAsync(1).Returns((Instance?)null);
var result = await _service.DeleteInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
// ── WP-8: Deployment comparison ──
[Fact]
public async Task GetDeploymentComparisonAsync_NoSnapshot_ReturnsFailure()
{
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns((DeployedConfigSnapshot?)null);
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsFailure);
Assert.Contains("No deployed snapshot", result.Error);
}
[Fact]
public async Task GetDeploymentComparisonAsync_SameHash_NotStale()
{
var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}")
{
InstanceId = 1,
DeployedAt = DateTimeOffset.UtcNow
};
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success())));
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsSuccess);
Assert.False(result.Value.IsStale);
}
[Fact]
public async Task GetDeploymentComparisonAsync_DifferentHash_IsStale()
{
var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}")
{
InstanceId = 1,
DeployedAt = DateTimeOffset.UtcNow
};
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, "sha256:xyz", ValidationResult.Success())));
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsSuccess);
Assert.True(result.Value.IsStale);
}
// ── WP-2: GetDeploymentStatusAsync ──
[Fact]
public async Task GetDeploymentStatusAsync_ReturnsRecordByDeploymentId()
{
var record = new DeploymentRecord("dep1", "admin")
{
Status = DeploymentStatus.Success
};
_repo.GetDeploymentByDeploymentIdAsync("dep1").Returns(record);
var result = await _service.GetDeploymentStatusAsync("dep1");
Assert.NotNull(result);
Assert.Equal("dep1", result!.DeploymentId);
Assert.Equal(DeploymentStatus.Success, result.Status);
}
// ── Audit logging ──
[Fact]
public async Task DeployInstanceAsync_AuditLogs()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
_pipeline.FlattenAndValidateAsync(1, Arg.Any())
.Returns(Result.Failure("Error"));
await _service.DeployInstanceAsync(1, "admin");
// Failure case does not reach audit (returns before communication)
// The audit is only logged after communication succeeds/fails
}
// ── DeploymentManager-006: query-the-site-before-redeploy idempotency ──
///
/// Builds a DeploymentService whose CommunicationService is backed by the
/// supplied actor, so the site query and deploy commands can be observed.
///
private DeploymentService CreateServiceWithCommActor(IActorRef commActor)
{
var comms = new CommunicationService(
Options.Create(new CommunicationOptions
{
QueryTimeout = TimeSpan.FromSeconds(5),
DeploymentTimeout = TimeSpan.FromSeconds(5)
}),
NullLogger.Instance);
comms.SetCommunicationActor(commActor);
var siteRepo = Substitute.For();
return new DeploymentService(
_repo, siteRepo, _pipeline, comms, _lockManager, _audit,
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
NullLogger.Instance);
}
private void SetupValidPipeline(int instanceId, string instanceName, string revisionHash)
{
var config = new FlattenedConfiguration { InstanceUniqueName = instanceName };
_pipeline.FlattenAndValidateAsync(instanceId, Arg.Any())
.Returns(Result.Success(
new FlatteningPipelineResult(config, revisionHash, ValidationResult.Success())));
}
[Fact]
public async Task DeployInstanceAsync_PriorInProgressRecord_SiteHasTargetHash_MarksSuccessWithoutRedeploy()
{
// Prior record stuck InProgress -> site is queried. The site reports it
// already has the TARGET revision hash, so the prior record is marked
// Success and NO new DeployInstanceCommand is sent.
var instance = new Instance("RedeployInst") { Id = 7, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(7, Arg.Any()).Returns(instance);
SetupValidPipeline(7, "RedeployInst", "sha256:target");
var prior = new DeploymentRecord("dep-prior", "admin")
{
InstanceId = 7,
Status = DeploymentStatus.InProgress,
RevisionHash = "sha256:target"
};
_repo.GetCurrentDeploymentStatusAsync(7, Arg.Any()).Returns(prior);
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(7, "admin");
Assert.True(result.IsSuccess);
Assert.Equal(DeploymentStatus.Success, prior.Status);
// The site query was issued, but no new deploy command was sent.
Assert.Equal(1, ReconcileProbeActor.QueryCount);
Assert.Equal(0, ReconcileProbeActor.DeployCount);
// No new deployment record was created — the prior one was reconciled.
await _repo.DidNotReceive().AddDeploymentRecordAsync(
Arg.Any(), Arg.Any());
}
[Fact]
public async Task DeployInstanceAsync_PriorInProgressRecord_SiteHasDifferentHash_ProceedsWithDeploy()
{
// Prior record stuck InProgress -> site is queried. The site has a
// DIFFERENT revision hash, so the normal deploy proceeds.
var instance = new Instance("RedeployInst2") { Id = 8, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(8, Arg.Any()).Returns(instance);
SetupValidPipeline(8, "RedeployInst2", "sha256:target");
var prior = new DeploymentRecord("dep-prior2", "admin")
{
InstanceId = 8,
Status = DeploymentStatus.InProgress,
RevisionHash = "sha256:old"
};
_repo.GetCurrentDeploymentStatusAsync(8, Arg.Any()).Returns(prior);
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:old", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(8, "admin");
Assert.True(result.IsSuccess);
Assert.Equal(1, ReconcileProbeActor.QueryCount);
// The normal deploy proceeded — a new command was sent.
Assert.Equal(1, ReconcileProbeActor.DeployCount);
await _repo.Received().AddDeploymentRecordAsync(
Arg.Any(), Arg.Any());
}
[Fact]
public async Task DeployInstanceAsync_PriorFailedTimeoutRecord_QueriesSite()
{
// A prior record Failed due to a timeout also triggers the site query.
var instance = new Instance("TimedOutInst") { Id = 9, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(9, Arg.Any()).Returns(instance);
SetupValidPipeline(9, "TimedOutInst", "sha256:target");
var prior = new DeploymentRecord("dep-prior3", "admin")
{
InstanceId = 9,
Status = DeploymentStatus.Failed,
RevisionHash = "sha256:target",
ErrorMessage = "Communication failure: deployment Ask timed out"
};
_repo.GetCurrentDeploymentStatusAsync(9, Arg.Any()).Returns(prior);
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(9, "admin");
Assert.True(result.IsSuccess);
Assert.Equal(1, ReconcileProbeActor.QueryCount);
Assert.Equal(0, ReconcileProbeActor.DeployCount);
Assert.Equal(DeploymentStatus.Success, prior.Status);
}
[Fact]
public async Task DeployInstanceAsync_PriorSuccessRecord_SkipsSiteQuery()
{
// A clean prior Success record must NOT trigger the extra round-trip.
var instance = new Instance("CleanInst") { Id = 10, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(10, Arg.Any()).Returns(instance);
SetupValidPipeline(10, "CleanInst", "sha256:target");
var prior = new DeploymentRecord("dep-clean", "admin")
{
InstanceId = 10,
Status = DeploymentStatus.Success,
RevisionHash = "sha256:old"
};
_repo.GetCurrentDeploymentStatusAsync(10, Arg.Any()).Returns(prior);
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(10, "admin");
Assert.True(result.IsSuccess);
// No site query — the prior deploy completed cleanly.
Assert.Equal(0, ReconcileProbeActor.QueryCount);
Assert.Equal(1, ReconcileProbeActor.DeployCount);
}
[Fact]
public async Task DeployInstanceAsync_FreshFirstTimeDeploy_SkipsSiteQuery()
{
// No prior record at all -> fresh deploy, no extra round-trip.
var instance = new Instance("FreshInst") { Id = 11, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(11, Arg.Any()).Returns(instance);
SetupValidPipeline(11, "FreshInst", "sha256:target");
_repo.GetCurrentDeploymentStatusAsync(11, Arg.Any())
.Returns((DeploymentRecord?)null);
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(11, "admin");
Assert.True(result.IsSuccess);
Assert.Equal(0, ReconcileProbeActor.QueryCount);
Assert.Equal(1, ReconcileProbeActor.DeployCount);
}
[Fact]
public async Task DeployInstanceAsync_PriorInProgressRecord_QueryFails_FallsThroughToDeploy()
{
// The site query fails (unreachable / times out). The deploy must NOT
// abort — it falls through to a normal deploy and relies on site-side
// stale-rejection as the safety net.
var instance = new Instance("UnreachableInst") { Id = 12, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(12, Arg.Any()).Returns(instance);
SetupValidPipeline(12, "UnreachableInst", "sha256:target");
var prior = new DeploymentRecord("dep-prior5", "admin")
{
InstanceId = 12,
Status = DeploymentStatus.InProgress,
RevisionHash = "sha256:target"
};
_repo.GetCurrentDeploymentStatusAsync(12, Arg.Any()).Returns(prior);
// The probe drops the query (no reply) -> the Ask times out.
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(siteHash: "sha256:target", failQuery: true)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(12, "admin");
// Did not abort — the deploy proceeded after the failed query.
Assert.True(result.IsSuccess);
Assert.Equal(1, ReconcileProbeActor.QueryCount);
Assert.Equal(1, ReconcileProbeActor.DeployCount);
}
///
/// Stand-in CentralCommunicationActor for reconciliation tests. Counts the
/// site queries and deploy commands it receives, answers queries with a
/// configurable applied revision hash, and (optionally) drops the query to
/// simulate an unreachable site so the central Ask times out.
///
private class ReconcileProbeActor : ReceiveActor
{
public static int QueryCount;
public static int DeployCount;
public ReconcileProbeActor(string siteHash, bool failQuery)
{
// Each test creates a fresh actor; reset the shared counters.
QueryCount = 0;
DeployCount = 0;
Receive(env =>
{
switch (env.Message)
{
case DeploymentStateQueryRequest q:
QueryCount++;
if (!failQuery)
{
Sender.Tell(new DeploymentStateQueryResponse(
q.CorrelationId, q.InstanceUniqueName, true,
"dep-applied", siteHash, DateTimeOffset.UtcNow));
}
// failQuery: drop the message -> caller's Ask times out.
break;
case DeployInstanceCommand d:
DeployCount++;
Sender.Tell(new DeploymentStatusResponse(
d.DeploymentId, d.InstanceUniqueName,
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
break;
}
});
}
}
}