using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ScadaLink.Commons.Entities.Deployment; using ScadaLink.Commons.Entities.Instances; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Messages.Deployment; using ScadaLink.Commons.Messages.Lifecycle; using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using ScadaLink.Commons.Types.Flattening; using ScadaLink.Communication; namespace ScadaLink.DeploymentManager.Tests; /// /// WP-1/2/4/5/6/8/16: Tests for central-side DeploymentService. /// public class DeploymentServiceTests : TestKit { private readonly IDeploymentManagerRepository _repo; private readonly IFlatteningPipeline _pipeline; private readonly CommunicationService _comms; private readonly OperationLockManager _lockManager; private readonly IAuditService _audit; private readonly DeploymentService _service; public DeploymentServiceTests() { _repo = Substitute.For(); _pipeline = Substitute.For(); _comms = new CommunicationService( Options.Create(new CommunicationOptions()), NullLogger.Instance); _lockManager = new OperationLockManager(); _audit = Substitute.For(); var options = Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }); var siteRepo = Substitute.For(); _service = new DeploymentService( _repo, siteRepo, _pipeline, _comms, _lockManager, _audit, options, NullLogger.Instance); } // ── WP-1: Deployment flow ── [Fact] public async Task DeployInstanceAsync_InstanceNotFound_ReturnsFailure() { _repo.GetInstanceByIdAsync(1).Returns((Instance?)null); var result = await _service.DeployInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("not found", result.Error); } [Fact] public async Task DeployInstanceAsync_ValidationFails_ReturnsFailure() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(1).Returns(instance); var validationResult = new ValidationResult { Errors = [ValidationEntry.Error(ValidationCategory.ScriptCompilation, "Compile error")] }; _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(new FlattenedConfiguration(), "hash1", validationResult))); var result = await _service.DeployInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("validation failed", result.Error); } [Fact] public async Task DeployInstanceAsync_FlatteningFails_ReturnsFailure() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(1).Returns(instance); _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Failure("Template chain empty")); var result = await _service.DeployInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("Validation failed", result.Error); } // ── WP-2: Deployment identity ── [Fact] public async Task DeployInstanceAsync_CreatesUniqueDeploymentId() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(1).Returns(instance); // Pipeline succeeds var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; var validResult = ValidationResult.Success(); _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "sha256:abc", validResult))); // Capture the deployment record DeploymentRecord? captured = null; await _repo.AddDeploymentRecordAsync(Arg.Do(r => captured = r), Arg.Any()); // CommunicationService will throw because actor not set -- this tests the flow up to that point try { await _service.DeployInstanceAsync(1, "admin"); } catch (InvalidOperationException) { // Expected -- CommunicationService not initialized } Assert.NotNull(captured); Assert.False(string.IsNullOrEmpty(captured!.DeploymentId)); Assert.Equal(32, captured.DeploymentId.Length); // GUID without hyphens Assert.Equal("sha256:abc", captured.RevisionHash); } // ── WP-4: State transition validation ── [Fact] public async Task DeployInstanceAsync_EnabledInstance_AllowsDeploy() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(1).Returns(instance); var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "hash", ValidationResult.Success()))); // Will fail at communication layer, but passes state validation try { await _service.DeployInstanceAsync(1, "admin"); } catch (InvalidOperationException) { } // If we got past state validation, the deployment record was created await _repo.Received().AddDeploymentRecordAsync(Arg.Any(), Arg.Any()); } // ── DeploymentManager-001: unexpected exception must not leave record InProgress ── [Fact] public async Task DeployInstanceAsync_CommunicationThrowsUnexpectedException_RecordMarkedFailed() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(1).Returns(instance); var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success()))); // Capture the deployment record so we can inspect its final state. DeploymentRecord? captured = null; await _repo.AddDeploymentRecordAsync( Arg.Do(r => captured = r), Arg.Any()); // _comms has no actor set, so DeployInstanceAsync throws // InvalidOperationException -- a non-timeout, non-cancellation exception. var result = await _service.DeployInstanceAsync(1, "admin"); // The exception must be handled, not escape. Assert.True(result.IsFailure); Assert.Contains("Deployment failed", result.Error); // The record must not be left stuck in InProgress. Assert.NotNull(captured); Assert.Equal(DeploymentStatus.Failed, captured!.Status); Assert.NotNull(captured.ErrorMessage); Assert.NotNull(captured.CompletedAt); } // ── DeploymentManager-002: failure write must not use a cancelled token ── [Fact] public async Task DeployInstanceAsync_FailureWrite_UsesNonCancellableToken() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(Arg.Any(), Arg.Any()).Returns(instance); var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; _pipeline.FlattenAndValidateAsync(Arg.Any(), Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success()))); DeploymentRecord? captured = null; await _repo.AddDeploymentRecordAsync( Arg.Do(r => captured = r), Arg.Any()); // Simulate a repository that rejects already-cancelled tokens (the // real EF Core behaviour when the operation token is cancelled). If the // catch block passes the operation's cancelled token, the Failed-status // write throws and the record stays InProgress -- the exact bug. _repo.UpdateDeploymentRecordAsync( Arg.Is(r => r.Status == DeploymentStatus.Failed), Arg.Is(ct => ct.IsCancellationRequested)) .Returns(_ => throw new OperationCanceledException()); _repo.SaveChangesAsync(Arg.Is(ct => ct.IsCancellationRequested)) .Returns>(_ => throw new OperationCanceledException()); // The communication call fails (no actor set). The catch block must // persist the Failed status with a non-cancellable token, so cleanup // succeeds even when the caller's token is cancelled. var result = await _service.DeployInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.NotNull(captured); Assert.Equal(DeploymentStatus.Failed, captured!.Status); // The Failed-status write happened with a non-cancelled token. await _repo.Received().UpdateDeploymentRecordAsync( Arg.Is(r => r.Status == DeploymentStatus.Failed), Arg.Is(ct => !ct.IsCancellationRequested)); } // ── WP-6: Lifecycle commands ── [Fact] public async Task DisableInstanceAsync_InstanceNotFound_ReturnsFailure() { _repo.GetInstanceByIdAsync(1).Returns((Instance?)null); var result = await _service.DisableInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("not found", result.Error); } [Fact] public async Task DisableInstanceAsync_WhenDisabled_ReturnsTransitionError() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Disabled }; _repo.GetInstanceByIdAsync(1).Returns(instance); var result = await _service.DisableInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("not allowed", result.Error); } [Fact] public async Task EnableInstanceAsync_WhenEnabled_ReturnsTransitionError() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(1).Returns(instance); var result = await _service.EnableInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("not allowed", result.Error); } [Fact] public async Task DeleteInstanceAsync_InstanceNotFound_ReturnsFailure() { _repo.GetInstanceByIdAsync(1).Returns((Instance?)null); var result = await _service.DeleteInstanceAsync(1, "admin"); Assert.True(result.IsFailure); Assert.Contains("not found", result.Error); } // ── WP-8: Deployment comparison ── [Fact] public async Task GetDeploymentComparisonAsync_NoSnapshot_ReturnsFailure() { _repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns((DeployedConfigSnapshot?)null); var result = await _service.GetDeploymentComparisonAsync(1); Assert.True(result.IsFailure); Assert.Contains("No deployed snapshot", result.Error); } [Fact] public async Task GetDeploymentComparisonAsync_SameHash_NotStale() { var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}") { InstanceId = 1, DeployedAt = DateTimeOffset.UtcNow }; _repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot); var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success()))); var result = await _service.GetDeploymentComparisonAsync(1); Assert.True(result.IsSuccess); Assert.False(result.Value.IsStale); } [Fact] public async Task GetDeploymentComparisonAsync_DifferentHash_IsStale() { var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}") { InstanceId = 1, DeployedAt = DateTimeOffset.UtcNow }; _repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot); var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" }; _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, "sha256:xyz", ValidationResult.Success()))); var result = await _service.GetDeploymentComparisonAsync(1); Assert.True(result.IsSuccess); Assert.True(result.Value.IsStale); } // ── WP-2: GetDeploymentStatusAsync ── [Fact] public async Task GetDeploymentStatusAsync_ReturnsRecordByDeploymentId() { var record = new DeploymentRecord("dep1", "admin") { Status = DeploymentStatus.Success }; _repo.GetDeploymentByDeploymentIdAsync("dep1").Returns(record); var result = await _service.GetDeploymentStatusAsync("dep1"); Assert.NotNull(result); Assert.Equal("dep1", result!.DeploymentId); Assert.Equal(DeploymentStatus.Success, result.Status); } // ── Audit logging ── [Fact] public async Task DeployInstanceAsync_AuditLogs() { var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(1).Returns(instance); _pipeline.FlattenAndValidateAsync(1, Arg.Any()) .Returns(Result.Failure("Error")); await _service.DeployInstanceAsync(1, "admin"); // Failure case does not reach audit (returns before communication) // The audit is only logged after communication succeeds/fails } // ── DeploymentManager-006: query-the-site-before-redeploy idempotency ── /// /// Builds a DeploymentService whose CommunicationService is backed by the /// supplied actor, so the site query and deploy commands can be observed. /// private DeploymentService CreateServiceWithCommActor(IActorRef commActor) { var comms = new CommunicationService( Options.Create(new CommunicationOptions { QueryTimeout = TimeSpan.FromSeconds(5), DeploymentTimeout = TimeSpan.FromSeconds(5) }), NullLogger.Instance); comms.SetCommunicationActor(commActor); var siteRepo = Substitute.For(); return new DeploymentService( _repo, siteRepo, _pipeline, comms, _lockManager, _audit, Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }), NullLogger.Instance); } private void SetupValidPipeline(int instanceId, string instanceName, string revisionHash) { var config = new FlattenedConfiguration { InstanceUniqueName = instanceName }; _pipeline.FlattenAndValidateAsync(instanceId, Arg.Any()) .Returns(Result.Success( new FlatteningPipelineResult(config, revisionHash, ValidationResult.Success()))); } [Fact] public async Task DeployInstanceAsync_PriorInProgressRecord_SiteHasTargetHash_MarksSuccessWithoutRedeploy() { // Prior record stuck InProgress -> site is queried. The site reports it // already has the TARGET revision hash, so the prior record is marked // Success and NO new DeployInstanceCommand is sent. var instance = new Instance("RedeployInst") { Id = 7, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(7, Arg.Any()).Returns(instance); SetupValidPipeline(7, "RedeployInst", "sha256:target"); var prior = new DeploymentRecord("dep-prior", "admin") { InstanceId = 7, Status = DeploymentStatus.InProgress, RevisionHash = "sha256:target" }; _repo.GetCurrentDeploymentStatusAsync(7, Arg.Any()).Returns(prior); var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(7, "admin"); Assert.True(result.IsSuccess); Assert.Equal(DeploymentStatus.Success, prior.Status); // The site query was issued, but no new deploy command was sent. Assert.Equal(1, ReconcileProbeActor.QueryCount); Assert.Equal(0, ReconcileProbeActor.DeployCount); // No new deployment record was created — the prior one was reconciled. await _repo.DidNotReceive().AddDeploymentRecordAsync( Arg.Any(), Arg.Any()); } [Fact] public async Task DeployInstanceAsync_PriorInProgressRecord_SiteHasDifferentHash_ProceedsWithDeploy() { // Prior record stuck InProgress -> site is queried. The site has a // DIFFERENT revision hash, so the normal deploy proceeds. var instance = new Instance("RedeployInst2") { Id = 8, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(8, Arg.Any()).Returns(instance); SetupValidPipeline(8, "RedeployInst2", "sha256:target"); var prior = new DeploymentRecord("dep-prior2", "admin") { InstanceId = 8, Status = DeploymentStatus.InProgress, RevisionHash = "sha256:old" }; _repo.GetCurrentDeploymentStatusAsync(8, Arg.Any()).Returns(prior); var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:old", failQuery: false))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(8, "admin"); Assert.True(result.IsSuccess); Assert.Equal(1, ReconcileProbeActor.QueryCount); // The normal deploy proceeded — a new command was sent. Assert.Equal(1, ReconcileProbeActor.DeployCount); await _repo.Received().AddDeploymentRecordAsync( Arg.Any(), Arg.Any()); } [Fact] public async Task DeployInstanceAsync_PriorFailedTimeoutRecord_QueriesSite() { // A prior record Failed due to a timeout also triggers the site query. var instance = new Instance("TimedOutInst") { Id = 9, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(9, Arg.Any()).Returns(instance); SetupValidPipeline(9, "TimedOutInst", "sha256:target"); var prior = new DeploymentRecord("dep-prior3", "admin") { InstanceId = 9, Status = DeploymentStatus.Failed, RevisionHash = "sha256:target", ErrorMessage = "Communication failure: deployment Ask timed out" }; _repo.GetCurrentDeploymentStatusAsync(9, Arg.Any()).Returns(prior); var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(9, "admin"); Assert.True(result.IsSuccess); Assert.Equal(1, ReconcileProbeActor.QueryCount); Assert.Equal(0, ReconcileProbeActor.DeployCount); Assert.Equal(DeploymentStatus.Success, prior.Status); } [Fact] public async Task DeployInstanceAsync_PriorSuccessRecord_SkipsSiteQuery() { // A clean prior Success record must NOT trigger the extra round-trip. var instance = new Instance("CleanInst") { Id = 10, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(10, Arg.Any()).Returns(instance); SetupValidPipeline(10, "CleanInst", "sha256:target"); var prior = new DeploymentRecord("dep-clean", "admin") { InstanceId = 10, Status = DeploymentStatus.Success, RevisionHash = "sha256:old" }; _repo.GetCurrentDeploymentStatusAsync(10, Arg.Any()).Returns(prior); var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(10, "admin"); Assert.True(result.IsSuccess); // No site query — the prior deploy completed cleanly. Assert.Equal(0, ReconcileProbeActor.QueryCount); Assert.Equal(1, ReconcileProbeActor.DeployCount); } [Fact] public async Task DeployInstanceAsync_FreshFirstTimeDeploy_SkipsSiteQuery() { // No prior record at all -> fresh deploy, no extra round-trip. var instance = new Instance("FreshInst") { Id = 11, SiteId = 1, State = InstanceState.NotDeployed }; _repo.GetInstanceByIdAsync(11, Arg.Any()).Returns(instance); SetupValidPipeline(11, "FreshInst", "sha256:target"); _repo.GetCurrentDeploymentStatusAsync(11, Arg.Any()) .Returns((DeploymentRecord?)null); var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:target", failQuery: false))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(11, "admin"); Assert.True(result.IsSuccess); Assert.Equal(0, ReconcileProbeActor.QueryCount); Assert.Equal(1, ReconcileProbeActor.DeployCount); } [Fact] public async Task DeployInstanceAsync_PriorInProgressRecord_QueryFails_FallsThroughToDeploy() { // The site query fails (unreachable / times out). The deploy must NOT // abort — it falls through to a normal deploy and relies on site-side // stale-rejection as the safety net. var instance = new Instance("UnreachableInst") { Id = 12, SiteId = 1, State = InstanceState.Enabled }; _repo.GetInstanceByIdAsync(12, Arg.Any()).Returns(instance); SetupValidPipeline(12, "UnreachableInst", "sha256:target"); var prior = new DeploymentRecord("dep-prior5", "admin") { InstanceId = 12, Status = DeploymentStatus.InProgress, RevisionHash = "sha256:target" }; _repo.GetCurrentDeploymentStatusAsync(12, Arg.Any()).Returns(prior); // The probe drops the query (no reply) -> the Ask times out. var commActor = Sys.ActorOf(Props.Create(() => new ReconcileProbeActor(siteHash: "sha256:target", failQuery: true))); var service = CreateServiceWithCommActor(commActor); var result = await service.DeployInstanceAsync(12, "admin"); // Did not abort — the deploy proceeded after the failed query. Assert.True(result.IsSuccess); Assert.Equal(1, ReconcileProbeActor.QueryCount); Assert.Equal(1, ReconcileProbeActor.DeployCount); } /// /// Stand-in CentralCommunicationActor for reconciliation tests. Counts the /// site queries and deploy commands it receives, answers queries with a /// configurable applied revision hash, and (optionally) drops the query to /// simulate an unreachable site so the central Ask times out. /// private class ReconcileProbeActor : ReceiveActor { public static int QueryCount; public static int DeployCount; public ReconcileProbeActor(string siteHash, bool failQuery) { // Each test creates a fresh actor; reset the shared counters. QueryCount = 0; DeployCount = 0; Receive(env => { switch (env.Message) { case DeploymentStateQueryRequest q: QueryCount++; if (!failQuery) { Sender.Tell(new DeploymentStateQueryResponse( q.CorrelationId, q.InstanceUniqueName, true, "dep-applied", siteHash, DateTimeOffset.UtcNow)); } // failQuery: drop the message -> caller's Ask times out. break; case DeployInstanceCommand d: DeployCount++; Sender.Tell(new DeploymentStatusResponse( d.DeploymentId, d.InstanceUniqueName, DeploymentStatus.Success, null, DateTimeOffset.UtcNow)); break; } }); } } }