Phase 3C: Deployment pipeline & Store-and-Forward engine

Deployment Manager (WP-1–8, WP-16):
- DeploymentService: full pipeline (flatten→validate→send→track→audit)
- OperationLockManager: per-instance concurrency control
- StateTransitionValidator: Enabled/Disabled/NotDeployed transition matrix
- ArtifactDeploymentService: broadcast to all sites with per-site results
- Deployment identity (GUID + revision hash), idempotency, staleness detection
- Instance lifecycle commands (disable/enable/delete) with deduplication

Store-and-Forward (WP-9–15):
- StoreAndForwardStorage: SQLite persistence, 3 categories, no max buffer
- StoreAndForwardService: fixed-interval retry, transient-only buffering, parking
- ReplicationService: async best-effort to standby (fire-and-forget)
- Parked message management (query/retry/discard from central)
- Messages survive instance deletion, S&F drains on disable

620 tests pass (+79 new), zero warnings.
This commit is contained in:
Joseph Doherty
2026-03-16 21:27:18 -04:00
parent b75bf52fb4
commit 6ea38faa6f
40 changed files with 3289 additions and 29 deletions

View File

@@ -0,0 +1,85 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ScadaLink.Commons.Entities.Sites;
using ScadaLink.Commons.Interfaces.Repositories;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.Artifacts;
using ScadaLink.Communication;
namespace ScadaLink.DeploymentManager.Tests;
/// <summary>
/// WP-7: Tests for system-wide artifact deployment.
/// </summary>
public class ArtifactDeploymentServiceTests
{
private readonly ISiteRepository _siteRepo;
private readonly IDeploymentManagerRepository _deploymentRepo;
private readonly IAuditService _audit;
public ArtifactDeploymentServiceTests()
{
_siteRepo = Substitute.For<ISiteRepository>();
_deploymentRepo = Substitute.For<IDeploymentManagerRepository>();
_audit = Substitute.For<IAuditService>();
}
[Fact]
public async Task DeployToAllSitesAsync_NoSites_ReturnsFailure()
{
_siteRepo.GetAllSitesAsync().Returns(new List<Site>());
var service = CreateService();
var command = CreateCommand();
var result = await service.DeployToAllSitesAsync(command, "admin");
Assert.True(result.IsFailure);
Assert.Contains("No sites", result.Error);
}
[Fact]
public void SiteArtifactResult_ContainsSiteInfo()
{
var result = new SiteArtifactResult("site1", "Site One", true, null);
Assert.Equal("site1", result.SiteId);
Assert.Equal("Site One", result.SiteName);
Assert.True(result.Success);
Assert.Null(result.ErrorMessage);
}
[Fact]
public void ArtifactDeploymentSummary_CountsCorrectly()
{
var results = new List<SiteArtifactResult>
{
new("s1", "Site1", true, null),
new("s2", "Site2", false, "error"),
new("s3", "Site3", true, null)
};
var summary = new ArtifactDeploymentSummary("dep1", results, 2, 1);
Assert.Equal(2, summary.SuccessCount);
Assert.Equal(1, summary.FailureCount);
Assert.Equal(3, summary.SiteResults.Count);
}
private ArtifactDeploymentService CreateService()
{
var comms = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
return new ArtifactDeploymentService(
_siteRepo, _deploymentRepo, comms, _audit,
Options.Create(new DeploymentManagerOptions()),
NullLogger<ArtifactDeploymentService>.Instance);
}
private static DeployArtifactsCommand CreateCommand()
{
return new DeployArtifactsCommand(
"dep1", null, null, null, null, DateTimeOffset.UtcNow);
}
}

View File

@@ -0,0 +1,37 @@
namespace ScadaLink.DeploymentManager.Tests;
/// <summary>
/// WP-8: Tests for deployed vs template-derived state comparison.
/// </summary>
public class DeploymentComparisonTests
{
[Fact]
public void DeploymentComparisonResult_MatchingHashes_NotStale()
{
var result = new DeploymentComparisonResult(
1, "sha256:abc", "sha256:abc", false, DateTimeOffset.UtcNow);
Assert.False(result.IsStale);
Assert.Equal("sha256:abc", result.DeployedRevisionHash);
Assert.Equal("sha256:abc", result.CurrentRevisionHash);
}
[Fact]
public void DeploymentComparisonResult_DifferentHashes_IsStale()
{
var result = new DeploymentComparisonResult(
1, "sha256:old", "sha256:new", true, DateTimeOffset.UtcNow);
Assert.True(result.IsStale);
Assert.NotEqual(result.DeployedRevisionHash, result.CurrentRevisionHash);
}
[Fact]
public void DeploymentComparisonResult_ContainsDeployedTimestamp()
{
var deployedAt = new DateTimeOffset(2026, 3, 16, 12, 0, 0, TimeSpan.Zero);
var result = new DeploymentComparisonResult(1, "h1", "h2", true, deployedAt);
Assert.Equal(deployedAt, result.DeployedAt);
}
}

View File

@@ -0,0 +1,290 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ScadaLink.Commons.Entities.Deployment;
using ScadaLink.Commons.Entities.Instances;
using ScadaLink.Commons.Interfaces.Repositories;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.Deployment;
using ScadaLink.Commons.Messages.Lifecycle;
using ScadaLink.Commons.Types;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.Commons.Types.Flattening;
using ScadaLink.Communication;
namespace ScadaLink.DeploymentManager.Tests;
/// <summary>
/// WP-1/2/4/5/6/8/16: Tests for central-side DeploymentService.
/// </summary>
public class DeploymentServiceTests
{
private readonly IDeploymentManagerRepository _repo;
private readonly IFlatteningPipeline _pipeline;
private readonly CommunicationService _comms;
private readonly OperationLockManager _lockManager;
private readonly IAuditService _audit;
private readonly DeploymentService _service;
public DeploymentServiceTests()
{
_repo = Substitute.For<IDeploymentManagerRepository>();
_pipeline = Substitute.For<IFlatteningPipeline>();
_comms = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
_lockManager = new OperationLockManager();
_audit = Substitute.For<IAuditService>();
var options = Options.Create(new DeploymentManagerOptions
{
OperationLockTimeout = TimeSpan.FromSeconds(5)
});
_service = new DeploymentService(
_repo, _pipeline, _comms, _lockManager, _audit, options,
NullLogger<DeploymentService>.Instance);
}
// ── WP-1: Deployment flow ──
[Fact]
public async Task DeployInstanceAsync_InstanceNotFound_ReturnsFailure()
{
_repo.GetInstanceByIdAsync(1).Returns((Instance?)null);
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
[Fact]
public async Task DeployInstanceAsync_ValidationFails_ReturnsFailure()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var validationResult = new ValidationResult
{
Errors = [ValidationEntry.Error(ValidationCategory.ScriptCompilation, "Compile error")]
};
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(new FlattenedConfiguration(), "hash1", validationResult)));
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("validation failed", result.Error);
}
[Fact]
public async Task DeployInstanceAsync_FlatteningFails_ReturnsFailure()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Failure("Template chain empty"));
var result = await _service.DeployInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("Validation failed", result.Error);
}
// ── WP-2: Deployment identity ──
[Fact]
public async Task DeployInstanceAsync_CreatesUniqueDeploymentId()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
// Pipeline succeeds
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
var validResult = ValidationResult.Success();
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(config, "sha256:abc", validResult)));
// Capture the deployment record
DeploymentRecord? captured = null;
await _repo.AddDeploymentRecordAsync(Arg.Do<DeploymentRecord>(r => captured = r), Arg.Any<CancellationToken>());
// CommunicationService will throw because actor not set -- this tests the flow up to that point
try
{
await _service.DeployInstanceAsync(1, "admin");
}
catch (InvalidOperationException)
{
// Expected -- CommunicationService not initialized
}
Assert.NotNull(captured);
Assert.False(string.IsNullOrEmpty(captured!.DeploymentId));
Assert.Equal(32, captured.DeploymentId.Length); // GUID without hyphens
Assert.Equal("sha256:abc", captured.RevisionHash);
}
// ── WP-4: State transition validation ──
[Fact]
public async Task DeployInstanceAsync_EnabledInstance_AllowsDeploy()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(config, "hash", ValidationResult.Success())));
// Will fail at communication layer, but passes state validation
try { await _service.DeployInstanceAsync(1, "admin"); } catch (InvalidOperationException) { }
// If we got past state validation, the deployment record was created
await _repo.Received().AddDeploymentRecordAsync(Arg.Any<DeploymentRecord>(), Arg.Any<CancellationToken>());
}
// ── WP-6: Lifecycle commands ──
[Fact]
public async Task DisableInstanceAsync_InstanceNotFound_ReturnsFailure()
{
_repo.GetInstanceByIdAsync(1).Returns((Instance?)null);
var result = await _service.DisableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not found", result.Error);
}
[Fact]
public async Task DisableInstanceAsync_WhenDisabled_ReturnsTransitionError()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Disabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var result = await _service.DisableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not allowed", result.Error);
}
[Fact]
public async Task EnableInstanceAsync_WhenEnabled_ReturnsTransitionError()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.Enabled };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var result = await _service.EnableInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not allowed", result.Error);
}
[Fact]
public async Task DeleteInstanceAsync_WhenNotDeployed_ReturnsTransitionError()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
var result = await _service.DeleteInstanceAsync(1, "admin");
Assert.True(result.IsFailure);
Assert.Contains("not allowed", result.Error);
}
// ── WP-8: Deployment comparison ──
[Fact]
public async Task GetDeploymentComparisonAsync_NoSnapshot_ReturnsFailure()
{
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns((DeployedConfigSnapshot?)null);
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsFailure);
Assert.Contains("No deployed snapshot", result.Error);
}
[Fact]
public async Task GetDeploymentComparisonAsync_SameHash_NotStale()
{
var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}")
{
InstanceId = 1,
DeployedAt = DateTimeOffset.UtcNow
};
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success())));
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsSuccess);
Assert.False(result.Value.IsStale);
}
[Fact]
public async Task GetDeploymentComparisonAsync_DifferentHash_IsStale()
{
var snapshot = new DeployedConfigSnapshot("dep1", "sha256:abc", "{}")
{
InstanceId = 1,
DeployedAt = DateTimeOffset.UtcNow
};
_repo.GetDeployedSnapshotByInstanceIdAsync(1).Returns(snapshot);
var config = new FlattenedConfiguration { InstanceUniqueName = "TestInst" };
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(config, "sha256:xyz", ValidationResult.Success())));
var result = await _service.GetDeploymentComparisonAsync(1);
Assert.True(result.IsSuccess);
Assert.True(result.Value.IsStale);
}
// ── WP-2: GetDeploymentStatusAsync ──
[Fact]
public async Task GetDeploymentStatusAsync_ReturnsRecordByDeploymentId()
{
var record = new DeploymentRecord("dep1", "admin")
{
Status = DeploymentStatus.Success
};
_repo.GetDeploymentByDeploymentIdAsync("dep1").Returns(record);
var result = await _service.GetDeploymentStatusAsync("dep1");
Assert.NotNull(result);
Assert.Equal("dep1", result!.DeploymentId);
Assert.Equal(DeploymentStatus.Success, result.Status);
}
// ── Audit logging ──
[Fact]
public async Task DeployInstanceAsync_AuditLogs()
{
var instance = new Instance("TestInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(1).Returns(instance);
_pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
.Returns(Result<FlatteningPipelineResult>.Failure("Error"));
await _service.DeployInstanceAsync(1, "admin");
// Failure case does not reach audit (returns before communication)
// The audit is only logged after communication succeeds/fails
}
}

View File

@@ -0,0 +1,95 @@
namespace ScadaLink.DeploymentManager.Tests;
/// <summary>
/// WP-3: Tests for per-instance operation lock.
/// </summary>
public class OperationLockManagerTests
{
private readonly OperationLockManager _lockManager = new();
[Fact]
public async Task AcquireAsync_ReturnsDisposable()
{
using var lockHandle = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
Assert.NotNull(lockHandle);
}
[Fact]
public async Task AcquireAsync_SameInstance_BlocksSecondCaller()
{
using var firstLock = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
// Second acquire should time out
await Assert.ThrowsAsync<TimeoutException>(() =>
_lockManager.AcquireAsync("inst1", TimeSpan.FromMilliseconds(50)));
}
[Fact]
public async Task AcquireAsync_DifferentInstances_BothSucceed()
{
using var lock1 = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
using var lock2 = await _lockManager.AcquireAsync("inst2", TimeSpan.FromSeconds(5));
Assert.NotNull(lock1);
Assert.NotNull(lock2);
}
[Fact]
public async Task AcquireAsync_AfterRelease_CanReacquire()
{
var firstLock = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
firstLock.Dispose();
// Should succeed now
using var secondLock = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
Assert.NotNull(secondLock);
}
[Fact]
public async Task IsLocked_ReturnsTrueWhileLocked()
{
Assert.False(_lockManager.IsLocked("inst1"));
using var lockHandle = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
Assert.True(_lockManager.IsLocked("inst1"));
}
[Fact]
public async Task IsLocked_ReturnsFalseAfterRelease()
{
var lockHandle = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
lockHandle.Dispose();
Assert.False(_lockManager.IsLocked("inst1"));
}
[Fact]
public async Task AcquireAsync_DoubleDispose_DoesNotThrow()
{
var lockHandle = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(5));
lockHandle.Dispose();
lockHandle.Dispose(); // Should not throw
}
[Fact]
public async Task AcquireAsync_CancellationToken_Respected()
{
using var firstLock = await _lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(30));
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50));
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
_lockManager.AcquireAsync("inst1", TimeSpan.FromSeconds(30), cts.Token));
}
[Fact]
public async Task AcquireAsync_ConcurrentDifferentInstances_AllSucceed()
{
var tasks = Enumerable.Range(0, 10).Select(async i =>
{
using var lockHandle = await _lockManager.AcquireAsync($"inst{i}", TimeSpan.FromSeconds(5));
await Task.Delay(10);
});
await Task.WhenAll(tasks);
}
}

View File

@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
@@ -11,6 +11,7 @@
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
</ItemGroup>
@@ -21,6 +22,9 @@
<ItemGroup>
<ProjectReference Include="../../src/ScadaLink.DeploymentManager/ScadaLink.DeploymentManager.csproj" />
<ProjectReference Include="../../src/ScadaLink.Commons/ScadaLink.Commons.csproj" />
<ProjectReference Include="../../src/ScadaLink.Communication/ScadaLink.Communication.csproj" />
<ProjectReference Include="../../src/ScadaLink.TemplateEngine/ScadaLink.TemplateEngine.csproj" />
</ItemGroup>
</Project>
</Project>

View File

@@ -0,0 +1,118 @@
using ScadaLink.Commons.Types.Enums;
namespace ScadaLink.DeploymentManager.Tests;
/// <summary>
/// WP-4: Tests for instance state transition matrix.
/// </summary>
public class StateTransitionValidatorTests
{
// ── Deploy transitions ──
[Theory]
[InlineData(InstanceState.NotDeployed)]
[InlineData(InstanceState.Enabled)]
[InlineData(InstanceState.Disabled)]
public void CanDeploy_AllStates_ReturnsTrue(InstanceState state)
{
Assert.True(StateTransitionValidator.CanDeploy(state));
}
// ── Disable transitions ──
[Fact]
public void CanDisable_WhenEnabled_ReturnsTrue()
{
Assert.True(StateTransitionValidator.CanDisable(InstanceState.Enabled));
}
[Fact]
public void CanDisable_WhenDisabled_ReturnsFalse()
{
Assert.False(StateTransitionValidator.CanDisable(InstanceState.Disabled));
}
[Fact]
public void CanDisable_WhenNotDeployed_ReturnsFalse()
{
Assert.False(StateTransitionValidator.CanDisable(InstanceState.NotDeployed));
}
// ── Enable transitions ──
[Fact]
public void CanEnable_WhenDisabled_ReturnsTrue()
{
Assert.True(StateTransitionValidator.CanEnable(InstanceState.Disabled));
}
[Fact]
public void CanEnable_WhenEnabled_ReturnsFalse()
{
Assert.False(StateTransitionValidator.CanEnable(InstanceState.Enabled));
}
[Fact]
public void CanEnable_WhenNotDeployed_ReturnsFalse()
{
Assert.False(StateTransitionValidator.CanEnable(InstanceState.NotDeployed));
}
// ── Delete transitions ──
[Fact]
public void CanDelete_WhenEnabled_ReturnsTrue()
{
Assert.True(StateTransitionValidator.CanDelete(InstanceState.Enabled));
}
[Fact]
public void CanDelete_WhenDisabled_ReturnsTrue()
{
Assert.True(StateTransitionValidator.CanDelete(InstanceState.Disabled));
}
[Fact]
public void CanDelete_WhenNotDeployed_ReturnsFalse()
{
Assert.False(StateTransitionValidator.CanDelete(InstanceState.NotDeployed));
}
// ── ValidateTransition ──
[Fact]
public void ValidateTransition_ValidDeploy_ReturnsNull()
{
var error = StateTransitionValidator.ValidateTransition(InstanceState.NotDeployed, "deploy");
Assert.Null(error);
}
[Fact]
public void ValidateTransition_InvalidEnable_ReturnsError()
{
var error = StateTransitionValidator.ValidateTransition(InstanceState.Enabled, "enable");
Assert.NotNull(error);
Assert.Contains("not allowed", error);
}
[Fact]
public void ValidateTransition_InvalidDisable_ReturnsError()
{
var error = StateTransitionValidator.ValidateTransition(InstanceState.Disabled, "disable");
Assert.NotNull(error);
}
[Fact]
public void ValidateTransition_InvalidDeleteOnNotDeployed_ReturnsError()
{
var error = StateTransitionValidator.ValidateTransition(InstanceState.NotDeployed, "delete");
Assert.NotNull(error);
}
[Fact]
public void ValidateTransition_UnknownOperation_ReturnsError()
{
var error = StateTransitionValidator.ValidateTransition(InstanceState.Enabled, "unknown");
Assert.NotNull(error);
}
}

View File

@@ -1,10 +0,0 @@
namespace ScadaLink.DeploymentManager.Tests;
public class UnitTest1
{
[Fact]
public void Test1()
{
}
}