Phase 3C: Deployment pipeline & Store-and-Forward engine
Deployment Manager (WP-1–8, WP-16): - DeploymentService: full pipeline (flatten→validate→send→track→audit) - OperationLockManager: per-instance concurrency control - StateTransitionValidator: Enabled/Disabled/NotDeployed transition matrix - ArtifactDeploymentService: broadcast to all sites with per-site results - Deployment identity (GUID + revision hash), idempotency, staleness detection - Instance lifecycle commands (disable/enable/delete) with deduplication Store-and-Forward (WP-9–15): - StoreAndForwardStorage: SQLite persistence, 3 categories, no max buffer - StoreAndForwardService: fixed-interval retry, transient-only buffering, parking - ReplicationService: async best-effort to standby (fire-and-forget) - Parked message management (query/retry/discard from central) - Messages survive instance deletion, S&F drains on disable 620 tests pass (+79 new), zero warnings.
This commit is contained in:
Joseph Doherty
178
src/ScadaLink.DeploymentManager/ArtifactDeploymentService.cs
Normal file
178
src/ScadaLink.DeploymentManager/ArtifactDeploymentService.cs
Normal file
@@ -0,0 +1,178 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ScadaLink.Commons.Entities.Deployment;
|
||||
using ScadaLink.Commons.Interfaces.Repositories;
|
||||
using ScadaLink.Commons.Interfaces.Services;
|
||||
using ScadaLink.Commons.Messages.Artifacts;
|
||||
using ScadaLink.Commons.Types;
|
||||
using ScadaLink.Communication;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// WP-7: System-wide artifact deployment.
|
||||
/// Broadcasts artifacts (shared scripts, external systems, notification lists, DB connections)
|
||||
/// to all sites with per-site tracking.
|
||||
///
|
||||
/// - Successful sites are NOT rolled back on other failures.
|
||||
/// - Failed sites are retryable individually.
|
||||
/// - 120s timeout per site.
|
||||
/// - Cross-site version skew is supported.
|
||||
/// </summary>
|
||||
public class ArtifactDeploymentService
|
||||
{
|
||||
private readonly ISiteRepository _siteRepo;
|
||||
private readonly IDeploymentManagerRepository _deploymentRepo;
|
||||
private readonly CommunicationService _communicationService;
|
||||
private readonly IAuditService _auditService;
|
||||
private readonly DeploymentManagerOptions _options;
|
||||
private readonly ILogger<ArtifactDeploymentService> _logger;
|
||||
|
||||
public ArtifactDeploymentService(
|
||||
ISiteRepository siteRepo,
|
||||
IDeploymentManagerRepository deploymentRepo,
|
||||
CommunicationService communicationService,
|
||||
IAuditService auditService,
|
||||
IOptions<DeploymentManagerOptions> options,
|
||||
ILogger<ArtifactDeploymentService> logger)
|
||||
{
|
||||
_siteRepo = siteRepo;
|
||||
_deploymentRepo = deploymentRepo;
|
||||
_communicationService = communicationService;
|
||||
_auditService = auditService;
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deploys artifacts to all sites. Returns per-site result matrix.
|
||||
/// </summary>
|
||||
public async Task<Result<ArtifactDeploymentSummary>> DeployToAllSitesAsync(
|
||||
DeployArtifactsCommand command,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var sites = await _siteRepo.GetAllSitesAsync(cancellationToken);
|
||||
if (sites.Count == 0)
|
||||
return Result<ArtifactDeploymentSummary>.Failure("No sites configured.");
|
||||
|
||||
var perSiteResults = new Dictionary<string, SiteArtifactResult>();
|
||||
|
||||
// Deploy to each site with per-site timeout
|
||||
var tasks = sites.Select(async site =>
|
||||
{
|
||||
try
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Deploying artifacts to site {SiteId} ({SiteName}), deploymentId={DeploymentId}",
|
||||
site.SiteIdentifier, site.Name, command.DeploymentId);
|
||||
|
||||
var response = await _communicationService.DeployArtifactsAsync(
|
||||
site.SiteIdentifier, command, cts.Token);
|
||||
|
||||
return new SiteArtifactResult(
|
||||
site.SiteIdentifier, site.Name, response.Success, response.ErrorMessage);
|
||||
}
|
||||
catch (Exception ex) when (ex is TimeoutException or OperationCanceledException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Artifact deployment to site {SiteId} timed out: {Error}",
|
||||
site.SiteIdentifier, ex.Message);
|
||||
|
||||
return new SiteArtifactResult(
|
||||
site.SiteIdentifier, site.Name, false, $"Timeout: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Artifact deployment to site {SiteId} failed",
|
||||
site.SiteIdentifier);
|
||||
|
||||
return new SiteArtifactResult(
|
||||
site.SiteIdentifier, site.Name, false, ex.Message);
|
||||
}
|
||||
}).ToList();
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
|
||||
foreach (var result in results)
|
||||
{
|
||||
perSiteResults[result.SiteId] = result;
|
||||
}
|
||||
|
||||
// Persist the system artifact deployment record
|
||||
var record = new SystemArtifactDeploymentRecord("Artifacts", user)
|
||||
{
|
||||
DeployedAt = DateTimeOffset.UtcNow,
|
||||
PerSiteStatus = JsonSerializer.Serialize(perSiteResults)
|
||||
};
|
||||
await _deploymentRepo.AddSystemArtifactDeploymentAsync(record, cancellationToken);
|
||||
await _deploymentRepo.SaveChangesAsync(cancellationToken);
|
||||
|
||||
var summary = new ArtifactDeploymentSummary(
|
||||
command.DeploymentId,
|
||||
results.ToList(),
|
||||
results.Count(r => r.Success),
|
||||
results.Count(r => !r.Success));
|
||||
|
||||
await _auditService.LogAsync(user, "DeployArtifacts", "SystemArtifact",
|
||||
command.DeploymentId, "Artifacts",
|
||||
new { summary.SuccessCount, summary.FailureCount },
|
||||
cancellationToken);
|
||||
|
||||
return Result<ArtifactDeploymentSummary>.Success(summary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-7: Retry artifact deployment to a specific site that previously failed.
|
||||
/// </summary>
|
||||
public async Task<Result<SiteArtifactResult>> RetryForSiteAsync(
|
||||
string siteId,
|
||||
DeployArtifactsCommand command,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);
|
||||
|
||||
var response = await _communicationService.DeployArtifactsAsync(siteId, command, cts.Token);
|
||||
|
||||
var result = new SiteArtifactResult(siteId, siteId, response.Success, response.ErrorMessage);
|
||||
|
||||
await _auditService.LogAsync(user, "RetryArtifactDeployment", "SystemArtifact",
|
||||
command.DeploymentId, siteId, new { response.Success }, cancellationToken);
|
||||
|
||||
return response.Success
|
||||
? Result<SiteArtifactResult>.Success(result)
|
||||
: Result<SiteArtifactResult>.Failure(response.ErrorMessage ?? "Retry failed.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Result<SiteArtifactResult>.Failure($"Retry failed for site {siteId}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-7: Per-site result for artifact deployment.
|
||||
/// </summary>
|
||||
public record SiteArtifactResult(
|
||||
string SiteId,
|
||||
string SiteName,
|
||||
bool Success,
|
||||
string? ErrorMessage);
|
||||
|
||||
/// <summary>
|
||||
/// WP-7: Summary of system-wide artifact deployment with per-site results.
|
||||
/// </summary>
|
||||
public record ArtifactDeploymentSummary(
|
||||
string DeploymentId,
|
||||
IReadOnlyList<SiteArtifactResult> SiteResults,
|
||||
int SuccessCount,
|
||||
int FailureCount);
|
||||
16
src/ScadaLink.DeploymentManager/DeploymentManagerOptions.cs
Normal file
16
src/ScadaLink.DeploymentManager/DeploymentManagerOptions.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the central-side Deployment Manager.
|
||||
/// </summary>
|
||||
public class DeploymentManagerOptions
|
||||
{
|
||||
/// <summary>Timeout for lifecycle commands sent to sites (disable, enable, delete).</summary>
|
||||
public TimeSpan LifecycleCommandTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>WP-7: Timeout per site for system-wide artifact deployment.</summary>
|
||||
public TimeSpan ArtifactDeploymentTimeoutPerSite { get; set; } = TimeSpan.FromSeconds(120);
|
||||
|
||||
/// <summary>WP-3: Timeout for acquiring an operation lock on an instance.</summary>
|
||||
public TimeSpan OperationLockTimeout { get; set; } = TimeSpan.FromSeconds(5);
|
||||
}
|
||||
393
src/ScadaLink.DeploymentManager/DeploymentService.cs
Normal file
393
src/ScadaLink.DeploymentManager/DeploymentService.cs
Normal file
@@ -0,0 +1,393 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ScadaLink.Commons.Entities.Deployment;
|
||||
using ScadaLink.Commons.Entities.Instances;
|
||||
using ScadaLink.Commons.Interfaces.Repositories;
|
||||
using ScadaLink.Commons.Interfaces.Services;
|
||||
using ScadaLink.Commons.Messages.Deployment;
|
||||
using ScadaLink.Commons.Messages.Lifecycle;
|
||||
using ScadaLink.Commons.Types;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
using ScadaLink.Commons.Types.Flattening;
|
||||
using ScadaLink.Communication;
|
||||
using ScadaLink.TemplateEngine.Flattening;
|
||||
using ScadaLink.TemplateEngine.Validation;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// WP-1: Central-side deployment orchestration service.
|
||||
/// Coordinates the full deployment pipeline:
|
||||
/// 1. Validate instance state transition (WP-4)
|
||||
/// 2. Acquire per-instance operation lock (WP-3)
|
||||
/// 3. Flatten configuration via TemplateEngine (captures template state at time of flatten -- WP-16)
|
||||
/// 4. Validate flattened configuration
|
||||
/// 5. Compute revision hash and diff
|
||||
/// 6. Send DeployInstanceCommand to site via CommunicationService
|
||||
/// 7. Track deployment status with optimistic concurrency (WP-4)
|
||||
/// 8. Store deployed config snapshot (WP-8)
|
||||
/// 9. Audit log all actions
|
||||
///
|
||||
/// WP-2: Each deployment has a unique deployment ID (GUID) + revision hash.
|
||||
/// WP-16: Template state captured at flatten time -- last-write-wins on templates is safe.
|
||||
/// </summary>
|
||||
public class DeploymentService
|
||||
{
|
||||
private readonly IDeploymentManagerRepository _repository;
|
||||
private readonly IFlatteningPipeline _flatteningPipeline;
|
||||
private readonly CommunicationService _communicationService;
|
||||
private readonly OperationLockManager _lockManager;
|
||||
private readonly IAuditService _auditService;
|
||||
private readonly DeploymentManagerOptions _options;
|
||||
private readonly ILogger<DeploymentService> _logger;
|
||||
|
||||
public DeploymentService(
|
||||
IDeploymentManagerRepository repository,
|
||||
IFlatteningPipeline flatteningPipeline,
|
||||
CommunicationService communicationService,
|
||||
OperationLockManager lockManager,
|
||||
IAuditService auditService,
|
||||
IOptions<DeploymentManagerOptions> options,
|
||||
ILogger<DeploymentService> logger)
|
||||
{
|
||||
_repository = repository;
|
||||
_flatteningPipeline = flatteningPipeline;
|
||||
_communicationService = communicationService;
|
||||
_lockManager = lockManager;
|
||||
_auditService = auditService;
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-1: Deploy an instance to its site.
|
||||
/// WP-2: Generates unique deployment ID, computes revision hash.
|
||||
/// WP-4: Validates state transitions, uses optimistic concurrency.
|
||||
/// WP-5: Site-side apply is all-or-nothing (handled by DeploymentManagerActor).
|
||||
/// WP-8: Stores deployed config snapshot on success.
|
||||
/// WP-16: Captures template state at time of flatten.
|
||||
/// </summary>
|
||||
public async Task<Result<DeploymentRecord>> DeployInstanceAsync(
|
||||
int instanceId,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Load instance
|
||||
var instance = await _repository.GetInstanceByIdAsync(instanceId, cancellationToken);
|
||||
if (instance == null)
|
||||
return Result<DeploymentRecord>.Failure($"Instance with ID {instanceId} not found.");
|
||||
|
||||
// WP-4: Validate state transition
|
||||
var transitionError = StateTransitionValidator.ValidateTransition(instance.State, "deploy");
|
||||
if (transitionError != null)
|
||||
return Result<DeploymentRecord>.Failure(transitionError);
|
||||
|
||||
// WP-3: Acquire per-instance operation lock
|
||||
using var lockHandle = await _lockManager.AcquireAsync(
|
||||
instance.UniqueName, _options.OperationLockTimeout, cancellationToken);
|
||||
|
||||
// WP-2: Generate unique deployment ID
|
||||
var deploymentId = Guid.NewGuid().ToString("N");
|
||||
|
||||
// WP-1/16: Flatten configuration (captures template state at this point in time)
|
||||
var flattenResult = await _flatteningPipeline.FlattenAndValidateAsync(instanceId, cancellationToken);
|
||||
if (flattenResult.IsFailure)
|
||||
return Result<DeploymentRecord>.Failure($"Validation failed: {flattenResult.Error}");
|
||||
|
||||
var flattenedConfig = flattenResult.Value.Configuration;
|
||||
var revisionHash = flattenResult.Value.RevisionHash;
|
||||
var validationResult = flattenResult.Value.Validation;
|
||||
|
||||
if (!validationResult.IsValid)
|
||||
{
|
||||
var errors = string.Join("; ", validationResult.Errors.Select(e => e.Message));
|
||||
return Result<DeploymentRecord>.Failure($"Pre-deployment validation failed: {errors}");
|
||||
}
|
||||
|
||||
// Serialize for transmission
|
||||
var configJson = JsonSerializer.Serialize(flattenedConfig);
|
||||
|
||||
// WP-4: Create deployment record with Pending status
|
||||
var record = new DeploymentRecord(deploymentId, user)
|
||||
{
|
||||
InstanceId = instanceId,
|
||||
Status = DeploymentStatus.Pending,
|
||||
RevisionHash = revisionHash,
|
||||
DeployedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
await _repository.AddDeploymentRecordAsync(record, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
// Update status to InProgress
|
||||
record.Status = DeploymentStatus.InProgress;
|
||||
await _repository.UpdateDeploymentRecordAsync(record, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
// WP-1: Send to site via CommunicationService
|
||||
var siteId = instance.SiteId.ToString();
|
||||
var command = new DeployInstanceCommand(
|
||||
deploymentId, instance.UniqueName, revisionHash, configJson, user, DateTimeOffset.UtcNow);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Sending deployment {DeploymentId} for instance {Instance} to site {SiteId}",
|
||||
deploymentId, instance.UniqueName, siteId);
|
||||
|
||||
var response = await _communicationService.DeployInstanceAsync(siteId, command, cancellationToken);
|
||||
|
||||
// WP-1: Update status based on site response
|
||||
record.Status = response.Status;
|
||||
record.ErrorMessage = response.ErrorMessage;
|
||||
record.CompletedAt = DateTimeOffset.UtcNow;
|
||||
await _repository.UpdateDeploymentRecordAsync(record, cancellationToken);
|
||||
|
||||
if (response.Status == DeploymentStatus.Success)
|
||||
{
|
||||
// WP-4: Update instance state to Enabled on successful deployment
|
||||
instance.State = InstanceState.Enabled;
|
||||
await _repository.UpdateInstanceAsync(instance, cancellationToken);
|
||||
|
||||
// WP-8: Store deployed config snapshot
|
||||
await StoreDeployedSnapshotAsync(instanceId, deploymentId, revisionHash, configJson, cancellationToken);
|
||||
}
|
||||
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
// Audit log
|
||||
await _auditService.LogAsync(user, "Deploy", "Instance", instanceId.ToString(),
|
||||
instance.UniqueName, new { DeploymentId = deploymentId, Status = record.Status.ToString() },
|
||||
cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Deployment {DeploymentId} for instance {Instance}: {Status}",
|
||||
deploymentId, instance.UniqueName, record.Status);
|
||||
|
||||
return record.Status == DeploymentStatus.Success
|
||||
? Result<DeploymentRecord>.Success(record)
|
||||
: Result<DeploymentRecord>.Failure(
|
||||
$"Deployment failed: {response.ErrorMessage ?? "Unknown error"}");
|
||||
}
|
||||
catch (Exception ex) when (ex is TimeoutException or OperationCanceledException)
|
||||
{
|
||||
record.Status = DeploymentStatus.Failed;
|
||||
record.ErrorMessage = $"Communication failure: {ex.Message}";
|
||||
record.CompletedAt = DateTimeOffset.UtcNow;
|
||||
await _repository.UpdateDeploymentRecordAsync(record, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
await _auditService.LogAsync(user, "DeployFailed", "Instance", instanceId.ToString(),
|
||||
instance.UniqueName, new { DeploymentId = deploymentId, Error = ex.Message },
|
||||
cancellationToken);
|
||||
|
||||
return Result<DeploymentRecord>.Failure($"Deployment timed out: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-6: Disable an instance. Stops Instance Actor, retains config, S&F drains.
|
||||
/// </summary>
|
||||
public async Task<Result<InstanceLifecycleResponse>> DisableInstanceAsync(
|
||||
int instanceId,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var instance = await _repository.GetInstanceByIdAsync(instanceId, cancellationToken);
|
||||
if (instance == null)
|
||||
return Result<InstanceLifecycleResponse>.Failure($"Instance with ID {instanceId} not found.");
|
||||
|
||||
var transitionError = StateTransitionValidator.ValidateTransition(instance.State, "disable");
|
||||
if (transitionError != null)
|
||||
return Result<InstanceLifecycleResponse>.Failure(transitionError);
|
||||
|
||||
using var lockHandle = await _lockManager.AcquireAsync(
|
||||
instance.UniqueName, _options.OperationLockTimeout, cancellationToken);
|
||||
|
||||
var commandId = Guid.NewGuid().ToString("N");
|
||||
var siteId = instance.SiteId.ToString();
|
||||
var command = new DisableInstanceCommand(commandId, instance.UniqueName, DateTimeOffset.UtcNow);
|
||||
|
||||
var response = await _communicationService.DisableInstanceAsync(siteId, command, cancellationToken);
|
||||
|
||||
if (response.Success)
|
||||
{
|
||||
instance.State = InstanceState.Disabled;
|
||||
await _repository.UpdateInstanceAsync(instance, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await _auditService.LogAsync(user, "Disable", "Instance", instanceId.ToString(),
|
||||
instance.UniqueName, new { CommandId = commandId, response.Success },
|
||||
cancellationToken);
|
||||
|
||||
return response.Success
|
||||
? Result<InstanceLifecycleResponse>.Success(response)
|
||||
: Result<InstanceLifecycleResponse>.Failure(response.ErrorMessage ?? "Disable failed.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-6: Enable an instance. Re-creates Instance Actor from stored config.
|
||||
/// </summary>
|
||||
public async Task<Result<InstanceLifecycleResponse>> EnableInstanceAsync(
|
||||
int instanceId,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var instance = await _repository.GetInstanceByIdAsync(instanceId, cancellationToken);
|
||||
if (instance == null)
|
||||
return Result<InstanceLifecycleResponse>.Failure($"Instance with ID {instanceId} not found.");
|
||||
|
||||
var transitionError = StateTransitionValidator.ValidateTransition(instance.State, "enable");
|
||||
if (transitionError != null)
|
||||
return Result<InstanceLifecycleResponse>.Failure(transitionError);
|
||||
|
||||
using var lockHandle = await _lockManager.AcquireAsync(
|
||||
instance.UniqueName, _options.OperationLockTimeout, cancellationToken);
|
||||
|
||||
var commandId = Guid.NewGuid().ToString("N");
|
||||
var siteId = instance.SiteId.ToString();
|
||||
var command = new EnableInstanceCommand(commandId, instance.UniqueName, DateTimeOffset.UtcNow);
|
||||
|
||||
var response = await _communicationService.EnableInstanceAsync(siteId, command, cancellationToken);
|
||||
|
||||
if (response.Success)
|
||||
{
|
||||
instance.State = InstanceState.Enabled;
|
||||
await _repository.UpdateInstanceAsync(instance, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await _auditService.LogAsync(user, "Enable", "Instance", instanceId.ToString(),
|
||||
instance.UniqueName, new { CommandId = commandId, response.Success },
|
||||
cancellationToken);
|
||||
|
||||
return response.Success
|
||||
? Result<InstanceLifecycleResponse>.Success(response)
|
||||
: Result<InstanceLifecycleResponse>.Failure(response.ErrorMessage ?? "Enable failed.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-6: Delete an instance. Stops actor, removes config. S&F NOT cleared.
|
||||
/// Delete fails if site unreachable (30s timeout via CommunicationOptions).
|
||||
/// </summary>
|
||||
public async Task<Result<InstanceLifecycleResponse>> DeleteInstanceAsync(
|
||||
int instanceId,
|
||||
string user,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var instance = await _repository.GetInstanceByIdAsync(instanceId, cancellationToken);
|
||||
if (instance == null)
|
||||
return Result<InstanceLifecycleResponse>.Failure($"Instance with ID {instanceId} not found.");
|
||||
|
||||
var transitionError = StateTransitionValidator.ValidateTransition(instance.State, "delete");
|
||||
if (transitionError != null)
|
||||
return Result<InstanceLifecycleResponse>.Failure(transitionError);
|
||||
|
||||
using var lockHandle = await _lockManager.AcquireAsync(
|
||||
instance.UniqueName, _options.OperationLockTimeout, cancellationToken);
|
||||
|
||||
var commandId = Guid.NewGuid().ToString("N");
|
||||
var siteId = instance.SiteId.ToString();
|
||||
var command = new DeleteInstanceCommand(commandId, instance.UniqueName, DateTimeOffset.UtcNow);
|
||||
|
||||
var response = await _communicationService.DeleteInstanceAsync(siteId, command, cancellationToken);
|
||||
|
||||
if (response.Success)
|
||||
{
|
||||
// Remove deployed snapshot
|
||||
await _repository.DeleteDeployedSnapshotAsync(instanceId, cancellationToken);
|
||||
|
||||
// Set state to NotDeployed (or the instance record could be deleted entirely by higher layers)
|
||||
instance.State = InstanceState.NotDeployed;
|
||||
await _repository.UpdateInstanceAsync(instance, cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await _auditService.LogAsync(user, "Delete", "Instance", instanceId.ToString(),
|
||||
instance.UniqueName, new { CommandId = commandId, response.Success },
|
||||
cancellationToken);
|
||||
|
||||
return response.Success
|
||||
? Result<InstanceLifecycleResponse>.Success(response)
|
||||
: Result<InstanceLifecycleResponse>.Failure(
|
||||
response.ErrorMessage ?? "Delete failed. Site may be unreachable.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-8: Get the deployed config snapshot and compare with current template-derived state.
|
||||
/// </summary>
|
||||
public async Task<Result<DeploymentComparisonResult>> GetDeploymentComparisonAsync(
|
||||
int instanceId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var snapshot = await _repository.GetDeployedSnapshotByInstanceIdAsync(instanceId, cancellationToken);
|
||||
if (snapshot == null)
|
||||
return Result<DeploymentComparisonResult>.Failure("No deployed snapshot found for this instance.");
|
||||
|
||||
// Compute current template-derived config
|
||||
var currentResult = await _flatteningPipeline.FlattenAndValidateAsync(instanceId, cancellationToken);
|
||||
if (currentResult.IsFailure)
|
||||
return Result<DeploymentComparisonResult>.Failure($"Cannot compute current config: {currentResult.Error}");
|
||||
|
||||
var currentHash = currentResult.Value.RevisionHash;
|
||||
var isStale = snapshot.RevisionHash != currentHash;
|
||||
|
||||
var result = new DeploymentComparisonResult(
|
||||
instanceId,
|
||||
snapshot.RevisionHash,
|
||||
currentHash,
|
||||
isStale,
|
||||
snapshot.DeployedAt);
|
||||
|
||||
return Result<DeploymentComparisonResult>.Success(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-2: After failover/timeout, query site for current deployment state before re-deploying.
|
||||
/// </summary>
|
||||
public async Task<DeploymentRecord?> GetDeploymentStatusAsync(
|
||||
string deploymentId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _repository.GetDeploymentByDeploymentIdAsync(deploymentId, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task StoreDeployedSnapshotAsync(
|
||||
int instanceId,
|
||||
string deploymentId,
|
||||
string revisionHash,
|
||||
string configJson,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await _repository.GetDeployedSnapshotByInstanceIdAsync(instanceId, cancellationToken);
|
||||
if (existing != null)
|
||||
{
|
||||
existing.DeploymentId = deploymentId;
|
||||
existing.RevisionHash = revisionHash;
|
||||
existing.ConfigurationJson = configJson;
|
||||
existing.DeployedAt = DateTimeOffset.UtcNow;
|
||||
await _repository.UpdateDeployedSnapshotAsync(existing, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
var snapshot = new DeployedConfigSnapshot(deploymentId, revisionHash, configJson)
|
||||
{
|
||||
InstanceId = instanceId,
|
||||
DeployedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
await _repository.AddDeployedSnapshotAsync(snapshot, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-8: Result of comparing deployed vs template-derived configuration.
|
||||
/// </summary>
|
||||
public record DeploymentComparisonResult(
|
||||
int InstanceId,
|
||||
string DeployedRevisionHash,
|
||||
string CurrentRevisionHash,
|
||||
bool IsStale,
|
||||
DateTimeOffset DeployedAt);
|
||||
121
src/ScadaLink.DeploymentManager/FlatteningPipeline.cs
Normal file
121
src/ScadaLink.DeploymentManager/FlatteningPipeline.cs
Normal file
@@ -0,0 +1,121 @@
|
||||
using ScadaLink.Commons.Entities.Sites;
|
||||
using ScadaLink.Commons.Interfaces.Repositories;
|
||||
using ScadaLink.Commons.Types;
|
||||
using ScadaLink.Commons.Types.Flattening;
|
||||
using ScadaLink.TemplateEngine.Flattening;
|
||||
using ScadaLink.TemplateEngine.Validation;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// Orchestrates the TemplateEngine services (FlatteningService, ValidationService, RevisionHashService)
|
||||
/// into a single pipeline for deployment use.
|
||||
///
|
||||
/// WP-16: This captures template state at the time of flatten, ensuring that concurrent template edits
|
||||
/// (last-write-wins) do not conflict with in-progress deployments.
|
||||
/// </summary>
|
||||
public class FlatteningPipeline : IFlatteningPipeline
|
||||
{
|
||||
private readonly ITemplateEngineRepository _templateRepo;
|
||||
private readonly ISiteRepository _siteRepo;
|
||||
private readonly FlatteningService _flatteningService;
|
||||
private readonly ValidationService _validationService;
|
||||
private readonly RevisionHashService _revisionHashService;
|
||||
|
||||
public FlatteningPipeline(
|
||||
ITemplateEngineRepository templateRepo,
|
||||
ISiteRepository siteRepo,
|
||||
FlatteningService flatteningService,
|
||||
ValidationService validationService,
|
||||
RevisionHashService revisionHashService)
|
||||
{
|
||||
_templateRepo = templateRepo;
|
||||
_siteRepo = siteRepo;
|
||||
_flatteningService = flatteningService;
|
||||
_validationService = validationService;
|
||||
_revisionHashService = revisionHashService;
|
||||
}
|
||||
|
||||
public async Task<Result<FlatteningPipelineResult>> FlattenAndValidateAsync(
|
||||
int instanceId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Load instance with full graph
|
||||
var instance = await _templateRepo.GetInstanceByIdAsync(instanceId, cancellationToken);
|
||||
if (instance == null)
|
||||
return Result<FlatteningPipelineResult>.Failure($"Instance with ID {instanceId} not found.");
|
||||
|
||||
// Build template chain
|
||||
var templateChain = await BuildTemplateChainAsync(instance.TemplateId, cancellationToken);
|
||||
if (templateChain.Count == 0)
|
||||
return Result<FlatteningPipelineResult>.Failure("Template chain is empty.");
|
||||
|
||||
// Build composition maps
|
||||
var compositionMap = new Dictionary<int, IReadOnlyList<Commons.Entities.Templates.TemplateComposition>>();
|
||||
var composedChains = new Dictionary<int, IReadOnlyList<Commons.Entities.Templates.Template>>();
|
||||
|
||||
foreach (var template in templateChain)
|
||||
{
|
||||
var compositions = await _templateRepo.GetCompositionsByTemplateIdAsync(template.Id, cancellationToken);
|
||||
if (compositions.Count > 0)
|
||||
{
|
||||
compositionMap[template.Id] = compositions;
|
||||
foreach (var comp in compositions)
|
||||
{
|
||||
if (!composedChains.ContainsKey(comp.ComposedTemplateId))
|
||||
{
|
||||
composedChains[comp.ComposedTemplateId] =
|
||||
await BuildTemplateChainAsync(comp.ComposedTemplateId, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load data connections for the site
|
||||
var dataConnections = await LoadDataConnectionsAsync(instance.SiteId, cancellationToken);
|
||||
|
||||
// Flatten
|
||||
var flattenResult = _flatteningService.Flatten(
|
||||
instance, templateChain, compositionMap, composedChains, dataConnections);
|
||||
|
||||
if (flattenResult.IsFailure)
|
||||
return Result<FlatteningPipelineResult>.Failure(flattenResult.Error);
|
||||
|
||||
var config = flattenResult.Value;
|
||||
|
||||
// Validate
|
||||
var validation = _validationService.Validate(config);
|
||||
|
||||
// Compute revision hash
|
||||
var hash = _revisionHashService.ComputeHash(config);
|
||||
|
||||
return Result<FlatteningPipelineResult>.Success(
|
||||
new FlatteningPipelineResult(config, hash, validation));
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<Commons.Entities.Templates.Template>> BuildTemplateChainAsync(
|
||||
int templateId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var chain = new List<Commons.Entities.Templates.Template>();
|
||||
var currentId = (int?)templateId;
|
||||
|
||||
while (currentId.HasValue)
|
||||
{
|
||||
var template = await _templateRepo.GetTemplateWithChildrenAsync(currentId.Value, cancellationToken);
|
||||
if (template == null) break;
|
||||
chain.Add(template);
|
||||
currentId = template.ParentTemplateId;
|
||||
}
|
||||
|
||||
return chain;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyDictionary<int, DataConnection>> LoadDataConnectionsAsync(
|
||||
int siteId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var connections = await _siteRepo.GetDataConnectionsBySiteIdAsync(siteId, cancellationToken);
|
||||
return connections.ToDictionary(c => c.Id);
|
||||
}
|
||||
}
|
||||
27
src/ScadaLink.DeploymentManager/IFlatteningPipeline.cs
Normal file
27
src/ScadaLink.DeploymentManager/IFlatteningPipeline.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
using ScadaLink.Commons.Types;
|
||||
using ScadaLink.Commons.Types.Flattening;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction over the TemplateEngine flattening + validation + hashing pipeline.
|
||||
/// Used by DeploymentService to obtain a validated, hashed FlattenedConfiguration.
|
||||
/// </summary>
|
||||
public interface IFlatteningPipeline
|
||||
{
|
||||
/// <summary>
|
||||
/// Flattens and validates an instance configuration, returning the configuration,
|
||||
/// revision hash, and validation result.
|
||||
/// </summary>
|
||||
Task<Result<FlatteningPipelineResult>> FlattenAndValidateAsync(
|
||||
int instanceId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of the flattening pipeline: configuration, hash, and validation.
|
||||
/// </summary>
|
||||
public record FlatteningPipelineResult(
|
||||
FlattenedConfiguration Configuration,
|
||||
string RevisionHash,
|
||||
ValidationResult Validation);
|
||||
58
src/ScadaLink.DeploymentManager/OperationLockManager.cs
Normal file
58
src/ScadaLink.DeploymentManager/OperationLockManager.cs
Normal file
@@ -0,0 +1,58 @@
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// WP-3: Per-instance operation lock. Only one mutating operation (deploy, disable, enable, delete)
|
||||
/// may be in progress per instance at a time. Different instances can proceed in parallel.
|
||||
///
|
||||
/// Implementation: ConcurrentDictionary of SemaphoreSlim(1,1) keyed by instance unique name.
|
||||
/// Lock released on completion, timeout, or failure.
|
||||
/// Lost on central failover (acceptable per design -- in-progress treated as failed).
|
||||
/// </summary>
|
||||
public class OperationLockManager
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _locks = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Acquires the operation lock for the given instance. Returns a disposable that releases the lock.
|
||||
/// Throws TimeoutException if the lock cannot be acquired within the timeout.
|
||||
/// </summary>
|
||||
public async Task<IDisposable> AcquireAsync(string instanceUniqueName, TimeSpan timeout, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var semaphore = _locks.GetOrAdd(instanceUniqueName, _ => new SemaphoreSlim(1, 1));
|
||||
|
||||
if (!await semaphore.WaitAsync(timeout, cancellationToken))
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Could not acquire operation lock for instance '{instanceUniqueName}' within {timeout.TotalSeconds}s. " +
|
||||
"Another mutating operation is in progress.");
|
||||
}
|
||||
|
||||
return new LockRelease(semaphore);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether a lock is currently held for the given instance (for diagnostics).
|
||||
/// </summary>
|
||||
public bool IsLocked(string instanceUniqueName)
|
||||
{
|
||||
return _locks.TryGetValue(instanceUniqueName, out var semaphore) && semaphore.CurrentCount == 0;
|
||||
}
|
||||
|
||||
private sealed class LockRelease : IDisposable
|
||||
{
|
||||
private readonly SemaphoreSlim _semaphore;
|
||||
private int _disposed;
|
||||
|
||||
public LockRelease(SemaphoreSlim semaphore) => _semaphore = semaphore;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _disposed, 1, 0) == 0)
|
||||
{
|
||||
_semaphore.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,14 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
|
||||
<ProjectReference Include="../ScadaLink.Communication/ScadaLink.Communication.csproj" />
|
||||
<ProjectReference Include="../ScadaLink.TemplateEngine/ScadaLink.TemplateEngine.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -6,13 +6,16 @@ public static class ServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddDeploymentManager(this IServiceCollection services)
|
||||
{
|
||||
// Phase 0: skeleton only
|
||||
services.AddSingleton<OperationLockManager>();
|
||||
services.AddScoped<IFlatteningPipeline, FlatteningPipeline>();
|
||||
services.AddScoped<DeploymentService>();
|
||||
services.AddScoped<ArtifactDeploymentService>();
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddDeploymentManagerActors(this IServiceCollection services)
|
||||
{
|
||||
// Phase 0: placeholder for Akka actor registration
|
||||
// Akka actor registration is handled by Host component during actor system startup
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
48
src/ScadaLink.DeploymentManager/StateTransitionValidator.cs
Normal file
48
src/ScadaLink.DeploymentManager/StateTransitionValidator.cs
Normal file
@@ -0,0 +1,48 @@
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.DeploymentManager;
|
||||
|
||||
/// <summary>
|
||||
/// WP-4: State transition matrix for instance lifecycle.
|
||||
///
|
||||
/// State | Deploy | Disable | Enable | Delete
|
||||
/// ----------|--------|---------|--------|-------
|
||||
/// NotDeploy | OK | NO | NO | NO
|
||||
/// Enabled | OK | OK | NO | OK
|
||||
/// Disabled | OK* | NO | OK | OK
|
||||
///
|
||||
/// * Deploy on a Disabled instance also enables it.
|
||||
/// </summary>
|
||||
public static class StateTransitionValidator
|
||||
{
|
||||
public static bool CanDeploy(InstanceState currentState) =>
|
||||
currentState is InstanceState.NotDeployed or InstanceState.Enabled or InstanceState.Disabled;
|
||||
|
||||
public static bool CanDisable(InstanceState currentState) =>
|
||||
currentState == InstanceState.Enabled;
|
||||
|
||||
public static bool CanEnable(InstanceState currentState) =>
|
||||
currentState == InstanceState.Disabled;
|
||||
|
||||
public static bool CanDelete(InstanceState currentState) =>
|
||||
currentState is InstanceState.Enabled or InstanceState.Disabled;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a human-readable error message if the transition is invalid, or null if valid.
|
||||
/// </summary>
|
||||
public static string? ValidateTransition(InstanceState currentState, string operation)
|
||||
{
|
||||
var allowed = operation.ToLowerInvariant() switch
|
||||
{
|
||||
"deploy" => CanDeploy(currentState),
|
||||
"disable" => CanDisable(currentState),
|
||||
"enable" => CanEnable(currentState),
|
||||
"delete" => CanDelete(currentState),
|
||||
_ => false
|
||||
};
|
||||
|
||||
if (allowed) return null;
|
||||
|
||||
return $"Operation '{operation}' is not allowed when instance is in state '{currentState}'.";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user