Phase 3C: Deployment pipeline & Store-and-Forward engine
Deployment Manager (WP-1–8, WP-16): - DeploymentService: full pipeline (flatten→validate→send→track→audit) - OperationLockManager: per-instance concurrency control - StateTransitionValidator: Enabled/Disabled/NotDeployed transition matrix - ArtifactDeploymentService: broadcast to all sites with per-site results - Deployment identity (GUID + revision hash), idempotency, staleness detection - Instance lifecycle commands (disable/enable/delete) with deduplication Store-and-Forward (WP-9–15): - StoreAndForwardStorage: SQLite persistence, 3 categories, no max buffer - StoreAndForwardService: fixed-interval retry, transient-only buffering, parking - ReplicationService: async best-effort to standby (fire-and-forget) - Parked message management (query/retry/discard from central) - Messages survive instance deletion, S&F drains on disable 620 tests pass (+79 new), zero warnings.
This commit is contained in:
136
src/ScadaLink.StoreAndForward/ReplicationService.cs
Normal file
136
src/ScadaLink.StoreAndForward/ReplicationService.cs
Normal file
@@ -0,0 +1,136 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Async replication of buffer operations to standby node.
|
||||
///
|
||||
/// - Forwards add/remove/park operations to standby via a replication handler.
|
||||
/// - No ack wait (fire-and-forget per design).
|
||||
/// - Standby applies operations to its own SQLite.
|
||||
/// - On failover, standby resumes delivery from its replicated state.
|
||||
/// </summary>
|
||||
public class ReplicationService
|
||||
{
|
||||
private readonly StoreAndForwardOptions _options;
|
||||
private readonly ILogger<ReplicationService> _logger;
|
||||
private Func<ReplicationOperation, Task>? _replicationHandler;
|
||||
|
||||
public ReplicationService(
|
||||
StoreAndForwardOptions options,
|
||||
ILogger<ReplicationService> logger)
|
||||
{
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the handler for forwarding replication operations to the standby node.
|
||||
/// Typically wraps Akka Tell to the standby's replication actor.
|
||||
/// </summary>
|
||||
public void SetReplicationHandler(Func<ReplicationOperation, Task> handler)
|
||||
{
|
||||
_replicationHandler = handler;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Replicates an enqueue operation to standby (fire-and-forget).
|
||||
/// </summary>
|
||||
public void ReplicateEnqueue(StoreAndForwardMessage message)
|
||||
{
|
||||
if (!_options.ReplicationEnabled || _replicationHandler == null) return;
|
||||
|
||||
FireAndForget(new ReplicationOperation(
|
||||
ReplicationOperationType.Add,
|
||||
message.Id,
|
||||
message));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Replicates a remove operation to standby (fire-and-forget).
|
||||
/// </summary>
|
||||
public void ReplicateRemove(string messageId)
|
||||
{
|
||||
if (!_options.ReplicationEnabled || _replicationHandler == null) return;
|
||||
|
||||
FireAndForget(new ReplicationOperation(
|
||||
ReplicationOperationType.Remove,
|
||||
messageId,
|
||||
null));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Replicates a park operation to standby (fire-and-forget).
|
||||
/// </summary>
|
||||
public void ReplicatePark(StoreAndForwardMessage message)
|
||||
{
|
||||
if (!_options.ReplicationEnabled || _replicationHandler == null) return;
|
||||
|
||||
FireAndForget(new ReplicationOperation(
|
||||
ReplicationOperationType.Park,
|
||||
message.Id,
|
||||
message));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Applies a replicated operation received from the active node.
|
||||
/// Used by the standby node to keep its SQLite in sync.
|
||||
/// </summary>
|
||||
public async Task ApplyReplicatedOperationAsync(
|
||||
ReplicationOperation operation,
|
||||
StoreAndForwardStorage storage)
|
||||
{
|
||||
switch (operation.OperationType)
|
||||
{
|
||||
case ReplicationOperationType.Add when operation.Message != null:
|
||||
await storage.EnqueueAsync(operation.Message);
|
||||
break;
|
||||
|
||||
case ReplicationOperationType.Remove:
|
||||
await storage.RemoveMessageAsync(operation.MessageId);
|
||||
break;
|
||||
|
||||
case ReplicationOperationType.Park when operation.Message != null:
|
||||
operation.Message.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await storage.UpdateMessageAsync(operation.Message);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void FireAndForget(ReplicationOperation operation)
|
||||
{
|
||||
Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _replicationHandler!.Invoke(operation);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// WP-11: No ack wait — log and move on
|
||||
_logger.LogDebug(ex,
|
||||
"Replication of {OpType} for message {MessageId} failed (best-effort)",
|
||||
operation.OperationType, operation.MessageId);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Represents a buffer operation to be replicated to standby.
|
||||
/// </summary>
|
||||
public record ReplicationOperation(
|
||||
ReplicationOperationType OperationType,
|
||||
string MessageId,
|
||||
StoreAndForwardMessage? Message);
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Types of buffer operations that are replicated.
|
||||
/// </summary>
|
||||
public enum ReplicationOperationType
|
||||
{
|
||||
Add,
|
||||
Remove,
|
||||
Park
|
||||
}
|
||||
@@ -8,7 +8,9 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -16,4 +18,8 @@
|
||||
<ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ScadaLink.StoreAndForward.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
@@ -6,13 +8,36 @@ public static class ServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddStoreAndForward(this IServiceCollection services)
|
||||
{
|
||||
// Phase 0: skeleton only
|
||||
services.AddSingleton<StoreAndForwardStorage>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<StoreAndForwardOptions>>().Value;
|
||||
var logger = sp.GetRequiredService<ILogger<StoreAndForwardStorage>>();
|
||||
return new StoreAndForwardStorage(
|
||||
$"Data Source={options.SqliteDbPath}",
|
||||
logger);
|
||||
});
|
||||
|
||||
services.AddSingleton<StoreAndForwardService>(sp =>
|
||||
{
|
||||
var storage = sp.GetRequiredService<StoreAndForwardStorage>();
|
||||
var options = sp.GetRequiredService<IOptions<StoreAndForwardOptions>>().Value;
|
||||
var logger = sp.GetRequiredService<ILogger<StoreAndForwardService>>();
|
||||
return new StoreAndForwardService(storage, options, logger);
|
||||
});
|
||||
|
||||
services.AddSingleton<ReplicationService>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<StoreAndForwardOptions>>().Value;
|
||||
var logger = sp.GetRequiredService<ILogger<ReplicationService>>();
|
||||
return new ReplicationService(options, logger);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddStoreAndForwardActors(this IServiceCollection services)
|
||||
{
|
||||
// Phase 0: placeholder for Akka actor registration
|
||||
// Akka actor registration handled by Host component during actor system startup
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
49
src/ScadaLink.StoreAndForward/StoreAndForwardMessage.cs
Normal file
49
src/ScadaLink.StoreAndForward/StoreAndForwardMessage.cs
Normal file
@@ -0,0 +1,49 @@
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9: Represents a single store-and-forward message as stored in SQLite.
|
||||
/// Maps to the sf_messages table.
|
||||
/// </summary>
|
||||
public class StoreAndForwardMessage
|
||||
{
|
||||
/// <summary>Unique message ID (GUID).</summary>
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>WP-9: Category: ExternalSystem, Notification, or CachedDbWrite.</summary>
|
||||
public StoreAndForwardCategory Category { get; set; }
|
||||
|
||||
/// <summary>Target system name (external system, notification list, or DB connection).</summary>
|
||||
public string Target { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>JSON-serialized payload containing the call details.</summary>
|
||||
public string PayloadJson { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Number of delivery attempts so far.</summary>
|
||||
public int RetryCount { get; set; }
|
||||
|
||||
/// <summary>Maximum retry attempts before parking (0 = no limit).</summary>
|
||||
public int MaxRetries { get; set; }
|
||||
|
||||
/// <summary>Retry interval in milliseconds.</summary>
|
||||
public long RetryIntervalMs { get; set; }
|
||||
|
||||
/// <summary>When this message was first enqueued.</summary>
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
/// <summary>When delivery was last attempted (null if never attempted).</summary>
|
||||
public DateTimeOffset? LastAttemptAt { get; set; }
|
||||
|
||||
/// <summary>Current status of the message.</summary>
|
||||
public StoreAndForwardMessageStatus Status { get; set; }
|
||||
|
||||
/// <summary>Last error message from a failed delivery attempt.</summary>
|
||||
public string? LastError { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Instance that originated this message (for S&F-survives-delete behavior).
|
||||
/// WP-13: Messages are NOT cleared when instance is deleted.
|
||||
/// </summary>
|
||||
public string? OriginInstanceName { get; set; }
|
||||
}
|
||||
@@ -1,7 +1,22 @@
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9/10: Configuration options for the Store-and-Forward Engine.
|
||||
/// </summary>
|
||||
public class StoreAndForwardOptions
|
||||
{
|
||||
/// <summary>Path to the SQLite database for S&F message persistence.</summary>
|
||||
public string SqliteDbPath { get; set; } = "./data/store-and-forward.db";
|
||||
|
||||
/// <summary>WP-11: Whether to replicate buffer operations to standby node.</summary>
|
||||
public bool ReplicationEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>WP-10: Default retry interval for messages without per-source settings.</summary>
|
||||
public TimeSpan DefaultRetryInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>WP-10: Default maximum retry count before parking.</summary>
|
||||
public int DefaultMaxRetries { get; set; } = 50;
|
||||
|
||||
/// <summary>WP-10: Interval for the background retry timer sweep.</summary>
|
||||
public TimeSpan RetryTimerInterval { get; set; } = TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
322
src/ScadaLink.StoreAndForward/StoreAndForwardService.cs
Normal file
322
src/ScadaLink.StoreAndForward/StoreAndForwardService.cs
Normal file
@@ -0,0 +1,322 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9/10: Core store-and-forward service.
|
||||
///
|
||||
/// Lifecycle:
|
||||
/// 1. Caller attempts immediate delivery via IDeliveryHandler
|
||||
/// 2. On transient failure → buffer in SQLite → retry loop
|
||||
/// 3. On success → remove from buffer
|
||||
/// 4. On max retries → park
|
||||
/// 5. Permanent failures are returned to caller immediately (never buffered)
|
||||
///
|
||||
/// WP-10: Fixed retry interval (not exponential). Per-source-entity retry settings.
|
||||
/// Background timer-based retry sweep.
|
||||
///
|
||||
/// WP-12: Parked messages queryable, retryable, and discardable.
|
||||
///
|
||||
/// WP-14: Buffer depth reported as health metric. Activity logged to site event log.
|
||||
///
|
||||
/// WP-15: CachedCall idempotency is the caller's responsibility.
|
||||
/// This service does not deduplicate — if the same message is enqueued twice,
|
||||
/// it will be delivered twice. Callers using ExternalSystem.CachedCall() must
|
||||
/// design their payloads to be idempotent (e.g., include unique request IDs
|
||||
/// and handle duplicate detection on the remote end).
|
||||
/// </summary>
|
||||
public class StoreAndForwardService
|
||||
{
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardOptions _options;
|
||||
private readonly ILogger<StoreAndForwardService> _logger;
|
||||
private Timer? _retryTimer;
|
||||
private int _retryInProgress;
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Delivery handler delegate. Returns true on success, throws on transient failure.
|
||||
/// Permanent failures should return false (message will NOT be buffered).
|
||||
/// </summary>
|
||||
private readonly Dictionary<StoreAndForwardCategory, Func<StoreAndForwardMessage, Task<bool>>> _deliveryHandlers = new();
|
||||
|
||||
/// <summary>
|
||||
/// WP-14: Event callback for logging S&F activity to site event log.
|
||||
/// </summary>
|
||||
public event Action<string, StoreAndForwardCategory, string>? OnActivity;
|
||||
|
||||
public StoreAndForwardService(
|
||||
StoreAndForwardStorage storage,
|
||||
StoreAndForwardOptions options,
|
||||
ILogger<StoreAndForwardService> logger)
|
||||
{
|
||||
_storage = storage;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a delivery handler for a given message category.
|
||||
/// </summary>
|
||||
public void RegisterDeliveryHandler(
|
||||
StoreAndForwardCategory category,
|
||||
Func<StoreAndForwardMessage, Task<bool>> handler)
|
||||
{
|
||||
_deliveryHandlers[category] = handler;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes storage and starts the background retry timer.
|
||||
/// </summary>
|
||||
public async Task StartAsync()
|
||||
{
|
||||
await _storage.InitializeAsync();
|
||||
_retryTimer = new Timer(
|
||||
_ => _ = RetryPendingMessagesAsync(),
|
||||
null,
|
||||
_options.RetryTimerInterval,
|
||||
_options.RetryTimerInterval);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Store-and-forward service started. Retry interval: {Interval}s",
|
||||
_options.DefaultRetryInterval.TotalSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the background retry timer.
|
||||
/// </summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
if (_retryTimer != null)
|
||||
{
|
||||
await _retryTimer.DisposeAsync();
|
||||
_retryTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Enqueues a message for store-and-forward delivery.
|
||||
/// Attempts immediate delivery first. On transient failure, buffers for retry.
|
||||
/// On permanent failure (handler returns false), returns false immediately.
|
||||
///
|
||||
/// WP-15: CachedCall idempotency note — this method does not deduplicate.
|
||||
/// The caller (e.g., ExternalSystem.CachedCall()) is responsible for ensuring
|
||||
/// that the remote system can handle duplicate deliveries safely.
|
||||
/// </summary>
|
||||
public async Task<StoreAndForwardResult> EnqueueAsync(
|
||||
StoreAndForwardCategory category,
|
||||
string target,
|
||||
string payloadJson,
|
||||
string? originInstanceName = null,
|
||||
int? maxRetries = null,
|
||||
TimeSpan? retryInterval = null)
|
||||
{
|
||||
var message = new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = category,
|
||||
Target = target,
|
||||
PayloadJson = payloadJson,
|
||||
RetryCount = 0,
|
||||
MaxRetries = maxRetries ?? _options.DefaultMaxRetries,
|
||||
RetryIntervalMs = (long)(retryInterval ?? _options.DefaultRetryInterval).TotalMilliseconds,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
OriginInstanceName = originInstanceName
|
||||
};
|
||||
|
||||
// Attempt immediate delivery
|
||||
if (_deliveryHandlers.TryGetValue(category, out var handler))
|
||||
{
|
||||
try
|
||||
{
|
||||
var success = await handler(message);
|
||||
if (success)
|
||||
{
|
||||
RaiseActivity("Delivered", category, $"Immediate delivery to {target}");
|
||||
return new StoreAndForwardResult(true, message.Id, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Permanent failure — do not buffer
|
||||
return new StoreAndForwardResult(false, message.Id, false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Transient failure — buffer for retry
|
||||
_logger.LogWarning(ex,
|
||||
"Immediate delivery to {Target} failed (transient), buffering for retry",
|
||||
target);
|
||||
|
||||
message.LastAttemptAt = DateTimeOffset.UtcNow;
|
||||
message.RetryCount = 1;
|
||||
message.LastError = ex.Message;
|
||||
await _storage.EnqueueAsync(message);
|
||||
|
||||
RaiseActivity("Queued", category, $"Buffered for retry: {target} ({ex.Message})");
|
||||
return new StoreAndForwardResult(true, message.Id, true);
|
||||
}
|
||||
}
|
||||
|
||||
// No handler registered — buffer for later
|
||||
await _storage.EnqueueAsync(message);
|
||||
RaiseActivity("Queued", category, $"No handler registered, buffered: {target}");
|
||||
return new StoreAndForwardResult(true, message.Id, true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Background retry sweep. Processes all pending messages that are due for retry.
|
||||
/// </summary>
|
||||
internal async Task RetryPendingMessagesAsync()
|
||||
{
|
||||
// Prevent overlapping retry sweeps
|
||||
if (Interlocked.CompareExchange(ref _retryInProgress, 1, 0) != 0)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var messages = await _storage.GetMessagesForRetryAsync();
|
||||
if (messages.Count == 0) return;
|
||||
|
||||
_logger.LogDebug("Retry sweep: {Count} messages due for retry", messages.Count);
|
||||
|
||||
foreach (var message in messages)
|
||||
{
|
||||
await RetryMessageAsync(message);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error during retry sweep");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Exchange(ref _retryInProgress, 0);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RetryMessageAsync(StoreAndForwardMessage message)
|
||||
{
|
||||
if (!_deliveryHandlers.TryGetValue(message.Category, out var handler))
|
||||
{
|
||||
_logger.LogWarning("No delivery handler for category {Category}", message.Category);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var success = await handler(message);
|
||||
if (success)
|
||||
{
|
||||
await _storage.RemoveMessageAsync(message.Id);
|
||||
RaiseActivity("Delivered", message.Category,
|
||||
$"Delivered to {message.Target} after {message.RetryCount} retries");
|
||||
return;
|
||||
}
|
||||
|
||||
// Permanent failure on retry — park immediately
|
||||
message.Status = StoreAndForwardMessageStatus.Parked;
|
||||
message.LastAttemptAt = DateTimeOffset.UtcNow;
|
||||
message.LastError = "Permanent failure (handler returned false)";
|
||||
await _storage.UpdateMessageAsync(message);
|
||||
RaiseActivity("Parked", message.Category,
|
||||
$"Permanent failure for {message.Target}: handler returned false");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Transient failure — increment retry, check max
|
||||
message.RetryCount++;
|
||||
message.LastAttemptAt = DateTimeOffset.UtcNow;
|
||||
message.LastError = ex.Message;
|
||||
|
||||
if (message.MaxRetries > 0 && message.RetryCount >= message.MaxRetries)
|
||||
{
|
||||
message.Status = StoreAndForwardMessageStatus.Parked;
|
||||
await _storage.UpdateMessageAsync(message);
|
||||
RaiseActivity("Parked", message.Category,
|
||||
$"Max retries ({message.MaxRetries}) reached for {message.Target}");
|
||||
_logger.LogWarning(
|
||||
"Message {MessageId} parked after {MaxRetries} retries to {Target}",
|
||||
message.Id, message.MaxRetries, message.Target);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _storage.UpdateMessageAsync(message);
|
||||
RaiseActivity("Retried", message.Category,
|
||||
$"Retry {message.RetryCount}/{message.MaxRetries} for {message.Target}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Gets parked messages for central query (Pattern 8).
|
||||
/// </summary>
|
||||
public async Task<(List<StoreAndForwardMessage> Messages, int TotalCount)> GetParkedMessagesAsync(
|
||||
StoreAndForwardCategory? category = null,
|
||||
int pageNumber = 1,
|
||||
int pageSize = 50)
|
||||
{
|
||||
return await _storage.GetParkedMessagesAsync(category, pageNumber, pageSize);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Retries a parked message (moves back to pending queue).
|
||||
/// </summary>
|
||||
public async Task<bool> RetryParkedMessageAsync(string messageId)
|
||||
{
|
||||
var success = await _storage.RetryParkedMessageAsync(messageId);
|
||||
if (success)
|
||||
{
|
||||
RaiseActivity("Retry", StoreAndForwardCategory.ExternalSystem,
|
||||
$"Parked message {messageId} moved back to queue");
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Permanently discards a parked message.
|
||||
/// </summary>
|
||||
public async Task<bool> DiscardParkedMessageAsync(string messageId)
|
||||
{
|
||||
var success = await _storage.DiscardParkedMessageAsync(messageId);
|
||||
if (success)
|
||||
{
|
||||
RaiseActivity("Discard", StoreAndForwardCategory.ExternalSystem,
|
||||
$"Parked message {messageId} discarded");
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-14: Gets buffer depth by category for health reporting.
|
||||
/// </summary>
|
||||
public async Task<Dictionary<StoreAndForwardCategory, int>> GetBufferDepthAsync()
|
||||
{
|
||||
return await _storage.GetBufferDepthByCategoryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-13: Gets count of S&F messages for a given instance (for verifying survival on deletion).
|
||||
/// </summary>
|
||||
public async Task<int> GetMessageCountForInstanceAsync(string instanceName)
|
||||
{
|
||||
return await _storage.GetMessageCountByOriginInstanceAsync(instanceName);
|
||||
}
|
||||
|
||||
private void RaiseActivity(string action, StoreAndForwardCategory category, string detail)
|
||||
{
|
||||
OnActivity?.Invoke(action, category, detail);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of an enqueue operation.
|
||||
/// </summary>
|
||||
public record StoreAndForwardResult(
|
||||
/// <summary>True if the message was accepted (either delivered immediately or buffered).</summary>
|
||||
bool Accepted,
|
||||
/// <summary>Unique message ID for tracking.</summary>
|
||||
string MessageId,
|
||||
/// <summary>True if the message was buffered (not delivered immediately).</summary>
|
||||
bool WasBuffered);
|
||||
339
src/ScadaLink.StoreAndForward/StoreAndForwardStorage.cs
Normal file
339
src/ScadaLink.StoreAndForward/StoreAndForwardStorage.cs
Normal file
@@ -0,0 +1,339 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.StoreAndForward;
|
||||
|
||||
/// <summary>
|
||||
/// WP-9: SQLite persistence layer for store-and-forward messages.
|
||||
/// Uses direct Microsoft.Data.Sqlite (not EF Core) for lightweight site-side storage.
|
||||
/// No max buffer size per design decision.
|
||||
/// </summary>
|
||||
public class StoreAndForwardStorage
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly ILogger<StoreAndForwardStorage> _logger;
|
||||
|
||||
public StoreAndForwardStorage(string connectionString, ILogger<StoreAndForwardStorage> logger)
|
||||
{
|
||||
_connectionString = connectionString;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the sf_messages table if it does not exist.
|
||||
/// </summary>
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = @"
|
||||
CREATE TABLE IF NOT EXISTS sf_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
category INTEGER NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_retries INTEGER NOT NULL DEFAULT 50,
|
||||
retry_interval_ms INTEGER NOT NULL DEFAULT 30000,
|
||||
created_at TEXT NOT NULL,
|
||||
last_attempt_at TEXT,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
origin_instance TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sf_messages_status ON sf_messages(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_sf_messages_category ON sf_messages(category);
|
||||
";
|
||||
await command.ExecuteNonQueryAsync();
|
||||
|
||||
_logger.LogInformation("Store-and-forward SQLite storage initialized");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-9: Enqueues a new message with Pending status.
|
||||
/// </summary>
|
||||
public async Task EnqueueAsync(StoreAndForwardMessage message)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
INSERT INTO sf_messages (id, category, target, payload_json, retry_count, max_retries,
|
||||
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance)
|
||||
VALUES (@id, @category, @target, @payload, @retryCount, @maxRetries,
|
||||
@retryIntervalMs, @createdAt, @lastAttempt, @status, @lastError, @origin)";
|
||||
|
||||
cmd.Parameters.AddWithValue("@id", message.Id);
|
||||
cmd.Parameters.AddWithValue("@category", (int)message.Category);
|
||||
cmd.Parameters.AddWithValue("@target", message.Target);
|
||||
cmd.Parameters.AddWithValue("@payload", message.PayloadJson);
|
||||
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
|
||||
cmd.Parameters.AddWithValue("@maxRetries", message.MaxRetries);
|
||||
cmd.Parameters.AddWithValue("@retryIntervalMs", message.RetryIntervalMs);
|
||||
cmd.Parameters.AddWithValue("@createdAt", message.CreatedAt.ToString("O"));
|
||||
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
|
||||
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@status", (int)message.Status);
|
||||
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@origin", (object?)message.OriginInstanceName ?? DBNull.Value);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Gets all messages that are due for retry (Pending status, last attempt older than retry interval).
|
||||
/// </summary>
|
||||
public async Task<List<StoreAndForwardMessage>> GetMessagesForRetryAsync()
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
SELECT id, category, target, payload_json, retry_count, max_retries,
|
||||
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance
|
||||
FROM sf_messages
|
||||
WHERE status = @pending
|
||||
AND (last_attempt_at IS NULL
|
||||
OR retry_interval_ms = 0
|
||||
OR (julianday('now') - julianday(last_attempt_at)) * 86400000 >= retry_interval_ms)
|
||||
ORDER BY created_at ASC";
|
||||
|
||||
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
|
||||
|
||||
return await ReadMessagesAsync(cmd);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Updates a message after a delivery attempt.
|
||||
/// </summary>
|
||||
public async Task UpdateMessageAsync(StoreAndForwardMessage message)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
UPDATE sf_messages
|
||||
SET retry_count = @retryCount,
|
||||
last_attempt_at = @lastAttempt,
|
||||
status = @status,
|
||||
last_error = @lastError
|
||||
WHERE id = @id";
|
||||
|
||||
cmd.Parameters.AddWithValue("@id", message.Id);
|
||||
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
|
||||
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
|
||||
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@status", (int)message.Status);
|
||||
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-10: Removes a successfully delivered message.
|
||||
/// </summary>
|
||||
public async Task RemoveMessageAsync(string messageId)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id";
|
||||
cmd.Parameters.AddWithValue("@id", messageId);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Gets all parked messages, optionally filtered by category, with pagination.
|
||||
/// </summary>
|
||||
public async Task<(List<StoreAndForwardMessage> Messages, int TotalCount)> GetParkedMessagesAsync(
|
||||
StoreAndForwardCategory? category = null,
|
||||
int pageNumber = 1,
|
||||
int pageSize = 50)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Count
|
||||
await using var countCmd = connection.CreateCommand();
|
||||
countCmd.CommandText = category.HasValue
|
||||
? "SELECT COUNT(*) FROM sf_messages WHERE status = @parked AND category = @category"
|
||||
: "SELECT COUNT(*) FROM sf_messages WHERE status = @parked";
|
||||
countCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
|
||||
if (category.HasValue) countCmd.Parameters.AddWithValue("@category", (int)category.Value);
|
||||
var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync());
|
||||
|
||||
// Page
|
||||
await using var pageCmd = connection.CreateCommand();
|
||||
var categoryFilter = category.HasValue ? " AND category = @category" : "";
|
||||
pageCmd.CommandText = $@"
|
||||
SELECT id, category, target, payload_json, retry_count, max_retries,
|
||||
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance
|
||||
FROM sf_messages
|
||||
WHERE status = @parked{categoryFilter}
|
||||
ORDER BY created_at ASC
|
||||
LIMIT @limit OFFSET @offset";
|
||||
|
||||
pageCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
|
||||
if (category.HasValue) pageCmd.Parameters.AddWithValue("@category", (int)category.Value);
|
||||
pageCmd.Parameters.AddWithValue("@limit", pageSize);
|
||||
pageCmd.Parameters.AddWithValue("@offset", (pageNumber - 1) * pageSize);
|
||||
|
||||
var messages = await ReadMessagesAsync(pageCmd);
|
||||
return (messages, totalCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Moves a parked message back to pending for retry.
|
||||
/// </summary>
|
||||
public async Task<bool> RetryParkedMessageAsync(string messageId)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
UPDATE sf_messages
|
||||
SET status = @pending, retry_count = 0, last_error = NULL
|
||||
WHERE id = @id AND status = @parked";
|
||||
|
||||
cmd.Parameters.AddWithValue("@id", messageId);
|
||||
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
|
||||
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
|
||||
|
||||
var rows = await cmd.ExecuteNonQueryAsync();
|
||||
return rows > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-12: Permanently discards a parked message.
|
||||
/// </summary>
|
||||
public async Task<bool> DiscardParkedMessageAsync(string messageId)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id AND status = @parked";
|
||||
cmd.Parameters.AddWithValue("@id", messageId);
|
||||
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
|
||||
|
||||
var rows = await cmd.ExecuteNonQueryAsync();
|
||||
return rows > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-14: Gets buffer depth by category (count of pending messages per category).
|
||||
/// </summary>
|
||||
public async Task<Dictionary<StoreAndForwardCategory, int>> GetBufferDepthByCategoryAsync()
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
SELECT category, COUNT(*) as cnt
|
||||
FROM sf_messages
|
||||
WHERE status = @pending
|
||||
GROUP BY category";
|
||||
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
|
||||
|
||||
var result = new Dictionary<StoreAndForwardCategory, int>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
var category = (StoreAndForwardCategory)reader.GetInt32(0);
|
||||
var count = reader.GetInt32(1);
|
||||
result[category] = count;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-13: Verifies messages are NOT deleted when an instance is deleted.
|
||||
/// Returns the count of messages for a given origin instance.
|
||||
/// </summary>
|
||||
public async Task<int> GetMessageCountByOriginInstanceAsync(string instanceName)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
SELECT COUNT(*)
|
||||
FROM sf_messages
|
||||
WHERE origin_instance = @origin";
|
||||
cmd.Parameters.AddWithValue("@origin", instanceName);
|
||||
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a message by ID.
|
||||
/// </summary>
|
||||
public async Task<StoreAndForwardMessage?> GetMessageByIdAsync(string messageId)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
SELECT id, category, target, payload_json, retry_count, max_retries,
|
||||
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance
|
||||
FROM sf_messages
|
||||
WHERE id = @id";
|
||||
cmd.Parameters.AddWithValue("@id", messageId);
|
||||
|
||||
var messages = await ReadMessagesAsync(cmd);
|
||||
return messages.FirstOrDefault();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets total message count by status.
|
||||
/// </summary>
|
||||
public async Task<int> GetMessageCountByStatusAsync(StoreAndForwardMessageStatus status)
|
||||
{
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM sf_messages WHERE status = @status";
|
||||
cmd.Parameters.AddWithValue("@status", (int)status);
|
||||
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync());
|
||||
}
|
||||
|
||||
private static async Task<List<StoreAndForwardMessage>> ReadMessagesAsync(SqliteCommand cmd)
|
||||
{
|
||||
var results = new List<StoreAndForwardMessage>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
results.Add(new StoreAndForwardMessage
|
||||
{
|
||||
Id = reader.GetString(0),
|
||||
Category = (StoreAndForwardCategory)reader.GetInt32(1),
|
||||
Target = reader.GetString(2),
|
||||
PayloadJson = reader.GetString(3),
|
||||
RetryCount = reader.GetInt32(4),
|
||||
MaxRetries = reader.GetInt32(5),
|
||||
RetryIntervalMs = reader.GetInt64(6),
|
||||
CreatedAt = DateTimeOffset.Parse(reader.GetString(7)),
|
||||
LastAttemptAt = reader.IsDBNull(8) ? null : DateTimeOffset.Parse(reader.GetString(8)),
|
||||
Status = (StoreAndForwardMessageStatus)reader.GetInt32(9),
|
||||
LastError = reader.IsDBNull(10) ? null : reader.GetString(10),
|
||||
OriginInstanceName = reader.IsDBNull(11) ? null : reader.GetString(11)
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user