Phase 3C: Deployment pipeline & Store-and-Forward engine
Deployment Manager (WP-1–8, WP-16): - DeploymentService: full pipeline (flatten→validate→send→track→audit) - OperationLockManager: per-instance concurrency control - StateTransitionValidator: Enabled/Disabled/NotDeployed transition matrix - ArtifactDeploymentService: broadcast to all sites with per-site results - Deployment identity (GUID + revision hash), idempotency, staleness detection - Instance lifecycle commands (disable/enable/delete) with deduplication Store-and-Forward (WP-9–15): - StoreAndForwardStorage: SQLite persistence, 3 categories, no max buffer - StoreAndForwardService: fixed-interval retry, transient-only buffering, parking - ReplicationService: async best-effort to standby (fire-and-forget) - Parked message management (query/retry/discard from central) - Messages survive instance deletion, S&F drains on disable 620 tests pass (+79 new), zero warnings.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,313 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.StoreAndForward.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-10/12/13/14: Tests for the StoreAndForwardService retry engine and management.
|
||||
/// </summary>
|
||||
public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _keepAlive;
|
||||
private readonly StoreAndForwardStorage _storage;
|
||||
private readonly StoreAndForwardService _service;
|
||||
private readonly StoreAndForwardOptions _options;
|
||||
|
||||
public StoreAndForwardServiceTests()
|
||||
{
|
||||
var dbName = $"SvcTests_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
_keepAlive = new SqliteConnection(connStr);
|
||||
_keepAlive.Open();
|
||||
|
||||
_storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
|
||||
_options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.Zero,
|
||||
DefaultMaxRetries = 3,
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10)
|
||||
};
|
||||
|
||||
_service = new StoreAndForwardService(
|
||||
_storage, _options, NullLogger<StoreAndForwardService>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() => await _storage.InitializeAsync();
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose() => _keepAlive.Dispose();
|
||||
|
||||
// ── WP-10: Immediate delivery ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_ImmediateDeliverySuccess_ReturnsAcceptedNotBuffered()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""", "Pump1");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.False(result.WasBuffered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_PermanentFailure_ReturnsNotAccepted()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(false));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""");
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.False(result.WasBuffered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_TransientFailure_BuffersForRetry()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("Connection refused"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api.example.com",
|
||||
"""{"method":"Test"}""", "Pump1");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
Assert.Equal(1, msg.RetryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_NoHandler_BuffersForLater()
|
||||
{
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification, "alerts@company.com",
|
||||
"""{"subject":"Alert"}""");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.True(result.WasBuffered);
|
||||
}
|
||||
|
||||
// ── WP-10: Retry engine ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_SuccessfulRetry_RemovesMessage()
|
||||
{
|
||||
int callCount = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 1) throw new HttpRequestException("fail");
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
Assert.True(result.WasBuffered);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_MaxRetriesReached_ParksMessage()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("always fails"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 2);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryPendingMessagesAsync_PermanentFailureOnRetry_ParksMessage()
|
||||
{
|
||||
int callCount = 0;
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 1) throw new HttpRequestException("transient");
|
||||
return Task.FromResult(false);
|
||||
});
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
}
|
||||
|
||||
// ── WP-12: Parked message management ──
|
||||
|
||||
[Fact]
|
||||
public async Task RetryParkedMessageAsync_MovesBackToQueue()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 1);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status);
|
||||
|
||||
var retried = await _service.RetryParkedMessageAsync(result.MessageId);
|
||||
Assert.True(retried);
|
||||
|
||||
msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
|
||||
Assert.Equal(0, msg.RetryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscardParkedMessageAsync_PermanentlyRemoves()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 1);
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var discarded = await _service.DiscardParkedMessageAsync(result.MessageId);
|
||||
Assert.True(discarded);
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Null(msg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetParkedMessagesAsync_ReturnsPaginatedResults()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, $"api{i}", """{}""",
|
||||
maxRetries: 1);
|
||||
}
|
||||
|
||||
await _service.RetryPendingMessagesAsync();
|
||||
|
||||
var (messages, total) = await _service.GetParkedMessagesAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, 1, 2);
|
||||
|
||||
Assert.Equal(2, messages.Count);
|
||||
Assert.True(total >= 3);
|
||||
}
|
||||
|
||||
// ── WP-13: Messages survive instance deletion ──
|
||||
|
||||
[Fact]
|
||||
public async Task MessagesForInstance_SurviveAfterDeletion()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""", "Pump1");
|
||||
await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api2", """{}""", "Pump1");
|
||||
|
||||
var count = await _service.GetMessageCountForInstanceAsync("Pump1");
|
||||
Assert.Equal(2, count);
|
||||
}
|
||||
|
||||
// ── WP-14: Health metrics ──
|
||||
|
||||
[Fact]
|
||||
public async Task GetBufferDepthAsync_ReturnsCorrectDepth()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api1", """{}""");
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api2", """{}""");
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.Notification, "email", """{}""");
|
||||
|
||||
var depth = await _service.GetBufferDepthAsync();
|
||||
Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.ExternalSystem) >= 2);
|
||||
Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.Notification) >= 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OnActivity_RaisedOnEnqueue()
|
||||
{
|
||||
var activities = new List<string>();
|
||||
_service.OnActivity += (action, _, _) => activities.Add(action);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => Task.FromResult(true));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
Assert.Contains("Delivered", activities);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OnActivity_RaisedOnBuffer()
|
||||
{
|
||||
var activities = new List<string>();
|
||||
_service.OnActivity += (action, _, _) => activities.Add(action);
|
||||
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}""");
|
||||
|
||||
Assert.Contains("Queued", activities);
|
||||
}
|
||||
|
||||
// ── WP-10: Per-source-entity retry settings ──
|
||||
|
||||
[Fact]
|
||||
public async Task EnqueueAsync_CustomRetrySettings_Respected()
|
||||
{
|
||||
_service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ => throw new HttpRequestException("fail"));
|
||||
|
||||
var result = await _service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem, "api", """{}""",
|
||||
maxRetries: 100,
|
||||
retryInterval: TimeSpan.FromSeconds(60));
|
||||
|
||||
var msg = await _storage.GetMessageByIdAsync(result.MessageId);
|
||||
Assert.Equal(100, msg!.MaxRetries);
|
||||
Assert.Equal(60000, msg.RetryIntervalMs);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user