using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging.Abstractions; using ScadaLink.Commons.Types.Enums; namespace ScadaLink.StoreAndForward.Tests; /// /// WP-10/12/13/14: Tests for the StoreAndForwardService retry engine and management. /// public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable { private readonly SqliteConnection _keepAlive; private readonly StoreAndForwardStorage _storage; private readonly StoreAndForwardService _service; private readonly StoreAndForwardOptions _options; public StoreAndForwardServiceTests() { var dbName = $"SvcTests_{Guid.NewGuid():N}"; var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared"; _keepAlive = new SqliteConnection(connStr); _keepAlive.Open(); _storage = new StoreAndForwardStorage(connStr, NullLogger.Instance); _options = new StoreAndForwardOptions { DefaultRetryInterval = TimeSpan.Zero, DefaultMaxRetries = 3, RetryTimerInterval = TimeSpan.FromMinutes(10) }; _service = new StoreAndForwardService( _storage, _options, NullLogger.Instance); } public async Task InitializeAsync() => await _storage.InitializeAsync(); public Task DisposeAsync() => Task.CompletedTask; public void Dispose() => _keepAlive.Dispose(); // ── WP-10: Immediate delivery ── [Fact] public async Task EnqueueAsync_ImmediateDeliverySuccess_ReturnsAcceptedNotBuffered() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}""", "Pump1"); Assert.True(result.Accepted); Assert.False(result.WasBuffered); } [Fact] public async Task EnqueueAsync_PermanentFailure_ReturnsNotAccepted() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(false)); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}"""); Assert.False(result.Accepted); Assert.False(result.WasBuffered); } [Fact] public async Task EnqueueAsync_TransientFailure_BuffersForRetry() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("Connection refused")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}""", "Pump1"); Assert.True(result.Accepted); Assert.True(result.WasBuffered); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status); Assert.Equal(1, msg.RetryCount); } [Fact] public async Task EnqueueAsync_NoHandler_BuffersForLater() { var result = await _service.EnqueueAsync( StoreAndForwardCategory.Notification, "alerts@company.com", """{"subject":"Alert"}"""); Assert.True(result.Accepted); Assert.True(result.WasBuffered); } // ── WP-10: Retry engine ── [Fact] public async Task RetryPendingMessagesAsync_SuccessfulRetry_RemovesMessage() { int callCount = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { callCount++; if (callCount == 1) throw new HttpRequestException("fail"); return Task.FromResult(true); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.True(result.WasBuffered); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); } [Fact] public async Task RetryPendingMessagesAsync_MaxRetriesReached_ParksMessage() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("always fails")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 2); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); } [Fact] public async Task RetryPendingMessagesAsync_PermanentFailureOnRetry_ParksMessage() { int callCount = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { callCount++; if (callCount == 1) throw new HttpRequestException("transient"); return Task.FromResult(false); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}"""); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); } // ── WP-12: Parked message management ── [Fact] public async Task RetryParkedMessageAsync_MovesBackToQueue() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); var retried = await _service.RetryParkedMessageAsync(result.MessageId); Assert.True(retried); msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status); Assert.Equal(0, msg.RetryCount); } [Fact] public async Task DiscardParkedMessageAsync_PermanentlyRemoves() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); var discarded = await _service.DiscardParkedMessageAsync(result.MessageId); Assert.True(discarded); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); } [Fact] public async Task GetParkedMessagesAsync_ReturnsPaginatedResults() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); for (int i = 0; i < 3; i++) { await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, $"api{i}", """{}""", maxRetries: 1); } await _service.RetryPendingMessagesAsync(); var (messages, total) = await _service.GetParkedMessagesAsync( StoreAndForwardCategory.ExternalSystem, 1, 2); Assert.Equal(2, messages.Count); Assert.True(total >= 3); } // ── WP-13: Messages survive instance deletion ── [Fact] public async Task MessagesForInstance_SurviveAfterDeletion() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", "Pump1"); await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api2", """{}""", "Pump1"); var count = await _service.GetMessageCountForInstanceAsync("Pump1"); Assert.Equal(2, count); } // ── WP-14: Health metrics ── [Fact] public async Task GetBufferDepthAsync_ReturnsCorrectDepth() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); _service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api1", """{}"""); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api2", """{}"""); await _service.EnqueueAsync(StoreAndForwardCategory.Notification, "email", """{}"""); var depth = await _service.GetBufferDepthAsync(); Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.ExternalSystem) >= 2); Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.Notification) >= 1); } [Fact] public async Task OnActivity_RaisedOnEnqueue() { var activities = new List(); _service.OnActivity += (action, _, _) => activities.Add(action); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.Contains("Delivered", activities); } [Fact] public async Task OnActivity_RaisedOnBuffer() { var activities = new List(); _service.OnActivity += (action, _, _) => activities.Add(action); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.Contains("Queued", activities); } // ── WP-10: Per-source-entity retry settings ── [Fact] public async Task EnqueueAsync_CustomRetrySettings_Respected() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 100, retryInterval: TimeSpan.FromSeconds(60)); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(100, msg!.MaxRetries); Assert.Equal(60000, msg.RetryIntervalMs); } }