using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests; /// /// WP-10/12/13/14: Tests for the StoreAndForwardService retry engine and management. /// public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable { private readonly SqliteConnection _keepAlive; private readonly StoreAndForwardStorage _storage; private readonly StoreAndForwardService _service; private readonly StoreAndForwardOptions _options; private readonly List _extraKeepAlives = new(); public StoreAndForwardServiceTests() { var dbName = $"SvcTests_{Guid.NewGuid():N}"; var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared"; _keepAlive = new SqliteConnection(connStr); _keepAlive.Open(); _storage = new StoreAndForwardStorage(connStr, NullLogger.Instance); _options = new StoreAndForwardOptions { DefaultRetryInterval = TimeSpan.Zero, DefaultMaxRetries = 3, RetryTimerInterval = TimeSpan.FromMinutes(10) }; _service = new StoreAndForwardService( _storage, _options, NullLogger.Instance); } public async Task InitializeAsync() => await _storage.InitializeAsync(); public Task DisposeAsync() => Task.CompletedTask; public void Dispose() { _keepAlive.Dispose(); foreach (var c in _extraKeepAlives) c.Dispose(); } /// /// Builds a fresh service over its own in-memory (shared-cache) SQLite so a test /// can call StartAsync without racing the shared _service's timer. The /// keep-alive connection is tracked for disposal. /// private StoreAndForwardService CreateService(TimeSpan? retryTimerInterval = null) { var connStr = $"Data Source=DeferTests_{Guid.NewGuid():N};Mode=Memory;Cache=Shared"; var keepAlive = new SqliteConnection(connStr); keepAlive.Open(); _extraKeepAlives.Add(keepAlive); var storage = new StoreAndForwardStorage(connStr, NullLogger.Instance); storage.InitializeAsync().GetAwaiter().GetResult(); var options = new StoreAndForwardOptions { DefaultRetryInterval = TimeSpan.Zero, DefaultMaxRetries = 3, RetryTimerInterval = retryTimerInterval ?? TimeSpan.FromMinutes(10), }; return new StoreAndForwardService(storage, options, NullLogger.Instance); } // ── WP-10: Immediate delivery ── [Fact] public async Task EnqueueAsync_ImmediateDeliverySuccess_ReturnsAcceptedNotBuffered() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}""", "Pump1"); Assert.True(result.Accepted); Assert.False(result.WasBuffered); } [Fact] public async Task EnqueueAsync_PermanentFailure_ReturnsNotAccepted() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(false)); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}"""); Assert.False(result.Accepted); Assert.False(result.WasBuffered); } [Fact] public async Task EnqueueAsync_TransientFailure_BuffersForRetry() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("Connection refused")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api.example.com", """{"method":"Test"}""", "Pump1"); Assert.True(result.Accepted); Assert.True(result.WasBuffered); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status); // StoreAndForward-003: RetryCount counts sweep retries only; the immediate // attempt is attempt 0, so a freshly buffered message has RetryCount 0. Assert.Equal(0, msg.RetryCount); } [Fact] public async Task EnqueueAsync_NoHandler_BuffersForLater() { var result = await _service.EnqueueAsync( StoreAndForwardCategory.Notification, "alerts@company.com", """{"subject":"Alert"}"""); Assert.True(result.Accepted); Assert.True(result.WasBuffered); } // ── WP-10: Retry engine ── [Fact] public async Task RetryPendingMessagesAsync_SuccessfulRetry_RemovesMessage() { int callCount = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { callCount++; if (callCount == 1) throw new HttpRequestException("fail"); return Task.FromResult(true); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.True(result.WasBuffered); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); } [Fact] public async Task RetryPendingMessagesAsync_MaxRetriesReached_ParksMessage() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("always fails")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 2); // StoreAndForward-003: MaxRetries bounds sweep retries (not the immediate // attempt), so a message with MaxRetries=2 needs two retry sweeps to park. await _service.RetryPendingMessagesAsync(); var afterFirst = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Pending, afterFirst!.Status); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); } // ── StoreAndForward-003: retry-count accounting ── [Fact] public async Task RetryPendingMessagesAsync_MaxRetriesOne_PerformsExactlyOneRetryBeforeParking() { // The immediate attempt is attempt 0; MaxRetries=1 must allow exactly one // retry sweep before parking. The pre-fix off-by-one parked with zero retries. var attempts = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { Interlocked.Increment(ref attempts); throw new HttpRequestException("always fails"); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1); // After the immediate failed attempt the message is buffered, not parked. var buffered = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Pending, buffered!.Status); Assert.Equal(1, attempts); // only the immediate attempt so far await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); Assert.Equal(2, attempts); // immediate attempt + exactly one retry Assert.Equal(1, msg.RetryCount); // one sweep retry recorded } // ── StoreAndForward-005: sweep-vs-management race hardening ── [Fact] public async Task RetryMessageAsync_StatusChangedDuringDelivery_SweepParkWriteIsSkipped() { // StoreAndForward-005: the retry sweep's state-changing writes must be // conditional on the status it observed, so a concurrent operator action that // moved the row out of Pending (e.g. between the sweep's snapshot load and its // park write) is not silently overwritten by the sweep's stale view. var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", attemptImmediateDelivery: false, maxRetries: 1); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async msg => { // Simulate an operator action winning the race: the row leaves Pending // (here: parked) while the sweep is still mid-delivery. The sweep would // otherwise unconditionally re-write this row from its stale snapshot. var parkedOutFromUnderTheSweep = new StoreAndForwardMessage { Id = msg.Id, Category = msg.Category, Target = msg.Target, PayloadJson = msg.PayloadJson, RetryCount = 7, MaxRetries = msg.MaxRetries, RetryIntervalMs = msg.RetryIntervalMs, CreatedAt = msg.CreatedAt, LastAttemptAt = DateTimeOffset.UtcNow, Status = StoreAndForwardMessageStatus.Parked, LastError = "operator/other writer" }; await _storage.UpdateMessageAsync(parkedOutFromUnderTheSweep); throw new HttpRequestException("transient — sweep will try to park"); }); await _service.RetryPendingMessagesAsync(); // The sweep observed Pending; the row is now Parked with the other writer's // RetryCount (7), not the sweep's (1). The sweep's conditional write was skipped. var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); Assert.Equal(7, msg.RetryCount); Assert.Equal("operator/other writer", msg.LastError); } [Fact] public async Task RetryPendingMessagesAsync_PermanentFailureOnRetry_ParksMessage() { int callCount = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { callCount++; if (callCount == 1) throw new HttpRequestException("transient"); return Task.FromResult(false); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}"""); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); } // ── WP-12: Parked message management ── [Fact] public async Task RetryParkedMessageAsync_MovesBackToQueue() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Parked, msg!.Status); var retried = await _service.RetryParkedMessageAsync(result.MessageId); Assert.True(retried); msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status); Assert.Equal(0, msg.RetryCount); } /// /// StoreAndForward-017: the Retry activity-log entry must carry the parked /// message's true category, not a hard-coded ExternalSystem. /// [Fact] public async Task RetryParkedMessageAsync_ActivityUsesMessageRealCategory() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.Notification, "ops-list", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); // -> parked var categories = new List(); _service.OnActivity += (action, category, _) => { if (action == "Retry") categories.Add(category); }; var retried = await _service.RetryParkedMessageAsync(result.MessageId); Assert.True(retried); Assert.Equal(new[] { StoreAndForwardCategory.Notification }, categories); } /// /// StoreAndForward-017: the Discard activity-log entry must carry the parked /// message's true category, not a hard-coded ExternalSystem. /// [Fact] public async Task DiscardParkedMessageAsync_ActivityUsesMessageRealCategory() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.CachedDbWrite, "site-db", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); // -> parked var categories = new List(); _service.OnActivity += (action, category, _) => { if (action == "Discard") categories.Add(category); }; var discarded = await _service.DiscardParkedMessageAsync(result.MessageId); Assert.True(discarded); Assert.Equal(new[] { StoreAndForwardCategory.CachedDbWrite }, categories); } [Fact] public async Task DiscardParkedMessageAsync_PermanentlyRemoves() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1); await _service.RetryPendingMessagesAsync(); var discarded = await _service.DiscardParkedMessageAsync(result.MessageId); Assert.True(discarded); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); } [Fact] public async Task GetParkedMessagesAsync_ReturnsPaginatedResults() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); for (int i = 0; i < 3; i++) { await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, $"api{i}", """{}""", maxRetries: 1); } await _service.RetryPendingMessagesAsync(); var (messages, total) = await _service.GetParkedMessagesAsync( StoreAndForwardCategory.ExternalSystem, 1, 2); Assert.Equal(2, messages.Count); Assert.True(total >= 3); } // ── WP-13: Messages survive instance deletion ── [Fact] public async Task MessagesForInstance_SurviveAfterDeletion() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", "Pump1"); await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api2", """{}""", "Pump1"); var count = await _service.GetMessageCountForInstanceAsync("Pump1"); Assert.Equal(2, count); } // ── WP-14: Health metrics ── [Fact] public async Task GetBufferDepthAsync_ReturnsCorrectDepth() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); _service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api1", """{}"""); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api2", """{}"""); await _service.EnqueueAsync(StoreAndForwardCategory.Notification, "email", """{}"""); var depth = await _service.GetBufferDepthAsync(); Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.ExternalSystem) >= 2); Assert.True(depth.GetValueOrDefault(StoreAndForwardCategory.Notification) >= 1); } [Fact] public async Task OnActivity_RaisedOnEnqueue() { var activities = new List(); _service.OnActivity += (action, _, _) => activities.Add(action); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.Contains("Delivered", activities); } [Fact] public async Task OnActivity_RaisedOnBuffer() { var activities = new List(); _service.OnActivity += (action, _, _) => activities.Add(action); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.Contains("Queued", activities); } // ── StoreAndForward-009: faulting activity subscriber must not corrupt delivery ── [Fact] public async Task EnqueueAsync_ImmediateDeliverySuccess_FaultingActivitySubscriber_StillReportsDelivered() { // StoreAndForward-009: a throwing OnActivity subscriber (e.g. the site event // log) must not be misclassified as a transient delivery failure. Pre-fix the // subscriber's exception escaped RaiseActivity, was caught by EnqueueAsync's // transient-failure handler, and a successfully delivered message was buffered. _service.OnActivity += (_, _, _) => throw new InvalidOperationException("logging blew up"); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}"""); Assert.True(result.Accepted); Assert.False(result.WasBuffered); // delivered, NOT buffered var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); // nothing left in the buffer } [Fact] public async Task RetryMessageAsync_FaultingActivitySubscriber_DoesNotIncrementRetryCount() { // StoreAndForward-009: a throwing subscriber raised after a successful retry // delivery must not be caught by the retry-failure handler and counted as a // transient failure. var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", attemptImmediateDelivery: false, maxRetries: 5); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => Task.FromResult(true)); _service.OnActivity += (_, _, _) => throw new InvalidOperationException("logging blew up"); await _service.RetryPendingMessagesAsync(); // The retry succeeded; the message must be gone, not re-buffered with a bumped count. var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Null(msg); } // ── WP-10: Per-source-entity retry settings ── [Fact] public async Task EnqueueAsync_CustomRetrySettings_Respected() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new HttpRequestException("fail")); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 100, retryInterval: TimeSpan.FromSeconds(60)); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.Equal(100, msg!.MaxRetries); Assert.Equal(60000, msg.RetryIntervalMs); } // ── attemptImmediateDelivery: false — caller already attempted delivery ── [Fact] public async Task EnqueueAsync_AttemptImmediateDeliveryFalse_BuffersWithoutInvokingHandler() { // A caller that has already made its own delivery attempt passes // attemptImmediateDelivery: false so the request is not dispatched twice. var handlerCalls = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { Interlocked.Increment(ref handlerCalls); return Task.FromResult(true); }); var result = await _service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", attemptImmediateDelivery: false); Assert.Equal(0, handlerCalls); // handler NOT invoked at enqueue time Assert.True(result.WasBuffered); var msg = await _storage.GetMessageByIdAsync(result.MessageId); Assert.NotNull(msg); Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status); // StoreAndForward-003: the caller's own attempt is attempt 0; RetryCount // counts only sweep retries, so a freshly buffered message has RetryCount 0. Assert.Equal(0, msg.RetryCount); } // ─── StoreAndForward-024: StopAsync waits for the in-flight sweep ─── /// /// StoreAndForward-024: must /// not return until any in-flight retry sweep has completed (or the bounded /// shutdown timeout fires). Pre-fix it disposed the timer and returned /// immediately, leaving a mid-flight sweep touching disposed dependencies. /// [Fact] public async Task StopAsync_AwaitsInFlightRetrySweep_BeforeReturning() { // Build a service whose timer fires almost immediately, with a handler // that pauses in the middle of delivery so we can observe StopAsync's // wait behaviour. var dbName = $"StopWait_{Guid.NewGuid():N}"; var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared"; using var keepAlive = new SqliteConnection(connStr); keepAlive.Open(); var storage = new StoreAndForwardStorage(connStr, NullLogger.Instance); await storage.InitializeAsync(); var options = new StoreAndForwardOptions { DefaultRetryInterval = TimeSpan.Zero, DefaultMaxRetries = 3, // Fire almost immediately so the sweep is in-flight by the time we call StopAsync. RetryTimerInterval = TimeSpan.FromMilliseconds(20), }; var service = new StoreAndForwardService( storage, options, NullLogger.Instance); // Pre-seed a buffered message so the sweep has work to do, and a // handler that blocks until we release it. var handlerEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseHandler = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var handlerCompleted = false; service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async _ => { handlerEntered.TrySetResult(); await releaseHandler.Task; handlerCompleted = true; return true; }); var seed = await service.EnqueueAsync( StoreAndForwardCategory.ExternalSystem, "api", """{}""", attemptImmediateDelivery: false); Assert.True(seed.WasBuffered); await service.StartAsync(); // Wait until the timer-driven sweep has called into the handler. var entered = await Task.WhenAny(handlerEntered.Task, Task.Delay(TimeSpan.FromSeconds(2))); Assert.Same(handlerEntered.Task, entered); Assert.False(handlerCompleted, "Handler should still be paused inside the sweep."); // Kick StopAsync — it must NOT return until the sweep finishes. Run the // release on a background task so we can prove StopAsync is awaiting. var stopTask = service.StopAsync(); Assert.False(stopTask.IsCompleted, "StopAsync returned before the in-flight sweep was given a chance to finish."); // Release the handler — StopAsync should now complete shortly. releaseHandler.SetResult(); await stopTask.WaitAsync(TimeSpan.FromSeconds(5)); Assert.True(handlerCompleted, "Sweep handler must have finished before StopAsync returned."); } // ── Task 3 (arch review 02, Stability #2): active-node delivery gate ── // The standby site node applies replicated buffer operations but must never // deliver; the sweep runs only when the gate reports the node is active. [Fact] public async Task RetrySweep_SkipsDelivery_WhenDeliveryGateReportsStandby() { var delivered = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { delivered++; return Task.FromResult(true); }); _service.SetDeliveryGate(() => false); // standby var result = await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "target-x", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.RetryPendingMessagesAsync(); Assert.Equal(0, delivered); var row = await _service.GetMessageByIdAsync(result.MessageId); Assert.NotNull(row); Assert.Equal(StoreAndForwardMessageStatus.Pending, row!.Status); // row untouched } [Fact] public async Task RetrySweep_ResumesDelivery_WhenGateFlipsActive() { var delivered = 0; var active = false; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { delivered++; return Task.FromResult(true); }); _service.SetDeliveryGate(() => active); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "target-x", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.RetryPendingMessagesAsync(); Assert.Equal(0, delivered); active = true; // failover: this node became active await _service.RetryPendingMessagesAsync(); Assert.Equal(1, delivered); } [Fact] public async Task RetrySweep_TreatsThrowingGateAsStandby() { var delivered = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => { delivered++; return Task.FromResult(true); }); _service.SetDeliveryGate(() => throw new InvalidOperationException("cluster not ready")); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "t", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.RetryPendingMessagesAsync(); // must not throw Assert.Equal(0, delivered); } // ── Task 8 (arch review 02, Performance): per-target short-circuit ── // After the first transient failure to a (category, target) the remaining // messages for that pair are skipped this sweep — N dead-target messages // cost one timeout, not N. Skipped rows keep their RetryCount. [Fact] public async Task RetrySweep_ShortCircuitsTarget_AfterFirstTransientFailure() { var attemptsByTarget = new Dictionary(); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, msg => { attemptsByTarget[msg.Target] = attemptsByTarget.GetValueOrDefault(msg.Target) + 1; if (msg.Target == "dead-target") throw new TimeoutException("down"); return Task.FromResult(true); }); for (var i = 0; i < 3; i++) await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "dead-target", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "healthy-target", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.RetryPendingMessagesAsync(); Assert.Equal(1, attemptsByTarget["dead-target"]); // pre-fix: 3 Assert.Equal(1, attemptsByTarget["healthy-target"]); // healthy lane unaffected } [Fact] public async Task RetrySweep_SkippedMessages_DoNotAccrueRetryCount() { _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => throw new TimeoutException("down")); var id1 = (await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "dead", "{}", maxRetries: 5, attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero)).MessageId; var id2 = (await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "dead", "{}", maxRetries: 5, attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero)).MessageId; await _service.RetryPendingMessagesAsync(); var r1 = (await _service.GetMessageByIdAsync(id1))!.RetryCount; var r2 = (await _service.GetMessageByIdAsync(id2))!.RetryCount; // Exactly one was attempted (RetryCount 1); the other was skipped by the // short-circuit, its RetryCount untouched (0). Order-independent so the // assertion doesn't depend on created_at tie-breaking. Assert.Equal(new[] { 0, 1 }, new[] { r1, r2 }.OrderBy(x => x).ToArray()); } // ── Task 9 (arch review 02, Performance): parallel per-target lanes ── // A slow/dead target no longer serializes the whole sweep; delivery within a // single (category, target) lane stays sequential (per-target FIFO). [Fact] public async Task RetrySweep_SlowTarget_DoesNotBlockOtherTargets() { var slowGate = new TaskCompletionSource(); var healthyDelivered = new TaskCompletionSource(); _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async msg => { if (msg.Target == "slow") { await slowGate.Task; return true; } healthyDelivered.TrySetResult(); return true; }); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "slow", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "healthy", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); var sweep = _service.RetryPendingMessagesAsync(); // Healthy lane completes while the slow lane is still blocked — pre-fix this // times out because delivery is strictly serial across targets. await healthyDelivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); slowGate.SetResult(); await sweep; } [Fact] public async Task RetrySweep_WithinTargetLane_StaysSequential() { var concurrent = 0; var maxConcurrent = 0; var delivered = 0; _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async _ => { var c = Interlocked.Increment(ref concurrent); InterlockedMax(ref maxConcurrent, c); await Task.Delay(20); Interlocked.Decrement(ref concurrent); Interlocked.Increment(ref delivered); return true; }); for (var i = 0; i < 4; i++) await _service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "same-target", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero, messageId: $"m{i}"); await _service.RetryPendingMessagesAsync(); Assert.Equal(4, delivered); Assert.Equal(1, maxConcurrent); // one (category,target) lane: strictly sequential } private static void InterlockedMax(ref int target, int value) { int current; do { current = Volatile.Read(ref target); if (value <= current) return; } while (Interlocked.CompareExchange(ref target, value, current) != current); } // ── Task 12: deferToSweep enqueue mode ── /// /// deferToSweep: true must NOT invoke the delivery handler inline (the /// caller is on a latency-sensitive thread), and the buffered row must be due on /// the very next sweep — LastAttemptAt stays null so it is not held back by /// RetryInterval. /// [Fact] public async Task Enqueue_DeferToSweep_DoesNotInvokeHandlerInline_AndRowIsDueImmediately() { var service = CreateService(); var inlineInvoked = false; service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, _ => { inlineInvoked = true; return Task.FromResult(true); }); var result = await service.EnqueueAsync(StoreAndForwardCategory.Notification, "ops", "{}", deferToSweep: true, messageId: "n1"); Assert.False(inlineInvoked); Assert.True(result.WasBuffered); var row = await service.GetMessageByIdAsync("n1"); Assert.Null(row!.LastAttemptAt); // due on the very next sweep, not after RetryInterval } /// /// After StartAsync, deferToSweep must kick an immediate background sweep so /// healthy-path latency is milliseconds, not one timer interval — proven with a /// timer set to an hour so only the kicked sweep can deliver. /// [Fact] public async Task Enqueue_DeferToSweep_AfterStart_KicksAnImmediateSweep() { var service = CreateService(retryTimerInterval: TimeSpan.FromHours(1)); // timer will never fire in-test var delivered = new TaskCompletionSource(); service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, _ => { delivered.TrySetResult(); return Task.FromResult(true); }); await service.StartAsync(); try { await service.EnqueueAsync(StoreAndForwardCategory.Notification, "ops", "{}", deferToSweep: true); await delivered.Task.WaitAsync(TimeSpan.FromSeconds(5)); // pre-fix: times out } finally { await service.StopAsync(); } } // ── R2 T8: _sweepTask clobber (N3) ── [Fact] public async Task TriggerSweep_WhileSweepInFlight_DoesNotClobberTheDrainHandle() { var service = CreateService(retryTimerInterval: TimeSpan.FromHours(1)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async _ => { entered.TrySetResult(); await release.Task; return true; }); await service.StartAsync(); try { await service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "t", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); service.TriggerSweep(); // real sweep, blocked in the handler await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); service.TriggerSweep(); // redundant kick — pre-fix clobbers _sweepTask var handle = service.CurrentSweepTaskForTest; Assert.NotNull(handle); Assert.False(handle!.IsCompleted); // pre-fix: true (a completed no-op replaced the real sweep) } finally { release.TrySetResult(); await service.StopAsync(); } } [Fact] public async Task StopAsync_WaitsForTheRealInFlightSweep_EvenAfterARedundantTrigger() { var service = CreateService(retryTimerInterval: TimeSpan.FromHours(1)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, async _ => { entered.TrySetResult(); await release.Task; return true; }); await service.StartAsync(); await service.EnqueueAsync(StoreAndForwardCategory.ExternalSystem, "t", "{}", attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero); service.TriggerSweep(); await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); service.TriggerSweep(); // the clobbering kick var stop = service.StopAsync(); await Task.Delay(300); Assert.False(stop.IsCompleted); // pre-fix: StopAsync already returned (awaited the no-op) release.TrySetResult(); await stop.WaitAsync(TimeSpan.FromSeconds(5)); // drains the real sweep promptly once released } }