fix(store-and-forward): resolve S&F delivery + replication wiring (3 Critical findings)

Resolves StoreAndForward-001, ExternalSystemGateway-001, NotificationService-001
— one systemic gap where buffered messages were persisted but never delivered,
and the active node never replicated its buffer to the standby.

Delivery handlers (ExternalSystemGateway-001 / NotificationService-001):
- AkkaHostedService registers delivery handlers for the ExternalSystem,
  CachedDbWrite and Notification categories after StoreAndForwardService starts;
  each resolves its scoped consumer in a fresh DI scope.
- ExternalSystemClient, DatabaseGateway and NotificationDeliveryService each
  gain a DeliverBufferedAsync method: re-resolve the target and re-attempt
  delivery, returning true/false/throwing per the transient-vs-permanent contract.
- EnqueueAsync gains an attemptImmediateDelivery flag; CachedCallAsync and
  NotificationDeliveryService.SendAsync pass false (they already attempted
  delivery themselves) so registering a handler does not dispatch twice.

Replication (StoreAndForward-001):
- ReplicationService is injected into StoreAndForwardService; a new BufferAsync
  helper replicates every enqueue, and successful-retry removes and parks are
  replicated too. Fire-and-forget, no-op when replication is disabled.

Tests: StoreAndForwardReplicationTests (Add/Remove/Park observed),
attemptImmediateDelivery behaviour, and DeliverBufferedAsync paths for each
consumer. Full solution builds; StoreAndForward/ExternalSystemGateway/
NotificationService suites green.

tests/ScadaLink.ExternalSystemGateway.Tests/DatabaseGatewayTests.cs

@@ -53,4 +53,26 @@ public class DatabaseGatewayTests
         await Assert.ThrowsAsync<InvalidOperationException>(
             () => gateway.CachedWriteAsync("nonexistent", "INSERT INTO t VALUES (1)"));
     }
+
+    // ── ExternalSystemGateway-001: buffered CachedDbWrite delivery handler ──
+
+    [Fact]
+    public async Task DeliverBuffered_ConnectionNoLongerExists_ReturnsFalseSoMessageParks()
+    {
+        _repository.GetAllDatabaseConnectionsAsync().Returns(new List<DatabaseConnectionDefinition>());
+        var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance);
+
+        var message = new ScadaLink.StoreAndForward.StoreAndForwardMessage
+        {
+            Id = Guid.NewGuid().ToString("N"),
+            Category = ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
+            Target = "gone-db",
+            PayloadJson =
+                """{"ConnectionName":"gone-db","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
+        };
+
+        var delivered = await gateway.DeliverBufferedAsync(message);
+
+        Assert.False(delivered); // permanent — the S&F engine parks the message
+    }
 }

tests/ScadaLink.ExternalSystemGateway.Tests/ExternalSystemClientTests.cs

@@ -153,6 +153,69 @@ public class ExternalSystemClientTests
         Assert.False(result.WasBuffered);
     }
 
+    // ── ExternalSystemGateway-001: buffered-call delivery handler ──
+
+    private static ScadaLink.StoreAndForward.StoreAndForwardMessage BufferedCall(
+        string systemName, string methodName) =>
+        new()
+        {
+            Id = Guid.NewGuid().ToString("N"),
+            Category = ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.ExternalSystem,
+            Target = systemName,
+            PayloadJson =
+                $$"""{"SystemName":"{{systemName}}","MethodName":"{{methodName}}","Parameters":null}""",
+        };
+
+    [Fact]
+    public async Task DeliverBuffered_SuccessfulHttp_ReturnsTrue()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1).Returns(new List<ExternalSystemMethod> { method });
+
+        var httpClient = new HttpClient(new MockHttpMessageHandler(HttpStatusCode.OK, "{\"ok\":true}"));
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        var delivered = await client.DeliverBufferedAsync(BufferedCall("TestAPI", "getData"));
+
+        Assert.True(delivered);
+    }
+
+    [Fact]
+    public async Task DeliverBuffered_SystemNoLongerExists_ReturnsFalseSoMessageParks()
+    {
+        _repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition>());
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        var delivered = await client.DeliverBufferedAsync(BufferedCall("GoneAPI", "method"));
+
+        Assert.False(delivered); // permanent — the S&F engine parks the message
+    }
+
+    [Fact]
+    public async Task DeliverBuffered_Transient500_ThrowsSoEngineRetries()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("failMethod", "POST", "/fail") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1).Returns(new List<ExternalSystemMethod> { method });
+
+        var httpClient = new HttpClient(new MockHttpMessageHandler(HttpStatusCode.InternalServerError, "boom"));
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await Assert.ThrowsAsync<TransientExternalSystemException>(
+            () => client.DeliverBufferedAsync(BufferedCall("TestAPI", "failMethod")));
+    }
+
     /// <summary>
     /// Test helper: mock HTTP message handler.
     /// </summary>

tests/ScadaLink.NotificationService.Tests/NotificationDeliveryServiceTests.cs

@@ -192,4 +192,37 @@ public class NotificationDeliveryServiceTests
         Assert.True(result.Success);
         Assert.True(result.WasBuffered);
     }
+
+    // ── NotificationService-001: buffered-notification delivery handler ──
+
+    private static StoreAndForward.StoreAndForwardMessage BufferedNotification(string listName) =>
+        new()
+        {
+            Id = Guid.NewGuid().ToString("N"),
+            Category = ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.Notification,
+            Target = listName,
+            PayloadJson = $$"""{"ListName":"{{listName}}","Subject":"Alert","Message":"Body"}""",
+        };
+
+    [Fact]
+    public async Task DeliverBuffered_HappyPath_ReturnsTrue()
+    {
+        SetupHappyPath();
+        var service = CreateService();
+
+        var delivered = await service.DeliverBufferedAsync(BufferedNotification("ops-team"));
+
+        Assert.True(delivered);
+    }
+
+    [Fact]
+    public async Task DeliverBuffered_ListNoLongerExists_ReturnsFalseSoMessageParks()
+    {
+        _repository.GetListByNameAsync("gone-list").Returns((NotificationList?)null);
+        var service = CreateService();
+
+        var delivered = await service.DeliverBufferedAsync(BufferedNotification("gone-list"));
+
+        Assert.False(delivered); // permanent — the S&F engine parks the message
+    }
 }

tests/ScadaLink.StoreAndForward.Tests/StoreAndForwardReplicationTests.cs

@@ -0,0 +1,108 @@
+using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging.Abstractions;
+using ScadaLink.Commons.Types.Enums;
+
+namespace ScadaLink.StoreAndForward.Tests;
+
+/// <summary>
+/// StoreAndForward-001: the active node must forward every buffer operation
+/// (add / remove / park) to the standby via the ReplicationService, so a
+/// failover does not lose the buffer.
+/// </summary>
+public class StoreAndForwardReplicationTests : IAsyncLifetime, IDisposable
+{
+    private readonly SqliteConnection _keepAlive;
+    private readonly StoreAndForwardStorage _storage;
+    private readonly StoreAndForwardService _service;
+    private readonly List<ReplicationOperation> _replicated = new();
+
+    public StoreAndForwardReplicationTests()
+    {
+        var connStr = $"Data Source=ReplTests_{Guid.NewGuid():N};Mode=Memory;Cache=Shared";
+        _keepAlive = new SqliteConnection(connStr);
+        _keepAlive.Open();
+
+        _storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
+
+        var options = new StoreAndForwardOptions
+        {
+            DefaultRetryInterval = TimeSpan.Zero,
+            DefaultMaxRetries = 1,
+            RetryTimerInterval = TimeSpan.FromMinutes(10),
+            ReplicationEnabled = true,
+        };
+
+        var replication = new ReplicationService(options, NullLogger<ReplicationService>.Instance);
+        replication.SetReplicationHandler(op =>
+        {
+            lock (_replicated) _replicated.Add(op);
+            return Task.CompletedTask;
+        });
+
+        _service = new StoreAndForwardService(
+            _storage, options, NullLogger<StoreAndForwardService>.Instance, replication);
+    }
+
+    public async Task InitializeAsync() => await _storage.InitializeAsync();
+    public Task DisposeAsync() => Task.CompletedTask;
+    public void Dispose() => _keepAlive.Dispose();
+
+    /// <summary>Replication is fire-and-forget (Task.Run); poll until the expected ops arrive.</summary>
+    private async Task<List<ReplicationOperation>> WaitForReplicationAsync(int count)
+    {
+        for (var i = 0; i < 100; i++)
+        {
+            lock (_replicated)
+                if (_replicated.Count >= count) return _replicated.ToList();
+            await Task.Delay(20);
+        }
+        lock (_replicated) return _replicated.ToList();
+    }
+
+    [Fact]
+    public async Task BufferingAMessage_ReplicatesAnAddOperation()
+    {
+        // No handler registered → message is buffered → an Add is replicated.
+        var result = await _service.EnqueueAsync(
+            StoreAndForwardCategory.ExternalSystem, "api", """{}""");
+        Assert.True(result.WasBuffered);
+
+        var ops = await WaitForReplicationAsync(1);
+        Assert.Contains(ops, o =>
+            o.OperationType == ReplicationOperationType.Add && o.MessageId == result.MessageId);
+    }
+
+    [Fact]
+    public async Task SuccessfulRetry_ReplicatesARemoveOperation()
+    {
+        var calls = 0;
+        _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+            _ => ++calls == 1
+                ? throw new HttpRequestException("transient")
+                : Task.FromResult(true));
+
+        var result = await _service.EnqueueAsync(
+            StoreAndForwardCategory.ExternalSystem, "api", """{}""");
+        await _service.RetryPendingMessagesAsync();
+
+        var ops = await WaitForReplicationAsync(2);
+        Assert.Contains(ops, o => o.OperationType == ReplicationOperationType.Add);
+        Assert.Contains(ops, o =>
+            o.OperationType == ReplicationOperationType.Remove && o.MessageId == result.MessageId);
+    }
+
+    [Fact]
+    public async Task ParkedMessage_ReplicatesAParkOperation()
+    {
+        _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+            _ => throw new HttpRequestException("always fails"));
+
+        var result = await _service.EnqueueAsync(
+            StoreAndForwardCategory.ExternalSystem, "api", """{}""", maxRetries: 1);
+        await _service.RetryPendingMessagesAsync();
+
+        var ops = await WaitForReplicationAsync(2);
+        Assert.Contains(ops, o =>
+            o.OperationType == ReplicationOperationType.Park && o.MessageId == result.MessageId);
+    }
+}

tests/ScadaLink.StoreAndForward.Tests/StoreAndForwardServiceTests.cs

@@ -310,4 +310,28 @@ public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable
         Assert.Equal(100, msg!.MaxRetries);
         Assert.Equal(60000, msg.RetryIntervalMs);
     }
+
+    // ── attemptImmediateDelivery: false — caller already attempted delivery ──
+
+    [Fact]
+    public async Task EnqueueAsync_AttemptImmediateDeliveryFalse_BuffersWithoutInvokingHandler()
+    {
+        // A caller that has already made its own delivery attempt passes
+        // attemptImmediateDelivery: false so the request is not dispatched twice.
+        var handlerCalls = 0;
+        _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+            _ => { Interlocked.Increment(ref handlerCalls); return Task.FromResult(true); });
+
+        var result = await _service.EnqueueAsync(
+            StoreAndForwardCategory.ExternalSystem, "api", """{}""",
+            attemptImmediateDelivery: false);
+
+        Assert.Equal(0, handlerCalls);   // handler NOT invoked at enqueue time
+        Assert.True(result.WasBuffered);
+
+        var msg = await _storage.GetMessageByIdAsync(result.MessageId);
+        Assert.NotNull(msg);
+        Assert.Equal(StoreAndForwardMessageStatus.Pending, msg!.Status);
+        Assert.Equal(1, msg.RetryCount);  // counts as the caller's first attempt
+    }
 }