fix(store-and-forward): resolve S&F delivery + replication wiring (3 Critical findings)

Resolves StoreAndForward-001, ExternalSystemGateway-001, NotificationService-001
— one systemic gap where buffered messages were persisted but never delivered,
and the active node never replicated its buffer to the standby.

Delivery handlers (ExternalSystemGateway-001 / NotificationService-001):
- AkkaHostedService registers delivery handlers for the ExternalSystem,
  CachedDbWrite and Notification categories after StoreAndForwardService starts;
  each resolves its scoped consumer in a fresh DI scope.
- ExternalSystemClient, DatabaseGateway and NotificationDeliveryService each
  gain a DeliverBufferedAsync method: re-resolve the target and re-attempt
  delivery, returning true/false/throwing per the transient-vs-permanent contract.
- EnqueueAsync gains an attemptImmediateDelivery flag; CachedCallAsync and
  NotificationDeliveryService.SendAsync pass false (they already attempted
  delivery themselves) so registering a handler does not dispatch twice.

Replication (StoreAndForward-001):
- ReplicationService is injected into StoreAndForwardService; a new BufferAsync
  helper replicates every enqueue, and successful-retry removes and parks are
  replicated too. Fire-and-forget, no-op when replication is disabled.

Tests: StoreAndForwardReplicationTests (Add/Remove/Park observed),
attemptImmediateDelivery behaviour, and DeliverBufferedAsync paths for each
consumer. Full solution builds; StoreAndForward/ExternalSystemGateway/
NotificationService suites green.
This commit is contained in:
Joseph Doherty
2026-05-16 18:58:11 -04:00
parent a9bd7ee37c
commit 61253e3269
15 changed files with 538 additions and 37 deletions

View File

@@ -53,4 +53,26 @@ public class DatabaseGatewayTests
await Assert.ThrowsAsync<InvalidOperationException>(
() => gateway.CachedWriteAsync("nonexistent", "INSERT INTO t VALUES (1)"));
}
// ── ExternalSystemGateway-001: buffered CachedDbWrite delivery handler ──
[Fact]
public async Task DeliverBuffered_ConnectionNoLongerExists_ReturnsFalseSoMessageParks()
{
_repository.GetAllDatabaseConnectionsAsync().Returns(new List<DatabaseConnectionDefinition>());
var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance);
var message = new ScadaLink.StoreAndForward.StoreAndForwardMessage
{
Id = Guid.NewGuid().ToString("N"),
Category = ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
Target = "gone-db",
PayloadJson =
"""{"ConnectionName":"gone-db","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
};
var delivered = await gateway.DeliverBufferedAsync(message);
Assert.False(delivered); // permanent — the S&F engine parks the message
}
}

View File

@@ -153,6 +153,69 @@ public class ExternalSystemClientTests
Assert.False(result.WasBuffered);
}
// ── ExternalSystemGateway-001: buffered-call delivery handler ──
private static ScadaLink.StoreAndForward.StoreAndForwardMessage BufferedCall(
string systemName, string methodName) =>
new()
{
Id = Guid.NewGuid().ToString("N"),
Category = ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.ExternalSystem,
Target = systemName,
PayloadJson =
$$"""{"SystemName":"{{systemName}}","MethodName":"{{methodName}}","Parameters":null}""",
};
[Fact]
public async Task DeliverBuffered_SuccessfulHttp_ReturnsTrue()
{
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
_repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition> { system });
_repository.GetMethodsByExternalSystemIdAsync(1).Returns(new List<ExternalSystemMethod> { method });
var httpClient = new HttpClient(new MockHttpMessageHandler(HttpStatusCode.OK, "{\"ok\":true}"));
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
var delivered = await client.DeliverBufferedAsync(BufferedCall("TestAPI", "getData"));
Assert.True(delivered);
}
[Fact]
public async Task DeliverBuffered_SystemNoLongerExists_ReturnsFalseSoMessageParks()
{
_repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition>());
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
var delivered = await client.DeliverBufferedAsync(BufferedCall("GoneAPI", "method"));
Assert.False(delivered); // permanent — the S&F engine parks the message
}
[Fact]
public async Task DeliverBuffered_Transient500_ThrowsSoEngineRetries()
{
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("failMethod", "POST", "/fail") { Id = 1, ExternalSystemDefinitionId = 1 };
_repository.GetAllExternalSystemsAsync().Returns(new List<ExternalSystemDefinition> { system });
_repository.GetMethodsByExternalSystemIdAsync(1).Returns(new List<ExternalSystemMethod> { method });
var httpClient = new HttpClient(new MockHttpMessageHandler(HttpStatusCode.InternalServerError, "boom"));
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await Assert.ThrowsAsync<TransientExternalSystemException>(
() => client.DeliverBufferedAsync(BufferedCall("TestAPI", "failMethod")));
}
/// <summary>
/// Test helper: mock HTTP message handler.
/// </summary>