fix(store-and-forward): resolve S&F delivery + replication wiring (3 Critical findings)
Resolves StoreAndForward-001, ExternalSystemGateway-001, NotificationService-001 — one systemic gap where buffered messages were persisted but never delivered, and the active node never replicated its buffer to the standby. Delivery handlers (ExternalSystemGateway-001 / NotificationService-001): - AkkaHostedService registers delivery handlers for the ExternalSystem, CachedDbWrite and Notification categories after StoreAndForwardService starts; each resolves its scoped consumer in a fresh DI scope. - ExternalSystemClient, DatabaseGateway and NotificationDeliveryService each gain a DeliverBufferedAsync method: re-resolve the target and re-attempt delivery, returning true/false/throwing per the transient-vs-permanent contract. - EnqueueAsync gains an attemptImmediateDelivery flag; CachedCallAsync and NotificationDeliveryService.SendAsync pass false (they already attempted delivery themselves) so registering a handler does not dispatch twice. Replication (StoreAndForward-001): - ReplicationService is injected into StoreAndForwardService; a new BufferAsync helper replicates every enqueue, and successful-retry removes and parks are replicated too. Fire-and-forget, no-op when replication is disabled. Tests: StoreAndForwardReplicationTests (Add/Remove/Park observed), attemptImmediateDelivery behaviour, and DeliverBufferedAsync paths for each consumer. Full solution builds; StoreAndForward/ExternalSystemGateway/ NotificationService suites green.
This commit is contained in:
@@ -93,18 +93,75 @@ public class NotificationDeliveryService : INotificationDeliveryService
|
||||
Message = message
|
||||
});
|
||||
|
||||
// attemptImmediateDelivery: false — DeliverAsync was already attempted
|
||||
// above; letting EnqueueAsync re-invoke the handler would send twice.
|
||||
await _storeAndForward.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification,
|
||||
listName,
|
||||
payload,
|
||||
originInstanceName,
|
||||
smtpConfig.MaxRetries > 0 ? smtpConfig.MaxRetries : null,
|
||||
smtpConfig.RetryDelay > TimeSpan.Zero ? smtpConfig.RetryDelay : null);
|
||||
smtpConfig.RetryDelay > TimeSpan.Zero ? smtpConfig.RetryDelay : null,
|
||||
attemptImmediateDelivery: false);
|
||||
|
||||
return new NotificationResult(true, null, WasBuffered: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WP-11/12: Delivers a buffered notification during a store-and-forward retry
|
||||
/// sweep — re-resolves the list, recipients and SMTP config and re-attempts
|
||||
/// delivery. Returns true on success, false on permanent failure (the message
|
||||
/// is parked); throws on a transient failure so the engine retries.
|
||||
/// </summary>
|
||||
public async Task<bool> DeliverBufferedAsync(
|
||||
StoreAndForwardMessage message, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var payload = JsonSerializer.Deserialize<BufferedNotification>(message.PayloadJson);
|
||||
if (payload == null || string.IsNullOrEmpty(payload.ListName))
|
||||
{
|
||||
_logger.LogError("Buffered notification message {Id} has an unreadable payload; parking.", message.Id);
|
||||
return false;
|
||||
}
|
||||
|
||||
var list = await _repository.GetListByNameAsync(payload.ListName, cancellationToken);
|
||||
if (list == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Buffered notification to list '{List}' cannot be delivered — the list no longer exists; parking.",
|
||||
payload.ListName);
|
||||
return false;
|
||||
}
|
||||
|
||||
var recipients = await _repository.GetRecipientsByListIdAsync(list.Id, cancellationToken);
|
||||
if (recipients.Count == 0)
|
||||
{
|
||||
_logger.LogError("Buffered notification to list '{List}' has no recipients; parking.", payload.ListName);
|
||||
return false;
|
||||
}
|
||||
|
||||
var smtpConfig = (await _repository.GetAllSmtpConfigurationsAsync(cancellationToken)).FirstOrDefault();
|
||||
if (smtpConfig == null)
|
||||
{
|
||||
_logger.LogError("Buffered notification cannot be delivered — no SMTP configuration available; parking.");
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await DeliverAsync(smtpConfig, recipients, payload.Subject, payload.Message, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
catch (SmtpPermanentException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Buffered notification to list '{List}' failed permanently; parking.", payload.ListName);
|
||||
return false;
|
||||
}
|
||||
// Transient SMTP errors propagate out of DeliverAsync — the S&F engine retries.
|
||||
}
|
||||
|
||||
private sealed record BufferedNotification(string ListName, string Subject, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// Delivers an email via SMTP. Throws on failure.
|
||||
/// </summary>
|
||||
|
||||
Reference in New Issue
Block a user