Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs
- WP-1-3: Central/site failover + dual-node recovery tests (17 tests) - WP-4: Performance testing framework for target scale (7 tests) - WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests) - WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs) - WP-7: Recovery drill test scaffolds (5 tests) - WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests) - WP-9: Message contract compatibility (forward/backward compat) (18 tests) - WP-10: Deployment packaging (installation guide, production checklist, topology) - WP-11: Operational runbooks (failover, troubleshooting, maintenance) 92 new tests, all passing. Zero warnings.
This commit is contained in:
Joseph Doherty
177
src/ScadaLink.NotificationService/NotificationDeliveryService.cs
Normal file
177
src/ScadaLink.NotificationService/NotificationDeliveryService.cs
Normal file
@@ -0,0 +1,177 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ScadaLink.Commons.Entities.Notifications;
|
||||
using ScadaLink.Commons.Interfaces.Repositories;
|
||||
using ScadaLink.Commons.Interfaces.Services;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
using ScadaLink.StoreAndForward;
|
||||
|
||||
namespace ScadaLink.NotificationService;
|
||||
|
||||
/// <summary>
|
||||
/// WP-11: Notification delivery via SMTP.
|
||||
/// WP-12: Error classification and S&F integration.
|
||||
/// Transient: connection refused, timeout, SMTP 4xx → hand to S&F.
|
||||
/// Permanent: SMTP 5xx → returned to script.
|
||||
/// </summary>
|
||||
public class NotificationDeliveryService : INotificationDeliveryService
|
||||
{
|
||||
private readonly INotificationRepository _repository;
|
||||
private readonly Func<ISmtpClientWrapper> _smtpClientFactory;
|
||||
private readonly OAuth2TokenService? _tokenService;
|
||||
private readonly StoreAndForwardService? _storeAndForward;
|
||||
private readonly ILogger<NotificationDeliveryService> _logger;
|
||||
|
||||
public NotificationDeliveryService(
|
||||
INotificationRepository repository,
|
||||
Func<ISmtpClientWrapper> smtpClientFactory,
|
||||
ILogger<NotificationDeliveryService> logger,
|
||||
OAuth2TokenService? tokenService = null,
|
||||
StoreAndForwardService? storeAndForward = null)
|
||||
{
|
||||
_repository = repository;
|
||||
_smtpClientFactory = smtpClientFactory;
|
||||
_logger = logger;
|
||||
_tokenService = tokenService;
|
||||
_storeAndForward = storeAndForward;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a notification to a named list. BCC delivery, plain text.
|
||||
/// </summary>
|
||||
public async Task<NotificationResult> SendAsync(
|
||||
string listName,
|
||||
string subject,
|
||||
string message,
|
||||
string? originInstanceName = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var list = await _repository.GetListByNameAsync(listName, cancellationToken);
|
||||
if (list == null)
|
||||
{
|
||||
return new NotificationResult(false, $"Notification list '{listName}' not found");
|
||||
}
|
||||
|
||||
var recipients = await _repository.GetRecipientsByListIdAsync(list.Id, cancellationToken);
|
||||
if (recipients.Count == 0)
|
||||
{
|
||||
return new NotificationResult(false, $"Notification list '{listName}' has no recipients");
|
||||
}
|
||||
|
||||
var smtpConfigs = await _repository.GetAllSmtpConfigurationsAsync(cancellationToken);
|
||||
var smtpConfig = smtpConfigs.FirstOrDefault();
|
||||
if (smtpConfig == null)
|
||||
{
|
||||
return new NotificationResult(false, "No SMTP configuration available");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await DeliverAsync(smtpConfig, recipients, subject, message, cancellationToken);
|
||||
return new NotificationResult(true, null);
|
||||
}
|
||||
catch (SmtpPermanentException ex)
|
||||
{
|
||||
// WP-12: Permanent SMTP failure — returned to script
|
||||
_logger.LogError(ex, "Permanent SMTP failure sending to list {List}", listName);
|
||||
return new NotificationResult(false, $"Permanent SMTP error: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex) when (IsTransientSmtpError(ex))
|
||||
{
|
||||
// WP-12: Transient SMTP failure — hand to S&F
|
||||
_logger.LogWarning(ex, "Transient SMTP failure sending to list {List}, buffering for retry", listName);
|
||||
|
||||
if (_storeAndForward == null)
|
||||
{
|
||||
return new NotificationResult(false, "Transient SMTP error and store-and-forward not available");
|
||||
}
|
||||
|
||||
var payload = JsonSerializer.Serialize(new
|
||||
{
|
||||
ListName = listName,
|
||||
Subject = subject,
|
||||
Message = message
|
||||
});
|
||||
|
||||
await _storeAndForward.EnqueueAsync(
|
||||
StoreAndForwardCategory.Notification,
|
||||
listName,
|
||||
payload,
|
||||
originInstanceName,
|
||||
smtpConfig.MaxRetries > 0 ? smtpConfig.MaxRetries : null,
|
||||
smtpConfig.RetryDelay > TimeSpan.Zero ? smtpConfig.RetryDelay : null);
|
||||
|
||||
return new NotificationResult(true, null, WasBuffered: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Delivers an email via SMTP. Throws on failure.
|
||||
/// </summary>
|
||||
internal async Task DeliverAsync(
|
||||
SmtpConfiguration config,
|
||||
IReadOnlyList<NotificationRecipient> recipients,
|
||||
string subject,
|
||||
string body,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var client = _smtpClientFactory() as IDisposable;
|
||||
var smtp = _smtpClientFactory();
|
||||
|
||||
try
|
||||
{
|
||||
var useTls = config.TlsMode?.Equals("starttls", StringComparison.OrdinalIgnoreCase) == true;
|
||||
await smtp.ConnectAsync(config.Host, config.Port, useTls, cancellationToken);
|
||||
|
||||
// Resolve credentials (OAuth2 token refresh if needed)
|
||||
var credentials = config.Credentials;
|
||||
if (config.AuthType.Equals("oauth2", StringComparison.OrdinalIgnoreCase) && _tokenService != null && credentials != null)
|
||||
{
|
||||
var token = await _tokenService.GetTokenAsync(credentials, cancellationToken);
|
||||
credentials = token;
|
||||
}
|
||||
|
||||
await smtp.AuthenticateAsync(config.AuthType, credentials, cancellationToken);
|
||||
|
||||
var bccAddresses = recipients.Select(r => r.EmailAddress).ToList();
|
||||
await smtp.SendAsync(config.FromAddress, bccAddresses, subject, body, cancellationToken);
|
||||
|
||||
await smtp.DisconnectAsync(cancellationToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is not SmtpPermanentException && !IsTransientSmtpError(ex))
|
||||
{
|
||||
// Classify unrecognized SMTP exceptions
|
||||
if (ex.Message.Contains("5.", StringComparison.Ordinal) ||
|
||||
ex.Message.Contains("550", StringComparison.Ordinal) ||
|
||||
ex.Message.Contains("553", StringComparison.Ordinal) ||
|
||||
ex.Message.Contains("554", StringComparison.Ordinal))
|
||||
{
|
||||
throw new SmtpPermanentException(ex.Message, ex);
|
||||
}
|
||||
|
||||
// Default: treat as transient
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientSmtpError(Exception ex)
|
||||
{
|
||||
return ex is TimeoutException
|
||||
or OperationCanceledException
|
||||
or System.Net.Sockets.SocketException
|
||||
or IOException
|
||||
|| ex.Message.Contains("4.", StringComparison.Ordinal)
|
||||
|| ex.Message.Contains("421", StringComparison.Ordinal)
|
||||
|| ex.Message.Contains("450", StringComparison.Ordinal)
|
||||
|| ex.Message.Contains("451", StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signals a permanent SMTP failure (5xx) that should not be retried.
|
||||
/// </summary>
|
||||
public class SmtpPermanentException : Exception
|
||||
{
|
||||
public SmtpPermanentException(string message, Exception? innerException = null)
|
||||
: base(message, innerException) { }
|
||||
}
|
||||
Reference in New Issue
Block a user