Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs

- WP-1-3: Central/site failover + dual-node recovery tests (17 tests) - WP-4: Performance testing framework for target scale (7 tests) - WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests) - WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs) - WP-7: Recovery drill test scaffolds (5 tests) - WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests) - WP-9: Message contract compatibility (forward/backward compat) (18 tests) - WP-10: Deployment packaging (installation guide, production checklist, topology) - WP-11: Operational runbooks (failover, troubleshooting, maintenance) 92 new tests, all passing. Zero warnings.
2026-03-16 22:12:31 -04:00
parent 3b2320bd35
commit b659978764
68 changed files with 6253 additions and 44 deletions
--- a/src/ScadaLink.NotificationService/ISmtpClientWrapper.cs
+++ b/src/ScadaLink.NotificationService/ISmtpClientWrapper.cs
@@ -0,0 +1,12 @@
+namespace ScadaLink.NotificationService;
+
+/// <summary>
+/// Abstraction over SMTP client for testability.
+/// </summary>
+public interface ISmtpClientWrapper
+{
+    Task ConnectAsync(string host, int port, bool useTls, CancellationToken cancellationToken = default);
+    Task AuthenticateAsync(string authType, string? credentials, CancellationToken cancellationToken = default);
+    Task SendAsync(string from, IEnumerable<string> bccRecipients, string subject, string body, CancellationToken cancellationToken = default);
+    Task DisconnectAsync(CancellationToken cancellationToken = default);
+}
--- a/src/ScadaLink.NotificationService/MailKitSmtpClientWrapper.cs
+++ b/src/ScadaLink.NotificationService/MailKitSmtpClientWrapper.cs
@@ -0,0 +1,73 @@
+using MailKit.Net.Smtp;
+using MailKit.Security;
+using MimeKit;
+
+namespace ScadaLink.NotificationService;
+
+/// <summary>
+/// WP-11: MailKit-based SMTP client wrapper.
+/// Supports OAuth2 Client Credentials (M365) and Basic Auth.
+/// BCC delivery, plain text.
+/// </summary>
+public class MailKitSmtpClientWrapper : ISmtpClientWrapper, IDisposable
+{
+    private readonly SmtpClient _client = new();
+
+    public async Task ConnectAsync(string host, int port, bool useTls, CancellationToken cancellationToken = default)
+    {
+        var secureSocket = useTls ? SecureSocketOptions.StartTls : SecureSocketOptions.Auto;
+        await _client.ConnectAsync(host, port, secureSocket, cancellationToken);
+    }
+
+    public async Task AuthenticateAsync(string authType, string? credentials, CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrEmpty(credentials))
+            return;
+
+        switch (authType.ToLowerInvariant())
+        {
+            case "basic":
+                var parts = credentials.Split(':', 2);
+                if (parts.Length == 2)
+                {
+                    await _client.AuthenticateAsync(parts[0], parts[1], cancellationToken);
+                }
+                break;
+
+            case "oauth2":
+                // OAuth2 token is passed directly as credentials (pre-fetched by token service)
+                var oauth2 = new SaslMechanismOAuth2("", credentials);
+                await _client.AuthenticateAsync(oauth2, cancellationToken);
+                break;
+        }
+    }
+
+    public async Task SendAsync(string from, IEnumerable<string> bccRecipients, string subject, string body, CancellationToken cancellationToken = default)
+    {
+        var message = new MimeMessage();
+        message.From.Add(MailboxAddress.Parse(from));
+
+        foreach (var recipient in bccRecipients)
+        {
+            message.Bcc.Add(MailboxAddress.Parse(recipient));
+        }
+
+        message.Subject = subject;
+        message.Body = new TextPart("plain") { Text = body };
+
+        await _client.SendAsync(message, cancellationToken);
+    }
+
+    public async Task DisconnectAsync(CancellationToken cancellationToken = default)
+    {
+        if (_client.IsConnected)
+        {
+            await _client.DisconnectAsync(true, cancellationToken);
+        }
+    }
+
+    public void Dispose()
+    {
+        _client.Dispose();
+    }
+}
--- a/src/ScadaLink.NotificationService/NotificationDeliveryService.cs
+++ b/src/ScadaLink.NotificationService/NotificationDeliveryService.cs
@@ -0,0 +1,177 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using ScadaLink.Commons.Entities.Notifications;
+using ScadaLink.Commons.Interfaces.Repositories;
+using ScadaLink.Commons.Interfaces.Services;
+using ScadaLink.Commons.Types.Enums;
+using ScadaLink.StoreAndForward;
+
+namespace ScadaLink.NotificationService;
+
+/// <summary>
+/// WP-11: Notification delivery via SMTP.
+/// WP-12: Error classification and S&amp;F integration.
+/// Transient: connection refused, timeout, SMTP 4xx → hand to S&amp;F.
+/// Permanent: SMTP 5xx → returned to script.
+/// </summary>
+public class NotificationDeliveryService : INotificationDeliveryService
+{
+    private readonly INotificationRepository _repository;
+    private readonly Func<ISmtpClientWrapper> _smtpClientFactory;
+    private readonly OAuth2TokenService? _tokenService;
+    private readonly StoreAndForwardService? _storeAndForward;
+    private readonly ILogger<NotificationDeliveryService> _logger;
+
+    public NotificationDeliveryService(
+        INotificationRepository repository,
+        Func<ISmtpClientWrapper> smtpClientFactory,
+        ILogger<NotificationDeliveryService> logger,
+        OAuth2TokenService? tokenService = null,
+        StoreAndForwardService? storeAndForward = null)
+    {
+        _repository = repository;
+        _smtpClientFactory = smtpClientFactory;
+        _logger = logger;
+        _tokenService = tokenService;
+        _storeAndForward = storeAndForward;
+    }
+
+    /// <summary>
+    /// Sends a notification to a named list. BCC delivery, plain text.
+    /// </summary>
+    public async Task<NotificationResult> SendAsync(
+        string listName,
+        string subject,
+        string message,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default)
+    {
+        var list = await _repository.GetListByNameAsync(listName, cancellationToken);
+        if (list == null)
+        {
+            return new NotificationResult(false, $"Notification list '{listName}' not found");
+        }
+
+        var recipients = await _repository.GetRecipientsByListIdAsync(list.Id, cancellationToken);
+        if (recipients.Count == 0)
+        {
+            return new NotificationResult(false, $"Notification list '{listName}' has no recipients");
+        }
+
+        var smtpConfigs = await _repository.GetAllSmtpConfigurationsAsync(cancellationToken);
+        var smtpConfig = smtpConfigs.FirstOrDefault();
+        if (smtpConfig == null)
+        {
+            return new NotificationResult(false, "No SMTP configuration available");
+        }
+
+        try
+        {
+            await DeliverAsync(smtpConfig, recipients, subject, message, cancellationToken);
+            return new NotificationResult(true, null);
+        }
+        catch (SmtpPermanentException ex)
+        {
+            // WP-12: Permanent SMTP failure — returned to script
+            _logger.LogError(ex, "Permanent SMTP failure sending to list {List}", listName);
+            return new NotificationResult(false, $"Permanent SMTP error: {ex.Message}");
+        }
+        catch (Exception ex) when (IsTransientSmtpError(ex))
+        {
+            // WP-12: Transient SMTP failure — hand to S&F
+            _logger.LogWarning(ex, "Transient SMTP failure sending to list {List}, buffering for retry", listName);
+
+            if (_storeAndForward == null)
+            {
+                return new NotificationResult(false, "Transient SMTP error and store-and-forward not available");
+            }
+
+            var payload = JsonSerializer.Serialize(new
+            {
+                ListName = listName,
+                Subject = subject,
+                Message = message
+            });
+
+            await _storeAndForward.EnqueueAsync(
+                StoreAndForwardCategory.Notification,
+                listName,
+                payload,
+                originInstanceName,
+                smtpConfig.MaxRetries > 0 ? smtpConfig.MaxRetries : null,
+                smtpConfig.RetryDelay > TimeSpan.Zero ? smtpConfig.RetryDelay : null);
+
+            return new NotificationResult(true, null, WasBuffered: true);
+        }
+    }
+
+    /// <summary>
+    /// Delivers an email via SMTP. Throws on failure.
+    /// </summary>
+    internal async Task DeliverAsync(
+        SmtpConfiguration config,
+        IReadOnlyList<NotificationRecipient> recipients,
+        string subject,
+        string body,
+        CancellationToken cancellationToken)
+    {
+        using var client = _smtpClientFactory() as IDisposable;
+        var smtp = _smtpClientFactory();
+
+        try
+        {
+            var useTls = config.TlsMode?.Equals("starttls", StringComparison.OrdinalIgnoreCase) == true;
+            await smtp.ConnectAsync(config.Host, config.Port, useTls, cancellationToken);
+
+            // Resolve credentials (OAuth2 token refresh if needed)
+            var credentials = config.Credentials;
+            if (config.AuthType.Equals("oauth2", StringComparison.OrdinalIgnoreCase) && _tokenService != null && credentials != null)
+            {
+                var token = await _tokenService.GetTokenAsync(credentials, cancellationToken);
+                credentials = token;
+            }
+
+            await smtp.AuthenticateAsync(config.AuthType, credentials, cancellationToken);
+
+            var bccAddresses = recipients.Select(r => r.EmailAddress).ToList();
+            await smtp.SendAsync(config.FromAddress, bccAddresses, subject, body, cancellationToken);
+
+            await smtp.DisconnectAsync(cancellationToken);
+        }
+        catch (Exception ex) when (ex is not SmtpPermanentException && !IsTransientSmtpError(ex))
+        {
+            // Classify unrecognized SMTP exceptions
+            if (ex.Message.Contains("5.", StringComparison.Ordinal) ||
+                ex.Message.Contains("550", StringComparison.Ordinal) ||
+                ex.Message.Contains("553", StringComparison.Ordinal) ||
+                ex.Message.Contains("554", StringComparison.Ordinal))
+            {
+                throw new SmtpPermanentException(ex.Message, ex);
+            }
+
+            // Default: treat as transient
+            throw;
+        }
+    }
+
+    private static bool IsTransientSmtpError(Exception ex)
+    {
+        return ex is TimeoutException
+            or OperationCanceledException
+            or System.Net.Sockets.SocketException
+            or IOException
+            || ex.Message.Contains("4.", StringComparison.Ordinal)
+            || ex.Message.Contains("421", StringComparison.Ordinal)
+            || ex.Message.Contains("450", StringComparison.Ordinal)
+            || ex.Message.Contains("451", StringComparison.Ordinal);
+    }
+}
+
+/// <summary>
+/// Signals a permanent SMTP failure (5xx) that should not be retried.
+/// </summary>
+public class SmtpPermanentException : Exception
+{
+    public SmtpPermanentException(string message, Exception? innerException = null)
+        : base(message, innerException) { }
+}
--- a/src/ScadaLink.NotificationService/NotificationOptions.cs
+++ b/src/ScadaLink.NotificationService/NotificationOptions.cs
@@ -1,6 +1,15 @@
 namespace ScadaLink.NotificationService;

+/// <summary>
+/// Configuration options for the Notification Service.
+/// Most SMTP configuration is stored in the database (SmtpConfiguration entity).
+/// This provides fallback defaults and operational limits.
+/// </summary>
 public class NotificationOptions
 {
-    // Phase 0: minimal POCO — most SMTP configuration is stored in the database
+    /// <summary>Default connection timeout for SMTP connections.</summary>
+    public int ConnectionTimeoutSeconds { get; set; } = 30;
+
+    /// <summary>Maximum concurrent SMTP connections.</summary>
+    public int MaxConcurrentConnections { get; set; } = 5;
 }
--- a/src/ScadaLink.NotificationService/OAuth2TokenService.cs
+++ b/src/ScadaLink.NotificationService/OAuth2TokenService.cs
@@ -0,0 +1,87 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+
+namespace ScadaLink.NotificationService;
+
+/// <summary>
+/// WP-11: OAuth2 Client Credentials token lifecycle — fetch, cache, refresh on expiry.
+/// Used for Microsoft 365 SMTP authentication.
+/// </summary>
+public class OAuth2TokenService
+{
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly ILogger<OAuth2TokenService> _logger;
+    private string? _cachedToken;
+    private DateTimeOffset _tokenExpiry = DateTimeOffset.MinValue;
+    private readonly SemaphoreSlim _lock = new(1, 1);
+
+    public OAuth2TokenService(
+        IHttpClientFactory httpClientFactory,
+        ILogger<OAuth2TokenService> logger)
+    {
+        _httpClientFactory = httpClientFactory;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Gets a valid access token, refreshing if expired.
+    /// Credentials format: "tenantId:clientId:clientSecret"
+    /// </summary>
+    public async Task<string> GetTokenAsync(string credentials, CancellationToken cancellationToken = default)
+    {
+        if (_cachedToken != null && DateTimeOffset.UtcNow < _tokenExpiry)
+        {
+            return _cachedToken;
+        }
+
+        await _lock.WaitAsync(cancellationToken);
+        try
+        {
+            // Double-check after acquiring lock
+            if (_cachedToken != null && DateTimeOffset.UtcNow < _tokenExpiry)
+            {
+                return _cachedToken;
+            }
+
+            var parts = credentials.Split(':', 3);
+            if (parts.Length < 3)
+            {
+                throw new InvalidOperationException("OAuth2 credentials must be 'tenantId:clientId:clientSecret'");
+            }
+
+            var tenantId = parts[0];
+            var clientId = parts[1];
+            var clientSecret = parts[2];
+
+            var client = _httpClientFactory.CreateClient("OAuth2");
+            var tokenUrl = $"https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token";
+
+            var form = new FormUrlEncodedContent(new Dictionary<string, string>
+            {
+                ["grant_type"] = "client_credentials",
+                ["client_id"] = clientId,
+                ["client_secret"] = clientSecret,
+                ["scope"] = "https://outlook.office365.com/.default"
+            });
+
+            var response = await client.PostAsync(tokenUrl, form, cancellationToken);
+            response.EnsureSuccessStatusCode();
+
+            var json = await response.Content.ReadAsStringAsync(cancellationToken);
+            using var doc = JsonDocument.Parse(json);
+
+            _cachedToken = doc.RootElement.GetProperty("access_token").GetString()
+                ?? throw new InvalidOperationException("No access_token in OAuth2 response");
+
+            var expiresIn = doc.RootElement.GetProperty("expires_in").GetInt32();
+            _tokenExpiry = DateTimeOffset.UtcNow.AddSeconds(expiresIn - 60); // Refresh 60s before expiry
+
+            _logger.LogInformation("OAuth2 token refreshed, expires in {ExpiresIn}s", expiresIn);
+            return _cachedToken;
+        }
+        finally
+        {
+            _lock.Release();
+        }
+    }
+}
--- a/src/ScadaLink.NotificationService/ScadaLink.NotificationService.csproj
+++ b/src/ScadaLink.NotificationService/ScadaLink.NotificationService.csproj
@@ -8,12 +8,20 @@
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="MailKit" Version="4.15.1" />
    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
    <PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
  </ItemGroup>

  <ItemGroup>
    <ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
+    <ProjectReference Include="../ScadaLink.StoreAndForward/ScadaLink.StoreAndForward.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="ScadaLink.NotificationService.Tests" />
  </ItemGroup>

 </Project>
--- a/src/ScadaLink.NotificationService/ServiceCollectionExtensions.cs
+++ b/src/ScadaLink.NotificationService/ServiceCollectionExtensions.cs
@@ -1,4 +1,5 @@
 using Microsoft.Extensions.DependencyInjection;
+using ScadaLink.Commons.Interfaces.Services;

 namespace ScadaLink.NotificationService;

@@ -6,13 +7,21 @@ public static class ServiceCollectionExtensions
 {
    public static IServiceCollection AddNotificationService(this IServiceCollection services)
    {
-        // Phase 0: skeleton only
+        services.AddOptions<NotificationOptions>()
+            .BindConfiguration("ScadaLink:Notification");
+
+        services.AddHttpClient();
+        services.AddSingleton<OAuth2TokenService>();
+        services.AddSingleton<Func<ISmtpClientWrapper>>(_ => () => new MailKitSmtpClientWrapper());
+        services.AddSingleton<NotificationDeliveryService>();
+        services.AddSingleton<INotificationDeliveryService>(sp => sp.GetRequiredService<NotificationDeliveryService>());
+
        return services;
    }

    public static IServiceCollection AddNotificationServiceActors(this IServiceCollection services)
    {
-        // Phase 0: placeholder for Akka actor registration
+        // Actor registration happens in AkkaHostedService.
        return services;
    }
 }