Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs

- WP-1-3: Central/site failover + dual-node recovery tests (17 tests) - WP-4: Performance testing framework for target scale (7 tests) - WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests) - WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs) - WP-7: Recovery drill test scaffolds (5 tests) - WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests) - WP-9: Message contract compatibility (forward/backward compat) (18 tests) - WP-10: Deployment packaging (installation guide, production checklist, topology) - WP-11: Operational runbooks (failover, troubleshooting, maintenance) 92 new tests, all passing. Zero warnings.
2026-03-16 22:12:31 -04:00
parent 3b2320bd35
commit b659978764
68 changed files with 6253 additions and 44 deletions
--- a/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs
+++ b/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs
@@ -0,0 +1,98 @@
+using System.Data.Common;
+using System.Text.Json;
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Logging;
+using ScadaLink.Commons.Entities.ExternalSystems;
+using ScadaLink.Commons.Interfaces.Repositories;
+using ScadaLink.Commons.Interfaces.Services;
+using ScadaLink.Commons.Types.Enums;
+using ScadaLink.StoreAndForward;
+
+namespace ScadaLink.ExternalSystemGateway;
+
+/// <summary>
+/// WP-9: Database access from scripts.
+/// Database.Connection("name") — returns ADO.NET SqlConnection (connection pooling).
+/// Database.CachedWrite("name", "sql", params) — submits to S&amp;F engine.
+/// </summary>
+public class DatabaseGateway : IDatabaseGateway
+{
+    private readonly IExternalSystemRepository _repository;
+    private readonly StoreAndForwardService? _storeAndForward;
+    private readonly ILogger<DatabaseGateway> _logger;
+
+    public DatabaseGateway(
+        IExternalSystemRepository repository,
+        ILogger<DatabaseGateway> logger,
+        StoreAndForwardService? storeAndForward = null)
+    {
+        _repository = repository;
+        _logger = logger;
+        _storeAndForward = storeAndForward;
+    }
+
+    /// <summary>
+    /// Returns an open SqlConnection from the named database connection definition.
+    /// Connection pooling is managed by the underlying ADO.NET provider.
+    /// </summary>
+    public async Task<DbConnection> GetConnectionAsync(
+        string connectionName,
+        CancellationToken cancellationToken = default)
+    {
+        var definition = await ResolveConnectionAsync(connectionName, cancellationToken);
+        if (definition == null)
+        {
+            throw new InvalidOperationException($"Database connection '{connectionName}' not found");
+        }
+
+        var connection = new SqlConnection(definition.ConnectionString);
+        await connection.OpenAsync(cancellationToken);
+        return connection;
+    }
+
+    /// <summary>
+    /// Submits a SQL write to the store-and-forward engine for reliable delivery.
+    /// </summary>
+    public async Task CachedWriteAsync(
+        string connectionName,
+        string sql,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default)
+    {
+        var definition = await ResolveConnectionAsync(connectionName, cancellationToken);
+        if (definition == null)
+        {
+            throw new InvalidOperationException($"Database connection '{connectionName}' not found");
+        }
+
+        if (_storeAndForward == null)
+        {
+            throw new InvalidOperationException("Store-and-forward service not available for cached writes");
+        }
+
+        var payload = JsonSerializer.Serialize(new
+        {
+            ConnectionName = connectionName,
+            Sql = sql,
+            Parameters = parameters
+        });
+
+        await _storeAndForward.EnqueueAsync(
+            StoreAndForwardCategory.CachedDbWrite,
+            connectionName,
+            payload,
+            originInstanceName,
+            definition.MaxRetries > 0 ? definition.MaxRetries : null,
+            definition.RetryDelay > TimeSpan.Zero ? definition.RetryDelay : null);
+    }
+
+    private async Task<DatabaseConnectionDefinition?> ResolveConnectionAsync(
+        string connectionName,
+        CancellationToken cancellationToken)
+    {
+        var connections = await _repository.GetAllDatabaseConnectionsAsync(cancellationToken);
+        return connections.FirstOrDefault(c =>
+            c.Name.Equals(connectionName, StringComparison.OrdinalIgnoreCase));
+    }
+}
--- a/src/ScadaLink.ExternalSystemGateway/ErrorClassifier.cs
+++ b/src/ScadaLink.ExternalSystemGateway/ErrorClassifier.cs
@@ -0,0 +1,62 @@
+using System.Net;
+
+namespace ScadaLink.ExternalSystemGateway;
+
+/// <summary>
+/// WP-8: Classifies HTTP errors as transient or permanent.
+/// Transient: connection refused, timeout, HTTP 408/429/5xx.
+/// Permanent: HTTP 4xx (except 408/429).
+/// </summary>
+public static class ErrorClassifier
+{
+    /// <summary>
+    /// Determines whether an HTTP status code represents a transient failure.
+    /// </summary>
+    public static bool IsTransient(HttpStatusCode statusCode)
+    {
+        var code = (int)statusCode;
+        return code >= 500 || code == 408 || code == 429;
+    }
+
+    /// <summary>
+    /// Determines whether an exception represents a transient failure.
+    /// </summary>
+    public static bool IsTransient(Exception exception)
+    {
+        return exception is HttpRequestException
+            or TaskCanceledException
+            or TimeoutException
+            or OperationCanceledException;
+    }
+
+    /// <summary>
+    /// Creates a TransientException for S&amp;F buffering.
+    /// </summary>
+    public static TransientExternalSystemException AsTransient(string message, Exception? inner = null)
+    {
+        return new TransientExternalSystemException(message, inner);
+    }
+}
+
+/// <summary>
+/// Exception type that signals a transient failure suitable for store-and-forward retry.
+/// </summary>
+public class TransientExternalSystemException : Exception
+{
+    public TransientExternalSystemException(string message, Exception? innerException = null)
+        : base(message, innerException) { }
+}
+
+/// <summary>
+/// Exception type that signals a permanent failure (should not be retried).
+/// </summary>
+public class PermanentExternalSystemException : Exception
+{
+    public int? HttpStatusCode { get; }
+
+    public PermanentExternalSystemException(string message, int? httpStatusCode = null, Exception? innerException = null)
+        : base(message, innerException)
+    {
+        HttpStatusCode = httpStatusCode;
+    }
+}
--- a/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs
+++ b/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs
@@ -0,0 +1,246 @@
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using ScadaLink.Commons.Entities.ExternalSystems;
+using ScadaLink.Commons.Interfaces.Repositories;
+using ScadaLink.Commons.Interfaces.Services;
+using ScadaLink.Commons.Types.Enums;
+using ScadaLink.StoreAndForward;
+
+namespace ScadaLink.ExternalSystemGateway;
+
+/// <summary>
+/// WP-6: HTTP/REST client that invokes external APIs.
+/// WP-7: Dual call modes — Call (synchronous) and CachedCall (S&amp;F on transient failure).
+/// WP-8: Error classification applied to HTTP responses and exceptions.
+/// </summary>
+public class ExternalSystemClient : IExternalSystemClient
+{
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly IExternalSystemRepository _repository;
+    private readonly StoreAndForwardService? _storeAndForward;
+    private readonly ILogger<ExternalSystemClient> _logger;
+
+    public ExternalSystemClient(
+        IHttpClientFactory httpClientFactory,
+        IExternalSystemRepository repository,
+        ILogger<ExternalSystemClient> logger,
+        StoreAndForwardService? storeAndForward = null)
+    {
+        _httpClientFactory = httpClientFactory;
+        _repository = repository;
+        _logger = logger;
+        _storeAndForward = storeAndForward;
+    }
+
+    /// <summary>
+    /// WP-7: Synchronous call — all failures returned to caller.
+    /// </summary>
+    public async Task<ExternalCallResult> CallAsync(
+        string systemName,
+        string methodName,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        CancellationToken cancellationToken = default)
+    {
+        var (system, method) = await ResolveSystemAndMethodAsync(systemName, methodName, cancellationToken);
+        if (system == null || method == null)
+        {
+            return new ExternalCallResult(false, null, $"External system '{systemName}' or method '{methodName}' not found");
+        }
+
+        try
+        {
+            var response = await InvokeHttpAsync(system, method, parameters, cancellationToken);
+            return new ExternalCallResult(true, response, null);
+        }
+        catch (TransientExternalSystemException ex)
+        {
+            return new ExternalCallResult(false, null, $"Transient error: {ex.Message}");
+        }
+        catch (PermanentExternalSystemException ex)
+        {
+            return new ExternalCallResult(false, null, $"Permanent error: {ex.Message}");
+        }
+    }
+
+    /// <summary>
+    /// WP-7: CachedCall — attempt immediate, transient failure goes to S&amp;F, permanent returned to script.
+    /// </summary>
+    public async Task<ExternalCallResult> CachedCallAsync(
+        string systemName,
+        string methodName,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default)
+    {
+        var (system, method) = await ResolveSystemAndMethodAsync(systemName, methodName, cancellationToken);
+        if (system == null || method == null)
+        {
+            return new ExternalCallResult(false, null, $"External system '{systemName}' or method '{methodName}' not found");
+        }
+
+        try
+        {
+            var response = await InvokeHttpAsync(system, method, parameters, cancellationToken);
+            return new ExternalCallResult(true, response, null);
+        }
+        catch (PermanentExternalSystemException ex)
+        {
+            // Permanent failures returned to script, never buffered
+            return new ExternalCallResult(false, null, $"Permanent error: {ex.Message}");
+        }
+        catch (TransientExternalSystemException)
+        {
+            // Transient failure — hand to S&F
+            if (_storeAndForward == null)
+            {
+                return new ExternalCallResult(false, null, "Transient error and store-and-forward not available");
+            }
+
+            var payload = JsonSerializer.Serialize(new
+            {
+                SystemName = systemName,
+                MethodName = methodName,
+                Parameters = parameters
+            });
+
+            var sfResult = await _storeAndForward.EnqueueAsync(
+                StoreAndForwardCategory.ExternalSystem,
+                systemName,
+                payload,
+                originInstanceName,
+                system.MaxRetries > 0 ? system.MaxRetries : null,
+                system.RetryDelay > TimeSpan.Zero ? system.RetryDelay : null);
+
+            return new ExternalCallResult(true, null, null, WasBuffered: true);
+        }
+    }
+
+    /// <summary>
+    /// WP-6: Executes the HTTP request against the external system.
+    /// </summary>
+    internal async Task<string?> InvokeHttpAsync(
+        ExternalSystemDefinition system,
+        ExternalSystemMethod method,
+        IReadOnlyDictionary<string, object?>? parameters,
+        CancellationToken cancellationToken)
+    {
+        var client = _httpClientFactory.CreateClient($"ExternalSystem_{system.Name}");
+
+        var url = BuildUrl(system.EndpointUrl, method.Path, parameters, method.HttpMethod);
+        var request = new HttpRequestMessage(new HttpMethod(method.HttpMethod), url);
+
+        // Apply authentication
+        ApplyAuth(request, system);
+
+        // For POST/PUT/PATCH, send parameters as JSON body
+        if (method.HttpMethod.Equals("POST", StringComparison.OrdinalIgnoreCase) ||
+            method.HttpMethod.Equals("PUT", StringComparison.OrdinalIgnoreCase) ||
+            method.HttpMethod.Equals("PATCH", StringComparison.OrdinalIgnoreCase))
+        {
+            if (parameters != null && parameters.Count > 0)
+            {
+                request.Content = new StringContent(
+                    JsonSerializer.Serialize(parameters),
+                    Encoding.UTF8,
+                    "application/json");
+            }
+        }
+
+        HttpResponseMessage response;
+        try
+        {
+            response = await client.SendAsync(request, cancellationToken);
+        }
+        catch (Exception ex) when (ErrorClassifier.IsTransient(ex))
+        {
+            throw ErrorClassifier.AsTransient($"Connection error to {system.Name}: {ex.Message}", ex);
+        }
+
+        if (response.IsSuccessStatusCode)
+        {
+            return await response.Content.ReadAsStringAsync(cancellationToken);
+        }
+
+        var errorBody = await response.Content.ReadAsStringAsync(cancellationToken);
+
+        if (ErrorClassifier.IsTransient(response.StatusCode))
+        {
+            throw ErrorClassifier.AsTransient(
+                $"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}");
+        }
+
+        throw new PermanentExternalSystemException(
+            $"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}",
+            (int)response.StatusCode);
+    }
+
+    private static string BuildUrl(string baseUrl, string path, IReadOnlyDictionary<string, object?>? parameters, string httpMethod)
+    {
+        var url = baseUrl.TrimEnd('/') + "/" + path.TrimStart('/');
+
+        // For GET/DELETE, append parameters as query string
+        if ((httpMethod.Equals("GET", StringComparison.OrdinalIgnoreCase) ||
+             httpMethod.Equals("DELETE", StringComparison.OrdinalIgnoreCase)) &&
+            parameters != null && parameters.Count > 0)
+        {
+            var queryString = string.Join("&",
+                parameters.Where(p => p.Value != null)
+                    .Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value?.ToString() ?? "")}"));
+            url += "?" + queryString;
+        }
+
+        return url;
+    }
+
+    private static void ApplyAuth(HttpRequestMessage request, ExternalSystemDefinition system)
+    {
+        if (string.IsNullOrEmpty(system.AuthConfiguration))
+            return;
+
+        switch (system.AuthType.ToLowerInvariant())
+        {
+            case "apikey":
+                // Auth config format: "HeaderName:KeyValue" or just "KeyValue" (default header: X-API-Key)
+                var parts = system.AuthConfiguration.Split(':', 2);
+                if (parts.Length == 2)
+                {
+                    request.Headers.TryAddWithoutValidation(parts[0], parts[1]);
+                }
+                else
+                {
+                    request.Headers.TryAddWithoutValidation("X-API-Key", system.AuthConfiguration);
+                }
+                break;
+
+            case "basic":
+                // Auth config format: "username:password"
+                var basicParts = system.AuthConfiguration.Split(':', 2);
+                if (basicParts.Length == 2)
+                {
+                    var encoded = Convert.ToBase64String(
+                        Encoding.UTF8.GetBytes($"{basicParts[0]}:{basicParts[1]}"));
+                    request.Headers.Authorization = new AuthenticationHeaderValue("Basic", encoded);
+                }
+                break;
+        }
+    }
+
+    private async Task<(ExternalSystemDefinition? system, ExternalSystemMethod? method)> ResolveSystemAndMethodAsync(
+        string systemName,
+        string methodName,
+        CancellationToken cancellationToken)
+    {
+        var systems = await _repository.GetAllExternalSystemsAsync(cancellationToken);
+        var system = systems.FirstOrDefault(s => s.Name.Equals(systemName, StringComparison.OrdinalIgnoreCase));
+        if (system == null)
+            return (null, null);
+
+        var methods = await _repository.GetMethodsByExternalSystemIdAsync(system.Id, cancellationToken);
+        var method = methods.FirstOrDefault(m => m.Name.Equals(methodName, StringComparison.OrdinalIgnoreCase));
+
+        return (system, method);
+    }
+}
--- a/src/ScadaLink.ExternalSystemGateway/ExternalSystemGatewayOptions.cs
+++ b/src/ScadaLink.ExternalSystemGateway/ExternalSystemGatewayOptions.cs
@@ -0,0 +1,13 @@
+namespace ScadaLink.ExternalSystemGateway;
+
+/// <summary>
+/// Configuration options for the External System Gateway component.
+/// </summary>
+public class ExternalSystemGatewayOptions
+{
+    /// <summary>Default HTTP request timeout per external system call.</summary>
+    public TimeSpan DefaultHttpTimeout { get; set; } = TimeSpan.FromSeconds(30);
+
+    /// <summary>Maximum number of concurrent HTTP connections per external system.</summary>
+    public int MaxConcurrentConnectionsPerSystem { get; set; } = 10;
+}
--- a/src/ScadaLink.ExternalSystemGateway/ScadaLink.ExternalSystemGateway.csproj
+++ b/src/ScadaLink.ExternalSystemGateway/ScadaLink.ExternalSystemGateway.csproj
@@ -8,12 +8,20 @@
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.2" />
    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
    <PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
  </ItemGroup>

  <ItemGroup>
    <ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
+    <ProjectReference Include="../ScadaLink.StoreAndForward/ScadaLink.StoreAndForward.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="ScadaLink.ExternalSystemGateway.Tests" />
  </ItemGroup>

 </Project>
--- a/src/ScadaLink.ExternalSystemGateway/ServiceCollectionExtensions.cs
+++ b/src/ScadaLink.ExternalSystemGateway/ServiceCollectionExtensions.cs
@@ -1,4 +1,5 @@
 using Microsoft.Extensions.DependencyInjection;
+using ScadaLink.Commons.Interfaces.Services;

 namespace ScadaLink.ExternalSystemGateway;

@@ -6,13 +7,22 @@ public static class ServiceCollectionExtensions
 {
    public static IServiceCollection AddExternalSystemGateway(this IServiceCollection services)
    {
-        // Phase 0: skeleton only
+        services.AddOptions<ExternalSystemGatewayOptions>()
+            .BindConfiguration("ScadaLink:ExternalSystemGateway");
+
+        services.AddHttpClient();
+        services.AddSingleton<ExternalSystemClient>();
+        services.AddSingleton<IExternalSystemClient>(sp => sp.GetRequiredService<ExternalSystemClient>());
+        services.AddSingleton<DatabaseGateway>();
+        services.AddSingleton<IDatabaseGateway>(sp => sp.GetRequiredService<DatabaseGateway>());
+
        return services;
    }

    public static IServiceCollection AddExternalSystemGatewayActors(this IServiceCollection services)
    {
-        // Phase 0: placeholder for Akka actor registration
+        // WP-10: Actor registration happens in AkkaHostedService.
+        // Script Execution Actors run on dedicated blocking I/O dispatcher.
        return services;
    }
 }