fix(external-system-gateway): resolve ExternalSystemGateway-012,013,014 — failure logging, connection-limit wiring, test coverage; ExternalSystemGateway-011 flagged

tests/ScadaLink.ExternalSystemGateway.Tests/DatabaseGatewayTests.cs

@@ -56,6 +56,98 @@ public class DatabaseGatewayTests
             () => gateway.CachedWriteAsync("nonexistent", "INSERT INTO t VALUES (1)"));
     }
 
+    // ── ExternalSystemGateway-014: CachedWrite happy-path buffering ──
+
+    [Fact]
+    public async Task CachedWrite_BuffersTheWriteWithConnectionRetrySettings()
+    {
+        var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test")
+        {
+            Id = 1,
+            MaxRetries = 5,
+            RetryDelay = TimeSpan.FromSeconds(12),
+        };
+        _repository.GetAllDatabaseConnectionsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<DatabaseConnectionDefinition> { conn });
+
+        var dbName = $"EsgCachedWrite_{Guid.NewGuid():N}";
+        var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
+        using var keepAlive = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
+        keepAlive.Open();
+        var storage = new ScadaLink.StoreAndForward.StoreAndForwardStorage(
+            connStr, NullLogger<ScadaLink.StoreAndForward.StoreAndForwardStorage>.Instance);
+        await storage.InitializeAsync();
+        var sfOptions = new ScadaLink.StoreAndForward.StoreAndForwardOptions
+        {
+            DefaultMaxRetries = 99,
+            DefaultRetryInterval = TimeSpan.FromMinutes(10),
+            RetryTimerInterval = TimeSpan.FromMinutes(10),
+        };
+        var sf = new ScadaLink.StoreAndForward.StoreAndForwardService(
+            storage, sfOptions, NullLogger<ScadaLink.StoreAndForward.StoreAndForwardService>.Instance);
+
+        var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
+
+        await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (@v)",
+            new Dictionary<string, object?> { ["v"] = 1 });
+
+        var depth = await storage.GetBufferDepthByCategoryAsync();
+        Assert.Equal(1, depth[ScadaLink.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite]);
+
+        var (maxRetries, retryIntervalMs) = ReadBufferedRetrySettings(connStr);
+        Assert.Equal(5, maxRetries);
+        Assert.Equal((long)TimeSpan.FromSeconds(12).TotalMilliseconds, retryIntervalMs);
+    }
+
+    [Fact]
+    public async Task CachedWrite_ZeroMaxRetriesIsHonouredNotTreatedAsUnset()
+    {
+        var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test")
+        {
+            Id = 1,
+            MaxRetries = 0,
+            RetryDelay = TimeSpan.FromSeconds(3),
+        };
+        _repository.GetAllDatabaseConnectionsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<DatabaseConnectionDefinition> { conn });
+
+        var dbName = $"EsgCachedWriteZero_{Guid.NewGuid():N}";
+        var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
+        using var keepAlive = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
+        keepAlive.Open();
+        var storage = new ScadaLink.StoreAndForward.StoreAndForwardStorage(
+            connStr, NullLogger<ScadaLink.StoreAndForward.StoreAndForwardStorage>.Instance);
+        await storage.InitializeAsync();
+        var sfOptions = new ScadaLink.StoreAndForward.StoreAndForwardOptions
+        {
+            DefaultMaxRetries = 99,
+            DefaultRetryInterval = TimeSpan.FromMinutes(10),
+            RetryTimerInterval = TimeSpan.FromMinutes(10),
+        };
+        var sf = new ScadaLink.StoreAndForward.StoreAndForwardService(
+            storage, sfOptions, NullLogger<ScadaLink.StoreAndForward.StoreAndForwardService>.Instance);
+
+        var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
+
+        await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
+
+        var (maxRetries, _) = ReadBufferedRetrySettings(connStr);
+        Assert.Equal(0, maxRetries); // honoured — not the S&F default of 99
+    }
+
+    private static (int MaxRetries, long RetryIntervalMs) ReadBufferedRetrySettings(string connStr)
+    {
+        using var conn = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
+        conn.Open();
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = "SELECT max_retries, retry_interval_ms FROM sf_messages";
+        using var reader = cmd.ExecuteReader();
+        Assert.True(reader.Read(), "expected exactly one buffered message");
+        var result = (reader.GetInt32(0), reader.GetInt64(1));
+        Assert.False(reader.Read(), "expected exactly one buffered message");
+        return result;
+    }
+
     // ── ExternalSystemGateway-001: buffered CachedDbWrite delivery handler ──
 
     [Fact]

tests/ScadaLink.ExternalSystemGateway.Tests/ExternalSystemClientTests.cs

@@ -1,5 +1,7 @@
 using System.Net;
+using System.Net.Http.Headers;
 using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using NSubstitute;
 using ScadaLink.Commons.Entities.ExternalSystems;
@@ -603,6 +605,237 @@ public class ExternalSystemClientTests
             "A caller-cancelled CachedCall must not be buffered for retry");
     }
 
+    // ── ExternalSystemGateway-014: BuildUrl query-string, ApplyAuth, connection errors ──
+
+    [Fact]
+    public async Task Call_GetWithParameters_AppendsEscapedQueryString()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("search", "GET", "/search") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await client.CallAsync("TestAPI", "search", new Dictionary<string, object?>
+        {
+            ["q"] = "a b&c",
+            ["page"] = 2,
+        });
+
+        // AbsoluteUri preserves percent-encoding; the '&' inside a value must be
+        // escaped so it is not mistaken for a parameter separator.
+        var uri = handler.LastUri!.AbsoluteUri;
+        Assert.StartsWith("https://api.example.com/search?", uri);
+        Assert.Contains("q=a%20b%26c", uri);
+        Assert.Contains("page=2", uri);
+    }
+
+    [Fact]
+    public async Task Call_PostWithParameters_SendsJsonBody()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("create", "POST", "/create") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await client.CallAsync("TestAPI", "create", new Dictionary<string, object?> { ["name"] = "widget" });
+
+        Assert.Equal("https://api.example.com/create", handler.LastUri!.ToString());
+        Assert.Contains("\"name\":\"widget\"", handler.LastBody);
+    }
+
+    [Fact]
+    public async Task Call_ApiKeyAuthWithDefaultHeader_SendsXApiKeyHeader()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "apikey")
+        {
+            Id = 1,
+            AuthConfiguration = "secret-key-123",
+        };
+        var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await client.CallAsync("TestAPI", "getData");
+
+        Assert.True(handler.LastHeaders!.TryGetValues("X-API-Key", out var values));
+        Assert.Equal("secret-key-123", values!.Single());
+    }
+
+    [Fact]
+    public async Task Call_ApiKeyAuthWithCustomHeader_SendsNamedHeader()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "apikey")
+        {
+            Id = 1,
+            AuthConfiguration = "Authorization-Token:abc",
+        };
+        var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await client.CallAsync("TestAPI", "getData");
+
+        Assert.True(handler.LastHeaders!.TryGetValues("Authorization-Token", out var values));
+        Assert.Equal("abc", values!.Single());
+    }
+
+    [Fact]
+    public async Task Call_BasicAuth_SendsBase64AuthorizationHeader()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "basic")
+        {
+            Id = 1,
+            AuthConfiguration = "alice:s3cret",
+        };
+        var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        await client.CallAsync("TestAPI", "getData");
+
+        var auth = handler.LastHeaders!.Authorization;
+        Assert.NotNull(auth);
+        Assert.Equal("Basic", auth!.Scheme);
+        var decoded = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(auth.Parameter!));
+        Assert.Equal("alice:s3cret", decoded);
+    }
+
+    [Fact]
+    public async Task Call_ConnectionError_IsClassifiedAsTransient()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        // A connection-level failure (e.g. host unreachable) surfaces as HttpRequestException.
+        var handler = new ThrowingHttpMessageHandler(new HttpRequestException("connection refused"));
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
+
+        var client = new ExternalSystemClient(
+            _httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
+
+        var result = await client.CallAsync("TestAPI", "getData");
+
+        Assert.False(result.Success);
+        Assert.Contains("Transient error", result.ErrorMessage);
+    }
+
+    // ── ExternalSystemGateway-012: permanent failures must be logged ──
+
+    [Fact]
+    public async Task Call_PermanentFailure_LogsAWarning()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("badMethod", "POST", "/bad") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new MockHttpMessageHandler(HttpStatusCode.BadRequest, "bad request");
+        var httpClient = new HttpClient(handler);
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
+
+        var logger = new CapturingLogger<ExternalSystemClient>();
+        var client = new ExternalSystemClient(_httpClientFactory, _repository, logger);
+
+        await client.CallAsync("TestAPI", "badMethod");
+
+        // The design doc requires permanent failures to be surfaced to Site Event
+        // Logging — the gateway must emit at least a warning, not stay silent.
+        Assert.Contains(logger.Entries, e =>
+            e.Level >= LogLevel.Warning && e.Message.Contains("TestAPI"));
+    }
+
+    [Fact]
+    public async Task Call_TransientFailure_DoesNotLogAtWarningOrAbove()
+    {
+        var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
+        var method = new ExternalSystemMethod("failMethod", "POST", "/fail") { Id = 1, ExternalSystemDefinitionId = 1 };
+        _repository.GetAllExternalSystemsAsync(Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemDefinition> { system });
+        _repository.GetMethodsByExternalSystemIdAsync(1, Arg.Any<CancellationToken>())
+            .Returns(new List<ExternalSystemMethod> { method });
+
+        var handler = new MockHttpMessageHandler(HttpStatusCode.InternalServerError, "boom");
+        var httpClient = new HttpClient(handler);
+        _httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
+
+        var logger = new CapturingLogger<ExternalSystemClient>();
+        var client = new ExternalSystemClient(_httpClientFactory, _repository, logger);
+
+        await client.CallAsync("TestAPI", "failMethod");
+
+        // A transient failure is normal operation handled by retry/S&F — it must not
+        // be logged at warning level (only permanent failures are).
+        Assert.DoesNotContain(logger.Entries, e => e.Level >= LogLevel.Warning);
+    }
+
+    /// <summary>Test helper: an ILogger that records every entry for assertions.</summary>
+    private sealed class CapturingLogger<T> : ILogger<T>
+    {
+        public List<(LogLevel Level, string Message)> Entries { get; } = new();
+
+        public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(
+            LogLevel logLevel, EventId eventId, TState state, Exception? exception,
+            Func<TState, Exception?, string> formatter)
+        {
+            Entries.Add((logLevel, formatter(state, exception)));
+        }
+
+        private sealed class NullScope : IDisposable
+        {
+            public static readonly NullScope Instance = new();
+            public void Dispose() { }
+        }
+    }
+
     /// <summary>
     /// Test helper: mock HTTP message handler.
     /// </summary>
@@ -667,7 +900,7 @@ public class ExternalSystemClientTests
     }
 
     /// <summary>
-    /// Test helper: captures the request URI of the last request.
+    /// Test helper: captures the URI, headers and body of the last request.
     /// </summary>
     private class RequestCapturingHandler : HttpMessageHandler
     {
@@ -681,17 +914,31 @@ public class ExternalSystemClientTests
         }
 
         public Uri? LastUri { get; private set; }
+        public HttpRequestHeaders? LastHeaders { get; private set; }
+        public string? LastBody { get; private set; }
 
-        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
         {
             LastUri = request.RequestUri;
-            return Task.FromResult(new HttpResponseMessage(_statusCode)
+            LastHeaders = request.Headers;
+            LastBody = request.Content == null ? null : await request.Content.ReadAsStringAsync(cancellationToken);
+            return new HttpResponseMessage(_statusCode)
             {
                 Content = new StringContent(_body)
-            });
+            };
         }
     }
 
+    /// <summary>Test helper: an HTTP handler that throws a connection-level exception.</summary>
+    private class ThrowingHttpMessageHandler : HttpMessageHandler
+    {
+        private readonly Exception _exception;
+        public ThrowingHttpMessageHandler(Exception exception) => _exception = exception;
+
+        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            => throw _exception;
+    }
+
     /// <summary>
     /// Test helper: an HTTP handler that hangs until cancelled (simulates a slow/hung system).
     /// </summary>

tests/ScadaLink.ExternalSystemGateway.Tests/ServiceWiringTests.cs

@@ -0,0 +1,52 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Http;
+using NSubstitute;
+using ScadaLink.Commons.Interfaces.Repositories;
+
+namespace ScadaLink.ExternalSystemGateway.Tests;
+
+/// <summary>
+/// ExternalSystemGateway-013: configuration options must actually influence the
+/// registered HTTP client — an operator setting them must not be silently ignored.
+/// </summary>
+public class ServiceWiringTests
+{
+    [Fact]
+    public void MaxConcurrentConnectionsPerSystem_IsAppliedToTheNamedHttpClientPrimaryHandler()
+    {
+        var config = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["ScadaLink:ExternalSystemGateway:MaxConcurrentConnectionsPerSystem"] = "4",
+            })
+            .Build();
+
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddSingleton<IConfiguration>(config);
+        services.AddSingleton(Substitute.For<IExternalSystemRepository>());
+        services.AddExternalSystemGateway();
+
+        using var provider = services.BuildServiceProvider();
+
+        // Resolve the per-system named client's message-handler chain and walk to the
+        // primary handler — the option must be reflected in MaxConnectionsPerServer.
+        var handlerFactory = provider.GetRequiredService<IHttpMessageHandlerFactory>();
+        var handler = handlerFactory.CreateHandler("ExternalSystem_AnySystem");
+
+        var primary = FindPrimaryHandler(handler);
+        var sockets = Assert.IsType<SocketsHttpHandler>(primary);
+        Assert.Equal(4, sockets.MaxConnectionsPerServer);
+    }
+
+    private static HttpMessageHandler FindPrimaryHandler(HttpMessageHandler handler)
+    {
+        var current = handler;
+        while (current is DelegatingHandler delegating && delegating.InnerHandler != null)
+        {
+            current = delegating.InnerHandler;
+        }
+        return current;
+    }
+}