Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs

- WP-1-3: Central/site failover + dual-node recovery tests (17 tests)
- WP-4: Performance testing framework for target scale (7 tests)
- WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests)
- WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs)
- WP-7: Recovery drill test scaffolds (5 tests)
- WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests)
- WP-9: Message contract compatibility (forward/backward compat) (18 tests)
- WP-10: Deployment packaging (installation guide, production checklist, topology)
- WP-11: Operational runbooks (failover, troubleshooting, maintenance)
92 new tests, all passing. Zero warnings.

tests/ScadaLink.InboundAPI.Tests/ApiKeyValidatorTests.cs

@@ -0,0 +1,101 @@
+using NSubstitute;
+using ScadaLink.Commons.Entities.InboundApi;
+using ScadaLink.Commons.Interfaces.Repositories;
+
+namespace ScadaLink.InboundAPI.Tests;
+
+/// <summary>
+/// WP-1: Tests for API key validation — X-API-Key header, enabled/disabled keys,
+/// method approval.
+/// </summary>
+public class ApiKeyValidatorTests
+{
+    private readonly IInboundApiRepository _repository = Substitute.For<IInboundApiRepository>();
+    private readonly ApiKeyValidator _validator;
+
+    public ApiKeyValidatorTests()
+    {
+        _validator = new ApiKeyValidator(_repository);
+    }
+
+    [Fact]
+    public async Task MissingApiKey_Returns401()
+    {
+        var result = await _validator.ValidateAsync(null, "testMethod");
+        Assert.False(result.IsValid);
+        Assert.Equal(401, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task EmptyApiKey_Returns401()
+    {
+        var result = await _validator.ValidateAsync("", "testMethod");
+        Assert.False(result.IsValid);
+        Assert.Equal(401, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task InvalidApiKey_Returns401()
+    {
+        _repository.GetApiKeyByValueAsync("bad-key").Returns((ApiKey?)null);
+
+        var result = await _validator.ValidateAsync("bad-key", "testMethod");
+        Assert.False(result.IsValid);
+        Assert.Equal(401, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task DisabledApiKey_Returns401()
+    {
+        var key = new ApiKey("test", "valid-key") { Id = 1, IsEnabled = false };
+        _repository.GetApiKeyByValueAsync("valid-key").Returns(key);
+
+        var result = await _validator.ValidateAsync("valid-key", "testMethod");
+        Assert.False(result.IsValid);
+        Assert.Equal(401, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task ValidKey_MethodNotFound_Returns400()
+    {
+        var key = new ApiKey("test", "valid-key") { Id = 1, IsEnabled = true };
+        _repository.GetApiKeyByValueAsync("valid-key").Returns(key);
+        _repository.GetMethodByNameAsync("nonExistent").Returns((ApiMethod?)null);
+
+        var result = await _validator.ValidateAsync("valid-key", "nonExistent");
+        Assert.False(result.IsValid);
+        Assert.Equal(400, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task ValidKey_NotApprovedForMethod_Returns403()
+    {
+        var key = new ApiKey("test", "valid-key") { Id = 1, IsEnabled = true };
+        var method = new ApiMethod("testMethod", "return 1;") { Id = 10 };
+
+        _repository.GetApiKeyByValueAsync("valid-key").Returns(key);
+        _repository.GetMethodByNameAsync("testMethod").Returns(method);
+        _repository.GetApprovedKeysForMethodAsync(10).Returns(new List<ApiKey>());
+
+        var result = await _validator.ValidateAsync("valid-key", "testMethod");
+        Assert.False(result.IsValid);
+        Assert.Equal(403, result.StatusCode);
+    }
+
+    [Fact]
+    public async Task ValidKey_ApprovedForMethod_ReturnsValid()
+    {
+        var key = new ApiKey("test", "valid-key") { Id = 1, IsEnabled = true };
+        var method = new ApiMethod("testMethod", "return 1;") { Id = 10 };
+
+        _repository.GetApiKeyByValueAsync("valid-key").Returns(key);
+        _repository.GetMethodByNameAsync("testMethod").Returns(method);
+        _repository.GetApprovedKeysForMethodAsync(10).Returns(new List<ApiKey> { key });
+
+        var result = await _validator.ValidateAsync("valid-key", "testMethod");
+        Assert.True(result.IsValid);
+        Assert.Equal(200, result.StatusCode);
+        Assert.Equal(key, result.ApiKey);
+        Assert.Equal(method, result.Method);
+    }
+}

tests/ScadaLink.InboundAPI.Tests/InboundScriptExecutorTests.cs

@@ -0,0 +1,119 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using ScadaLink.Commons.Entities.InboundApi;
+using ScadaLink.Commons.Interfaces.Services;
+using ScadaLink.Communication;
+
+namespace ScadaLink.InboundAPI.Tests;
+
+/// <summary>
+/// WP-3: Tests for script execution on central — timeout, handler dispatch, error handling.
+/// WP-5: Safe error messages.
+/// </summary>
+public class InboundScriptExecutorTests
+{
+    private readonly InboundScriptExecutor _executor;
+    private readonly RouteHelper _route;
+
+    public InboundScriptExecutorTests()
+    {
+        _executor = new InboundScriptExecutor(NullLogger<InboundScriptExecutor>.Instance);
+        var locator = Substitute.For<IInstanceLocator>();
+        var commService = Substitute.For<CommunicationService>(
+            Microsoft.Extensions.Options.Options.Create(new CommunicationOptions()),
+            NullLogger<CommunicationService>.Instance);
+        _route = new RouteHelper(locator, commService);
+    }
+
+    [Fact]
+    public async Task RegisteredHandler_ExecutesSuccessfully()
+    {
+        var method = new ApiMethod("test", "return 42;") { Id = 1, TimeoutSeconds = 10 };
+        _executor.RegisterHandler("test", async ctx =>
+        {
+            await Task.CompletedTask;
+            return new { result = 42 };
+        });
+
+        var result = await _executor.ExecuteAsync(
+            method,
+            new Dictionary<string, object?>(),
+            _route,
+            TimeSpan.FromSeconds(10));
+
+        Assert.True(result.Success);
+        Assert.NotNull(result.ResultJson);
+        Assert.Contains("42", result.ResultJson);
+    }
+
+    [Fact]
+    public async Task UnregisteredHandler_ReturnsFailure()
+    {
+        var method = new ApiMethod("unknown", "return 1;") { Id = 1, TimeoutSeconds = 10 };
+
+        var result = await _executor.ExecuteAsync(
+            method,
+            new Dictionary<string, object?>(),
+            _route,
+            TimeSpan.FromSeconds(10));
+
+        Assert.False(result.Success);
+        Assert.Contains("not compiled", result.ErrorMessage);
+    }
+
+    [Fact]
+    public async Task HandlerThrows_ReturnsSafeErrorMessage()
+    {
+        var method = new ApiMethod("failing", "throw new Exception();") { Id = 1, TimeoutSeconds = 10 };
+        _executor.RegisterHandler("failing", _ => throw new InvalidOperationException("internal detail leak"));
+
+        var result = await _executor.ExecuteAsync(
+            method,
+            new Dictionary<string, object?>(),
+            _route,
+            TimeSpan.FromSeconds(10));
+
+        Assert.False(result.Success);
+        // WP-5: Safe error message — should NOT contain "internal detail leak"
+        Assert.Equal("Internal script error", result.ErrorMessage);
+    }
+
+    [Fact]
+    public async Task HandlerTimesOut_ReturnsTimeoutError()
+    {
+        var method = new ApiMethod("slow", "Thread.Sleep(60000);") { Id = 1, TimeoutSeconds = 1 };
+        _executor.RegisterHandler("slow", async ctx =>
+        {
+            await Task.Delay(TimeSpan.FromSeconds(60), ctx.CancellationToken);
+            return "never";
+        });
+
+        var result = await _executor.ExecuteAsync(
+            method,
+            new Dictionary<string, object?>(),
+            _route,
+            TimeSpan.FromMilliseconds(100));
+
+        Assert.False(result.Success);
+        Assert.Contains("timed out", result.ErrorMessage);
+    }
+
+    [Fact]
+    public async Task HandlerAccessesParameters()
+    {
+        var method = new ApiMethod("echo", "return params;") { Id = 1, TimeoutSeconds = 10 };
+        _executor.RegisterHandler("echo", async ctx =>
+        {
+            await Task.CompletedTask;
+            return ctx.Parameters["name"];
+        });
+
+        var parameters = new Dictionary<string, object?> { { "name", "ScadaLink" } };
+
+        var result = await _executor.ExecuteAsync(
+            method, parameters, _route, TimeSpan.FromSeconds(10));
+
+        Assert.True(result.Success);
+        Assert.Contains("ScadaLink", result.ResultJson!);
+    }
+}

tests/ScadaLink.InboundAPI.Tests/ParameterValidatorTests.cs

@@ -0,0 +1,137 @@
+using System.Text.Json;
+
+namespace ScadaLink.InboundAPI.Tests;
+
+/// <summary>
+/// WP-2: Tests for parameter validation — type checking, required fields, extended type system.
+/// </summary>
+public class ParameterValidatorTests
+{
+    [Fact]
+    public void NoDefinitions_NoBody_ReturnsValid()
+    {
+        var result = ParameterValidator.Validate(null, null);
+        Assert.True(result.IsValid);
+        Assert.Empty(result.Parameters);
+    }
+
+    [Fact]
+    public void EmptyDefinitions_ReturnsValid()
+    {
+        var result = ParameterValidator.Validate(null, "[]");
+        Assert.True(result.IsValid);
+    }
+
+    [Fact]
+    public void RequiredParameterMissing_ReturnsInvalid()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "value", Type = "Integer", Required = true }
+        });
+
+        var result = ParameterValidator.Validate(null, definitions);
+        Assert.False(result.IsValid);
+        Assert.Contains("Missing required parameter", result.ErrorMessage);
+    }
+
+    [Fact]
+    public void BodyNotObject_ReturnsInvalid()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "value", Type = "String", Required = true }
+        });
+
+        using var doc = JsonDocument.Parse("\"just a string\"");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.False(result.IsValid);
+        Assert.Contains("must be a JSON object", result.ErrorMessage);
+    }
+
+    [Theory]
+    [InlineData("Boolean", "true", true)]
+    [InlineData("Integer", "42", (long)42)]
+    [InlineData("Float", "3.14", 3.14)]
+    [InlineData("String", "\"hello\"", "hello")]
+    public void ValidTypeCoercion_Succeeds(string type, string jsonValue, object expected)
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "val", Type = type, Required = true }
+        });
+
+        using var doc = JsonDocument.Parse($"{{\"val\": {jsonValue}}}");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.True(result.IsValid);
+        Assert.Equal(expected, result.Parameters["val"]);
+    }
+
+    [Fact]
+    public void WrongType_ReturnsInvalid()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "count", Type = "Integer", Required = true }
+        });
+
+        using var doc = JsonDocument.Parse("{\"count\": \"not a number\"}");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.False(result.IsValid);
+        Assert.Contains("must be an Integer", result.ErrorMessage);
+    }
+
+    [Fact]
+    public void ObjectType_Parsed()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "data", Type = "Object", Required = true }
+        });
+
+        using var doc = JsonDocument.Parse("{\"data\": {\"key\": \"value\"}}");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.True(result.IsValid);
+        Assert.IsType<Dictionary<string, object?>>(result.Parameters["data"]);
+    }
+
+    [Fact]
+    public void ListType_Parsed()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "items", Type = "List", Required = true }
+        });
+
+        using var doc = JsonDocument.Parse("{\"items\": [1, 2, 3]}");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.True(result.IsValid);
+        Assert.IsType<List<object?>>(result.Parameters["items"]);
+    }
+
+    [Fact]
+    public void OptionalParameter_MissingBody_ReturnsValid()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "optional", Type = "String", Required = false }
+        });
+
+        var result = ParameterValidator.Validate(null, definitions);
+        Assert.True(result.IsValid);
+    }
+
+    [Fact]
+    public void UnknownType_ReturnsInvalid()
+    {
+        var definitions = JsonSerializer.Serialize(new[]
+        {
+            new { Name = "val", Type = "CustomType", Required = true }
+        });
+
+        using var doc = JsonDocument.Parse("{\"val\": \"test\"}");
+        var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
+        Assert.False(result.IsValid);
+        Assert.Contains("Unknown parameter type", result.ErrorMessage);
+    }
+}

tests/ScadaLink.InboundAPI.Tests/ScadaLink.InboundAPI.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
@@ -11,6 +11,7 @@
   <ItemGroup>
     <PackageReference Include="coverlet.collector" Version="6.0.4" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
+    <PackageReference Include="NSubstitute" Version="5.3.0" />
     <PackageReference Include="xunit" Version="2.9.3" />
     <PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
   </ItemGroup>
@@ -21,6 +22,7 @@
 
   <ItemGroup>
     <ProjectReference Include="../../src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj" />
+    <ProjectReference Include="../../src/ScadaLink.Commons/ScadaLink.Commons.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

tests/ScadaLink.InboundAPI.Tests/UnitTest1.cs

@@ -1,10 +0,0 @@
-namespace ScadaLink.InboundAPI.Tests;
-
-public class UnitTest1
-{
-    [Fact]
-    public void Test1()
-    {
-
-    }
-}