Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs

- WP-1-3: Central/site failover + dual-node recovery tests (17 tests)
- WP-4: Performance testing framework for target scale (7 tests)
- WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests)
- WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs)
- WP-7: Recovery drill test scaffolds (5 tests)
- WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests)
- WP-9: Message contract compatibility (forward/backward compat) (18 tests)
- WP-10: Deployment packaging (installation guide, production checklist, topology)
- WP-11: Operational runbooks (failover, troubleshooting, maintenance)
92 new tests, all passing. Zero warnings.

src/ScadaLink.InboundAPI/ApiKeyValidator.cs

@@ -0,0 +1,80 @@
+using ScadaLink.Commons.Entities.InboundApi;
+using ScadaLink.Commons.Interfaces.Repositories;
+
+namespace ScadaLink.InboundAPI;
+
+/// <summary>
+/// WP-1: Validates API keys from X-API-Key header.
+/// Checks that the key exists, is enabled, and is approved for the requested method.
+/// </summary>
+public class ApiKeyValidator
+{
+    private readonly IInboundApiRepository _repository;
+
+    public ApiKeyValidator(IInboundApiRepository repository)
+    {
+        _repository = repository;
+    }
+
+    /// <summary>
+    /// Validates an API key for a given method.
+    /// Returns (isValid, apiKey, statusCode, errorMessage).
+    /// </summary>
+    public async Task<ApiKeyValidationResult> ValidateAsync(
+        string? apiKeyValue,
+        string methodName,
+        CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrEmpty(apiKeyValue))
+        {
+            return ApiKeyValidationResult.Unauthorized("Missing X-API-Key header");
+        }
+
+        var apiKey = await _repository.GetApiKeyByValueAsync(apiKeyValue, cancellationToken);
+        if (apiKey == null || !apiKey.IsEnabled)
+        {
+            return ApiKeyValidationResult.Unauthorized("Invalid or disabled API key");
+        }
+
+        var method = await _repository.GetMethodByNameAsync(methodName, cancellationToken);
+        if (method == null)
+        {
+            return ApiKeyValidationResult.NotFound($"Method '{methodName}' not found");
+        }
+
+        // Check if this key is approved for the method
+        var approvedKeys = await _repository.GetApprovedKeysForMethodAsync(method.Id, cancellationToken);
+        var isApproved = approvedKeys.Any(k => k.Id == apiKey.Id);
+
+        if (!isApproved)
+        {
+            return ApiKeyValidationResult.Forbidden("API key not approved for this method");
+        }
+
+        return ApiKeyValidationResult.Valid(apiKey, method);
+    }
+}
+
+/// <summary>
+/// Result of API key validation.
+/// </summary>
+public class ApiKeyValidationResult
+{
+    public bool IsValid { get; private init; }
+    public int StatusCode { get; private init; }
+    public string? ErrorMessage { get; private init; }
+    public ApiKey? ApiKey { get; private init; }
+    public ApiMethod? Method { get; private init; }
+
+    public static ApiKeyValidationResult Valid(ApiKey apiKey, ApiMethod method) =>
+        new() { IsValid = true, StatusCode = 200, ApiKey = apiKey, Method = method };
+
+    public static ApiKeyValidationResult Unauthorized(string message) =>
+        new() { IsValid = false, StatusCode = 401, ErrorMessage = message };
+
+    public static ApiKeyValidationResult Forbidden(string message) =>
+        new() { IsValid = false, StatusCode = 403, ErrorMessage = message };
+
+    public static ApiKeyValidationResult NotFound(string message) =>
+        new() { IsValid = false, StatusCode = 400, ErrorMessage = message };
+}

src/ScadaLink.InboundAPI/EndpointExtensions.cs

@@ -1,12 +1,107 @@
+using System.Text.Json;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace ScadaLink.InboundAPI;
 
+/// <summary>
+/// WP-1: POST /api/{methodName} endpoint registration.
+/// WP-2: Method routing and parameter validation.
+/// WP-3: Script execution on central.
+/// WP-5: Error handling — 401, 403, 400, 500.
+/// </summary>
 public static class EndpointExtensions
 {
     public static IEndpointRouteBuilder MapInboundAPI(this IEndpointRouteBuilder endpoints)
     {
-        // Phase 0: skeleton only
+        endpoints.MapPost("/api/{methodName}", HandleInboundApiRequest);
         return endpoints;
     }
+
+    private static async Task<IResult> HandleInboundApiRequest(
+        HttpContext httpContext,
+        string methodName)
+    {
+        var logger = httpContext.RequestServices.GetRequiredService<ILogger<ApiKeyValidator>>();
+        var validator = httpContext.RequestServices.GetRequiredService<ApiKeyValidator>();
+        var executor = httpContext.RequestServices.GetRequiredService<InboundScriptExecutor>();
+        var routeHelper = httpContext.RequestServices.GetRequiredService<RouteHelper>();
+        var options = httpContext.RequestServices.GetRequiredService<IOptions<InboundApiOptions>>().Value;
+
+        // WP-1: Extract and validate API key
+        var apiKeyValue = httpContext.Request.Headers["X-API-Key"].FirstOrDefault();
+        var validationResult = await validator.ValidateAsync(apiKeyValue, methodName, httpContext.RequestAborted);
+
+        if (!validationResult.IsValid)
+        {
+            // WP-5: Failures-only logging
+            logger.LogWarning(
+                "Inbound API auth failure for method {Method}: {Error} (status {StatusCode})",
+                methodName, validationResult.ErrorMessage, validationResult.StatusCode);
+
+            return Results.Json(
+                new { error = validationResult.ErrorMessage },
+                statusCode: validationResult.StatusCode);
+        }
+
+        var method = validationResult.Method!;
+
+        // WP-2: Deserialize and validate parameters
+        JsonElement? body = null;
+        try
+        {
+            if (httpContext.Request.ContentLength > 0 || httpContext.Request.ContentType?.Contains("json") == true)
+            {
+                using var doc = await JsonDocument.ParseAsync(
+                    httpContext.Request.Body, cancellationToken: httpContext.RequestAborted);
+                body = doc.RootElement.Clone();
+            }
+        }
+        catch (JsonException)
+        {
+            return Results.Json(
+                new { error = "Invalid JSON in request body" },
+                statusCode: 400);
+        }
+
+        var paramResult = ParameterValidator.Validate(body, method.ParameterDefinitions);
+        if (!paramResult.IsValid)
+        {
+            return Results.Json(
+                new { error = paramResult.ErrorMessage },
+                statusCode: 400);
+        }
+
+        // WP-3: Execute the method's script
+        var timeout = method.TimeoutSeconds > 0
+            ? TimeSpan.FromSeconds(method.TimeoutSeconds)
+            : options.DefaultMethodTimeout;
+
+        var scriptResult = await executor.ExecuteAsync(
+            method, paramResult.Parameters, routeHelper, timeout, httpContext.RequestAborted);
+
+        if (!scriptResult.Success)
+        {
+            // WP-5: 500 for script failures, safe error message
+            logger.LogWarning(
+                "Inbound API script failure for method {Method}: {Error}",
+                methodName, scriptResult.ErrorMessage);
+
+            return Results.Json(
+                new { error = scriptResult.ErrorMessage ?? "Internal server error" },
+                statusCode: 500);
+        }
+
+        // Return the script result as JSON
+        if (scriptResult.ResultJson != null)
+        {
+            return Results.Text(scriptResult.ResultJson, "application/json", statusCode: 200);
+        }
+
+        return Results.Ok();
+    }
 }

src/ScadaLink.InboundAPI/InboundScriptExecutor.cs

@@ -0,0 +1,109 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using ScadaLink.Commons.Entities.InboundApi;
+
+namespace ScadaLink.InboundAPI;
+
+/// <summary>
+/// WP-3: Executes the C# script associated with an inbound API method.
+/// The script receives input parameters and a route helper, and returns a result
+/// that is serialized as the JSON response.
+///
+/// In a full implementation this would use Roslyn scripting. For now, scripts
+/// are a simple dispatch table so the rest of the pipeline can be tested end-to-end.
+/// </summary>
+public class InboundScriptExecutor
+{
+    private readonly ILogger<InboundScriptExecutor> _logger;
+    private readonly Dictionary<string, Func<InboundScriptContext, Task<object?>>> _scriptHandlers = new();
+
+    public InboundScriptExecutor(ILogger<InboundScriptExecutor> logger)
+    {
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Registers a compiled script handler for a method name.
+    /// In production, this would be called after Roslyn compilation of the method's Script property.
+    /// </summary>
+    public void RegisterHandler(string methodName, Func<InboundScriptContext, Task<object?>> handler)
+    {
+        _scriptHandlers[methodName] = handler;
+    }
+
+    /// <summary>
+    /// Executes the script for the given method with the provided context.
+    /// </summary>
+    public async Task<InboundScriptResult> ExecuteAsync(
+        ApiMethod method,
+        IReadOnlyDictionary<string, object?> parameters,
+        RouteHelper route,
+        TimeSpan timeout,
+        CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            cts.CancelAfter(timeout);
+
+            var context = new InboundScriptContext(parameters, route, cts.Token);
+
+            object? result;
+            if (_scriptHandlers.TryGetValue(method.Name, out var handler))
+            {
+                result = await handler(context).WaitAsync(cts.Token);
+            }
+            else
+            {
+                // No compiled handler — this means the script hasn't been registered.
+                // In production, we'd compile the method.Script and cache it.
+                return new InboundScriptResult(false, null, "Script not compiled or registered for this method");
+            }
+
+            var resultJson = result != null
+                ? JsonSerializer.Serialize(result)
+                : null;
+
+            return new InboundScriptResult(true, resultJson, null);
+        }
+        catch (OperationCanceledException)
+        {
+            _logger.LogWarning("Script execution timed out for method {Method}", method.Name);
+            return new InboundScriptResult(false, null, "Script execution timed out");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Script execution failed for method {Method}", method.Name);
+            // WP-5: Safe error message, no internal details
+            return new InboundScriptResult(false, null, "Internal script error");
+        }
+    }
+}
+
+/// <summary>
+/// Context provided to inbound API scripts.
+/// </summary>
+public class InboundScriptContext
+{
+    public IReadOnlyDictionary<string, object?> Parameters { get; }
+    public RouteHelper Route { get; }
+    public CancellationToken CancellationToken { get; }
+
+    public InboundScriptContext(
+        IReadOnlyDictionary<string, object?> parameters,
+        RouteHelper route,
+        CancellationToken cancellationToken = default)
+    {
+        Parameters = parameters;
+        Route = route;
+        CancellationToken = cancellationToken;
+    }
+}
+
+/// <summary>
+/// Result of executing an inbound API script.
+/// </summary>
+public record InboundScriptResult(
+    bool Success,
+    string? ResultJson,
+    string? ErrorMessage);

src/ScadaLink.InboundAPI/ParameterValidator.cs

@@ -0,0 +1,149 @@
+using System.Text.Json;
+
+namespace ScadaLink.InboundAPI;
+
+/// <summary>
+/// WP-2: Validates and deserializes JSON request body against method parameter definitions.
+/// Extended type system: Boolean, Integer, Float, String, Object, List.
+/// </summary>
+public static class ParameterValidator
+{
+    /// <summary>
+    /// Validates the request body against the method's parameter definitions.
+    /// Returns deserialized parameters or an error message.
+    /// </summary>
+    public static ParameterValidationResult Validate(
+        JsonElement? body,
+        string? parameterDefinitions)
+    {
+        if (string.IsNullOrEmpty(parameterDefinitions))
+        {
+            // No parameters defined — body should be empty or null
+            return ParameterValidationResult.Valid(new Dictionary<string, object?>());
+        }
+
+        List<ParameterDefinition> definitions;
+        try
+        {
+            definitions = JsonSerializer.Deserialize<List<ParameterDefinition>>(
+                parameterDefinitions,
+                new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
+                ?? [];
+        }
+        catch (JsonException)
+        {
+            return ParameterValidationResult.Invalid("Invalid parameter definitions in method configuration");
+        }
+
+        if (definitions.Count == 0)
+        {
+            return ParameterValidationResult.Valid(new Dictionary<string, object?>());
+        }
+
+        if (body == null || body.Value.ValueKind == JsonValueKind.Null || body.Value.ValueKind == JsonValueKind.Undefined)
+        {
+            // Check if all parameters are optional
+            var required = definitions.Where(d => d.Required).ToList();
+            if (required.Count > 0)
+            {
+                return ParameterValidationResult.Invalid(
+                    $"Missing required parameters: {string.Join(", ", required.Select(r => r.Name))}");
+            }
+
+            return ParameterValidationResult.Valid(new Dictionary<string, object?>());
+        }
+
+        if (body.Value.ValueKind != JsonValueKind.Object)
+        {
+            return ParameterValidationResult.Invalid("Request body must be a JSON object");
+        }
+
+        var result = new Dictionary<string, object?>();
+        var errors = new List<string>();
+
+        foreach (var def in definitions)
+        {
+            if (body.Value.TryGetProperty(def.Name, out var prop))
+            {
+                var (value, error) = CoerceValue(prop, def.Type, def.Name);
+                if (error != null)
+                {
+                    errors.Add(error);
+                }
+                else
+                {
+                    result[def.Name] = value;
+                }
+            }
+            else if (def.Required)
+            {
+                errors.Add($"Missing required parameter: {def.Name}");
+            }
+        }
+
+        if (errors.Count > 0)
+        {
+            return ParameterValidationResult.Invalid(string.Join("; ", errors));
+        }
+
+        return ParameterValidationResult.Valid(result);
+    }
+
+    private static (object? value, string? error) CoerceValue(JsonElement element, string expectedType, string paramName)
+    {
+        return expectedType.ToLowerInvariant() switch
+        {
+            "boolean" => element.ValueKind == JsonValueKind.True || element.ValueKind == JsonValueKind.False
+                ? (element.GetBoolean(), null)
+                : (null, $"Parameter '{paramName}' must be a Boolean"),
+
+            "integer" => element.ValueKind == JsonValueKind.Number && element.TryGetInt64(out var intVal)
+                ? (intVal, null)
+                : (null, $"Parameter '{paramName}' must be an Integer"),
+
+            "float" => element.ValueKind == JsonValueKind.Number
+                ? (element.GetDouble(), null)
+                : (null, $"Parameter '{paramName}' must be a Float"),
+
+            "string" => element.ValueKind == JsonValueKind.String
+                ? (element.GetString(), null)
+                : (null, $"Parameter '{paramName}' must be a String"),
+
+            "object" => element.ValueKind == JsonValueKind.Object
+                ? (JsonSerializer.Deserialize<Dictionary<string, object?>>(element.GetRawText()), null)
+                : (null, $"Parameter '{paramName}' must be an Object"),
+
+            "list" => element.ValueKind == JsonValueKind.Array
+                ? (JsonSerializer.Deserialize<List<object?>>(element.GetRawText()), null)
+                : (null, $"Parameter '{paramName}' must be a List"),
+
+            _ => (null, $"Unknown parameter type '{expectedType}' for parameter '{paramName}'")
+        };
+    }
+}
+
+/// <summary>
+/// Defines a parameter in a method's parameter definitions.
+/// </summary>
+public class ParameterDefinition
+{
+    public string Name { get; set; } = string.Empty;
+    public string Type { get; set; } = "String";
+    public bool Required { get; set; } = true;
+}
+
+/// <summary>
+/// Result of parameter validation.
+/// </summary>
+public class ParameterValidationResult
+{
+    public bool IsValid { get; private init; }
+    public string? ErrorMessage { get; private init; }
+    public IReadOnlyDictionary<string, object?> Parameters { get; private init; } = new Dictionary<string, object?>();
+
+    public static ParameterValidationResult Valid(Dictionary<string, object?> parameters) =>
+        new() { IsValid = true, Parameters = parameters };
+
+    public static ParameterValidationResult Invalid(string message) =>
+        new() { IsValid = false, ErrorMessage = message };
+}

src/ScadaLink.InboundAPI/RouteHelper.cs

@@ -0,0 +1,162 @@
+using ScadaLink.Commons.Interfaces.Services;
+using ScadaLink.Commons.Messages.InboundApi;
+using ScadaLink.Communication;
+
+namespace ScadaLink.InboundAPI;
+
+/// <summary>
+/// WP-4: Route.To() helper for cross-site calls from inbound API scripts.
+/// Resolves instance to site, routes via CommunicationService, blocks until response or timeout.
+/// Site unreachable returns error (no store-and-forward).
+/// </summary>
+public class RouteHelper
+{
+    private readonly IInstanceLocator _instanceLocator;
+    private readonly CommunicationService _communicationService;
+
+    public RouteHelper(
+        IInstanceLocator instanceLocator,
+        CommunicationService communicationService)
+    {
+        _instanceLocator = instanceLocator;
+        _communicationService = communicationService;
+    }
+
+    /// <summary>
+    /// Creates a route target for the specified instance.
+    /// </summary>
+    public RouteTarget To(string instanceCode)
+    {
+        return new RouteTarget(instanceCode, _instanceLocator, _communicationService);
+    }
+}
+
+/// <summary>
+/// WP-4: Represents a route target (an instance) for cross-site calls.
+/// </summary>
+public class RouteTarget
+{
+    private readonly string _instanceCode;
+    private readonly IInstanceLocator _instanceLocator;
+    private readonly CommunicationService _communicationService;
+
+    internal RouteTarget(
+        string instanceCode,
+        IInstanceLocator instanceLocator,
+        CommunicationService communicationService)
+    {
+        _instanceCode = instanceCode;
+        _instanceLocator = instanceLocator;
+        _communicationService = communicationService;
+    }
+
+    /// <summary>
+    /// Calls a script on the remote instance. Synchronous from API caller's perspective.
+    /// </summary>
+    public async Task<object?> Call(
+        string scriptName,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        CancellationToken cancellationToken = default)
+    {
+        var siteId = await ResolveSiteAsync(cancellationToken);
+        var correlationId = Guid.NewGuid().ToString();
+
+        var request = new RouteToCallRequest(
+            correlationId, _instanceCode, scriptName, parameters, DateTimeOffset.UtcNow);
+
+        var response = await _communicationService.RouteToCallAsync(
+            siteId, request, cancellationToken);
+
+        if (!response.Success)
+        {
+            throw new InvalidOperationException(
+                response.ErrorMessage ?? "Remote script call failed");
+        }
+
+        return response.ReturnValue;
+    }
+
+    /// <summary>
+    /// Gets a single attribute value from the remote instance.
+    /// </summary>
+    public async Task<object?> GetAttribute(
+        string attributeName,
+        CancellationToken cancellationToken = default)
+    {
+        var result = await GetAttributes(new[] { attributeName }, cancellationToken);
+        return result.TryGetValue(attributeName, out var value) ? value : null;
+    }
+
+    /// <summary>
+    /// Gets multiple attribute values from the remote instance (batch read).
+    /// </summary>
+    public async Task<IReadOnlyDictionary<string, object?>> GetAttributes(
+        IEnumerable<string> attributeNames,
+        CancellationToken cancellationToken = default)
+    {
+        var siteId = await ResolveSiteAsync(cancellationToken);
+        var correlationId = Guid.NewGuid().ToString();
+
+        var request = new RouteToGetAttributesRequest(
+            correlationId, _instanceCode, attributeNames.ToList(), DateTimeOffset.UtcNow);
+
+        var response = await _communicationService.RouteToGetAttributesAsync(
+            siteId, request, cancellationToken);
+
+        if (!response.Success)
+        {
+            throw new InvalidOperationException(
+                response.ErrorMessage ?? "Remote attribute read failed");
+        }
+
+        return response.Values;
+    }
+
+    /// <summary>
+    /// Sets a single attribute value on the remote instance.
+    /// </summary>
+    public async Task SetAttribute(
+        string attributeName,
+        string value,
+        CancellationToken cancellationToken = default)
+    {
+        await SetAttributes(
+            new Dictionary<string, string> { { attributeName, value } },
+            cancellationToken);
+    }
+
+    /// <summary>
+    /// Sets multiple attribute values on the remote instance (batch write).
+    /// </summary>
+    public async Task SetAttributes(
+        IReadOnlyDictionary<string, string> attributeValues,
+        CancellationToken cancellationToken = default)
+    {
+        var siteId = await ResolveSiteAsync(cancellationToken);
+        var correlationId = Guid.NewGuid().ToString();
+
+        var request = new RouteToSetAttributesRequest(
+            correlationId, _instanceCode, attributeValues, DateTimeOffset.UtcNow);
+
+        var response = await _communicationService.RouteToSetAttributesAsync(
+            siteId, request, cancellationToken);
+
+        if (!response.Success)
+        {
+            throw new InvalidOperationException(
+                response.ErrorMessage ?? "Remote attribute write failed");
+        }
+    }
+
+    private async Task<string> ResolveSiteAsync(CancellationToken cancellationToken)
+    {
+        var siteId = await _instanceLocator.GetSiteIdForInstanceAsync(_instanceCode, cancellationToken);
+        if (siteId == null)
+        {
+            throw new InvalidOperationException(
+                $"Instance '{_instanceCode}' not found or has no assigned site");
+        }
+
+        return siteId;
+    }
+}

src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj

@@ -13,6 +13,11 @@
 
   <ItemGroup>
     <ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
+    <ProjectReference Include="../ScadaLink.Communication/ScadaLink.Communication.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="ScadaLink.InboundAPI.Tests" />
   </ItemGroup>
 
 </Project>

src/ScadaLink.InboundAPI/ServiceCollectionExtensions.cs

@@ -6,7 +6,10 @@ public static class ServiceCollectionExtensions
 {
     public static IServiceCollection AddInboundAPI(this IServiceCollection services)
     {
-        // Phase 0: skeleton only
+        services.AddScoped<ApiKeyValidator>();
+        services.AddSingleton<InboundScriptExecutor>();
+        services.AddScoped<RouteHelper>();
+
         return services;
     }
 }