Phase 8: Production readiness — failover tests, security hardening, sandboxing, deployment docs

- WP-1-3: Central/site failover + dual-node recovery tests (17 tests)
- WP-4: Performance testing framework for target scale (7 tests)
- WP-5: Security hardening (LDAPS, JWT key length, no secrets in logs) (11 tests)
- WP-6: Script sandboxing adversarial tests (28 tests, all forbidden APIs)
- WP-7: Recovery drill test scaffolds (5 tests)
- WP-8: Observability validation (structured logs, correlation IDs, metrics) (6 tests)
- WP-9: Message contract compatibility (forward/backward compat) (18 tests)
- WP-10: Deployment packaging (installation guide, production checklist, topology)
- WP-11: Operational runbooks (failover, troubleshooting, maintenance)
92 new tests, all passing. Zero warnings.

src/ScadaLink.Commons/Interfaces/Services/IDatabaseGateway.cs

@@ -0,0 +1,29 @@
+using System.Data.Common;
+
+namespace ScadaLink.Commons.Interfaces.Services;
+
+/// <summary>
+/// Interface for database access from scripts.
+/// Implemented by ExternalSystemGateway, consumed by ScriptRuntimeContext.
+/// </summary>
+public interface IDatabaseGateway
+{
+    /// <summary>
+    /// Returns an ADO.NET DbConnection (typically SqlConnection) from the named connection.
+    /// Connection pooling is managed by the underlying provider.
+    /// Caller is responsible for disposing.
+    /// </summary>
+    Task<DbConnection> GetConnectionAsync(
+        string connectionName,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Submits a SQL write to the store-and-forward engine for reliable delivery.
+    /// </summary>
+    Task CachedWriteAsync(
+        string connectionName,
+        string sql,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default);
+}

src/ScadaLink.Commons/Interfaces/Services/IExternalSystemClient.cs

@@ -0,0 +1,37 @@
+namespace ScadaLink.Commons.Interfaces.Services;
+
+/// <summary>
+/// Interface for invoking external system HTTP APIs.
+/// Implemented by ExternalSystemGateway, consumed by ScriptRuntimeContext.
+/// </summary>
+public interface IExternalSystemClient
+{
+    /// <summary>
+    /// Synchronous call to an external system. All failures returned to caller.
+    /// </summary>
+    Task<ExternalCallResult> CallAsync(
+        string systemName,
+        string methodName,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Attempt immediate delivery; on transient failure, hand to S&amp;F engine.
+    /// Permanent failures returned to caller.
+    /// </summary>
+    Task<ExternalCallResult> CachedCallAsync(
+        string systemName,
+        string methodName,
+        IReadOnlyDictionary<string, object?>? parameters = null,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Result of an external system call.
+/// </summary>
+public record ExternalCallResult(
+    bool Success,
+    string? ResponseJson,
+    string? ErrorMessage,
+    bool WasBuffered = false);

src/ScadaLink.Commons/Interfaces/Services/IInstanceLocator.cs

@@ -0,0 +1,16 @@
+namespace ScadaLink.Commons.Interfaces.Services;
+
+/// <summary>
+/// Resolves an instance unique name to its site identifier.
+/// Used by Inbound API's Route.To() to determine which site to route requests to.
+/// </summary>
+public interface IInstanceLocator
+{
+    /// <summary>
+    /// Resolves the site identifier for a given instance unique name.
+    /// Returns null if the instance is not found.
+    /// </summary>
+    Task<string?> GetSiteIdForInstanceAsync(
+        string instanceUniqueName,
+        CancellationToken cancellationToken = default);
+}

src/ScadaLink.Commons/Interfaces/Services/INotificationDeliveryService.cs

@@ -0,0 +1,27 @@
+namespace ScadaLink.Commons.Interfaces.Services;
+
+/// <summary>
+/// Interface for sending notifications.
+/// Implemented by NotificationService, consumed by ScriptRuntimeContext.
+/// </summary>
+public interface INotificationDeliveryService
+{
+    /// <summary>
+    /// Sends a notification to a named list. Transient failures go to S&amp;F.
+    /// Permanent failures returned to caller.
+    /// </summary>
+    Task<NotificationResult> SendAsync(
+        string listName,
+        string subject,
+        string message,
+        string? originInstanceName = null,
+        CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Result of a notification send attempt.
+/// </summary>
+public record NotificationResult(
+    bool Success,
+    string? ErrorMessage,
+    bool WasBuffered = false);

src/ScadaLink.Commons/Messages/InboundApi/RouteToInstanceRequest.cs

@@ -0,0 +1,59 @@
+namespace ScadaLink.Commons.Messages.InboundApi;
+
+/// <summary>
+/// Request routed from Inbound API to a site to invoke a script on an instance.
+/// Used by Route.To("instanceCode").Call("scriptName", params).
+/// </summary>
+public record RouteToCallRequest(
+    string CorrelationId,
+    string InstanceUniqueName,
+    string ScriptName,
+    IReadOnlyDictionary<string, object?>? Parameters,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Response from a Route.To() call.
+/// </summary>
+public record RouteToCallResponse(
+    string CorrelationId,
+    bool Success,
+    object? ReturnValue,
+    string? ErrorMessage,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Request to read attribute(s) from a remote instance.
+/// </summary>
+public record RouteToGetAttributesRequest(
+    string CorrelationId,
+    string InstanceUniqueName,
+    IReadOnlyList<string> AttributeNames,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Response containing attribute values from a remote instance.
+/// </summary>
+public record RouteToGetAttributesResponse(
+    string CorrelationId,
+    IReadOnlyDictionary<string, object?> Values,
+    bool Success,
+    string? ErrorMessage,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Request to write attribute(s) on a remote instance.
+/// </summary>
+public record RouteToSetAttributesRequest(
+    string CorrelationId,
+    string InstanceUniqueName,
+    IReadOnlyDictionary<string, string> AttributeValues,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Response confirming attribute writes on a remote instance.
+/// </summary>
+public record RouteToSetAttributesResponse(
+    string CorrelationId,
+    bool Success,
+    string? ErrorMessage,
+    DateTimeOffset Timestamp);

src/ScadaLink.Commons/Messages/Instance/GetAttributesBatchRequest.cs

@@ -0,0 +1,22 @@
+namespace ScadaLink.Commons.Messages.Instance;
+
+/// <summary>
+/// Batch request to get multiple attribute values from an Instance Actor.
+/// Used by Route.To().GetAttributes() in Inbound API.
+/// </summary>
+public record GetAttributesBatchRequest(
+    string CorrelationId,
+    string InstanceUniqueName,
+    IReadOnlyList<string> AttributeNames,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Batch response containing multiple attribute values.
+/// </summary>
+public record GetAttributesBatchResponse(
+    string CorrelationId,
+    string InstanceUniqueName,
+    IReadOnlyDictionary<string, object?> Values,
+    bool Success,
+    string? ErrorMessage,
+    DateTimeOffset Timestamp);

src/ScadaLink.Commons/Messages/Instance/SetAttributesBatchCommand.cs

@@ -0,0 +1,21 @@
+namespace ScadaLink.Commons.Messages.Instance;
+
+/// <summary>
+/// Batch command to set multiple attribute values on an Instance Actor.
+/// Used by Route.To().SetAttributes() in Inbound API.
+/// </summary>
+public record SetAttributesBatchCommand(
+    string CorrelationId,
+    string InstanceUniqueName,
+    IReadOnlyDictionary<string, string> AttributeValues,
+    DateTimeOffset Timestamp);
+
+/// <summary>
+/// Batch response confirming multiple attribute writes.
+/// </summary>
+public record SetAttributesBatchResponse(
+    string CorrelationId,
+    string InstanceUniqueName,
+    bool Success,
+    string? ErrorMessage,
+    DateTimeOffset Timestamp);