Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -17,9 +17,10 @@ namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
 /// first response that is not itself a standby refusal. A peer that is also a standby
 /// (mid-failover, or a misconfigured list that includes this node) answers 503 with the
 /// <see cref="StandbyResponseHeader"/> marker, which the forwarder treats like a
-/// connection failure: try the next peer. Connection errors and per-attempt timeouts
-/// (<see cref="InboundApiOptions.StandbyForwardTimeout"/>) also move to the next peer.
-/// When every peer fails the caller gets 503 <c>ACTIVE_NODE_UNREACHABLE</c>.</para>
+/// known pre-execution rejection: try the next peer. Only proven name-resolution or
+/// connection-refused failures also permit retry. A timeout or ambiguous transport failure
+/// returns 502 OUTCOME_UNKNOWN without replay. One budget spans every attempt. When all
+/// permitted attempts fail before execution, returns 503 ACTIVE_NODE_UNREACHABLE.</para>
 ///
 /// <para><b>Loop guard.</b> Every relayed request carries
 /// <see cref="ForwardedRequestHeader"/>. <see cref="InboundApiEndpointFilter"/> never
@@ -94,33 +95,31 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder
        // Buffer the body once so it can be replayed to a second peer. The filter has
        // already enforced MaxRequestBodyBytes (Content-Length up front, and the
        // per-request Kestrel cap for chunked bodies), so this is bounded.
-        byte[] body;
-        using (var buffer = new MemoryStream())
-        {
-            await context.Request.Body.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
-            body = buffer.ToArray();
-        }
+        var body = await RelayProvenance.ReadBoundedAsync(context.Request.Body, opts.MaxRequestBodyBytes, cancellationToken);
+        var provenance = RelayProvenance.Sign(context, body, opts.ApiKeyPepper);

        var client = _httpClientFactory.CreateClient(HttpClientName);
        var pathAndQuery = context.Request.Path.Value + context.Request.QueryString.Value;
        var forwardedBy = string.IsNullOrEmpty(context.Request.Host.Host) ? "1" : context.Request.Host.Host;

+        // A single caller-visible budget spans all known-preexecution attempts.
+        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        deadline.CancelAfter(opts.StandbyForwardTimeout);
        foreach (var peer in peers)
        {
            cancellationToken.ThrowIfCancellationRequested();

-            using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-            attemptCts.CancelAfter(opts.StandbyForwardTimeout);
+            if (deadline.IsCancellationRequested) return Unreachable();

            try
            {
-                using var request = BuildRequest(context, peer, pathAndQuery, body, forwardedBy);
-                var response = await client
-                    .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, attemptCts.Token)
+                using var request = BuildRequest(context, peer, pathAndQuery, body, forwardedBy, provenance);
+                using var response = await client
+                    .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, deadline.Token)
                    .ConfigureAwait(false);

                if (response.StatusCode == HttpStatusCode.ServiceUnavailable
-                    && response.Headers.Contains(StandbyResponseHeader))
+                    && response.Headers.TryGetValues(StandbyResponseHeader, out var marker) && marker.SequenceEqual(new[] { "1" }))
                {
                    response.Dispose();
                    _logger.LogInformation(
@@ -129,7 +128,7 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder
                }

                var responseBody = await response.Content
-                    .ReadAsByteArrayAsync(attemptCts.Token)
+                    .ReadAsByteArrayAsync(deadline.Token)
                    .ConfigureAwait(false);
                var contentType = response.Content.Headers.ContentType?.ToString();
                var status = (int)response.StatusCode;
@@ -146,14 +145,17 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder
            }
            catch (OperationCanceledException)
            {
-                _logger.LogWarning(
-                    "Inbound API relay: peer {Peer} did not answer within {Timeout} — trying the next peer",
-                    peer, opts.StandbyForwardTimeout);
+                _logger.LogWarning("Inbound relay deadline expired after dispatch to {Peer}; outcome uncertain", peer);
+                return OutcomeUnknown();
+            }
+            catch (HttpRequestException ex) when (KnownNotDispatched(ex))
+            {
+                _logger.LogWarning(ex, "Inbound relay could not connect to {Peer}; trying next peer", peer);
            }
            catch (HttpRequestException ex)
            {
-                _logger.LogWarning(ex,
-                    "Inbound API relay: peer {Peer} unreachable — trying the next peer", peer);
+                _logger.LogWarning(ex, "Inbound relay lost response from {Peer}; outcome uncertain", peer);
+                return OutcomeUnknown();
            }
        }

@@ -164,7 +166,7 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder
    }

    private static HttpRequestMessage BuildRequest(
-        HttpContext context, string peer, string pathAndQuery, byte[] body, string forwardedBy)
+        HttpContext context, string peer, string pathAndQuery, byte[] body, string forwardedBy, string? provenance)
    {
        var request = new HttpRequestMessage(
            new HttpMethod(context.Request.Method),
@@ -174,7 +176,9 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder
        foreach (var header in context.Request.Headers)
        {
            if (HopByHopRequestHeaders.Contains(header.Key)
-                || string.Equals(header.Key, ForwardedRequestHeader, StringComparison.OrdinalIgnoreCase))
+                || string.Equals(header.Key, ForwardedRequestHeader, StringComparison.OrdinalIgnoreCase)
+                || string.Equals(header.Key, RelayProvenance.Header, StringComparison.OrdinalIgnoreCase)
+                || string.Equals(header.Key, "X-Forwarded-For", StringComparison.OrdinalIgnoreCase))
                continue;

            var values = header.Value.ToArray();
@@ -188,9 +192,20 @@ public sealed class HttpStandbyRequestForwarder : IStandbyRequestForwarder

        request.Content = content;
        request.Headers.TryAddWithoutValidation(ForwardedRequestHeader, forwardedBy);
+        if (provenance is not null) request.Headers.TryAddWithoutValidation(RelayProvenance.Header, provenance);
        return request;
    }

+    private static bool KnownNotDispatched(HttpRequestException error)
+        => error.HttpRequestError == HttpRequestError.NameResolutionError
+            || error.InnerException is System.Net.Sockets.SocketException { SocketErrorCode: System.Net.Sockets.SocketError.ConnectionRefused };
+
+    private static IResult OutcomeUnknown() => Results.Json(new
+    {
+        error = "The peer may have applied this request. Verify its result before retrying.",
+        code = "OUTCOME_UNKNOWN"
+    }, statusCode: StatusCodes.Status502BadGateway);
+
    private static IResult Unreachable() => Results.Json(
        new
        {
@@ -94,16 +94,20 @@ public sealed class InboundApiEndpointFilter : IEndpointFilter

            if (!alreadyForwarded && forwarder is { IsConfigured: true })
            {
-                httpContext.Items[RelayedToActiveItemKey] = true;
                _logger.LogDebug("Inbound API request on a standby node — relaying to the active peer");
-                return await forwarder.ForwardAsync(httpContext, httpContext.RequestAborted)
-                    .ConfigureAwait(false);
+                var result = await forwarder.ForwardAsync(httpContext, httpContext.RequestAborted).ConfigureAwait(false);
+                if (result is HttpStandbyRequestForwarder.RelayedResult)
+                    httpContext.Items[RelayedToActiveItemKey] = true;
+                return result;
            }

            _logger.LogWarning(
                alreadyForwarded
                    ? "Inbound API request rejected — relayed here by a peer, but this node is also a standby (loop guard)"
                    : "Inbound API request rejected — this node is a standby (not the active central node) and no relay peers are configured");
+            // A verified internal refusal has not served the call. Its originating relay
+            // owns the single failure audit if all peers refuse; unsigned callers cannot suppress it.
+            if (RelayProvenance.IsVerified(httpContext)) httpContext.Items[RelayedToActiveItemKey] = true;
            httpContext.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader] = "1";
            return Results.Json(
                new { error = "Inbound API is only available on the active central node", code = "STANDBY_NODE" },
@@ -63,10 +63,10 @@ public class InboundApiOptions
    public List<string> StandbyForwardPeerEndpoints { get; set; } = new();

    /// <summary>
-    /// Per-peer deadline for one relay attempt (connect + the active node's full method
-    /// execution). Must comfortably exceed <see cref="DefaultMethodTimeout"/> and the
-    /// longest per-method timeout in use, otherwise the standby gives up on a peer that
-    /// is still legitimately working and moves on to the next one. Default 60 s.
+    /// End-to-end relay budget spanning connection, permitted standby retries and response
+    /// consumption. Size it for the longest method execution plus network overhead. A send
+    /// whose response is lost or times out returns OUTCOME_UNKNOWN and is never replayed.
+    /// Default 60 s.
    /// </summary>
    public TimeSpan StandbyForwardTimeout { get; set; } = TimeSpan.FromSeconds(60);

@@ -107,6 +107,7 @@ public sealed class AuditWriteMiddleware
    private readonly IOptionsMonitor<AuditLogOptions> _options;
    private readonly IAuditActorAccessor? _actorAccessor;
    private readonly IAuditInboundCeilingHitsCounter _ceilingHitsCounter;
+    private readonly IOptionsMonitor<InboundApiOptions>? _inboundOptions;

    /// <summary>
    /// Initializes the middleware with its required dependencies.
@@ -134,13 +135,15 @@ public sealed class AuditWriteMiddleware
        ILogger<AuditWriteMiddleware> logger,
        IOptionsMonitor<AuditLogOptions> options,
        IAuditActorAccessor? actorAccessor = null,
-        IAuditInboundCeilingHitsCounter? ceilingHitsCounter = null)
+        IAuditInboundCeilingHitsCounter? ceilingHitsCounter = null,
+        IOptionsMonitor<InboundApiOptions>? inboundOptions = null)
    {
        _next = next ?? throw new ArgumentNullException(nameof(next));
        _auditWriter = auditWriter ?? throw new ArgumentNullException(nameof(auditWriter));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _options = options ?? throw new ArgumentNullException(nameof(options));
        _actorAccessor = actorAccessor;
+        _inboundOptions = inboundOptions;
        _ceilingHitsCounter = ceilingHitsCounter ?? new NoOpAuditInboundCeilingHitsCounter();
    }

@@ -151,6 +154,15 @@ public sealed class AuditWriteMiddleware
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async Task InvokeAsync(HttpContext ctx)
    {
+        // Install the route cap before EnableBuffering or any audit read makes Kestrel readonly.
+        var maxBytes = _inboundOptions?.CurrentValue.MaxRequestBodyBytes ?? new InboundApiOptions().MaxRequestBodyBytes;
+        var size = ctx.Features.Get<Microsoft.AspNetCore.Http.Features.IHttpMaxRequestBodySizeFeature>();
+        if (size is { IsReadOnly: false }) size.MaxRequestBodySize = maxBytes;
+        if (ctx.Request.ContentLength is { } declared && declared > maxBytes)
+        {
+            ctx.Response.StatusCode = StatusCodes.Status413PayloadTooLarge;
+            return;
+        }
        var sw = Stopwatch.StartNew();

        // Per-request hot read of the options snapshot so a live config change
@@ -169,6 +181,15 @@ public sealed class AuditWriteMiddleware
        //       ExecutionId (the row stays top-level — its ParentExecutionId is
        //       never set).
        ctx.Items[InboundExecutionIdItemKey] = Guid.NewGuid();
+        try
+        {
+            if (_inboundOptions is not null) await RelayProvenance.VerifyAsync(ctx, _inboundOptions.CurrentValue);
+        }
+        catch (BadHttpRequestException ex) when (ex.StatusCode == StatusCodes.Status413PayloadTooLarge)
+        {
+            ctx.Response.StatusCode = StatusCodes.Status413PayloadTooLarge;
+            return;
+        }

        // Buffer the request body up front so we can both audit it and let the
        // downstream handler still parse it. EnableBuffering swaps the request
@@ -201,8 +222,15 @@ public sealed class AuditWriteMiddleware
        if (!skipBody && RequestHasBody(ctx.Request))
        {
            ctx.Request.EnableBuffering();
-            (requestBody, requestTruncated) =
-                await ReadBufferedRequestBodyAsync(ctx.Request, cap).ConfigureAwait(false);
+            try
+            {
+                (requestBody, requestTruncated) = await ReadBufferedRequestBodyAsync(ctx.Request, cap).ConfigureAwait(false);
+            }
+            catch (BadHttpRequestException ex) when (ex.StatusCode == StatusCodes.Status413PayloadTooLarge)
+            {
+                ctx.Response.StatusCode = StatusCodes.Status413PayloadTooLarge;
+                return;
+            }
        }

        // Response body — wrap Response.Body in a forwarding stream that mirrors
@@ -322,6 +350,7 @@ public sealed class AuditWriteMiddleware
            {
                // Redact headers whose name appears in the HeaderRedactList —
                // the same "<redacted>" marker used by ScadaBridgeAuditRedactor.
+                if (string.Equals(header.Key, RelayProvenance.Header, StringComparison.OrdinalIgnoreCase)) continue;
                var value = redactSet.Contains(header.Key)
                    ? "<redacted>"
                    : header.Value.ToString();
@@ -332,8 +361,9 @@ public sealed class AuditWriteMiddleware
            // pipeline is the authoritative failure, so it wins; otherwise the endpoint may
            // have stashed the method script's exception (which the executor keeps out of the
            // HTTP body). Absent both, the row carries no detail.
-            var errorDetail = thrown?.ToString();
-            if (errorDetail is null
+            var suppressErrors = methodName is not null && opts.PerTargetOverrides.TryGetValue(methodName, out var privacy) && privacy.SkipBodyCapture;
+            var errorDetail = suppressErrors ? null : thrown?.ToString();
+            if (!suppressErrors && errorDetail is null
                && ctx.Items.TryGetValue(ErrorDetailItemKey, out var stashedErrorDetail))
            {
                errorDetail = stashedErrorDetail as string;
@@ -341,7 +371,7 @@ public sealed class AuditWriteMiddleware

            var extra = JsonSerializer.Serialize(new
            {
-                remoteIp = ctx.Connection.RemoteIpAddress?.ToString(),
+                remoteIp = RelayProvenance.CallerIp(ctx),
                userAgent = ctx.Request.Headers.UserAgent.ToString(),
                requestHeaders = headerDict,
            });
@@ -367,7 +397,7 @@ public sealed class AuditWriteMiddleware
                correlationId: null,
                httpStatus: statusCode,
                durationMs: (int)Math.Min(durationMs, int.MaxValue),
-                errorMessage: thrown?.Message,
+                errorMessage: suppressErrors ? null : thrown?.Message,
                errorDetail: errorDetail,
                requestSummary: requestBody,
                responseSummary: responseBody,
@@ -512,6 +542,10 @@ public sealed class AuditWriteMiddleware
            var content = DecodeUtf8Bounded(buffer, bytesForString, cutAtValidBytes: truncated);
            return (string.IsNullOrEmpty(content) ? null : content, truncated);
        }
+        catch (BadHttpRequestException ex) when (ex.StatusCode == StatusCodes.Status413PayloadTooLarge)
+        {
+            throw; // The route limit is authoritative, even when audit capture reads first.
+        }
        catch
        {
            // A failed body read must not abort the request — fall through
@@ -0,0 +1,87 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using Microsoft.AspNetCore.Http;
+using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
+
+namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
+
+/// <summary>One-hop caller provenance authenticated by the central pair's server-side pepper.</summary>
+internal static class RelayProvenance
+{
+    internal const string Header = "X-ScadaBridge-Relay";
+    private static readonly object VerifiedKey = new();
+    private sealed record Proof(long Time, Guid ExecutionId, string Ip, string Binding);
+    private static readonly byte[] Domain = "ScadaBridge.InboundRelay.v1"u8.ToArray();
+
+    internal static bool IsVerified(HttpContext context) => context.Items.ContainsKey(VerifiedKey);
+    internal static string? CallerIp(HttpContext context)
+        => context.Items.TryGetValue(VerifiedKey, out var ip) ? ip as string : context.Connection.RemoteIpAddress?.ToString();
+
+    internal static string? Sign(HttpContext context, byte[] body, string pepper)
+    {
+        if (pepper.Length < 16 || context.Connection.RemoteIpAddress is null) return null;
+        var executionId = context.Items.TryGetValue(AuditWriteMiddleware.InboundExecutionIdItemKey, out var value) && value is Guid id
+            ? id : Guid.NewGuid();
+        var proof = new Proof(DateTimeOffset.UtcNow.ToUnixTimeSeconds(), executionId,
+            context.Connection.RemoteIpAddress.ToString(), Binding(context.Request, body));
+        var bytes = JsonSerializer.SerializeToUtf8Bytes(proof);
+        return Convert.ToBase64String(bytes) + "." + Convert.ToHexString(Mac(bytes, pepper));
+    }
+
+    internal static async Task VerifyAsync(HttpContext context, InboundApiOptions options)
+    {
+        var header = context.Request.Headers[Header].ToString();
+        if (options.ApiKeyPepper.Length < 16 || header.Length is 0 or > 2048) return;
+        Proof? proof;
+        try
+        {
+            var parts = header.Split('.');
+            if (parts.Length != 2) return;
+            var payload = Convert.FromBase64String(parts[0]);
+            var signature = Convert.FromHexString(parts[1]);
+            if (!CryptographicOperations.FixedTimeEquals(signature, Mac(payload, options.ApiKeyPepper))) return;
+            proof = JsonSerializer.Deserialize<Proof>(payload);
+            if (proof is null || proof.ExecutionId == Guid.Empty || !IPAddress.TryParse(proof.Ip, out _)
+                || Math.Abs((double)DateTimeOffset.UtcNow.ToUnixTimeSeconds() - proof.Time) > 300) return;
+        }
+        catch (Exception ex) when (ex is FormatException or JsonException or ArgumentException) { return; }
+
+        // Authenticate small metadata before reading; bound even non-Kestrel request streams.
+        context.Request.EnableBuffering();
+        var body = await ReadBoundedAsync(context.Request.Body, options.MaxRequestBodyBytes, context.RequestAborted);
+        context.Request.Body.Position = 0;
+        if (!string.Equals(proof.Binding, Binding(context.Request, body), StringComparison.Ordinal)) return;
+        context.Items[VerifiedKey] = proof.Ip;
+        context.Items[AuditWriteMiddleware.InboundExecutionIdItemKey] = proof.ExecutionId;
+    }
+
+    internal static async Task<byte[]> ReadBoundedAsync(Stream body, long limit, CancellationToken ct)
+    {
+        using var buffer = new MemoryStream();
+        var bytes = new byte[8192];
+        while (true)
+        {
+            var count = await body.ReadAsync(bytes.AsMemory(0, (int)Math.Min(bytes.Length, limit - buffer.Length + 1)), ct);
+            if (count == 0) return buffer.ToArray();
+            if (buffer.Length + count > limit)
+                throw new BadHttpRequestException("Request body too large", StatusCodes.Status413PayloadTooLarge);
+            await buffer.WriteAsync(bytes.AsMemory(0, count), ct);
+        }
+    }
+
+    private static string Binding(HttpRequest request, byte[] body)
+    {
+        var fields = new[] { request.Method, request.Path.Value + request.QueryString.Value,
+            request.ContentType ?? "", request.Headers.Authorization.ToString(), request.Headers["X-API-Key"].ToString(),
+            Convert.ToHexString(SHA256.HashData(body)) };
+        return Convert.ToHexString(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(fields)));
+    }
+
+    private static byte[] Mac(byte[] payload, string pepper)
+    {
+        var key = HMACSHA256.HashData(Encoding.UTF8.GetBytes(pepper), Domain);
+        return HMACSHA256.HashData(key, payload);
+    }
+}
@@ -240,14 +240,14 @@ public class RouteHelper

    /// <summary>
    /// Arch-review F11: returns a <see cref="RouteHelper"/> whose always-on <c>RouteCall</c> row
-    /// carries no request/response summary when <paramref name="captureBodies"/> is <c>false</c>.
+    /// carries no request/response summary or free-text error when <paramref name="captureBodies"/> is <c>false</c>.
    /// <see cref="InboundScriptExecutor"/> sets this from the executing method's
    /// <c>AuditLogOptions.PerTargetOverrides[method.Name].SkipBodyCapture</c> — the same override
    /// <c>AuditWriteMiddleware</c> honours for the inbound row — so an operator who suppressed a
    /// method's body from the audit log doesn't get it back via the routed call's arguments
    /// (which are derived from that same body). Default <c>true</c> (unchanged behaviour).
    /// </summary>
-    /// <param name="captureBodies">False to omit the request/response summary on the <c>RouteCall</c> row.</param>
+    /// <param name="captureBodies">False to omit request/response summaries and free-text errors on the <c>RouteCall</c> row.</param>
    /// <returns>A new <see cref="RouteHelper"/> carrying the flag.</returns>
    public RouteHelper WithBodyCapture(bool captureBodies) =>
        new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
@@ -416,7 +416,7 @@ public class RouteTarget
                executionId: _parentExecutionId,
                parentExecutionId: null,
                durationMs: (int)Math.Min(elapsedMs, int.MaxValue),
-                errorMessage: error,
+                errorMessage: _captureBodies ? error : null,
                // Arch-review F11: a method with SkipBodyCapture set must not leak the routed
                // call's arguments (derived from the same request body) or its return value.
                requestSummary: _captureBodies ? TraceJson(args) : null,