Resolve all 33 medium-severity architecture review findings

Second slice of the 2026-09-09 whole-system review remediation (docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323. Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush; SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump. Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC. Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11 error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5 authenticated relay provenance; EI-10 body cap before audit read. Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation; DL-2 DataConnection protocol config encrypted at rest (migration 20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging; DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test. Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code; DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs. Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
@@ -6,7 +6,9 @@ namespace ZB.MOM.WW.ScadaBridge.DelmiaNotifier;
 /// <summary>
 /// <see cref="IRecipeSender"/> over a real <see cref="HttpClient"/>: POSTs the recipe payload to
 /// <c>{baseUrl}/api/DelmiaRecipeDownload</c> with the <c>X-API-Key</c> header. A reachable server
-/// (any status) is <see cref="AttemptKind.Connected"/>; a connection/timeout failure is <see cref="AttemptKind.ConnectFailed"/>.
+/// response is final except for the explicit pre-execution standby marker. Only proven
+/// connection refusal/name resolution failures permit another URL; timeouts and other
+/// transport failures have an uncertain application outcome.
 /// </summary>
 internal sealed class HttpRecipeSender(HttpClient http, string apiKey) : IRecipeSender
 {
@@ -25,6 +27,9 @@ internal sealed class HttpRecipeSender(HttpClient http, string apiKey) : IRecipe

            using var response = await http.SendAsync(request, ct);
            var status = (int)response.StatusCode;
+            if (status == 503 && response.Headers.TryGetValues("X-ScadaBridge-Standby", out var marker)
+                && marker.SequenceEqual(new[] { "1" }))
+                return new AttemptOutcome(AttemptKind.StandbyRejected, status, null, "standby rejected before execution");
            RecipeDownloadResult? body = null;
            if (response.IsSuccessStatusCode)
            {
@@ -36,12 +41,14 @@ internal sealed class HttpRecipeSender(HttpClient http, string apiKey) : IRecipe
        }
        catch (HttpRequestException ex)
        {
-            return new AttemptOutcome(AttemptKind.ConnectFailed, 0, null, ex.Message);
+            var knownNotSent = ex.HttpRequestError == HttpRequestError.NameResolutionError
+                || ex.InnerException is System.Net.Sockets.SocketException { SocketErrorCode: System.Net.Sockets.SocketError.ConnectionRefused };
+            return new AttemptOutcome(knownNotSent ? AttemptKind.ConnectFailed : AttemptKind.OutcomeUnknown, 0, null, ex.Message);
        }
        catch (OperationCanceledException ex) when (!ct.IsCancellationRequested)
        {
-            // No external cancellation requested → this is an HttpClient.Timeout, i.e. the node never answered.
-            return new AttemptOutcome(AttemptKind.ConnectFailed, 0, null, "timeout: " + ex.Message);
+            // A missing response does not prove the node failed to apply the request.
+            return new AttemptOutcome(AttemptKind.OutcomeUnknown, 0, null, "timeout: " + ex.Message);
        }
    }

@@ -5,6 +5,8 @@ internal enum AttemptKind
 {
    Connected,
    ConnectFailed,
+    StandbyRejected,
+    OutcomeUnknown,
 }

 /// <summary>Result of one POST attempt against a single base URL.</summary>
@@ -4,14 +4,13 @@ namespace ZB.MOM.WW.ScadaBridge.DelmiaNotifier;
 internal sealed record NotifyResult(bool Ok, string Reason);

 /// <summary>
-/// Connect-failure-only failover loop: tries each base URL in order. A node that responds at all is
-/// authoritative — its answer is final (success, business rejection, or HTTP error alike). Only a
-/// failure to connect rolls over to the next URL; if every URL fails to connect the last error is reported.
+/// Tries another URL only after proven pre-execution connection failure or explicit standby
+/// rejection. Business responses are final; ambiguous response loss reports OUTCOME_UNKNOWN.
 /// </summary>
 internal static class Notifier
 {
    /// <summary>
-    /// Runs the connect-failure-only failover loop across <paramref name="baseUrls"/>,
+    /// Runs the pre-execution-only failover loop across <paramref name="baseUrls"/>,
    /// sending <paramref name="payload"/> via <paramref name="sender"/> until a node
    /// responds (connects) or every URL fails to connect.
    /// </summary>
@@ -29,12 +28,15 @@ internal static class Notifier
        {
            var outcome = await sender.SendAsync(baseUrl, payload, ct);

-            if (outcome.Kind == AttemptKind.ConnectFailed)
+            if (outcome.Kind is AttemptKind.ConnectFailed or AttemptKind.StandbyRejected)
            {
                lastError = outcome.Error ?? "connection failed";
                continue; // unreachable node → try the next
            }

+            if (outcome.Kind == AttemptKind.OutcomeUnknown)
+                return new NotifyResult(false, "OUTCOME_UNKNOWN: the request may have been applied; verify before retrying.");
+
            // Connected — this node's answer is authoritative; never fail over past it.
            if (IsSuccessStatus(outcome.StatusCode))
            {
@@ -54,12 +54,11 @@ WWNotifier.exe -m Z28061 -d "C:\recipes\job.nc" -w WO12345 -p PN-7788 -s 0100 -u

 ## Delivery semantics

-The `BaseUrls` failover loop is at-least-once, not exactly-once: a connection reset *after* the
-target already processed the POST, or a slow response that exceeds `TimeoutSeconds`, both look like
-an unreachable node to this client and cause a re-POST to the next URL — even though the first
-attempt may already have succeeded. The deployed `DelmiaRecipeDownload` inbound method must be
-idempotent on `(MachineCode, DownloadPath, WorkOrderNumber)` so a duplicate delivery is harmless;
-see the design doc's "Delivery semantics: at-least-once" section for details.
+The notifier retries only when a request is known not to have executed: connection refusal,
+name-resolution failure, or the server's explicit standby rejection. A connection reset after
+sending or a response timeout returns `OUTCOME_UNKNOWN` without replay. Operators must verify
+an uncertain result before manually retrying. Idempotent method implementations remain useful
+because callers can deliberately retry, but the notifier does not blindly repeat ambiguous sends.

 ## Configuration — `appsettings.json`

@@ -77,10 +76,9 @@ Placed next to the exe (copied to the output on build/publish):
 ```

 - **`BaseUrls`** — comma-separated failover list of **base URLs only**; the app appends
-  `/api/DelmiaRecipeDownload`. URLs are tried in order with **connect-failure-only failover**: a node
-  that responds at all (even `4xx`/`5xx`) is authoritative and its answer is final; only a node that
-  fails to connect (refused/reset, DNS, TLS, timeout) rolls over to the next. If every URL fails to
-  connect → `NO` + last connection error.
+  `/api/DelmiaRecipeDownload`. URLs are tried in order after proven DNS/connection refusal or
+  explicit standby rejection. Other HTTP responses are final; ambiguous transport errors return
+  `NO` + `OUTCOME_UNKNOWN` without trying another URL.
 - **`TimeoutSeconds`** — per-attempt HTTP timeout (default `30`).
 - **`LogPath`** — optional diagnostic log file, relative to the exe. Omit to log to stderr only.

@@ -150,7 +148,13 @@ bash docker/verify-delmia-notifier.sh     # runs the managed build from a scratc
 | A | active node first in `BaseUrls` | `YES` |
 | B | standby first (standby relays to the active) | `YES` |
 | C | bad key | `NO` + `HTTP 401` (an answer is final — no failover) |
-| D | active `docker kill`ed, attempt inside the takeover window | `NO` + `HTTP 503` — expected: connect-failure-only failover, the survivor is not yet active |
+| D | active `docker kill`ed, attempt inside the takeover window | historical baseline: `NO` + `HTTP 503`; the updated client retries explicit standby refusals and still fails if no node is active |
 | E | survivor `/health/active` = 200 (~27 s) | `YES` (first URL refused → second URL) |
 | F | victim restarted and rejoined as standby | `YES` (victim relays) |
 | G | `DownloadPath` ending in `.bad` | `NO` + `Simulated reject: bad path or missing work order` |
+
+### Safe failover
+
+The notifier advances to the next configured URL after a proven connection refusal/name-resolution failure, or HTTP 503 carrying the server's exact `X-ScadaBridge-Standby: 1` pre-execution marker. A generic 503 or business rejection is final. Both centrals may therefore be listed even when their internal standby-forwarding peer lists are absent.
+
+A timeout or other response loss is reported as `OUTCOME_UNKNOWN` with a failed result and no automatic replay: the first node may already have applied the download. Verify the operation before manually retrying. Configure trusted central endpoints; the standby marker is a server protocol contract, not permission to retry arbitrary HTTP errors.