fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable

SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA
else NodeB as primary (a NodeB-only site is no longer skipped) and carries a
distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial
the primary first and, on a transport fault (Unavailable/DeadlineExceeded/
Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to
NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail
over. Keeps the reconciliation loss-recovery net available during a NodeA outage.

src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/GrpcPullAuditEventsClient.cs

@@ -76,7 +76,7 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
         int batchSize,
         CancellationToken ct)
     {
-        var endpoint = await ResolveEndpointAsync(siteId, ct).ConfigureAwait(false);
+        var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false);
         if (endpoint is null)
         {
             // No gRPC address registered for the site — absence of an address is
@@ -95,44 +95,27 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
             BatchSize = batchSize,
         };
 
-        ProtoPullResponse reply;
-        try
+        var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct)
+            .ConfigureAwait(false);
+
+        // NodeB failover (Task 18): a transport fault against the primary (NodeA)
+        // dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA
+        // outage doesn't take the reconciliation loss-recovery net offline. Only
+        // the transport-fault set triggers this (not a mapping/unexpected fault),
+        // and not when the caller's token is already cancelled (host shutdown).
+        if (reply is null && transportFault
+            && !string.IsNullOrWhiteSpace(fallback)
+            && !string.Equals(fallback, endpoint, StringComparison.Ordinal)
+            && !ct.IsCancellationRequested)
         {
-            reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
+            _logger.LogInformation(
+                "PullAuditEvents failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.",
+                endpoint, fallback, siteId);
+            (reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false);
         }
-        catch (RpcException ex) when (IsTolerable(ex.StatusCode))
+
+        if (reply is null)
         {
-            _logger.LogDebug(ex,
-                "PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}. Returning empty batch.",
-                siteId, endpoint, ex.StatusCode);
-            return Empty;
-        }
-        catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
-        {
-            _logger.LogDebug(ex,
-                "PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}). Returning empty batch.",
-                siteId, endpoint);
-            return Empty;
-        }
-        catch (OperationCanceledException)
-        {
-            // Reconciliation tick was cancelled — either the caller's token
-            // (host shutdown / scope dispose) or an internal gRPC deadline /
-            // linked-CTS cancellation. Both are tolerable for a best-effort
-            // pull; collapse to empty rather than letting an internal
-            // cancellation land noisily in the catch-all below.
-            return Empty;
-        }
-        catch (Exception ex)
-        {
-            // Any other fault (e.g. a malformed reply that fails DTO mapping
-            // below would actually surface here only if mapping moved inline,
-            // but a non-RpcException transport fault wrapper lands here too).
-            // Audit reconciliation is best-effort; swallow to empty rather than
-            // throw — the actor's per-site guard would only re-catch it.
-            _logger.LogWarning(ex,
-                "PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
-                siteId, endpoint);
             return Empty;
         }
 
@@ -147,7 +130,60 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
         return new PullAuditEventsResponse(events, reply.MoreAvailable);
     }
 
-    private async Task<string?> ResolveEndpointAsync(string siteId, CancellationToken ct)
+    /// <summary>
+    /// Issues one pull against <paramref name="endpoint"/>, classifying faults.
+    /// Returns <c>(reply, false)</c> on success; <c>(null, true)</c> on a
+    /// tolerable TRANSPORT fault (the failover-eligible set:
+    /// <see cref="StatusCode.Unavailable"/> / <see cref="StatusCode.DeadlineExceeded"/> /
+    /// <see cref="StatusCode.Cancelled"/> / <see cref="HttpRequestException"/> /
+    /// <c>SocketException</c> / <see cref="OperationCanceledException"/>); and
+    /// <c>(null, false)</c> on any other (mapping/unexpected) fault, which
+    /// collapses to empty WITHOUT a fallback dial.
+    /// </summary>
+    private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync(
+        string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct)
+    {
+        try
+        {
+            var reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
+            return (reply, false);
+        }
+        catch (RpcException ex) when (IsTolerable(ex.StatusCode))
+        {
+            _logger.LogDebug(ex,
+                "PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.",
+                siteId, endpoint, ex.StatusCode);
+            return (null, true);
+        }
+        catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
+        {
+            _logger.LogDebug(ex,
+                "PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}).",
+                siteId, endpoint);
+            return (null, true);
+        }
+        catch (OperationCanceledException)
+        {
+            // Reconciliation tick was cancelled — either the caller's token
+            // (host shutdown / scope dispose) or an internal gRPC deadline /
+            // linked-CTS cancellation. Tolerable for a best-effort pull.
+            return (null, true);
+        }
+        catch (Exception ex)
+        {
+            // Any other fault (e.g. a malformed reply that fails DTO mapping).
+            // Audit reconciliation is best-effort; swallow to empty rather than
+            // throw — and do NOT fail over, since a second node would hit the
+            // same non-transport fault.
+            _logger.LogWarning(ex,
+                "PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
+                siteId, endpoint);
+            return (null, false);
+        }
+    }
+
+    private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync(
+        string siteId, CancellationToken ct)
     {
         var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false);
         foreach (var site in sites)
@@ -155,10 +191,10 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
             if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) &&
                 !string.IsNullOrWhiteSpace(site.GrpcEndpoint))
             {
-                return site.GrpcEndpoint;
+                return (site.GrpcEndpoint, site.FallbackGrpcEndpoint);
             }
         }
-        return null;
+        return (null, null);
     }
 
     private static readonly PullAuditEventsResponse Empty =