fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable

SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA
else NodeB as primary (a NodeB-only site is no longer skipped) and carries a
distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial
the primary first and, on a transport fault (Unavailable/DeadlineExceeded/
Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to
NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail
over. Keeps the reconciliation loss-recovery net available during a NodeA outage.

src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/GrpcPullSiteCallsClient.cs

@@ -84,7 +84,7 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
         int batchSize,
         CancellationToken ct)
     {
-        var endpoint = await ResolveEndpointAsync(siteId, ct).ConfigureAwait(false);
+        var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false);
         if (endpoint is null)
         {
             // No gRPC address registered for the site — a configuration decision
@@ -107,41 +107,27 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
             AfterId = afterId ?? string.Empty,
         };
 
-        ProtoPullResponse reply;
-        try
+        var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct)
+            .ConfigureAwait(false);
+
+        // NodeB failover (Task 18): a transport fault against the primary (NodeA)
+        // dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA
+        // outage doesn't take the reconciliation loss-recovery net offline. Only
+        // the transport-fault set triggers this (not a mapping/unexpected fault),
+        // and not when the caller's token is already cancelled (host shutdown).
+        if (reply is null && transportFault
+            && !string.IsNullOrWhiteSpace(fallback)
+            && !string.Equals(fallback, endpoint, StringComparison.Ordinal)
+            && !ct.IsCancellationRequested)
         {
-            reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
+            _logger.LogInformation(
+                "PullSiteCalls failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.",
+                endpoint, fallback, siteId);
+            (reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false);
         }
-        catch (RpcException ex) when (IsTolerable(ex.StatusCode))
+
+        if (reply is null)
         {
-            _logger.LogDebug(ex,
-                "PullSiteCalls tolerable transport fault for site {SiteId} ({Endpoint}): {Status}. Returning empty batch.",
-                siteId, endpoint, ex.StatusCode);
-            return Empty;
-        }
-        catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
-        {
-            _logger.LogDebug(ex,
-                "PullSiteCalls connection-layer fault for site {SiteId} ({Endpoint}). Returning empty batch.",
-                siteId, endpoint);
-            return Empty;
-        }
-        catch (OperationCanceledException)
-        {
-            // Reconciliation tick cancelled — caller token (host shutdown) or an
-            // internal gRPC deadline / linked-CTS cancellation. Both tolerable for
-            // a best-effort pull; collapse to empty rather than landing noisily in
-            // the catch-all below.
-            return Empty;
-        }
-        catch (Exception ex)
-        {
-            // Any other fault. Reconciliation is best-effort; swallow to empty
-            // rather than throw — the (future) actor's per-site guard would only
-            // re-catch it.
-            _logger.LogWarning(ex,
-                "PullSiteCalls unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
-                siteId, endpoint);
             return Empty;
         }
 
@@ -173,7 +159,59 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
         return new PullSiteCallsResponse(siteCalls, reply.MoreAvailable);
     }
 
-    private async Task<string?> ResolveEndpointAsync(string siteId, CancellationToken ct)
+    /// <summary>
+    /// Issues one pull against <paramref name="endpoint"/>, classifying faults.
+    /// Returns <c>(reply, false)</c> on success; <c>(null, true)</c> on a
+    /// tolerable TRANSPORT fault (the failover-eligible set:
+    /// <see cref="StatusCode.Unavailable"/> / <see cref="StatusCode.DeadlineExceeded"/> /
+    /// <see cref="StatusCode.Cancelled"/> / <see cref="HttpRequestException"/> /
+    /// <c>SocketException</c> / <see cref="OperationCanceledException"/>); and
+    /// <c>(null, false)</c> on any other (mapping/unexpected) fault, which
+    /// collapses to empty WITHOUT a fallback dial.
+    /// </summary>
+    private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync(
+        string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct)
+    {
+        try
+        {
+            var reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
+            return (reply, false);
+        }
+        catch (RpcException ex) when (IsTolerable(ex.StatusCode))
+        {
+            _logger.LogDebug(ex,
+                "PullSiteCalls tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.",
+                siteId, endpoint, ex.StatusCode);
+            return (null, true);
+        }
+        catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
+        {
+            _logger.LogDebug(ex,
+                "PullSiteCalls connection-layer fault for site {SiteId} ({Endpoint}).",
+                siteId, endpoint);
+            return (null, true);
+        }
+        catch (OperationCanceledException)
+        {
+            // Reconciliation tick cancelled — caller token (host shutdown) or an
+            // internal gRPC deadline / linked-CTS cancellation. Tolerable for a
+            // best-effort pull.
+            return (null, true);
+        }
+        catch (Exception ex)
+        {
+            // Any other fault. Reconciliation is best-effort; swallow to empty
+            // rather than throw — and do NOT fail over, since a second node would
+            // hit the same non-transport fault.
+            _logger.LogWarning(ex,
+                "PullSiteCalls unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
+                siteId, endpoint);
+            return (null, false);
+        }
+    }
+
+    private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync(
+        string siteId, CancellationToken ct)
     {
         var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false);
         foreach (var site in sites)
@@ -181,10 +219,10 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
             if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) &&
                 !string.IsNullOrWhiteSpace(site.GrpcEndpoint))
             {
-                return site.GrpcEndpoint;
+                return (site.GrpcEndpoint, site.FallbackGrpcEndpoint);
             }
         }
-        return null;
+        return (null, null);
     }
 
     private static readonly PullSiteCallsResponse Empty =