fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable

SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA
else NodeB as primary (a NodeB-only site is no longer skipped) and carries a
distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial
the primary first and, on a transport fault (Unavailable/DeadlineExceeded/
Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to
NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail
over. Keeps the reconciliation loss-recovery net available during a NodeA outage.

tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/GrpcPullAuditEventsClientTests.cs

@@ -71,6 +71,26 @@ public class GrpcPullAuditEventsClientTests
         }
     }
 
+    /// <summary>
+    /// Per-endpoint fake invoker (Task 18): routes dials by endpoint string so a
+    /// NodeA-down / NodeB-up failover can be scripted. Records dialed endpoints.
+    /// </summary>
+    private sealed class PerEndpointInvoker : GrpcPullAuditEventsClient.IPullAuditEventsInvoker
+    {
+        private readonly Dictionary<string, Func<ProtoPullResponse>> _byEndpoint;
+        public List<string> Dialed { get; } = new();
+
+        public PerEndpointInvoker(Dictionary<string, Func<ProtoPullResponse>> byEndpoint) =>
+            _byEndpoint = byEndpoint;
+
+        public Task<ProtoPullResponse> InvokeAsync(
+            string endpoint, ProtoPullRequest request, CancellationToken ct)
+        {
+            Dialed.Add(endpoint);
+            return Task.FromResult(_byEndpoint[endpoint]());
+        }
+    }
+
     private static AuditEventDto Dto(Guid id, DateTime occurredAtUtc) =>
         AuditEventDtoMapper.ToDto(ScadaBridgeAuditEventFactory.Create(
             eventId: id,
@@ -212,4 +232,48 @@ public class GrpcPullAuditEventsClientTests
         Assert.Empty(result.Events);
         Assert.False(result.MoreAvailable);
     }
+
+    [Fact]
+    public async Task PullAsync_fails_over_to_NodeB_when_primary_raises_transport_fault()
+    {
+        // Task 18: NodeA (primary) is unreachable, NodeB (fallback) answers — the
+        // pull must return NodeB's events rather than collapsing to empty.
+        var id = Guid.NewGuid();
+        var fallbackResponse = new ProtoPullResponse { MoreAvailable = false };
+        fallbackResponse.Events.Add(Dto(id, BaseTime));
+
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+            ["http://node-b:8083"] = () => fallbackResponse,
+        });
+        var sut = new GrpcPullAuditEventsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083", "http://node-b:8083")),
+            invoker,
+            NullLogger<GrpcPullAuditEventsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
+
+        Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
+        var evt = Assert.Single(result.Events);
+        Assert.Equal(id, evt.EventId);
+    }
+
+    [Fact]
+    public async Task PullAsync_does_not_fail_over_when_no_fallback_configured()
+    {
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+        });
+        var sut = new GrpcPullAuditEventsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083")), // no fallback
+            invoker,
+            NullLogger<GrpcPullAuditEventsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
+
+        Assert.Empty(result.Events);
+        Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
+    }
 }

tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/GrpcPullSiteCallsClientTests.cs

@@ -63,6 +63,28 @@ public class GrpcPullSiteCallsClientTests
         }
     }
 
+    /// <summary>
+    /// Per-endpoint fake invoker (Task 18): dials are routed by endpoint string so
+    /// a NodeA-down / NodeB-up failover can be scripted. Records every dialed
+    /// endpoint in order.
+    /// </summary>
+    private sealed class PerEndpointInvoker : GrpcPullSiteCallsClient.IPullSiteCallsInvoker
+    {
+        private readonly Dictionary<string, Func<ProtoPullResponse>> _byEndpoint;
+        public List<string> Dialed { get; } = new();
+
+        public PerEndpointInvoker(Dictionary<string, Func<ProtoPullResponse>> byEndpoint) =>
+            _byEndpoint = byEndpoint;
+
+        public Task<ProtoPullResponse> InvokeAsync(
+            string endpoint, ProtoPullRequest request, CancellationToken ct)
+        {
+            Dialed.Add(endpoint);
+            // The Func may throw (transport fault) — the client's try/catch handles it.
+            return Task.FromResult(_byEndpoint[endpoint]());
+        }
+    }
+
     // The site leaves SourceSite empty (it is not a tracking-store column); the
     // client re-stamps it from the dialed siteId. Mint DTOs with empty SourceSite
     // to prove that re-stamp.
@@ -248,4 +270,52 @@ public class GrpcPullSiteCallsClientTests
         Assert.Empty(result.SiteCalls);
         Assert.False(result.MoreAvailable);
     }
+
+    [Fact]
+    public async Task PullAsync_fails_over_to_NodeB_when_primary_raises_transport_fault()
+    {
+        // Task 18: NodeA (primary) is unreachable, NodeB (fallback) answers — the
+        // pull must return NodeB's rows rather than collapsing to empty.
+        var id = Guid.NewGuid();
+        var fallbackResponse = new ProtoPullResponse();
+        fallbackResponse.Operationals.Add(Dto(id, BaseTime.AddMinutes(1)));
+
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+            ["http://node-b:8083"] = () => fallbackResponse,
+        });
+        var sut = new GrpcPullSiteCallsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083", "http://node-b:8083")),
+            invoker,
+            NullLogger<GrpcPullSiteCallsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
+
+        // Primary dialed first, then the NodeB fallback once.
+        Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
+        var row = Assert.Single(result.SiteCalls);
+        Assert.Equal(id, row.TrackedOperationId.Value);
+        Assert.Equal("site-a", row.SourceSite);
+    }
+
+    [Fact]
+    public async Task PullAsync_does_not_fail_over_when_no_fallback_configured()
+    {
+        // A transport fault with no NodeB fallback collapses to empty and dials
+        // only the primary once (no phantom second dial).
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+        });
+        var sut = new GrpcPullSiteCallsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083")), // no fallback
+            invoker,
+            NullLogger<GrpcPullSiteCallsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
+
+        Assert.Empty(result.SiteCalls);
+        Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
+    }
 }

tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/SiteEnumeratorTests.cs

@@ -76,6 +76,44 @@ public class SiteEnumeratorTests
         Assert.Empty(result);
     }
 
+    [Fact]
+    public async Task EnumerateAsync_BothAddresses_ProjectsNodeBAsFallback()
+    {
+        // Task 18: a site with both NodeA and a DISTINCT NodeB yields
+        // SiteEntry(primary=NodeA, FallbackGrpcEndpoint=NodeB).
+        var repository = Substitute.For<ISiteRepository>();
+        repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
+        {
+            SiteWith("site-ab", grpcNodeA: "http://site-a:8083", grpcNodeB: "http://site-b:8083"),
+        });
+
+        var enumerator = new SiteEnumerator(BuildProvider(repository));
+
+        var entry = Assert.Single(await enumerator.EnumerateAsync());
+        Assert.Equal("site-ab", entry.SiteId);
+        Assert.Equal("http://site-a:8083", entry.GrpcEndpoint);
+        Assert.Equal("http://site-b:8083", entry.FallbackGrpcEndpoint);
+    }
+
+    [Fact]
+    public async Task EnumerateAsync_OnlyNodeB_IsNotSkipped_AndHasNoFallback()
+    {
+        // Task 18: a site with a blank NodeA but a valid NodeB is no longer
+        // skipped — NodeB becomes the primary and there is no separate fallback.
+        var repository = Substitute.For<ISiteRepository>();
+        repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
+        {
+            SiteWith("site-bonly", grpcNodeA: "   ", grpcNodeB: "http://site-b:8083"),
+        });
+
+        var enumerator = new SiteEnumerator(BuildProvider(repository));
+
+        var entry = Assert.Single(await enumerator.EnumerateAsync());
+        Assert.Equal("site-bonly", entry.SiteId);
+        Assert.Equal("http://site-b:8083", entry.GrpcEndpoint);
+        Assert.Null(entry.FallbackGrpcEndpoint);
+    }
+
     [Fact]
     public async Task EnumerateAsync_ReturnsEmpty_WhenNoSites()
     {