fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable

SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA else NodeB as primary (a NodeB-only site is no longer skipped) and carries a distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial the primary first and, on a transport fault (Unavailable/DeadlineExceeded/ Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail over. Keeps the reconciliation loss-recovery net available during a NodeA outage.
2026-07-09 08:57:06 -04:00
parent 0385942c3f
commit bfd2cc7e6f
8 changed files with 382 additions and 100 deletions
@@ -71,6 +71,26 @@ public class GrpcPullAuditEventsClientTests
        }
    }

+    /// <summary>
+    /// Per-endpoint fake invoker (Task 18): routes dials by endpoint string so a
+    /// NodeA-down / NodeB-up failover can be scripted. Records dialed endpoints.
+    /// </summary>
+    private sealed class PerEndpointInvoker : GrpcPullAuditEventsClient.IPullAuditEventsInvoker
+    {
+        private readonly Dictionary<string, Func<ProtoPullResponse>> _byEndpoint;
+        public List<string> Dialed { get; } = new();
+
+        public PerEndpointInvoker(Dictionary<string, Func<ProtoPullResponse>> byEndpoint) =>
+            _byEndpoint = byEndpoint;
+
+        public Task<ProtoPullResponse> InvokeAsync(
+            string endpoint, ProtoPullRequest request, CancellationToken ct)
+        {
+            Dialed.Add(endpoint);
+            return Task.FromResult(_byEndpoint[endpoint]());
+        }
+    }
+
    private static AuditEventDto Dto(Guid id, DateTime occurredAtUtc) =>
        AuditEventDtoMapper.ToDto(ScadaBridgeAuditEventFactory.Create(
            eventId: id,
@@ -212,4 +232,48 @@ public class GrpcPullAuditEventsClientTests
        Assert.Empty(result.Events);
        Assert.False(result.MoreAvailable);
    }
+
+    [Fact]
+    public async Task PullAsync_fails_over_to_NodeB_when_primary_raises_transport_fault()
+    {
+        // Task 18: NodeA (primary) is unreachable, NodeB (fallback) answers — the
+        // pull must return NodeB's events rather than collapsing to empty.
+        var id = Guid.NewGuid();
+        var fallbackResponse = new ProtoPullResponse { MoreAvailable = false };
+        fallbackResponse.Events.Add(Dto(id, BaseTime));
+
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+            ["http://node-b:8083"] = () => fallbackResponse,
+        });
+        var sut = new GrpcPullAuditEventsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083", "http://node-b:8083")),
+            invoker,
+            NullLogger<GrpcPullAuditEventsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
+
+        Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
+        var evt = Assert.Single(result.Events);
+        Assert.Equal(id, evt.EventId);
+    }
+
+    [Fact]
+    public async Task PullAsync_does_not_fail_over_when_no_fallback_configured()
+    {
+        var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
+        {
+            ["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
+        });
+        var sut = new GrpcPullAuditEventsClient(
+            new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083")), // no fallback
+            invoker,
+            NullLogger<GrpcPullAuditEventsClient>.Instance);
+
+        var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
+
+        Assert.Empty(result.Events);
+        Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
+    }
 }