fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable

SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA
else NodeB as primary (a NodeB-only site is no longer skipped) and carries a
distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial
the primary first and, on a transport fault (Unavailable/DeadlineExceeded/
Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to
NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail
over. Keeps the reconciliation loss-recovery net available during a NodeA outage.
This commit is contained in:
Joseph Doherty
2026-07-09 08:57:06 -04:00
parent 0385942c3f
commit bfd2cc7e6f
8 changed files with 382 additions and 100 deletions
@@ -76,6 +76,44 @@ public class SiteEnumeratorTests
Assert.Empty(result);
}
[Fact]
public async Task EnumerateAsync_BothAddresses_ProjectsNodeBAsFallback()
{
// Task 18: a site with both NodeA and a DISTINCT NodeB yields
// SiteEntry(primary=NodeA, FallbackGrpcEndpoint=NodeB).
var repository = Substitute.For<ISiteRepository>();
repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
{
SiteWith("site-ab", grpcNodeA: "http://site-a:8083", grpcNodeB: "http://site-b:8083"),
});
var enumerator = new SiteEnumerator(BuildProvider(repository));
var entry = Assert.Single(await enumerator.EnumerateAsync());
Assert.Equal("site-ab", entry.SiteId);
Assert.Equal("http://site-a:8083", entry.GrpcEndpoint);
Assert.Equal("http://site-b:8083", entry.FallbackGrpcEndpoint);
}
[Fact]
public async Task EnumerateAsync_OnlyNodeB_IsNotSkipped_AndHasNoFallback()
{
// Task 18: a site with a blank NodeA but a valid NodeB is no longer
// skipped — NodeB becomes the primary and there is no separate fallback.
var repository = Substitute.For<ISiteRepository>();
repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
{
SiteWith("site-bonly", grpcNodeA: " ", grpcNodeB: "http://site-b:8083"),
});
var enumerator = new SiteEnumerator(BuildProvider(repository));
var entry = Assert.Single(await enumerator.EnumerateAsync());
Assert.Equal("site-bonly", entry.SiteId);
Assert.Equal("http://site-b:8083", entry.GrpcEndpoint);
Assert.Null(entry.FallbackGrpcEndpoint);
}
[Fact]
public async Task EnumerateAsync_ReturnsEmpty_WhenNoSites()
{