fix(audit-log): reconciliation pulls fail over to site NodeB when NodeA is unreachable
SiteEntry gains an additive FallbackGrpcEndpoint; SiteEnumerator now picks NodeA
else NodeB as primary (a NodeB-only site is no longer skipped) and carries a
distinct NodeB as the fallback. Both GrpcPull{AuditEvents,SiteCalls} clients dial
the primary first and, on a transport fault (Unavailable/DeadlineExceeded/
Cancelled/HttpRequestException/SocketException/OperationCanceled), fail over to
NodeB once before collapsing to empty — a mapping/unexpected fault does NOT fail
over. Keeps the reconciliation loss-recovery net available during a NodeA outage.
This commit is contained in:
@@ -279,6 +279,18 @@ non-draining (e.g., telemetry actor wedged), central issues a
|
||||
are flipped to `ForwardState = 'Reconciled'` site-side. Same self-healing
|
||||
pattern as Site Call Audit's reconciliation of `SiteCalls`.
|
||||
|
||||
**Endpoint resolution & NodeB failover.** Each pull dials the site's `NodeA`
|
||||
gRPC address first; if `NodeA` is blank the site's `NodeB` address becomes the
|
||||
primary (a NodeB-only site is no longer skipped — only a site with *both*
|
||||
addresses blank is). When a distinct `NodeB` address also exists it is carried
|
||||
as a fallback: a transport fault against the primary (site offline, deadline
|
||||
exceeded, cancelled, connection error) fails over to `NodeB` ONCE before
|
||||
collapsing to an empty batch. So during a NodeA outage the loss-recovery net
|
||||
stays available. The failover is scoped to transport faults — a
|
||||
mapping/unexpected fault collapses to empty without a second dial (the other
|
||||
node would hit the same fault). The same resolution + failover applies to Site
|
||||
Call Audit's `PullSiteCalls`.
|
||||
|
||||
> **Cursor keyset (tracked follow-up).** The `PullAuditEvents` cursor is still a
|
||||
> single `sinceUtc` timestamp. Site Call Audit's pull now uses a composite
|
||||
> `(UpdatedAtUtc, TrackedOperationId)` keyset to avoid a single-timestamp pin
|
||||
|
||||
@@ -76,7 +76,7 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
|
||||
int batchSize,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var endpoint = await ResolveEndpointAsync(siteId, ct).ConfigureAwait(false);
|
||||
var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false);
|
||||
if (endpoint is null)
|
||||
{
|
||||
// No gRPC address registered for the site — absence of an address is
|
||||
@@ -95,44 +95,27 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
|
||||
BatchSize = batchSize,
|
||||
};
|
||||
|
||||
ProtoPullResponse reply;
|
||||
try
|
||||
var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// NodeB failover (Task 18): a transport fault against the primary (NodeA)
|
||||
// dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA
|
||||
// outage doesn't take the reconciliation loss-recovery net offline. Only
|
||||
// the transport-fault set triggers this (not a mapping/unexpected fault),
|
||||
// and not when the caller's token is already cancelled (host shutdown).
|
||||
if (reply is null && transportFault
|
||||
&& !string.IsNullOrWhiteSpace(fallback)
|
||||
&& !string.Equals(fallback, endpoint, StringComparison.Ordinal)
|
||||
&& !ct.IsCancellationRequested)
|
||||
{
|
||||
reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"PullAuditEvents failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.",
|
||||
endpoint, fallback, siteId);
|
||||
(reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RpcException ex) when (IsTolerable(ex.StatusCode))
|
||||
|
||||
if (reply is null)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}. Returning empty batch.",
|
||||
siteId, endpoint, ex.StatusCode);
|
||||
return Empty;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return Empty;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Reconciliation tick was cancelled — either the caller's token
|
||||
// (host shutdown / scope dispose) or an internal gRPC deadline /
|
||||
// linked-CTS cancellation. Both are tolerable for a best-effort
|
||||
// pull; collapse to empty rather than letting an internal
|
||||
// cancellation land noisily in the catch-all below.
|
||||
return Empty;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Any other fault (e.g. a malformed reply that fails DTO mapping
|
||||
// below would actually surface here only if mapping moved inline,
|
||||
// but a non-RpcException transport fault wrapper lands here too).
|
||||
// Audit reconciliation is best-effort; swallow to empty rather than
|
||||
// throw — the actor's per-site guard would only re-catch it.
|
||||
_logger.LogWarning(ex,
|
||||
"PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return Empty;
|
||||
}
|
||||
|
||||
@@ -147,7 +130,60 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
|
||||
return new PullAuditEventsResponse(events, reply.MoreAvailable);
|
||||
}
|
||||
|
||||
private async Task<string?> ResolveEndpointAsync(string siteId, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Issues one pull against <paramref name="endpoint"/>, classifying faults.
|
||||
/// Returns <c>(reply, false)</c> on success; <c>(null, true)</c> on a
|
||||
/// tolerable TRANSPORT fault (the failover-eligible set:
|
||||
/// <see cref="StatusCode.Unavailable"/> / <see cref="StatusCode.DeadlineExceeded"/> /
|
||||
/// <see cref="StatusCode.Cancelled"/> / <see cref="HttpRequestException"/> /
|
||||
/// <c>SocketException</c> / <see cref="OperationCanceledException"/>); and
|
||||
/// <c>(null, false)</c> on any other (mapping/unexpected) fault, which
|
||||
/// collapses to empty WITHOUT a fallback dial.
|
||||
/// </summary>
|
||||
private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync(
|
||||
string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
|
||||
return (reply, false);
|
||||
}
|
||||
catch (RpcException ex) when (IsTolerable(ex.StatusCode))
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.",
|
||||
siteId, endpoint, ex.StatusCode);
|
||||
return (null, true);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}).",
|
||||
siteId, endpoint);
|
||||
return (null, true);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Reconciliation tick was cancelled — either the caller's token
|
||||
// (host shutdown / scope dispose) or an internal gRPC deadline /
|
||||
// linked-CTS cancellation. Tolerable for a best-effort pull.
|
||||
return (null, true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Any other fault (e.g. a malformed reply that fails DTO mapping).
|
||||
// Audit reconciliation is best-effort; swallow to empty rather than
|
||||
// throw — and do NOT fail over, since a second node would hit the
|
||||
// same non-transport fault.
|
||||
_logger.LogWarning(ex,
|
||||
"PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return (null, false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync(
|
||||
string siteId, CancellationToken ct)
|
||||
{
|
||||
var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false);
|
||||
foreach (var site in sites)
|
||||
@@ -155,10 +191,10 @@ public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
|
||||
if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) &&
|
||||
!string.IsNullOrWhiteSpace(site.GrpcEndpoint))
|
||||
{
|
||||
return site.GrpcEndpoint;
|
||||
return (site.GrpcEndpoint, site.FallbackGrpcEndpoint);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return (null, null);
|
||||
}
|
||||
|
||||
private static readonly PullAuditEventsResponse Empty =
|
||||
|
||||
@@ -84,7 +84,7 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
int batchSize,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var endpoint = await ResolveEndpointAsync(siteId, ct).ConfigureAwait(false);
|
||||
var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false);
|
||||
if (endpoint is null)
|
||||
{
|
||||
// No gRPC address registered for the site — a configuration decision
|
||||
@@ -107,41 +107,27 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
AfterId = afterId ?? string.Empty,
|
||||
};
|
||||
|
||||
ProtoPullResponse reply;
|
||||
try
|
||||
var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// NodeB failover (Task 18): a transport fault against the primary (NodeA)
|
||||
// dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA
|
||||
// outage doesn't take the reconciliation loss-recovery net offline. Only
|
||||
// the transport-fault set triggers this (not a mapping/unexpected fault),
|
||||
// and not when the caller's token is already cancelled (host shutdown).
|
||||
if (reply is null && transportFault
|
||||
&& !string.IsNullOrWhiteSpace(fallback)
|
||||
&& !string.Equals(fallback, endpoint, StringComparison.Ordinal)
|
||||
&& !ct.IsCancellationRequested)
|
||||
{
|
||||
reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"PullSiteCalls failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.",
|
||||
endpoint, fallback, siteId);
|
||||
(reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RpcException ex) when (IsTolerable(ex.StatusCode))
|
||||
|
||||
if (reply is null)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullSiteCalls tolerable transport fault for site {SiteId} ({Endpoint}): {Status}. Returning empty batch.",
|
||||
siteId, endpoint, ex.StatusCode);
|
||||
return Empty;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullSiteCalls connection-layer fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return Empty;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Reconciliation tick cancelled — caller token (host shutdown) or an
|
||||
// internal gRPC deadline / linked-CTS cancellation. Both tolerable for
|
||||
// a best-effort pull; collapse to empty rather than landing noisily in
|
||||
// the catch-all below.
|
||||
return Empty;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Any other fault. Reconciliation is best-effort; swallow to empty
|
||||
// rather than throw — the (future) actor's per-site guard would only
|
||||
// re-catch it.
|
||||
_logger.LogWarning(ex,
|
||||
"PullSiteCalls unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return Empty;
|
||||
}
|
||||
|
||||
@@ -173,7 +159,59 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
return new PullSiteCallsResponse(siteCalls, reply.MoreAvailable);
|
||||
}
|
||||
|
||||
private async Task<string?> ResolveEndpointAsync(string siteId, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Issues one pull against <paramref name="endpoint"/>, classifying faults.
|
||||
/// Returns <c>(reply, false)</c> on success; <c>(null, true)</c> on a
|
||||
/// tolerable TRANSPORT fault (the failover-eligible set:
|
||||
/// <see cref="StatusCode.Unavailable"/> / <see cref="StatusCode.DeadlineExceeded"/> /
|
||||
/// <see cref="StatusCode.Cancelled"/> / <see cref="HttpRequestException"/> /
|
||||
/// <c>SocketException</c> / <see cref="OperationCanceledException"/>); and
|
||||
/// <c>(null, false)</c> on any other (mapping/unexpected) fault, which
|
||||
/// collapses to empty WITHOUT a fallback dial.
|
||||
/// </summary>
|
||||
private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync(
|
||||
string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reply = await _invoker.InvokeAsync(endpoint, request, ct).ConfigureAwait(false);
|
||||
return (reply, false);
|
||||
}
|
||||
catch (RpcException ex) when (IsTolerable(ex.StatusCode))
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullSiteCalls tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.",
|
||||
siteId, endpoint, ex.StatusCode);
|
||||
return (null, true);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"PullSiteCalls connection-layer fault for site {SiteId} ({Endpoint}).",
|
||||
siteId, endpoint);
|
||||
return (null, true);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Reconciliation tick cancelled — caller token (host shutdown) or an
|
||||
// internal gRPC deadline / linked-CTS cancellation. Tolerable for a
|
||||
// best-effort pull.
|
||||
return (null, true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Any other fault. Reconciliation is best-effort; swallow to empty
|
||||
// rather than throw — and do NOT fail over, since a second node would
|
||||
// hit the same non-transport fault.
|
||||
_logger.LogWarning(ex,
|
||||
"PullSiteCalls unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
|
||||
siteId, endpoint);
|
||||
return (null, false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync(
|
||||
string siteId, CancellationToken ct)
|
||||
{
|
||||
var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false);
|
||||
foreach (var site in sites)
|
||||
@@ -181,10 +219,10 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) &&
|
||||
!string.IsNullOrWhiteSpace(site.GrpcEndpoint))
|
||||
{
|
||||
return site.GrpcEndpoint;
|
||||
return (site.GrpcEndpoint, site.FallbackGrpcEndpoint);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return (null, null);
|
||||
}
|
||||
|
||||
private static readonly PullSiteCallsResponse Empty =
|
||||
|
||||
@@ -8,13 +8,13 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The production implementation wraps <c>ISiteRepository.GetAllSitesAsync</c>
|
||||
/// and projects each <c>Site</c> to a <see cref="SiteEntry"/> using the
|
||||
/// site's configured <c>GrpcNodeAAddress</c>. This is a NodeA-only first cut:
|
||||
/// sites with a blank <c>GrpcNodeAAddress</c> are silently SKIPPED — the
|
||||
/// reconciliation pull cannot reach them, but absence of an address is a
|
||||
/// configuration decision, not a runtime error. NodeB-fallback endpoint
|
||||
/// selection (dial NodeB when NodeA is unset/unreachable) is a follow-up
|
||||
/// (mirrors the comment in <c>SiteEnumerator.cs</c>).
|
||||
/// and projects each <c>Site</c> to a <see cref="SiteEntry"/>. The primary dial
|
||||
/// target is <c>GrpcNodeAAddress</c> when set, otherwise <c>GrpcNodeBAddress</c>;
|
||||
/// when both a distinct NodeA and NodeB address exist, NodeB rides along as
|
||||
/// <see cref="SiteEntry.FallbackGrpcEndpoint"/> so the reconciliation pull can
|
||||
/// fail over per call (Task 18). A site is skipped only when BOTH addresses are
|
||||
/// blank — the reconciliation pull cannot reach it, but absence of any address
|
||||
/// is a configuration decision, not a runtime error.
|
||||
/// </remarks>
|
||||
public interface ISiteEnumerator
|
||||
{
|
||||
@@ -30,8 +30,19 @@ public interface ISiteEnumerator
|
||||
|
||||
/// <summary>
|
||||
/// One reconciliation target: the site identifier the actor uses as the
|
||||
/// cursor key and the gRPC endpoint <see cref="IPullAuditEventsClient"/> dials
|
||||
/// to issue the pull. Endpoint is the bare authority (e.g. <c>http://siteA:8083</c>);
|
||||
/// transport selection (TLS, keepalive, etc.) is the client's concern.
|
||||
/// cursor key and the gRPC endpoint the pull clients dial to issue the pull.
|
||||
/// Endpoints are bare authorities (e.g. <c>http://siteA:8083</c>); transport
|
||||
/// selection (TLS, keepalive, etc.) is the client's concern.
|
||||
/// </summary>
|
||||
public sealed record SiteEntry(string SiteId, string GrpcEndpoint);
|
||||
/// <param name="SiteId">The site identifier used as the reconciliation cursor key.</param>
|
||||
/// <param name="GrpcEndpoint">The primary gRPC authority to dial (NodeA where configured, else NodeB).</param>
|
||||
/// <param name="FallbackGrpcEndpoint">
|
||||
/// Optional secondary gRPC authority (NodeB) dialed once when the primary raises
|
||||
/// a transport fault during a pull (Task 18). Null when no distinct second
|
||||
/// address is configured. Additive — absent value preserves the prior
|
||||
/// single-endpoint behaviour.
|
||||
/// </param>
|
||||
public sealed record SiteEntry(
|
||||
string SiteId,
|
||||
string GrpcEndpoint,
|
||||
string? FallbackGrpcEndpoint = null);
|
||||
|
||||
@@ -21,17 +21,18 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// per-tick scope pattern in <see cref="SiteAuditReconciliationActor.OnTickAsync"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Blank-address skip.</b> Sites with no <c>GrpcNodeAAddress</c> configured
|
||||
/// are silently skipped: the reconciliation pull cannot dial them, but absence
|
||||
/// of an address is a configuration decision, not a runtime error (per the
|
||||
/// <b>Blank-address skip.</b> A site is skipped only when BOTH
|
||||
/// <c>GrpcNodeAAddress</c> and <c>GrpcNodeBAddress</c> are blank: the
|
||||
/// reconciliation pull cannot dial it, but absence of any address is a
|
||||
/// configuration decision, not a runtime error (per the
|
||||
/// <see cref="ISiteEnumerator"/> contract).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>NodeA-only first cut.</b> This implementation always uses NodeA's gRPC
|
||||
/// address. NodeA/NodeB failover endpoint selection (dial NodeB when NodeA is
|
||||
/// unreachable) is a follow-up — the <see cref="SiteEntry"/> shape already
|
||||
/// carries a single endpoint, so failover will live in the puller/client, not
|
||||
/// here.
|
||||
/// <b>NodeA/NodeB failover (Task 18).</b> The primary dial target is NodeA when
|
||||
/// configured, otherwise NodeB (a site with only NodeB is no longer skipped).
|
||||
/// When a distinct NodeB address also exists it rides along as
|
||||
/// <see cref="SiteEntry.FallbackGrpcEndpoint"/> so the pull clients can fail over
|
||||
/// to it once per call when the primary raises a transport fault.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SiteEnumerator : ISiteEnumerator
|
||||
@@ -60,16 +61,28 @@ public sealed class SiteEnumerator : ISiteEnumerator
|
||||
var entries = new List<SiteEntry>(sites.Count);
|
||||
foreach (var site in sites)
|
||||
{
|
||||
// First cut: NodeA's gRPC address is the dial target. NodeA/NodeB
|
||||
// failover endpoint selection is a follow-up.
|
||||
if (string.IsNullOrWhiteSpace(site.GrpcNodeAAddress))
|
||||
// Primary dial target: NodeA when configured, otherwise NodeB (Task 18) —
|
||||
// a site with only a NodeB address is no longer skipped.
|
||||
var nodeA = site.GrpcNodeAAddress;
|
||||
var nodeB = site.GrpcNodeBAddress;
|
||||
var primary = !string.IsNullOrWhiteSpace(nodeA) ? nodeA : nodeB;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(primary))
|
||||
{
|
||||
// Both addresses blank — nothing to dial (a configuration
|
||||
// decision, not a runtime error).
|
||||
continue;
|
||||
}
|
||||
|
||||
// The IsNullOrWhiteSpace guard above proves GrpcNodeAAddress is
|
||||
// non-null here; explicit null-forgiving for clarity.
|
||||
entries.Add(new SiteEntry(site.SiteIdentifier, site.GrpcNodeAAddress!));
|
||||
// NodeB rides along as the failover endpoint only when it is a
|
||||
// distinct, non-blank address (i.e. NodeA is the primary and NodeB
|
||||
// differs); when NodeB IS the primary there is no separate fallback.
|
||||
var fallback =
|
||||
!string.IsNullOrWhiteSpace(nodeB) && !string.Equals(nodeB, primary, StringComparison.Ordinal)
|
||||
? nodeB
|
||||
: null;
|
||||
|
||||
entries.Add(new SiteEntry(site.SiteIdentifier, primary!, fallback));
|
||||
}
|
||||
|
||||
return entries;
|
||||
|
||||
@@ -71,6 +71,26 @@ public class GrpcPullAuditEventsClientTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-endpoint fake invoker (Task 18): routes dials by endpoint string so a
|
||||
/// NodeA-down / NodeB-up failover can be scripted. Records dialed endpoints.
|
||||
/// </summary>
|
||||
private sealed class PerEndpointInvoker : GrpcPullAuditEventsClient.IPullAuditEventsInvoker
|
||||
{
|
||||
private readonly Dictionary<string, Func<ProtoPullResponse>> _byEndpoint;
|
||||
public List<string> Dialed { get; } = new();
|
||||
|
||||
public PerEndpointInvoker(Dictionary<string, Func<ProtoPullResponse>> byEndpoint) =>
|
||||
_byEndpoint = byEndpoint;
|
||||
|
||||
public Task<ProtoPullResponse> InvokeAsync(
|
||||
string endpoint, ProtoPullRequest request, CancellationToken ct)
|
||||
{
|
||||
Dialed.Add(endpoint);
|
||||
return Task.FromResult(_byEndpoint[endpoint]());
|
||||
}
|
||||
}
|
||||
|
||||
private static AuditEventDto Dto(Guid id, DateTime occurredAtUtc) =>
|
||||
AuditEventDtoMapper.ToDto(ScadaBridgeAuditEventFactory.Create(
|
||||
eventId: id,
|
||||
@@ -212,4 +232,48 @@ public class GrpcPullAuditEventsClientTests
|
||||
Assert.Empty(result.Events);
|
||||
Assert.False(result.MoreAvailable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAsync_fails_over_to_NodeB_when_primary_raises_transport_fault()
|
||||
{
|
||||
// Task 18: NodeA (primary) is unreachable, NodeB (fallback) answers — the
|
||||
// pull must return NodeB's events rather than collapsing to empty.
|
||||
var id = Guid.NewGuid();
|
||||
var fallbackResponse = new ProtoPullResponse { MoreAvailable = false };
|
||||
fallbackResponse.Events.Add(Dto(id, BaseTime));
|
||||
|
||||
var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
|
||||
{
|
||||
["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
|
||||
["http://node-b:8083"] = () => fallbackResponse,
|
||||
});
|
||||
var sut = new GrpcPullAuditEventsClient(
|
||||
new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083", "http://node-b:8083")),
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
|
||||
var evt = Assert.Single(result.Events);
|
||||
Assert.Equal(id, evt.EventId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAsync_does_not_fail_over_when_no_fallback_configured()
|
||||
{
|
||||
var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
|
||||
{
|
||||
["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
|
||||
});
|
||||
var sut = new GrpcPullAuditEventsClient(
|
||||
new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083")), // no fallback
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,28 @@ public class GrpcPullSiteCallsClientTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-endpoint fake invoker (Task 18): dials are routed by endpoint string so
|
||||
/// a NodeA-down / NodeB-up failover can be scripted. Records every dialed
|
||||
/// endpoint in order.
|
||||
/// </summary>
|
||||
private sealed class PerEndpointInvoker : GrpcPullSiteCallsClient.IPullSiteCallsInvoker
|
||||
{
|
||||
private readonly Dictionary<string, Func<ProtoPullResponse>> _byEndpoint;
|
||||
public List<string> Dialed { get; } = new();
|
||||
|
||||
public PerEndpointInvoker(Dictionary<string, Func<ProtoPullResponse>> byEndpoint) =>
|
||||
_byEndpoint = byEndpoint;
|
||||
|
||||
public Task<ProtoPullResponse> InvokeAsync(
|
||||
string endpoint, ProtoPullRequest request, CancellationToken ct)
|
||||
{
|
||||
Dialed.Add(endpoint);
|
||||
// The Func may throw (transport fault) — the client's try/catch handles it.
|
||||
return Task.FromResult(_byEndpoint[endpoint]());
|
||||
}
|
||||
}
|
||||
|
||||
// The site leaves SourceSite empty (it is not a tracking-store column); the
|
||||
// client re-stamps it from the dialed siteId. Mint DTOs with empty SourceSite
|
||||
// to prove that re-stamp.
|
||||
@@ -248,4 +270,52 @@ public class GrpcPullSiteCallsClientTests
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.False(result.MoreAvailable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAsync_fails_over_to_NodeB_when_primary_raises_transport_fault()
|
||||
{
|
||||
// Task 18: NodeA (primary) is unreachable, NodeB (fallback) answers — the
|
||||
// pull must return NodeB's rows rather than collapsing to empty.
|
||||
var id = Guid.NewGuid();
|
||||
var fallbackResponse = new ProtoPullResponse();
|
||||
fallbackResponse.Operationals.Add(Dto(id, BaseTime.AddMinutes(1)));
|
||||
|
||||
var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
|
||||
{
|
||||
["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
|
||||
["http://node-b:8083"] = () => fallbackResponse,
|
||||
});
|
||||
var sut = new GrpcPullSiteCallsClient(
|
||||
new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083", "http://node-b:8083")),
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
// Primary dialed first, then the NodeB fallback once.
|
||||
Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
|
||||
var row = Assert.Single(result.SiteCalls);
|
||||
Assert.Equal(id, row.TrackedOperationId.Value);
|
||||
Assert.Equal("site-a", row.SourceSite);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAsync_does_not_fail_over_when_no_fallback_configured()
|
||||
{
|
||||
// A transport fault with no NodeB fallback collapses to empty and dials
|
||||
// only the primary once (no phantom second dial).
|
||||
var invoker = new PerEndpointInvoker(new Dictionary<string, Func<ProtoPullResponse>>
|
||||
{
|
||||
["http://node-a:8083"] = () => throw new RpcException(new Status(StatusCode.Unavailable, "node A down")),
|
||||
});
|
||||
var sut = new GrpcPullSiteCallsClient(
|
||||
new StaticEnumerator(new SiteEntry("site-a", "http://node-a:8083")), // no fallback
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +76,44 @@ public class SiteEnumeratorTests
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_BothAddresses_ProjectsNodeBAsFallback()
|
||||
{
|
||||
// Task 18: a site with both NodeA and a DISTINCT NodeB yields
|
||||
// SiteEntry(primary=NodeA, FallbackGrpcEndpoint=NodeB).
|
||||
var repository = Substitute.For<ISiteRepository>();
|
||||
repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
|
||||
{
|
||||
SiteWith("site-ab", grpcNodeA: "http://site-a:8083", grpcNodeB: "http://site-b:8083"),
|
||||
});
|
||||
|
||||
var enumerator = new SiteEnumerator(BuildProvider(repository));
|
||||
|
||||
var entry = Assert.Single(await enumerator.EnumerateAsync());
|
||||
Assert.Equal("site-ab", entry.SiteId);
|
||||
Assert.Equal("http://site-a:8083", entry.GrpcEndpoint);
|
||||
Assert.Equal("http://site-b:8083", entry.FallbackGrpcEndpoint);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_OnlyNodeB_IsNotSkipped_AndHasNoFallback()
|
||||
{
|
||||
// Task 18: a site with a blank NodeA but a valid NodeB is no longer
|
||||
// skipped — NodeB becomes the primary and there is no separate fallback.
|
||||
var repository = Substitute.For<ISiteRepository>();
|
||||
repository.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<SiteEntity>
|
||||
{
|
||||
SiteWith("site-bonly", grpcNodeA: " ", grpcNodeB: "http://site-b:8083"),
|
||||
});
|
||||
|
||||
var enumerator = new SiteEnumerator(BuildProvider(repository));
|
||||
|
||||
var entry = Assert.Single(await enumerator.EnumerateAsync());
|
||||
Assert.Equal("site-bonly", entry.SiteId);
|
||||
Assert.Equal("http://site-b:8083", entry.GrpcEndpoint);
|
||||
Assert.Null(entry.FallbackGrpcEndpoint);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_ReturnsEmpty_WhenNoSites()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user