using System.Collections.Concurrent;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
using ProtoPullRequest = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest;
using ProtoPullResponse = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsResponse;
using PullAuditEventsResponse = ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
///
/// Production that the
/// central uses to pull the next
/// reconciliation batch from a site over the PullAuditEvents unary gRPC
/// RPC served by SiteStreamGrpcServer.
///
///
///
/// Endpoint resolution. The actor passes only a siteId; this
/// client resolves it to a gRPC authority via
/// () on every call so a NodeA→NodeB
/// failover flip or an edited site address takes effect on the next tick — the
/// same liveness guarantee SiteStreamGrpcClientFactory gives the
/// real-time stream. A site with no registered endpoint yields an empty
/// response (no dial); reconciliation simply has nothing to pull from it.
///
///
/// Fault tolerance. Per the
/// contract, tolerable transport faults (connection refused / site offline =
/// , slow site = ,
/// shutdown = , plus bare
/// / SocketException before a gRPC
/// status is established) are caught and collapsed to an empty response — one
/// offline site must never sink the rest of the reconciliation tick. Any other
/// fault (e.g. a malformed reply that fails DTO mapping) is also swallowed to
/// empty: audit reconciliation is best-effort and a throw would only get
/// re-caught by the actor's own per-site guard.
///
///
/// Testability. The unary call is reached through the
/// seam. Production binds
/// (one cached
/// per endpoint, keepalive from ); unit tests
/// inject a fake invoker so no real HTTP/2 endpoint is required.
///
///
public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient
{
private readonly ISiteEnumerator _sites;
private readonly IPullAuditEventsInvoker _invoker;
private readonly ILogger _logger;
///
/// Creates the client over the given site enumerator and unary-call invoker.
///
/// Resolves a siteId to its gRPC endpoint.
/// Seam that issues the PullAuditEvents unary RPC against a resolved endpoint.
/// Logger for transport-fault diagnostics.
public GrpcPullAuditEventsClient(
ISiteEnumerator sites,
IPullAuditEventsInvoker invoker,
ILogger logger)
{
_sites = sites ?? throw new ArgumentNullException(nameof(sites));
_invoker = invoker ?? throw new ArgumentNullException(nameof(invoker));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
///
public async Task PullAsync(
string siteId,
DateTime sinceUtc,
string? afterId,
int batchSize,
CancellationToken ct)
{
var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false);
if (endpoint is null)
{
// No gRPC address registered for the site — absence of an address is
// a configuration decision (mirrors ISiteEnumerator's own contract),
// not a runtime error, so there is simply nothing to pull.
_logger.LogDebug(
"PullAuditEvents skipped: no gRPC endpoint registered for site {SiteId}.", siteId);
return Empty;
}
var request = new ProtoPullRequest
{
// ReadPendingSinceAsync treats DateTime.MinValue as "from the start";
// EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind).
SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)),
BatchSize = batchSize,
// Composite-keyset tiebreak (proto field 3), mirroring PullSiteCalls exactly.
// proto3 has no nullable string — an unset/empty AfterId is the site's signal to
// keep the legacy inclusive-timestamp contract (also what a first pull sends).
AfterId = afterId ?? string.Empty,
};
var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct)
.ConfigureAwait(false);
// NodeB failover: a transport fault against the primary (NodeA)
// dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA
// outage doesn't take the reconciliation loss-recovery net offline. Only
// the transport-fault set triggers this (not a mapping/unexpected fault),
// and not when the caller's token is already cancelled (host shutdown).
if (reply is null && transportFault
&& !string.IsNullOrWhiteSpace(fallback)
&& !string.Equals(fallback, endpoint, StringComparison.Ordinal)
&& !ct.IsCancellationRequested)
{
_logger.LogInformation(
"PullAuditEvents failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.",
endpoint, fallback, siteId);
(reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false);
}
if (reply is null)
{
return Empty;
}
// Map proto DTOs to canonical AuditEvent records and order oldest-first
// (the wire is already ordered by the site queue, but the
// IPullAuditEventsClient contract is explicit, so sort defensively). The EventId
// tiebreak matches the site's own composite ordering — ordinal over the "D" GUID
// text, which is what SQLite's BINARY collation compares.
var events = reply.Events
.Select(AuditEventDtoMapper.FromDto)
.OrderBy(e => e.OccurredAtUtc)
.ThenBy(e => e.EventId.ToString(), StringComparer.Ordinal)
.ToList();
return new PullAuditEventsResponse(events, reply.MoreAvailable);
}
///
/// Issues one pull against , classifying faults.
/// Returns (reply, false) on success; (null, true) on a
/// tolerable TRANSPORT fault (the failover-eligible set:
/// / /
/// / /
/// SocketException / ); and
/// (null, false) on any other (mapping/unexpected) fault, which
/// collapses to empty WITHOUT a fallback dial.
///
private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync(
string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct)
{
try
{
var reply = await _invoker.InvokeAsync(siteId, endpoint, request, ct).ConfigureAwait(false);
return (reply, false);
}
catch (RpcException ex) when (IsTolerable(ex.StatusCode))
{
_logger.LogDebug(ex,
"PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.",
siteId, endpoint, ex.StatusCode);
return (null, true);
}
catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException)
{
_logger.LogDebug(ex,
"PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}).",
siteId, endpoint);
return (null, true);
}
catch (OperationCanceledException)
{
// Reconciliation tick was cancelled — either the caller's token
// (host shutdown / scope dispose) or an internal gRPC deadline /
// linked-CTS cancellation. Tolerable for a best-effort pull.
return (null, true);
}
catch (Exception ex)
{
// Any other fault (e.g. a malformed reply that fails DTO mapping).
// Audit reconciliation is best-effort; swallow to empty rather than
// throw — and do NOT fail over, since a second node would hit the
// same non-transport fault.
_logger.LogWarning(ex,
"PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.",
siteId, endpoint);
return (null, false);
}
}
private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync(
string siteId, CancellationToken ct)
{
var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false);
foreach (var site in sites)
{
if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) &&
!string.IsNullOrWhiteSpace(site.GrpcEndpoint))
{
return (site.GrpcEndpoint, site.FallbackGrpcEndpoint);
}
}
return (null, null);
}
private static readonly PullAuditEventsResponse Empty =
new(Array.Empty(), MoreAvailable: false);
private static bool IsTolerable(StatusCode code) => code is
StatusCode.Unavailable or
StatusCode.DeadlineExceeded or
StatusCode.Cancelled;
// All ScadaBridge timestamps are UTC by invariant. A non-UTC cursor (the
// reconciliation cursor starts at DateTime.MinValue, Kind=Unspecified) is
// therefore treated AS UTC — never ToUniversalTime()-converted: on a host
// with a positive UTC offset MinValue.ToUniversalTime() underflows and
// Timestamp.FromDateTime throws, crashing the first pull for every site.
private static DateTime EnsureUtc(DateTime value) =>
value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc);
///
/// Seam over the PullAuditEvents unary gRPC call against a resolved
/// site endpoint. Extracted so can
/// be unit-tested without a real . Production binds
/// .
///
public interface IPullAuditEventsInvoker
{
///
/// Issues the PullAuditEvents unary RPC against .
/// May throw /
/// on transport faults — the caller classifies and swallows tolerable ones.
///
///
/// The site being pulled from. Selects which preshared key the call presents —
/// PullAuditEvents is gated by the site's ControlPlaneAuthInterceptor, and
/// keys are per-site, so the endpoint alone is not enough to authenticate.
///
/// The site gRPC authority (e.g. http://site-a:8083).
/// The wire-format pull request.
/// Cancellation token.
/// The wire-format pull response.
Task InvokeAsync(
string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct);
}
}
///
/// Production :
/// caches one per endpoint (keepalive from
/// , mirroring SiteStreamGrpcClient)
/// and issues the unary PullAuditEventsAsync call. The cache is keyed by
/// endpoint string, so a changed site address (NodeA→NodeB failover flip / an
/// edited gRPC address) is reached as soon as the resolver hands the new
/// endpoint to — it creates a fresh channel for the
/// new address. Unlike SiteStreamGrpcClientFactory (keyed by siteId,
/// which actively evicts a re-keyed client), the channel for the previous
/// address is NOT actively evicted here; it lingers idle until
/// . Idle channels hold no streams, so this is a minor
/// cache footprint cost, not a correctness or liveness gap.
///
public sealed class GrpcPullAuditEventsInvoker
: GrpcPullAuditEventsClient.IPullAuditEventsInvoker, IDisposable
{
private readonly ConcurrentDictionary<(string Site, string Endpoint), GrpcChannel> _channels = new();
private readonly CommunicationOptions _options;
private readonly ISitePskProvider? _pskProvider;
///
/// Creates the invoker using default .
///
public GrpcPullAuditEventsInvoker()
: this(new CommunicationOptions())
{
}
///
/// Creates the invoker, applying the configured gRPC keepalive settings to
/// every channel it opens.
///
/// Communication options supplying gRPC keepalive timings.
public GrpcPullAuditEventsInvoker(CommunicationOptions options)
: this(options, pskProvider: null)
{
}
///
/// Creates the invoker with per-site call credentials, the production shape: the site's
/// ControlPlaneAuthInterceptor refuses an unauthenticated PullAuditEvents.
///
/// Communication options supplying gRPC keepalive timings.
/// Resolves each site's preshared key; null dials unauthenticated.
public GrpcPullAuditEventsInvoker(CommunicationOptions options, ISitePskProvider? pskProvider)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_pskProvider = pskProvider;
}
///
public async Task InvokeAsync(
string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct)
{
var channel = GetOrCreateChannel(siteId, endpoint);
var client = new SiteStreamService.SiteStreamServiceClient(channel);
using var call = client.PullAuditEventsAsync(request, cancellationToken: ct);
return await call.ResponseAsync.ConfigureAwait(false);
}
// Race-safe channel cache. ConcurrentDictionary.GetOrAdd(key, valueFactory)
// does NOT serialize the factory, so two concurrent first dials of the same
// endpoint can both build a GrpcChannel (each holds an HTTP/2 connection
// pool) and the loser would leak. Create-then-GetOrAdd-then-dispose-if-lost
// mirrors SiteStreamGrpcClientFactory: only the channel actually installed
// survives; a channel that lost the race is disposed immediately.
private GrpcChannel GetOrCreateChannel(string siteId, string endpoint)
{
var key = (siteId, endpoint);
if (!_channels.TryGetValue(key, out var channel))
{
var created = CreateChannel(siteId, endpoint);
channel = _channels.GetOrAdd(key, created);
if (!ReferenceEquals(channel, created))
{
created.Dispose();
}
}
return channel;
}
// Keyed by (site, endpoint) rather than endpoint alone: the call credentials are bound to
// the channel, and they are per-site, so two sites sharing an endpoint string would
// otherwise share one channel carrying the first site's key.
private GrpcChannel CreateChannel(string siteId, string endpoint) =>
GrpcChannel.ForAddress(endpoint, new GrpcChannelOptions
{
HttpHandler = new SocketsHttpHandler
{
KeepAlivePingDelay = _options.GrpcKeepAlivePingDelay,
KeepAlivePingTimeout = _options.GrpcKeepAlivePingTimeout,
KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always,
},
}.WithSiteCredentials(_pskProvider, siteId));
/// Disposes all cached channels.
public void Dispose()
{
foreach (var channel in _channels.Values)
{
channel.Dispose();
}
_channels.Clear();
}
}