using System.Collections.Concurrent; using Google.Protobuf.WellKnownTypes; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; using ProtoPullRequest = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest; using ProtoPullResponse = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsResponse; using PullAuditEventsResponse = ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central; /// /// Production that the /// central uses to pull the next /// reconciliation batch from a site over the PullAuditEvents unary gRPC /// RPC served by SiteStreamGrpcServer. /// /// /// /// Endpoint resolution. The actor passes only a siteId; this /// client resolves it to a gRPC authority via /// () on every call so a NodeA→NodeB /// failover flip or an edited site address takes effect on the next tick — the /// same liveness guarantee SiteStreamGrpcClientFactory gives the /// real-time stream. A site with no registered endpoint yields an empty /// response (no dial); reconciliation simply has nothing to pull from it. /// /// /// Fault tolerance. Per the /// contract, tolerable transport faults (connection refused / site offline = /// , slow site = , /// shutdown = , plus bare /// / SocketException before a gRPC /// status is established) are caught and collapsed to an empty response — one /// offline site must never sink the rest of the reconciliation tick. Any other /// fault (e.g. a malformed reply that fails DTO mapping) is also swallowed to /// empty: audit reconciliation is best-effort and a throw would only get /// re-caught by the actor's own per-site guard. /// /// /// Testability. The unary call is reached through the /// seam. Production binds /// (one cached /// per endpoint, keepalive from ); unit tests /// inject a fake invoker so no real HTTP/2 endpoint is required. /// /// public sealed class GrpcPullAuditEventsClient : IPullAuditEventsClient { private readonly ISiteEnumerator _sites; private readonly IPullAuditEventsInvoker _invoker; private readonly ILogger _logger; /// /// Creates the client over the given site enumerator and unary-call invoker. /// /// Resolves a siteId to its gRPC endpoint. /// Seam that issues the PullAuditEvents unary RPC against a resolved endpoint. /// Logger for transport-fault diagnostics. public GrpcPullAuditEventsClient( ISiteEnumerator sites, IPullAuditEventsInvoker invoker, ILogger logger) { _sites = sites ?? throw new ArgumentNullException(nameof(sites)); _invoker = invoker ?? throw new ArgumentNullException(nameof(invoker)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// public async Task PullAsync( string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct) { var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false); if (endpoint is null) { // No gRPC address registered for the site — absence of an address is // a configuration decision (mirrors ISiteEnumerator's own contract), // not a runtime error, so there is simply nothing to pull. _logger.LogDebug( "PullAuditEvents skipped: no gRPC endpoint registered for site {SiteId}.", siteId); return Empty; } var request = new ProtoPullRequest { // ReadPendingSinceAsync treats DateTime.MinValue as "from the start"; // EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind). SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)), BatchSize = batchSize, // Composite-keyset tiebreak (proto field 3), mirroring PullSiteCalls exactly. // proto3 has no nullable string — an unset/empty AfterId is the site's signal to // keep the legacy inclusive-timestamp contract (also what a first pull sends). AfterId = afterId ?? string.Empty, }; var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct) .ConfigureAwait(false); // NodeB failover: a transport fault against the primary (NodeA) // dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA // outage doesn't take the reconciliation loss-recovery net offline. Only // the transport-fault set triggers this (not a mapping/unexpected fault), // and not when the caller's token is already cancelled (host shutdown). if (reply is null && transportFault && !string.IsNullOrWhiteSpace(fallback) && !string.Equals(fallback, endpoint, StringComparison.Ordinal) && !ct.IsCancellationRequested) { _logger.LogInformation( "PullAuditEvents failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.", endpoint, fallback, siteId); (reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false); } if (reply is null) { return Empty; } // Map proto DTOs to canonical AuditEvent records and order oldest-first // (the wire is already ordered by the site queue, but the // IPullAuditEventsClient contract is explicit, so sort defensively). The EventId // tiebreak matches the site's own composite ordering — ordinal over the "D" GUID // text, which is what SQLite's BINARY collation compares. var events = reply.Events .Select(AuditEventDtoMapper.FromDto) .OrderBy(e => e.OccurredAtUtc) .ThenBy(e => e.EventId.ToString(), StringComparer.Ordinal) .ToList(); return new PullAuditEventsResponse(events, reply.MoreAvailable); } /// /// Issues one pull against , classifying faults. /// Returns (reply, false) on success; (null, true) on a /// tolerable TRANSPORT fault (the failover-eligible set: /// / / /// / / /// SocketException / ); and /// (null, false) on any other (mapping/unexpected) fault, which /// collapses to empty WITHOUT a fallback dial. /// private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync( string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct) { try { var reply = await _invoker.InvokeAsync(siteId, endpoint, request, ct).ConfigureAwait(false); return (reply, false); } catch (RpcException ex) when (IsTolerable(ex.StatusCode)) { _logger.LogDebug(ex, "PullAuditEvents tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.", siteId, endpoint, ex.StatusCode); return (null, true); } catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException) { _logger.LogDebug(ex, "PullAuditEvents connection-layer fault for site {SiteId} ({Endpoint}).", siteId, endpoint); return (null, true); } catch (OperationCanceledException) { // Reconciliation tick was cancelled — either the caller's token // (host shutdown / scope dispose) or an internal gRPC deadline / // linked-CTS cancellation. Tolerable for a best-effort pull. return (null, true); } catch (Exception ex) { // Any other fault (e.g. a malformed reply that fails DTO mapping). // Audit reconciliation is best-effort; swallow to empty rather than // throw — and do NOT fail over, since a second node would hit the // same non-transport fault. _logger.LogWarning(ex, "PullAuditEvents unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.", siteId, endpoint); return (null, false); } } private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync( string siteId, CancellationToken ct) { var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false); foreach (var site in sites) { if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) && !string.IsNullOrWhiteSpace(site.GrpcEndpoint)) { return (site.GrpcEndpoint, site.FallbackGrpcEndpoint); } } return (null, null); } private static readonly PullAuditEventsResponse Empty = new(Array.Empty(), MoreAvailable: false); private static bool IsTolerable(StatusCode code) => code is StatusCode.Unavailable or StatusCode.DeadlineExceeded or StatusCode.Cancelled; // All ScadaBridge timestamps are UTC by invariant. A non-UTC cursor (the // reconciliation cursor starts at DateTime.MinValue, Kind=Unspecified) is // therefore treated AS UTC — never ToUniversalTime()-converted: on a host // with a positive UTC offset MinValue.ToUniversalTime() underflows and // Timestamp.FromDateTime throws, crashing the first pull for every site. private static DateTime EnsureUtc(DateTime value) => value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc); /// /// Seam over the PullAuditEvents unary gRPC call against a resolved /// site endpoint. Extracted so can /// be unit-tested without a real . Production binds /// . /// public interface IPullAuditEventsInvoker { /// /// Issues the PullAuditEvents unary RPC against . /// May throw / /// on transport faults — the caller classifies and swallows tolerable ones. /// /// /// The site being pulled from. Selects which preshared key the call presents — /// PullAuditEvents is gated by the site's ControlPlaneAuthInterceptor, and /// keys are per-site, so the endpoint alone is not enough to authenticate. /// /// The site gRPC authority (e.g. http://site-a:8083). /// The wire-format pull request. /// Cancellation token. /// The wire-format pull response. Task InvokeAsync( string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct); } } /// /// Production : /// caches one per endpoint (keepalive from /// , mirroring SiteStreamGrpcClient) /// and issues the unary PullAuditEventsAsync call. The cache is keyed by /// endpoint string, so a changed site address (NodeA→NodeB failover flip / an /// edited gRPC address) is reached as soon as the resolver hands the new /// endpoint to — it creates a fresh channel for the /// new address. Unlike SiteStreamGrpcClientFactory (keyed by siteId, /// which actively evicts a re-keyed client), the channel for the previous /// address is NOT actively evicted here; it lingers idle until /// . Idle channels hold no streams, so this is a minor /// cache footprint cost, not a correctness or liveness gap. /// public sealed class GrpcPullAuditEventsInvoker : GrpcPullAuditEventsClient.IPullAuditEventsInvoker, IDisposable { private readonly ConcurrentDictionary<(string Site, string Endpoint), GrpcChannel> _channels = new(); private readonly CommunicationOptions _options; private readonly ISitePskProvider? _pskProvider; /// /// Creates the invoker using default . /// public GrpcPullAuditEventsInvoker() : this(new CommunicationOptions()) { } /// /// Creates the invoker, applying the configured gRPC keepalive settings to /// every channel it opens. /// /// Communication options supplying gRPC keepalive timings. public GrpcPullAuditEventsInvoker(CommunicationOptions options) : this(options, pskProvider: null) { } /// /// Creates the invoker with per-site call credentials, the production shape: the site's /// ControlPlaneAuthInterceptor refuses an unauthenticated PullAuditEvents. /// /// Communication options supplying gRPC keepalive timings. /// Resolves each site's preshared key; null dials unauthenticated. public GrpcPullAuditEventsInvoker(CommunicationOptions options, ISitePskProvider? pskProvider) { _options = options ?? throw new ArgumentNullException(nameof(options)); _pskProvider = pskProvider; } /// public async Task InvokeAsync( string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct) { var channel = GetOrCreateChannel(siteId, endpoint); var client = new SiteStreamService.SiteStreamServiceClient(channel); using var call = client.PullAuditEventsAsync(request, cancellationToken: ct); return await call.ResponseAsync.ConfigureAwait(false); } // Race-safe channel cache. ConcurrentDictionary.GetOrAdd(key, valueFactory) // does NOT serialize the factory, so two concurrent first dials of the same // endpoint can both build a GrpcChannel (each holds an HTTP/2 connection // pool) and the loser would leak. Create-then-GetOrAdd-then-dispose-if-lost // mirrors SiteStreamGrpcClientFactory: only the channel actually installed // survives; a channel that lost the race is disposed immediately. private GrpcChannel GetOrCreateChannel(string siteId, string endpoint) { var key = (siteId, endpoint); if (!_channels.TryGetValue(key, out var channel)) { var created = CreateChannel(siteId, endpoint); channel = _channels.GetOrAdd(key, created); if (!ReferenceEquals(channel, created)) { created.Dispose(); } } return channel; } // Keyed by (site, endpoint) rather than endpoint alone: the call credentials are bound to // the channel, and they are per-site, so two sites sharing an endpoint string would // otherwise share one channel carrying the first site's key. private GrpcChannel CreateChannel(string siteId, string endpoint) => GrpcChannel.ForAddress(endpoint, new GrpcChannelOptions { HttpHandler = new SocketsHttpHandler { KeepAlivePingDelay = _options.GrpcKeepAlivePingDelay, KeepAlivePingTimeout = _options.GrpcKeepAlivePingTimeout, KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always, }, }.WithSiteCredentials(_pskProvider, siteId)); /// Disposes all cached channels. public void Dispose() { foreach (var channel in _channels.Values) { channel.Dispose(); } _channels.Clear(); } }