using System.Collections.Concurrent; using Google.Protobuf.WellKnownTypes; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; using ProtoPullRequest = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsRequest; using ProtoPullResponse = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsResponse; using PullSiteCallsResponse = ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullSiteCallsResponse; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central; /// /// Production (Site Call Audit) that the /// central reconciliation tick (a separate follow-up component) uses to pull the /// next batch of cached-call operational rows from a site over the /// PullSiteCalls unary gRPC RPC served by SiteStreamGrpcServer. /// A near-exact sibling of . /// /// /// /// Endpoint resolution. The caller passes only a siteId; this /// client resolves it to a gRPC authority via /// () on every call so a NodeA→NodeB /// failover flip or an edited site address takes effect on the next tick. A site /// with no registered endpoint yields an empty response (no dial). /// /// /// SourceSite re-stamp. The site leaves /// SiteCallOperationalDto.SourceSite empty (the tracking store has no /// site-id column). This client is the authority that knows which site it /// dialed, so it re-stamps the mapped from /// siteId — the same "re-stamp from the forwarder's own id" pattern the /// site push path uses. /// /// /// Fault tolerance. Per the contract, /// tolerable transport faults (, /// , , /// bare / SocketException) are caught /// and collapsed to an empty response so one offline site never sinks the rest /// of the reconciliation tick. Any other transport/protocol fault is also /// swallowed to empty: reconciliation is best-effort. Per-row DTO mapping faults /// (e.g. a single unparseable TrackedOperationId) are narrower still — /// the offending row is skipped+logged and the rest of the batch is returned. /// /// /// Testability. The unary call is reached through the /// seam. Production binds /// (one cached /// per endpoint, keepalive from ); unit tests /// inject a fake invoker so no real HTTP/2 endpoint is required. /// /// public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient { private readonly ISiteEnumerator _sites; private readonly IPullSiteCallsInvoker _invoker; private readonly ILogger _logger; /// /// Creates the client over the given site enumerator and unary-call invoker. /// /// Resolves a siteId to its gRPC endpoint. /// Seam that issues the PullSiteCalls unary RPC against a resolved endpoint. /// Logger for transport-fault diagnostics. public GrpcPullSiteCallsClient( ISiteEnumerator sites, IPullSiteCallsInvoker invoker, ILogger logger) { _sites = sites ?? throw new ArgumentNullException(nameof(sites)); _invoker = invoker ?? throw new ArgumentNullException(nameof(invoker)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// public async Task PullAsync( string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct) { var (endpoint, fallback) = await ResolveEndpointsAsync(siteId, ct).ConfigureAwait(false); if (endpoint is null) { // No gRPC address registered for the site — a configuration decision // (mirrors ISiteEnumerator's own contract), not a runtime error, so // there is simply nothing to pull. _logger.LogDebug( "PullSiteCalls skipped: no gRPC endpoint registered for site {SiteId}.", siteId); return Empty; } var request = new ProtoPullRequest { // ReadChangedSinceAsync treats DateTime.MinValue as "from the start"; // EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind). SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)), BatchSize = batchSize, // Composite-keyset tiebreak. proto3 has no nullable string — // an unset/empty AfterId is the site's signal to keep the legacy // inclusive-timestamp contract (also what a first pull sends). AfterId = afterId ?? string.Empty, }; var (reply, transportFault) = await TryInvokeAsync(endpoint, request, siteId, ct) .ConfigureAwait(false); // NodeB failover: a transport fault against the primary (NodeA) // dials the fallback (NodeB) ONCE before collapsing to empty, so a NodeA // outage doesn't take the reconciliation loss-recovery net offline. Only // the transport-fault set triggers this (not a mapping/unexpected fault), // and not when the caller's token is already cancelled (host shutdown). if (reply is null && transportFault && !string.IsNullOrWhiteSpace(fallback) && !string.Equals(fallback, endpoint, StringComparison.Ordinal) && !ct.IsCancellationRequested) { _logger.LogInformation( "PullSiteCalls failing over from primary {Primary} to NodeB {Fallback} for site {SiteId}.", endpoint, fallback, siteId); (reply, _) = await TryInvokeAsync(fallback, request, siteId, ct).ConfigureAwait(false); } if (reply is null) { return Empty; } // Map proto DTOs to central SiteCall entities PER-ROW so one malformed // operational (e.g. an unparseable TrackedOperationId) is skipped+logged // rather than sinking the whole batch through the outer catch-all. Each // survivor is re-stamped with SourceSite from the dialed siteId (the site // leaves it empty). var siteCalls = new List(reply.Operationals.Count); foreach (var dto in reply.Operationals) { try { var sc = SiteCallDtoMapper.FromDto(dto) with { SourceSite = siteId }; siteCalls.Add(sc); } catch (Exception ex) { _logger.LogWarning(ex, "PullSiteCalls dropped a malformed operational row from site {SiteId} (id='{Id}'); continuing with the rest of the batch.", siteId, dto.TrackedOperationId); } } // Order oldest-first by UpdatedAtUtc (the wire is already ordered by the // site read, but the contract is explicit, so sort defensively). siteCalls.Sort((a, b) => a.UpdatedAtUtc.CompareTo(b.UpdatedAtUtc)); return new PullSiteCallsResponse(siteCalls, reply.MoreAvailable); } /// /// Issues one pull against , classifying faults. /// Returns (reply, false) on success; (null, true) on a /// tolerable TRANSPORT fault (the failover-eligible set: /// / / /// / / /// SocketException / ); and /// (null, false) on any other (mapping/unexpected) fault, which /// collapses to empty WITHOUT a fallback dial. /// private async Task<(ProtoPullResponse? Reply, bool TransportFault)> TryInvokeAsync( string endpoint, ProtoPullRequest request, string siteId, CancellationToken ct) { try { var reply = await _invoker.InvokeAsync(siteId, endpoint, request, ct).ConfigureAwait(false); return (reply, false); } catch (RpcException ex) when (IsTolerable(ex.StatusCode)) { _logger.LogDebug(ex, "PullSiteCalls tolerable transport fault for site {SiteId} ({Endpoint}): {Status}.", siteId, endpoint, ex.StatusCode); return (null, true); } catch (Exception ex) when (ex is HttpRequestException or System.Net.Sockets.SocketException) { _logger.LogDebug(ex, "PullSiteCalls connection-layer fault for site {SiteId} ({Endpoint}).", siteId, endpoint); return (null, true); } catch (OperationCanceledException) { // Reconciliation tick cancelled — caller token (host shutdown) or an // internal gRPC deadline / linked-CTS cancellation. Tolerable for a // best-effort pull. return (null, true); } catch (Exception ex) { // Any other fault. Reconciliation is best-effort; swallow to empty // rather than throw — and do NOT fail over, since a second node would // hit the same non-transport fault. _logger.LogWarning(ex, "PullSiteCalls unexpected fault for site {SiteId} ({Endpoint}). Returning empty batch.", siteId, endpoint); return (null, false); } } private async Task<(string? Primary, string? Fallback)> ResolveEndpointsAsync( string siteId, CancellationToken ct) { var sites = await _sites.EnumerateAsync(ct).ConfigureAwait(false); foreach (var site in sites) { if (string.Equals(site.SiteId, siteId, StringComparison.Ordinal) && !string.IsNullOrWhiteSpace(site.GrpcEndpoint)) { return (site.GrpcEndpoint, site.FallbackGrpcEndpoint); } } return (null, null); } private static readonly PullSiteCallsResponse Empty = new(Array.Empty(), MoreAvailable: false); private static bool IsTolerable(StatusCode code) => code is StatusCode.Unavailable or StatusCode.DeadlineExceeded or StatusCode.Cancelled; // All ScadaBridge timestamps are UTC by invariant. A non-UTC cursor (the // reconciliation cursor starts at DateTime.MinValue, Kind=Unspecified) is // treated AS UTC — never ToUniversalTime()-converted: on a host with a // positive UTC offset MinValue.ToUniversalTime() underflows and // Timestamp.FromDateTime throws, crashing the first pull for every site. private static DateTime EnsureUtc(DateTime value) => value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc); /// /// Seam over the PullSiteCalls unary gRPC call against a resolved site /// endpoint. Extracted so can be /// unit-tested without a real . Production binds /// . /// public interface IPullSiteCallsInvoker { /// /// Issues the PullSiteCalls unary RPC against . /// May throw / /// on transport faults — the caller classifies and swallows tolerable ones. /// /// /// The site being pulled from. Selects which preshared key the call presents — /// PullSiteCalls is gated by the site's ControlPlaneAuthInterceptor, and /// keys are per-site, so the endpoint alone is not enough to authenticate. /// /// The site gRPC authority (e.g. http://site-a:8083). /// The wire-format pull request. /// Cancellation token. /// The wire-format pull response. Task InvokeAsync( string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct); } } /// /// Production : caches /// one per endpoint (keepalive from /// , mirroring SiteStreamGrpcClient) and /// issues the unary PullSiteCallsAsync call. The cache is keyed by /// endpoint string, so a changed site address (NodeA→NodeB failover flip / an /// edited gRPC address) is reached as soon as the resolver hands the new endpoint /// to . The channel for a previous address lingers idle /// until (idle channels hold no streams — a minor cache /// footprint cost, not a correctness or liveness gap). Sibling of /// . /// public sealed class GrpcPullSiteCallsInvoker : GrpcPullSiteCallsClient.IPullSiteCallsInvoker, IDisposable { private readonly ConcurrentDictionary<(string Site, string Endpoint), GrpcChannel> _channels = new(); private readonly CommunicationOptions _options; private readonly ISitePskProvider? _pskProvider; /// Creates the invoker using default . public GrpcPullSiteCallsInvoker() : this(new CommunicationOptions()) { } /// /// Creates the invoker, applying the configured gRPC keepalive settings to /// every channel it opens. /// /// Communication options supplying gRPC keepalive timings. public GrpcPullSiteCallsInvoker(CommunicationOptions options) : this(options, pskProvider: null) { } /// /// Creates the invoker with per-site call credentials, the production shape: the site's /// ControlPlaneAuthInterceptor refuses an unauthenticated PullSiteCalls. /// /// Communication options supplying gRPC keepalive timings. /// Resolves each site's preshared key; null dials unauthenticated. public GrpcPullSiteCallsInvoker(CommunicationOptions options, ISitePskProvider? pskProvider) { _options = options ?? throw new ArgumentNullException(nameof(options)); _pskProvider = pskProvider; } /// public async Task InvokeAsync( string siteId, string endpoint, ProtoPullRequest request, CancellationToken ct) { var channel = GetOrCreateChannel(siteId, endpoint); var client = new SiteStreamService.SiteStreamServiceClient(channel); using var call = client.PullSiteCallsAsync(request, cancellationToken: ct); return await call.ResponseAsync.ConfigureAwait(false); } // Race-safe channel cache (create-then-GetOrAdd-then-dispose-if-lost): two // concurrent first dials of the same endpoint can both build a GrpcChannel; // only the channel actually installed survives, the loser is disposed. // Mirrors SiteStreamGrpcClientFactory / GrpcPullAuditEventsInvoker. private GrpcChannel GetOrCreateChannel(string siteId, string endpoint) { var key = (siteId, endpoint); if (!_channels.TryGetValue(key, out var channel)) { var created = CreateChannel(siteId, endpoint); channel = _channels.GetOrAdd(key, created); if (!ReferenceEquals(channel, created)) { created.Dispose(); } } return channel; } // Keyed by (site, endpoint) rather than endpoint alone: the call credentials are bound to // the channel, and they are per-site, so two sites sharing an endpoint string would // otherwise share one channel carrying the first site's key. private GrpcChannel CreateChannel(string siteId, string endpoint) => GrpcChannel.ForAddress(endpoint, new GrpcChannelOptions { HttpHandler = new SocketsHttpHandler { KeepAlivePingDelay = _options.GrpcKeepAlivePingDelay, KeepAlivePingTimeout = _options.GrpcKeepAlivePingTimeout, KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always, }, }.WithSiteCredentials(_pskProvider, siteId)); /// Disposes all cached channels. public void Dispose() { foreach (var channel in _channels.Values) { channel.Dispose(); } _channels.Clear(); } }