using System.Collections.Concurrent; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc; /// /// Raised when a site has no usable gRPC channel — an unknown site, or a site with neither /// GrpcNodeAAddress nor GrpcNodeBAddress configured. The gRPC transport treats this /// the way the Akka path treats "no ClusterClient for site": warn and drop, so the caller's Ask /// times out (central never buffers). /// public sealed class SiteChannelUnavailableException(string siteId) : Exception($"No gRPC channel is configured for site '{siteId}'.") { /// The site with no configured channel. public string SiteId { get; } = siteId; } /// /// Shared, per-site gRPC channel PAIR provider with sticky failover + failback (design §3.5). /// Central holds one per site node (NodeA/NodeB) rather than one channel /// with re-dial logic; NodeA is the preferred node (config order). Every call goes to the current /// sticky channel; on connect failure / / readiness rejection /// it flips to the other node and stays there. A background probe returns to the preferred node /// once it answers again. Reconnect probing backs off 1 s → doubling → 60 s cap. /// /// /// /// Retry safety. retries on the other node ONLY when the first /// attempt failed with (provably never reached a server — /// connect refused or readiness-rejected before response headers). A /// , or any other status, is rethrown WITHOUT a cross-node /// retry: a WriteTag/DeployArtifacts/TriggerSiteFailover may already have /// executed, and the layers above tolerate the one-shot ambiguity exactly as ClusterClient's Ask /// did — the migration must not turn one write into two. /// /// /// Addresses. Fed by / from the single DB /// refresh loop (the GrpcNodeAAddress/GrpcNodeBAddress columns). PSK credentials and /// the x-scadabridge-site header ride every channel via /// ; removing a site invalidates its /// cached key. /// /// public sealed class SitePairChannelProvider : IDisposable { private static readonly TimeSpan InitialBackoff = TimeSpan.FromSeconds(1); private static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(60); private static readonly TimeSpan FailbackSweepInterval = TimeSpan.FromSeconds(1); private readonly ISitePskProvider _pskProvider; private readonly CommunicationOptions _options; private readonly ILogger _logger; private readonly Func? _handlerFactory; private readonly Func> _reachabilityProbe; private readonly ConcurrentDictionary _sites = new(StringComparer.Ordinal); private readonly CancellationTokenSource _shutdown = new(); private Task? _failbackLoop; private readonly object _loopGate = new(); private bool _disposed; /// Creates the provider. /// Resolves each site's preshared key for channel credentials. /// Communication options (keepalive settings applied to production channels). /// Logger for failover/failback diagnostics. /// /// Test seam mapping an endpoint to the its channel should use /// (e.g. an in-process TestServer handler). Null in production, where each channel builds /// a keepalive-configured . /// /// /// Test seam deciding whether a preferred node is reachable during a failback sweep. Null in /// production, where is used. /// public SitePairChannelProvider( ISitePskProvider pskProvider, IOptions options, ILogger logger, Func? handlerFactory = null, Func>? reachabilityProbe = null) { ArgumentNullException.ThrowIfNull(pskProvider); ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(logger); _pskProvider = pskProvider; _options = options.Value; _logger = logger; _handlerFactory = handlerFactory; _reachabilityProbe = reachabilityProbe ?? DefaultReachabilityProbe; } /// /// (Re)builds a site's channel pair from its gRPC node endpoints. A no-op when neither endpoint /// changed. Disposes and rebuilds a channel whose endpoint changed, and resets stickiness to the /// preferred node when the pair is (re)created. Called on the DB refresh tick. /// /// Site identifier. /// NodeA gRPC base address, or null when unconfigured. /// NodeB gRPC base address, or null when unconfigured. public void UpdateSite(string siteId, string? nodeAEndpoint, string? nodeBEndpoint) { ArgumentException.ThrowIfNullOrWhiteSpace(siteId); var pair = _sites.GetOrAdd(siteId, id => new SitePair(id)); lock (pair.Gate) { var changedA = !string.Equals(pair.NodeAEndpoint, nodeAEndpoint, StringComparison.Ordinal); var changedB = !string.Equals(pair.NodeBEndpoint, nodeBEndpoint, StringComparison.Ordinal); if (!changedA && !changedB) { return; } if (changedA) { pair.ChannelA?.Dispose(); pair.ChannelA = string.IsNullOrWhiteSpace(nodeAEndpoint) ? null : BuildChannel(nodeAEndpoint, siteId); pair.NodeAEndpoint = nodeAEndpoint; } if (changedB) { pair.ChannelB?.Dispose(); pair.ChannelB = string.IsNullOrWhiteSpace(nodeBEndpoint) ? null : BuildChannel(nodeBEndpoint, siteId); pair.NodeBEndpoint = nodeBEndpoint; } // Addresses changed — return to the preferred node and reset failback backoff. pair.CurrentIsA = pair.ChannelA is not null; pair.ResetFailback(); _logger.LogInformation( "gRPC channel pair for site {SiteId} refreshed (nodeA={HasA}, nodeB={HasB})", siteId, pair.ChannelA is not null, pair.ChannelB is not null); } } /// Disposes a removed site's channels and invalidates its cached preshared key. /// Site identifier to remove. public void RemoveSite(string siteId) { if (string.IsNullOrWhiteSpace(siteId)) { return; } if (_sites.TryRemove(siteId, out var pair)) { lock (pair.Gate) { pair.ChannelA?.Dispose(); pair.ChannelB?.Dispose(); pair.ChannelA = null; pair.ChannelB = null; } _pskProvider.Invalidate(siteId); _logger.LogInformation("gRPC channel pair for site {SiteId} removed", siteId); } } /// /// Runs against the site's current sticky channel, failing over to the /// other node once on (never on /// ). Ensures the background failback loop is running. /// /// The RPC reply type. /// Target site. /// The RPC to run against a given channel. /// Cancellation token. /// The RPC reply. /// The site has no configured channel. public async Task ExecuteAsync( string siteId, Func> call, CancellationToken ct) { ArgumentException.ThrowIfNullOrWhiteSpace(siteId); ArgumentNullException.ThrowIfNull(call); EnsureFailbackLoop(); if (!_sites.TryGetValue(siteId, out var pair)) { throw new SiteChannelUnavailableException(siteId); } GrpcChannel primary; GrpcChannel? secondary; bool primaryIsA; lock (pair.Gate) { primaryIsA = pair.CurrentIsA; primary = (primaryIsA ? pair.ChannelA : pair.ChannelB) ?? pair.ChannelA ?? pair.ChannelB ?? throw new SiteChannelUnavailableException(siteId); // Recompute which node `primary` actually is (the current side may be null). primaryIsA = ReferenceEquals(primary, pair.ChannelA); secondary = primaryIsA ? pair.ChannelB : pair.ChannelA; } try { return await call(primary, ct).ConfigureAwait(false); } catch (RpcException ex) when (ex.StatusCode == StatusCode.Unavailable && secondary is not null) { // Provably never reached a server → safe to try the other node, even for a write. _logger.LogWarning( "Site {SiteId} node {FromNode} unavailable; failing over to node {ToNode}", siteId, primaryIsA ? "A" : "B", primaryIsA ? "B" : "A"); FlipTo(pair, toIsA: !primaryIsA); return await call(secondary, ct).ConfigureAwait(false); } } /// /// Probes a site's preferred node (NodeA) and, if reachable, returns stickiness to it. The /// background loop calls this per due site; tests call it to drive failback deterministically. /// /// Site identifier. /// Cancellation token. /// True when the site is now (or already) pointed at its preferred node. internal async Task TryFailbackAsync(string siteId, CancellationToken ct) { if (!_sites.TryGetValue(siteId, out var pair)) { return false; } GrpcChannel? preferred; lock (pair.Gate) { if (pair.CurrentIsA || pair.ChannelA is null) { return true; // already on preferred (or no preferred channel to fail back to) } preferred = pair.ChannelA; } var reachable = await _reachabilityProbe(preferred, ct).ConfigureAwait(false); lock (pair.Gate) { if (reachable) { pair.CurrentIsA = true; pair.ResetFailback(); _logger.LogInformation("Site {SiteId} preferred node A reachable again; failing back", siteId); return true; } pair.BumpFailbackBackoff(); return false; } } /// Test/diagnostic accessor: true when the site currently targets its preferred node (A). /// Site identifier. /// True when on NodeA (or when the site is unknown). internal bool IsOnPreferredNode(string siteId) => !_sites.TryGetValue(siteId, out var pair) || pair.CurrentIsA; /// Starts the background failback loop if not already running. Idempotent. public void EnsureFailbackLoop() { if (_failbackLoop is not null || _disposed) { return; } lock (_loopGate) { if (_failbackLoop is null && !_disposed) { _failbackLoop = Task.Run(() => FailbackLoopAsync(_shutdown.Token)); } } } private async Task FailbackLoopAsync(CancellationToken ct) { using var timer = new PeriodicTimer(FailbackSweepInterval); try { while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) { var now = DateTime.UtcNow; foreach (var kvp in _sites) { var pair = kvp.Value; bool due; lock (pair.Gate) { due = !pair.CurrentIsA && pair.ChannelA is not null && now >= pair.NextProbeUtc; } if (due) { try { await TryFailbackAsync(kvp.Key, ct).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch (Exception ex) { _logger.LogDebug(ex, "Failback probe for site {SiteId} faulted", kvp.Key); } } } } } catch (OperationCanceledException) { // Shutting down. } } private void FlipTo(SitePair pair, bool toIsA) { lock (pair.Gate) { pair.CurrentIsA = toIsA; if (!toIsA) { // Failed over off the preferred node → schedule the first failback probe. pair.ScheduleFailback(InitialBackoff); } else { pair.ResetFailback(); } } } private GrpcChannel BuildChannel(string endpoint, string siteId) { var channelOptions = new GrpcChannelOptions { HttpHandler = _handlerFactory is not null ? _handlerFactory(endpoint) : new SocketsHttpHandler { KeepAlivePingDelay = _options.GrpcKeepAlivePingDelay, KeepAlivePingTimeout = _options.GrpcKeepAlivePingTimeout, KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always, EnableMultipleHttp2Connections = true } }.WithSiteCredentials(_pskProvider, siteId); return GrpcChannel.ForAddress(endpoint, channelOptions); } private static async Task DefaultReachabilityProbe(GrpcChannel channel, CancellationToken ct) { try { using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); timeout.CancelAfter(TimeSpan.FromSeconds(5)); await channel.ConnectAsync(timeout.Token).ConfigureAwait(false); return true; } catch { return false; } } /// public void Dispose() { if (_disposed) { return; } _disposed = true; _shutdown.Cancel(); try { _failbackLoop?.Wait(TimeSpan.FromSeconds(2)); } catch { // Best-effort loop drain on shutdown. } _shutdown.Dispose(); foreach (var pair in _sites.Values) { lock (pair.Gate) { pair.ChannelA?.Dispose(); pair.ChannelB?.Dispose(); } } _sites.Clear(); } /// Per-site mutable channel state, guarded by . private sealed class SitePair(string siteId) { public string SiteId { get; } = siteId; public readonly object Gate = new(); public string? NodeAEndpoint; public string? NodeBEndpoint; public GrpcChannel? ChannelA; public GrpcChannel? ChannelB; /// Sticky current node; NodeA is preferred. True = pointing at NodeA. public bool CurrentIsA = true; /// Next time a failback probe is due (while failed over). MaxValue = not scheduled. public DateTime NextProbeUtc = DateTime.MaxValue; private TimeSpan _backoff = InitialBackoff; public void ScheduleFailback(TimeSpan delay) { _backoff = delay; NextProbeUtc = DateTime.UtcNow + _backoff; } public void BumpFailbackBackoff() { var next = TimeSpan.FromTicks(Math.Min(_backoff.Ticks * 2, MaxBackoff.Ticks)); _backoff = next; NextProbeUtc = DateTime.UtcNow + _backoff; } public void ResetFailback() { _backoff = InitialBackoff; NextProbeUtc = DateTime.MaxValue; } } }