using Google.Protobuf.WellKnownTypes; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Logging; namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc; /// /// A pair (or more) of gRPC channels to the central cluster's control-plane nodes, with the /// sticky-failover + background-failback policy of the design (§3.5). The site holds one channel /// per central endpoint, prefers the first, and stays on it until a call proves it unreachable — /// only then flipping to the next, and only / connect /// failures count (a never flips or retries, because the /// call may have run). /// /// /// /// Sticky. All calls go to the current channel; a healthy preferred endpoint never /// ping-pongs. flips to the next endpoint (round-robin) when the /// caller sees the current one refuse a connection. /// /// /// Failback. While off the preferred endpoint a background probe (a cheap Heartbeat /// ping — always answered, even by a not-yet-ready node) re-checks the preferred one. On the first /// success new calls return to it; in-flight calls finish where they are. The probe is /// event-driven: it arms on a flip and stops the moment we are back on the preferred endpoint, so /// a steady healthy pair spends no cycles. Its cadence backs off exponentially — 1 s, doubling, /// capped at 60 s — while the preferred endpoint stays down, so a genuinely dead node is not /// probed hard. /// /// /// Auth. Every channel carries the site's own preshared key and its /// x-scadabridge-site identity via — the same /// insecure-h2c call-credentials shape the streaming client uses. The /// seam lets a test point a channel at an in-process TestServer; production uses a /// keepalive-configured . /// /// public sealed class CentralChannelProvider : IDisposable { private static readonly TimeSpan DefaultBackoffBase = TimeSpan.FromSeconds(1); private static readonly TimeSpan DefaultBackoffCap = TimeSpan.FromSeconds(60); private static readonly TimeSpan DefaultProbeDeadline = TimeSpan.FromSeconds(5); private readonly IReadOnlyList _endpoints; private readonly GrpcChannel[] _channels; private readonly CentralControlService.CentralControlServiceClient[] _clients; private readonly ILogger _logger; private readonly string _siteId; private readonly TimeSpan _backoffBase; private readonly TimeSpan _backoffCap; private readonly TimeSpan _probeDeadline; private readonly Timer? _failbackTimer; private readonly object _gate = new(); private volatile int _current; // preferred == 0 private int _consecutiveProbeFailures; private bool _disposed; /// Creates the provider and opens one channel per endpoint. /// Central control-plane endpoints, preferred first (index 0). Must be non-empty. /// Resolves this site's preshared key (site-side: a single-key provider). /// This site's identity, sent as the x-scadabridge-site header. /// Communication options supplying gRPC keepalive settings. /// Logger for flip/failback diagnostics. /// Test seam: per-endpoint ; null uses a production socket handler. /// Deadline for a failback probe. Null uses 5 s. /// Initial failback-probe backoff. Null uses 1 s. /// Maximum failback-probe backoff. Null uses 60 s. public CentralChannelProvider( IReadOnlyList endpoints, ISitePskProvider pskProvider, string siteId, CommunicationOptions options, ILogger logger, Func? handlerFactory = null, TimeSpan? probeDeadline = null, TimeSpan? backoffBase = null, TimeSpan? backoffCap = null) { ArgumentNullException.ThrowIfNull(endpoints); ArgumentNullException.ThrowIfNull(pskProvider); ArgumentException.ThrowIfNullOrWhiteSpace(siteId); ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(logger); if (endpoints.Count == 0) { throw new ArgumentException("At least one central gRPC endpoint is required.", nameof(endpoints)); } _endpoints = endpoints; _logger = logger; _siteId = siteId; _backoffBase = backoffBase ?? DefaultBackoffBase; _backoffCap = backoffCap ?? DefaultBackoffCap; _probeDeadline = probeDeadline ?? DefaultProbeDeadline; _channels = new GrpcChannel[endpoints.Count]; _clients = new CentralControlService.CentralControlServiceClient[endpoints.Count]; for (var i = 0; i < endpoints.Count; i++) { var channelOptions = new GrpcChannelOptions { HttpHandler = handlerFactory?.Invoke(endpoints[i]) ?? new SocketsHttpHandler { KeepAlivePingDelay = options.GrpcKeepAlivePingDelay, KeepAlivePingTimeout = options.GrpcKeepAlivePingTimeout, KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always, EnableMultipleHttp2Connections = true, }, }.WithSiteCredentials(pskProvider, siteId); _channels[i] = GrpcChannel.ForAddress(endpoints[i], channelOptions); _clients[i] = new CentralControlService.CentralControlServiceClient(_channels[i]); } // Only a multi-endpoint pair can ever fail over, so a lone endpoint needs no probe. if (endpoints.Count > 1) { _failbackTimer = new Timer(_ => _ = FailbackTickAsync(), null, Timeout.Infinite, Timeout.Infinite); } } /// The number of endpoints in the pair. public int EndpointCount => _endpoints.Count; /// The index of the endpoint calls are currently routed to (preferred == 0). public int CurrentIndex => _current; /// The endpoint address calls are currently routed to. public string CurrentEndpoint => _endpoints[_current]; /// /// The endpoint index and client calls should use right now. Captured together so a caller can /// tell exactly which endpoint failed even if a concurrent flip /// has already moved . /// /// The current endpoint index and its client. public (int Index, CentralControlService.CentralControlServiceClient Client) Current() { var idx = _current; return (idx, _clients[idx]); } /// /// Reports that the endpoint at refused a connection (an /// / connect failure). If it is still the current endpoint /// and another exists, flips to the next one and — when now off the preferred endpoint — arms /// the failback probe. Idempotent under a concurrent flip: a stale index is ignored. /// /// The endpoint index the caller's failed call used. public void ReportUnavailable(int failedIndex) { if (_endpoints.Count < 2) { return; // nothing to fail over to } lock (_gate) { if (_disposed || failedIndex != _current) { return; // a concurrent flip already moved us; do not double-flip } var next = (failedIndex + 1) % _endpoints.Count; _current = next; _logger.LogWarning( "Central control-plane endpoint {Failed} is unavailable; site {SiteId} failed over to {Next}.", _endpoints[failedIndex], _siteId, _endpoints[next]); if (_current != 0) { _consecutiveProbeFailures = 0; ArmFailback(_backoffBase); } } } private void ArmFailback(TimeSpan due) { if (_disposed) { return; } _failbackTimer?.Change(due, Timeout.InfiniteTimeSpan); } private async Task FailbackTickAsync() { int currentAtTick = _current; if (_disposed || currentAtTick == 0) { return; // already back on the preferred endpoint (or shutting down) } var preferred = _clients[0]; try { await preferred.HeartbeatAsync( new HeartbeatDto { SiteId = _siteId, NodeHostname = "failback-probe", IsActive = false, Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), }, deadline: DateTime.UtcNow.Add(_probeDeadline)).ConfigureAwait(false); // The preferred endpoint answered — return new calls to it. lock (_gate) { if (_disposed) { return; } _current = 0; _consecutiveProbeFailures = 0; } _logger.LogInformation( "Central control-plane preferred endpoint {Preferred} is reachable again; site {SiteId} failed back.", _endpoints[0], _siteId); } catch (Exception ex) { lock (_gate) { if (_disposed || _current == 0) { return; } _consecutiveProbeFailures++; var backoff = NextBackoff(_consecutiveProbeFailures); _logger.LogDebug(ex, "Failback probe of preferred central endpoint {Preferred} failed; re-probing in {Backoff}.", _endpoints[0], backoff); ArmFailback(backoff); } } } private TimeSpan NextBackoff(int failures) { // 1 s, doubling, capped at 60 s. Guard the shift against overflow for a long outage. var exponent = Math.Min(failures - 1, 20); var scaled = _backoffBase.Ticks * (1L << exponent); var cap = _backoffCap.Ticks; return TimeSpan.FromTicks(scaled >= cap || scaled < 0 ? cap : scaled); } /// public void Dispose() { lock (_gate) { if (_disposed) { return; } _disposed = true; } _failbackTimer?.Dispose(); foreach (var channel in _channels) { channel.Dispose(); } } }