ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralChannelProvider.cs

using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;

namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;

/// <summary>
/// A pair (or more) of gRPC channels to the central cluster's control-plane nodes, with the
/// sticky-failover + background-failback policy of the design (§3.5). The site holds one channel
/// per central endpoint, prefers the first, and stays on it until a call proves it unreachable —
/// only then flipping to the next, and only <see cref="StatusCode.Unavailable"/> / connect
/// failures count (a <see cref="StatusCode.DeadlineExceeded"/> never flips or retries, because the
/// call may have run).
/// </summary>
/// <remarks>
/// <para>
/// <b>Sticky.</b> All calls go to the current channel; a healthy preferred endpoint never
/// ping-pongs. <see cref="ReportUnavailable"/> flips to the next endpoint (round-robin) when the
/// caller sees the current one refuse a connection.
/// </para>
/// <para>
/// <b>Failback.</b> While off the preferred endpoint a background probe (a cheap <c>Heartbeat</c>
/// ping — always answered, even by a not-yet-ready node) re-checks the preferred one. On the first
/// success new calls return to it; in-flight calls finish where they are. The probe is
/// event-driven: it arms on a flip and stops the moment we are back on the preferred endpoint, so
/// a steady healthy pair spends no cycles. Its cadence backs off exponentially — 1 s, doubling,
/// capped at 60 s — while the preferred endpoint stays down, so a genuinely dead node is not
/// probed hard.
/// </para>
/// <para>
/// <b>Auth.</b> Every channel carries the site's own preshared key and its
/// <c>x-scadabridge-site</c> identity via <see cref="ControlPlaneCredentials"/> — the same
/// insecure-h2c call-credentials shape the streaming client uses. The <paramref name="handlerFactory"/>
/// seam lets a test point a channel at an in-process <c>TestServer</c>; production uses a
/// keepalive-configured <see cref="SocketsHttpHandler"/>.
/// </para>
/// </remarks>
public sealed class CentralChannelProvider : IDisposable
{
    private static readonly TimeSpan DefaultBackoffBase = TimeSpan.FromSeconds(1);
    private static readonly TimeSpan DefaultBackoffCap = TimeSpan.FromSeconds(60);
    private static readonly TimeSpan DefaultProbeDeadline = TimeSpan.FromSeconds(5);

    private readonly IReadOnlyList<string> _endpoints;
    private readonly GrpcChannel[] _channels;
    private readonly CentralControlService.CentralControlServiceClient[] _clients;
    private readonly ILogger _logger;
    private readonly string _siteId;
    private readonly TimeSpan _backoffBase;
    private readonly TimeSpan _backoffCap;
    private readonly TimeSpan _probeDeadline;
    private readonly Timer? _failbackTimer;
    private readonly object _gate = new();

    private volatile int _current; // preferred == 0
    private int _consecutiveProbeFailures;
    private bool _disposed;

    /// <summary>Creates the provider and opens one channel per endpoint.</summary>
    /// <param name="endpoints">Central control-plane endpoints, preferred first (index 0). Must be non-empty.</param>
    /// <param name="pskProvider">Resolves this site's preshared key (site-side: a single-key provider).</param>
    /// <param name="siteId">This site's identity, sent as the <c>x-scadabridge-site</c> header.</param>
    /// <param name="options">Communication options supplying gRPC keepalive settings.</param>
    /// <param name="logger">Logger for flip/failback diagnostics.</param>
    /// <param name="handlerFactory">Test seam: per-endpoint <see cref="HttpMessageHandler"/>; null uses a production socket handler.</param>
    /// <param name="probeDeadline">Deadline for a failback probe. Null uses 5 s.</param>
    /// <param name="backoffBase">Initial failback-probe backoff. Null uses 1 s.</param>
    /// <param name="backoffCap">Maximum failback-probe backoff. Null uses 60 s.</param>
    public CentralChannelProvider(
        IReadOnlyList<string> endpoints,
        ISitePskProvider pskProvider,
        string siteId,
        CommunicationOptions options,
        ILogger logger,
        Func<string, HttpMessageHandler>? handlerFactory = null,
        TimeSpan? probeDeadline = null,
        TimeSpan? backoffBase = null,
        TimeSpan? backoffCap = null)
    {
        ArgumentNullException.ThrowIfNull(endpoints);
        ArgumentNullException.ThrowIfNull(pskProvider);
        ArgumentException.ThrowIfNullOrWhiteSpace(siteId);
        ArgumentNullException.ThrowIfNull(options);
        ArgumentNullException.ThrowIfNull(logger);
        if (endpoints.Count == 0)
        {
            throw new ArgumentException("At least one central gRPC endpoint is required.", nameof(endpoints));
        }

        _endpoints = endpoints;
        _logger = logger;
        _siteId = siteId;
        _backoffBase = backoffBase ?? DefaultBackoffBase;
        _backoffCap = backoffCap ?? DefaultBackoffCap;
        _probeDeadline = probeDeadline ?? DefaultProbeDeadline;

        _channels = new GrpcChannel[endpoints.Count];
        _clients = new CentralControlService.CentralControlServiceClient[endpoints.Count];
        for (var i = 0; i < endpoints.Count; i++)
        {
            var channelOptions = new GrpcChannelOptions
            {
                HttpHandler = handlerFactory?.Invoke(endpoints[i]) ?? new SocketsHttpHandler
                {
                    KeepAlivePingDelay = options.GrpcKeepAlivePingDelay,
                    KeepAlivePingTimeout = options.GrpcKeepAlivePingTimeout,
                    KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always,
                    EnableMultipleHttp2Connections = true,
                },
            }.WithSiteCredentials(pskProvider, siteId);

            _channels[i] = GrpcChannel.ForAddress(endpoints[i], channelOptions);
            _clients[i] = new CentralControlService.CentralControlServiceClient(_channels[i]);
        }

        // Only a multi-endpoint pair can ever fail over, so a lone endpoint needs no probe.
        if (endpoints.Count > 1)
        {
            _failbackTimer = new Timer(_ => _ = FailbackTickAsync(), null, Timeout.Infinite, Timeout.Infinite);
        }
    }

    /// <summary>The number of endpoints in the pair.</summary>
    public int EndpointCount => _endpoints.Count;

    /// <summary>The index of the endpoint calls are currently routed to (preferred == 0).</summary>
    public int CurrentIndex => _current;

    /// <summary>The endpoint address calls are currently routed to.</summary>
    public string CurrentEndpoint => _endpoints[_current];

    /// <summary>
    /// The endpoint index and client calls should use right now. Captured together so a caller can
    /// tell <see cref="ReportUnavailable"/> exactly which endpoint failed even if a concurrent flip
    /// has already moved <see cref="CurrentIndex"/>.
    /// </summary>
    /// <returns>The current endpoint index and its client.</returns>
    public (int Index, CentralControlService.CentralControlServiceClient Client) Current()
    {
        var idx = _current;
        return (idx, _clients[idx]);
    }

    /// <summary>
    /// Reports that the endpoint at <paramref name="failedIndex"/> refused a connection (an
    /// <see cref="StatusCode.Unavailable"/> / connect failure). If it is still the current endpoint
    /// and another exists, flips to the next one and — when now off the preferred endpoint — arms
    /// the failback probe. Idempotent under a concurrent flip: a stale index is ignored.
    /// </summary>
    /// <param name="failedIndex">The endpoint index the caller's failed call used.</param>
    public void ReportUnavailable(int failedIndex)
    {
        if (_endpoints.Count < 2)
        {
            return; // nothing to fail over to
        }

        lock (_gate)
        {
            if (_disposed || failedIndex != _current)
            {
                return; // a concurrent flip already moved us; do not double-flip
            }

            var next = (failedIndex + 1) % _endpoints.Count;
            _current = next;
            _logger.LogWarning(
                "Central control-plane endpoint {Failed} is unavailable; site {SiteId} failed over to {Next}.",
                _endpoints[failedIndex], _siteId, _endpoints[next]);

            if (_current != 0)
            {
                _consecutiveProbeFailures = 0;
                ArmFailback(_backoffBase);
            }
        }
    }

    private void ArmFailback(TimeSpan due)
    {
        if (_disposed)
        {
            return;
        }

        _failbackTimer?.Change(due, Timeout.InfiniteTimeSpan);
    }

    private async Task FailbackTickAsync()
    {
        int currentAtTick = _current;
        if (_disposed || currentAtTick == 0)
        {
            return; // already back on the preferred endpoint (or shutting down)
        }

        var preferred = _clients[0];
        try
        {
            await preferred.HeartbeatAsync(
                new HeartbeatDto
                {
                    SiteId = _siteId,
                    NodeHostname = "failback-probe",
                    IsActive = false,
                    Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
                },
                deadline: DateTime.UtcNow.Add(_probeDeadline)).ConfigureAwait(false);

            // The preferred endpoint answered — return new calls to it.
            lock (_gate)
            {
                if (_disposed)
                {
                    return;
                }

                _current = 0;
                _consecutiveProbeFailures = 0;
            }

            _logger.LogInformation(
                "Central control-plane preferred endpoint {Preferred} is reachable again; site {SiteId} failed back.",
                _endpoints[0], _siteId);
        }
        catch (Exception ex)
        {
            lock (_gate)
            {
                if (_disposed || _current == 0)
                {
                    return;
                }

                _consecutiveProbeFailures++;
                var backoff = NextBackoff(_consecutiveProbeFailures);
                _logger.LogDebug(ex,
                    "Failback probe of preferred central endpoint {Preferred} failed; re-probing in {Backoff}.",
                    _endpoints[0], backoff);
                ArmFailback(backoff);
            }
        }
    }

    private TimeSpan NextBackoff(int failures)
    {
        // 1 s, doubling, capped at 60 s. Guard the shift against overflow for a long outage.
        var exponent = Math.Min(failures - 1, 20);
        var scaled = _backoffBase.Ticks * (1L << exponent);
        var cap = _backoffCap.Ticks;
        return TimeSpan.FromTicks(scaled >= cap || scaled < 0 ? cap : scaled);
    }

    /// <inheritdoc />
    public void Dispose()
    {
        lock (_gate)
        {
            if (_disposed)
            {
                return;
            }

            _disposed = true;
        }

        _failbackTimer?.Dispose();
        foreach (var channel in _channels)
        {
            channel.Dispose();
        }
    }
}