feat(secrets-grpc): FallbackEndpoints + sticky failover hub reader (scadaproj#4)

SecretsGrpcHubClientOptions gains a getter-only FallbackEndpoints list: additional hub addresses tried in order when a call fails with a transport-level RpcException. Validation fails startup on blank/relative entries and on any duplicate of the primary or another fallback (a duplicated endpoint silently halves the configured failover). New FailoverSecretsHubReader composes one GrpcSecretsHubClient per endpoint: the answering endpoint is sticky (a sweep is GetManifest + N GetSecrets batches — without stickiness every call after a primary outage re-pays a full CallDeadline before reaching the healthy fallback), rotation wraps so a recovered primary is retried when the stuck-to endpoint fails, only RpcException triggers failover (cancellation and defects propagate untouched), and when every endpoint fails the last RpcException is rethrown so the sweep degrades to last-known-good exactly as today. Failover warnings name only the two endpoint URIs and the status code — never any credential material. DI: zero fallbacks keeps the registration graph byte-identical to 0.4.x (single keyed channel under ChannelServiceKey, plain GrpcSecretsHubClient); with fallbacks, one container-owned keyed channel per endpoint (fallback i under "{ChannelServiceKey}:fallback:{i}") wired into the failover reader. GrpcSecretsHubClient itself is unchanged. Safe only when every listed endpoint serves the SAME store — for ScadaBridge, the central pair over one shared SQL Server store; the options doc and CHANGELOG carry the warning. Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
2026-08-07 09:37:22 -04:00
parent 7df05ccc27
commit 0d4978953f
7 changed files with 695 additions and 8 deletions
@@ -45,6 +45,13 @@ public static class SecretsGrpcHubClientExtensions
    /// means local writes never leave the node, so there is nothing to publish and the store stays
    /// exactly as the application configured it. A host that forgets <c>AddZbSecrets</c> fails loudly
    /// at startup when the sweep cannot resolve the store, rather than silently replicating nothing.
+    /// <para>
+    /// When <see cref="SecretsGrpcHubClientOptions.FallbackEndpoints"/> is non-empty, one keyed
+    /// channel and one client are registered per endpoint and the reader becomes a
+    /// <see cref="FailoverSecretsHubReader"/> composed over them, primary first. With the list
+    /// empty — the default — the registration graph is identical to releases before the list
+    /// existed: one keyed channel, one <see cref="GrpcSecretsHubClient"/>, no failover wrapper.
+    /// </para>
    /// </remarks>
    /// <param name="services">The service collection to add to.</param>
    /// <param name="configuration">The application configuration.</param>
@@ -82,15 +89,79 @@ public static class SecretsGrpcHubClientExtensions
        services.TryAddKeyedSingleton(ChannelServiceKey, (sp, _) => GrpcChannel.ForAddress(
            sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>().Value.Endpoint));

-        services.TryAddSingleton<ISecretsHubReader>(sp => new GrpcSecretsHubClient(
-            new SecretsHub.SecretsHubClient(
-                sp.GetRequiredKeyedService<GrpcChannel>(ChannelServiceKey)),
-            sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>(),
-            sp.GetRequiredService<ILogger<GrpcSecretsHubClient>>(),
-            sp.GetService<TimeProvider>()));
+        if (eager.FallbackEndpoints.Count == 0)
+        {
+            // No fallbacks: the graph is exactly the pre-0.5.0 one — the single keyed channel above
+            // plus one GrpcSecretsHubClient as the reader. Kept as its own branch rather than a
+            // one-element failover composition so existing consumers see an identical container
+            // shape and zero added indirection on every call.
+            services.TryAddSingleton<ISecretsHubReader>(sp => new GrpcSecretsHubClient(
+                new SecretsHub.SecretsHubClient(
+                    sp.GetRequiredKeyedService<GrpcChannel>(ChannelServiceKey)),
+                sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>(),
+                sp.GetRequiredService<ILogger<GrpcSecretsHubClient>>(),
+                sp.GetService<TimeProvider>()));
+        }
+        else
+        {
+            // One keyed channel per fallback, each a container-owned singleton for the same
+            // disposal reason as the primary. The loop variable is copied because the factory
+            // closes over it and runs long after the loop has finished.
+            for (int i = 0; i < eager.FallbackEndpoints.Count; i++)
+            {
+                int index = i;
+                services.TryAddKeyedSingleton(
+                    FallbackChannelServiceKey(index),
+                    (sp, _) => GrpcChannel.ForAddress(sp
+                        .GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>()
+                        .Value.FallbackEndpoints[index]));
+            }
+
+            services.TryAddSingleton<ISecretsHubReader>(sp =>
+            {
+                IOptions<SecretsGrpcHubClientOptions> options =
+                    sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>();
+                ILogger<GrpcSecretsHubClient> clientLogger =
+                    sp.GetRequiredService<ILogger<GrpcSecretsHubClient>>();
+                TimeProvider? timeProvider = sp.GetService<TimeProvider>();
+
+                GrpcSecretsHubClient ReaderOver(object channelKey) => new(
+                    new SecretsHub.SecretsHubClient(
+                        sp.GetRequiredKeyedService<GrpcChannel>(channelKey)),
+                    options,
+                    clientLogger,
+                    timeProvider);
+
+                IList<string> fallbacks = options.Value.FallbackEndpoints;
+                var readers = new List<ISecretsHubReader>(1 + fallbacks.Count)
+                {
+                    ReaderOver(ChannelServiceKey),
+                };
+                var labels = new List<string>(1 + fallbacks.Count) { options.Value.Endpoint };
+
+                for (int index = 0; index < fallbacks.Count; index++)
+                {
+                    readers.Add(ReaderOver(FallbackChannelServiceKey(index)));
+                    labels.Add(fallbacks[index]);
+                }
+
+                return new FailoverSecretsHubReader(
+                    readers, labels, sp.GetRequiredService<ILogger<FailoverSecretsHubReader>>());
+            });
+        }

        services.AddHostedService<GrpcSecretSyncService>();

        return services;
    }
+
+    /// <summary>
+    /// Service key for the keyed <see cref="GrpcChannel"/> dialing the fallback endpoint at
+    /// <paramref name="index"/> in <see cref="SecretsGrpcHubClientOptions.FallbackEndpoints"/>.
+    /// The primary endpoint's channel keeps <see cref="ChannelServiceKey"/> unchanged.
+    /// </summary>
+    /// <param name="index">Zero-based index into the fallback list.</param>
+    /// <returns>The service key.</returns>
+    internal static string FallbackChannelServiceKey(int index) =>
+        $"{ChannelServiceKey}:fallback:{index}";
 }
@@ -0,0 +1,141 @@
+using Grpc.Core;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
+
+/// <summary>
+/// An <see cref="ISecretsHubReader"/> over several single-endpoint readers: tries each endpoint in
+/// rotation order until one answers, remembering the one that did.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>Safe only when every endpoint serves the same store.</b> This type exists for ScadaBridge's
+/// central pair over one shared SQL Server store, where either central answers with the truth.
+/// Composed over endpoints with independent stores it would turn a primary outage into a silent
+/// convergence stop — the fallback "succeeds" against an emptier inventory (scadaproj#4). The DI
+/// extension's options doc carries the same warning where operators author the list.
+/// </para>
+/// <para>
+/// <b>The answering endpoint is sticky.</b> A sweep is one <c>GetManifest</c> plus N
+/// <c>GetSecrets</c> batches; without stickiness every call after a primary outage would re-pay the
+/// primary's failure — a full <c>CallDeadline</c> of latency — before reaching the healthy fallback.
+/// So each call starts at the last endpoint that answered (initially the primary) and rotates from
+/// there, which also means a recovered primary is retried naturally the next time the stuck-to
+/// endpoint fails, rather than being probed on a timer.
+/// </para>
+/// <para>
+/// <b>Only <see cref="RpcException"/> triggers failover.</b> It is the one signal that means "this
+/// endpoint did not usefully answer". <see cref="OperationCanceledException"/> is the host stopping
+/// — dialing another endpoint with a cancelled token would only delay shutdown — and anything else
+/// is a defect, which retrying elsewhere would mask. Both propagate untouched. When every endpoint
+/// fails, the <em>last</em> <see cref="RpcException"/> is rethrown so the sweep's catch-all handles
+/// it exactly as it handles a single-endpoint outage today.
+/// </para>
+/// </remarks>
+internal sealed partial class FailoverSecretsHubReader : ISecretsHubReader
+{
+    private readonly IReadOnlyList<ISecretsHubReader> _readers;
+    private readonly IReadOnlyList<string> _endpointLabels;
+    private readonly ILogger<FailoverSecretsHubReader> _logger;
+
+    /// <summary>
+    /// Index of the last endpoint that answered; every call starts its rotation here. Read and
+    /// written with <see cref="Volatile"/> — a torn or stale value costs at worst one extra failed
+    /// dial, never a wrong result, so a lock would buy nothing.
+    /// </summary>
+    private int _stickyIndex;
+
+    /// <summary>Creates the failover reader.</summary>
+    /// <param name="readers">One reader per endpoint, primary first — rotation order.</param>
+    /// <param name="endpointLabels">
+    /// The endpoint URI each reader dials, index-aligned with <paramref name="readers"/>. Used only
+    /// in log messages: a URI is not a secret, and naming both sides of a failover is what makes the
+    /// warning actionable.
+    /// </param>
+    /// <param name="logger">Logger for failover warnings.</param>
+    /// <exception cref="ArgumentException">
+    /// <paramref name="readers"/> is empty or the two lists differ in length.
+    /// </exception>
+    public FailoverSecretsHubReader(
+        IReadOnlyList<ISecretsHubReader> readers,
+        IReadOnlyList<string> endpointLabels,
+        ILogger<FailoverSecretsHubReader> logger)
+    {
+        ArgumentNullException.ThrowIfNull(readers);
+        ArgumentNullException.ThrowIfNull(endpointLabels);
+
+        if (readers.Count == 0)
+        {
+            throw new ArgumentException("At least one hub reader is required.", nameof(readers));
+        }
+
+        if (readers.Count != endpointLabels.Count)
+        {
+            throw new ArgumentException(
+                $"Endpoint labels ({endpointLabels.Count}) must align one-to-one with readers " +
+                $"({readers.Count}).",
+                nameof(endpointLabels));
+        }
+
+        _readers = readers;
+        _endpointLabels = endpointLabels;
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    /// <inheritdoc />
+    public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
+        ExecuteWithFailoverAsync(static (reader, token) => reader.GetManifestAsync(token), ct);
+
+    /// <inheritdoc />
+    public Task<IReadOnlyList<StoredSecret>> GetManyAsync(
+        IReadOnlyList<SecretName> names, CancellationToken ct) =>
+        ExecuteWithFailoverAsync((reader, token) => reader.GetManyAsync(names, token), ct);
+
+    // The one rotation, shared by both RPCs so their failover behaviour cannot drift apart.
+    private async Task<T> ExecuteWithFailoverAsync<T>(
+        Func<ISecretsHubReader, CancellationToken, Task<T>> call, CancellationToken ct)
+    {
+        int start = Volatile.Read(ref _stickyIndex);
+        RpcException? lastFailure = null;
+
+        for (int attempt = 0; attempt < _readers.Count; attempt++)
+        {
+            int index = (start + attempt) % _readers.Count;
+
+            try
+            {
+                T result = await call(_readers[index], ct).ConfigureAwait(false);
+                Volatile.Write(ref _stickyIndex, index);
+                return result;
+            }
+            catch (RpcException ex)
+            {
+                lastFailure = ex;
+
+                if (attempt < _readers.Count - 1)
+                {
+                    // The message carries the two endpoint URIs and the status code — nothing from
+                    // the exception detail, which is server-authored text no credential belongs in
+                    // but this node cannot vouch for.
+                    LogEndpointFailedOver(
+                        ex,
+                        _endpointLabels[index],
+                        ex.StatusCode,
+                        _endpointLabels[(start + attempt + 1) % _readers.Count]);
+                }
+            }
+        }
+
+        // Every endpoint failed. Rethrowing the LAST failure hands the sweep's catch-all the
+        // freshest evidence; it logs and retries next tick exactly as for a single endpoint.
+        throw lastFailure!;
+    }
+
+    [LoggerMessage(
+        EventId = 1,
+        Level = LogLevel.Warning,
+        Message = "Secrets hub endpoint {FailedEndpoint} did not answer ({StatusCode}); failing over to {NextEndpoint}.")]
+    private partial void LogEndpointFailedOver(
+        Exception exception, string failedEndpoint, StatusCode statusCode, string nextEndpoint);
+}
@@ -28,6 +28,21 @@ public sealed class SecretsGrpcHubClientOptions
    /// </remarks>
    public string Endpoint { get; set; } = string.Empty;

+    /// <summary>
+    /// Additional hub addresses tried, in order, when a call to <see cref="Endpoint"/> (or an
+    /// earlier fallback) fails with a transport-level gRPC error. Optional; empty by default.
+    /// Meaningful only when every listed endpoint serves the SAME store — for ScadaBridge that
+    /// is the central pair over one shared SQL Server store. Do NOT list endpoints backed by
+    /// independent stores: failing over to an emptier hub is a silent convergence stop, which is
+    /// the exact defect (scadaproj#4) the shared store exists to prevent.
+    /// </summary>
+    /// <remarks>
+    /// Getter-only so the configuration binder adds entries to this instance rather than needing a
+    /// setter. Each entry carries the same http-cleartext caveat as <see cref="Endpoint"/>: a plain
+    /// <c>http</c> fallback sends <see cref="BearerToken"/> in cleartext on every failed-over call.
+    /// </remarks>
+    public IList<string> FallbackEndpoints { get; } = [];
+
    /// <summary>
    /// Shared bearer token presented on every call. Required — an unset token is a hard startup
    /// failure rather than an anonymous call, because the hub authenticates on this alone.
@@ -69,8 +84,9 @@ public sealed class SecretsGrpcHubClientOptions
    /// operator actually has to edit rather than the default one.
    /// </param>
    /// <exception cref="InvalidOperationException">
-    /// The endpoint is missing or is not an absolute http/https URI, the bearer token is missing, or
-    /// an interval is not positive.
+    /// The endpoint is missing or is not an absolute http/https URI, a fallback endpoint is blank,
+    /// not an absolute http/https URI, or duplicates another endpoint, the bearer token is missing,
+    /// or an interval is not positive.
    /// </exception>
    public void Validate(string sectionPath = DefaultSectionPath)
    {
@@ -92,6 +108,35 @@ public sealed class SecretsGrpcHubClientOptions
                $"({sectionPath}:Endpoint, was '{Endpoint}').");
        }

+        var seenEndpoints = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { Endpoint };
+
+        foreach (string fallback in FallbackEndpoints)
+        {
+            if (string.IsNullOrWhiteSpace(fallback))
+            {
+                throw new InvalidOperationException(
+                    $"gRPC secrets hub client fallback endpoints must not be blank " +
+                    $"({sectionPath}:FallbackEndpoints).");
+            }
+
+            if (!Uri.TryCreate(fallback, UriKind.Absolute, out Uri? fallbackUri) ||
+                (fallbackUri.Scheme != Uri.UriSchemeHttp && fallbackUri.Scheme != Uri.UriSchemeHttps))
+            {
+                throw new InvalidOperationException(
+                    $"gRPC secrets hub client fallback endpoints must be absolute http or https " +
+                    $"URIs ({sectionPath}:FallbackEndpoints, entry '{fallback}').");
+            }
+
+            if (!seenEndpoints.Add(fallback))
+            {
+                // A duplicated endpoint silently halves the failover the operator thinks they
+                // configured: the rotation would try the same address twice and call it two tries.
+                throw new InvalidOperationException(
+                    $"gRPC secrets hub client fallback endpoints must not repeat the Endpoint or " +
+                    $"each other ({sectionPath}:FallbackEndpoints, entry '{fallback}').");
+            }
+        }
+
        if (string.IsNullOrWhiteSpace(BearerToken))
        {
            // Fail closed. An empty token would otherwise be presented as `Bearer `, which the hub