feat(secrets-grpc): FallbackEndpoints + sticky failover hub reader (scadaproj#4)

SecretsGrpcHubClientOptions gains a getter-only FallbackEndpoints list: additional hub addresses tried in order when a call fails with a transport-level RpcException. Validation fails startup on blank/relative entries and on any duplicate of the primary or another fallback (a duplicated endpoint silently halves the configured failover). New FailoverSecretsHubReader composes one GrpcSecretsHubClient per endpoint: the answering endpoint is sticky (a sweep is GetManifest + N GetSecrets batches — without stickiness every call after a primary outage re-pays a full CallDeadline before reaching the healthy fallback), rotation wraps so a recovered primary is retried when the stuck-to endpoint fails, only RpcException triggers failover (cancellation and defects propagate untouched), and when every endpoint fails the last RpcException is rethrown so the sweep degrades to last-known-good exactly as today. Failover warnings name only the two endpoint URIs and the status code — never any credential material. DI: zero fallbacks keeps the registration graph byte-identical to 0.4.x (single keyed channel under ChannelServiceKey, plain GrpcSecretsHubClient); with fallbacks, one container-owned keyed channel per endpoint (fallback i under "{ChannelServiceKey}:fallback:{i}") wired into the failover reader. GrpcSecretsHubClient itself is unchanged. Safe only when every listed endpoint serves the SAME store — for ScadaBridge, the central pair over one shared SQL Server store; the options doc and CHANGELOG carry the warning. Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
2026-08-07 09:37:22 -04:00
parent 7df05ccc27
commit 0d4978953f
7 changed files with 695 additions and 8 deletions
@@ -452,6 +452,82 @@ public sealed class GrpcSecretSyncServiceTests : IDisposable
                SecretsGrpcHubClientExtensions.ChannelServiceKey));
    }

+    [Fact]
+    public void Registration_with_no_fallbacks_resolves_the_plain_single_endpoint_client()
+    {
+        // The zero-fallback registration graph is the pre-0.5.0 one, unchanged: one keyed channel,
+        // one GrpcSecretsHubClient as the reader. Consumers wired before FallbackEndpoints existed
+        // must see an identical container shape.
+        ServiceCollection services = new();
+        services.AddLogging();
+        services.AddZbSecretsGrpcHubClient(HubConfig());
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+
+        Assert.IsType<GrpcSecretsHubClient>(provider.GetRequiredService<ISecretsHubReader>());
+        Assert.NotNull(provider.GetRequiredKeyedService<GrpcChannel>(
+            SecretsGrpcHubClientExtensions.ChannelServiceKey));
+        Assert.Null(provider.GetKeyedService<GrpcChannel>(
+            $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:0"));
+    }
+
+    [Fact]
+    public void Registration_with_fallbacks_resolves_the_failover_reader_over_one_channel_per_endpoint()
+    {
+        ServiceCollection services = new();
+        services.AddLogging();
+        services.AddZbSecretsGrpcHubClient(HubConfig(new Dictionary<string, string?>
+        {
+            ["Secrets:GrpcHub:FallbackEndpoints:0"] = "https://central-b:5001",
+            ["Secrets:GrpcHub:FallbackEndpoints:1"] = "https://central-c:5001",
+        }));
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+
+        Assert.IsType<FailoverSecretsHubReader>(provider.GetRequiredService<ISecretsHubReader>());
+
+        GrpcChannel primary = provider.GetRequiredKeyedService<GrpcChannel>(
+            SecretsGrpcHubClientExtensions.ChannelServiceKey);
+        GrpcChannel fallback0 = provider.GetRequiredKeyedService<GrpcChannel>(
+            $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:0");
+        GrpcChannel fallback1 = provider.GetRequiredKeyedService<GrpcChannel>(
+            $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:1");
+
+        // Each channel dials its own endpoint — three distinct singletons, container-owned.
+        Assert.Equal("central-a:5001", primary.Target);
+        Assert.Equal("central-b:5001", fallback0.Target);
+        Assert.Equal("central-c:5001", fallback1.Target);
+    }
+
+    [Fact]
+    public void Registration_refuses_a_malformed_fallback_endpoint_at_startup()
+    {
+        InvalidOperationException ex = Assert.Throws<InvalidOperationException>(() =>
+            new ServiceCollection().AddZbSecretsGrpcHubClient(HubConfig(new Dictionary<string, string?>
+            {
+                ["Secrets:GrpcHub:FallbackEndpoints:0"] = "not a uri",
+            })));
+
+        Assert.Contains("Secrets:GrpcHub:FallbackEndpoints", ex.Message, StringComparison.Ordinal);
+    }
+
+    // A minimal valid hub-client configuration, extendable per test.
+    private static IConfiguration HubConfig(Dictionary<string, string?>? extra = null)
+    {
+        var values = new Dictionary<string, string?>
+        {
+            ["Secrets:GrpcHub:Endpoint"] = "https://central-a:5001",
+            ["Secrets:GrpcHub:BearerToken"] = "token",
+        };
+
+        foreach ((string key, string? value) in extra ?? [])
+        {
+            values[key] = value;
+        }
+
+        return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
+    }
+
    // Runs the hosted service through one startup sweep and shuts it down, asserting that nothing
    // escaped ExecuteAsync. The long sync interval means exactly one sweep happens.
    private async Task RunOneStartupSweepAsync(Func<bool> sweepObserved)