feat(secrets-grpc): FallbackEndpoints + sticky failover hub reader (scadaproj#4)
SecretsGrpcHubClientOptions gains a getter-only FallbackEndpoints list:
additional hub addresses tried in order when a call fails with a
transport-level RpcException. Validation fails startup on blank/relative
entries and on any duplicate of the primary or another fallback (a
duplicated endpoint silently halves the configured failover).
New FailoverSecretsHubReader composes one GrpcSecretsHubClient per
endpoint: the answering endpoint is sticky (a sweep is GetManifest + N
GetSecrets batches — without stickiness every call after a primary
outage re-pays a full CallDeadline before reaching the healthy
fallback), rotation wraps so a recovered primary is retried when the
stuck-to endpoint fails, only RpcException triggers failover
(cancellation and defects propagate untouched), and when every endpoint
fails the last RpcException is rethrown so the sweep degrades to
last-known-good exactly as today. Failover warnings name only the two
endpoint URIs and the status code — never any credential material.
DI: zero fallbacks keeps the registration graph byte-identical to 0.4.x
(single keyed channel under ChannelServiceKey, plain
GrpcSecretsHubClient); with fallbacks, one container-owned keyed channel
per endpoint (fallback i under "{ChannelServiceKey}:fallback:{i}") wired
into the failover reader. GrpcSecretsHubClient itself is unchanged.
Safe only when every listed endpoint serves the SAME store — for
ScadaBridge, the central pair over one shared SQL Server store; the
options doc and CHANGELOG carry the warning.
Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
This commit is contained in:
+47
-2
@@ -28,6 +28,21 @@ public sealed class SecretsGrpcHubClientOptions
|
||||
/// </remarks>
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Additional hub addresses tried, in order, when a call to <see cref="Endpoint"/> (or an
|
||||
/// earlier fallback) fails with a transport-level gRPC error. Optional; empty by default.
|
||||
/// Meaningful only when every listed endpoint serves the SAME store — for ScadaBridge that
|
||||
/// is the central pair over one shared SQL Server store. Do NOT list endpoints backed by
|
||||
/// independent stores: failing over to an emptier hub is a silent convergence stop, which is
|
||||
/// the exact defect (scadaproj#4) the shared store exists to prevent.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Getter-only so the configuration binder adds entries to this instance rather than needing a
|
||||
/// setter. Each entry carries the same http-cleartext caveat as <see cref="Endpoint"/>: a plain
|
||||
/// <c>http</c> fallback sends <see cref="BearerToken"/> in cleartext on every failed-over call.
|
||||
/// </remarks>
|
||||
public IList<string> FallbackEndpoints { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Shared bearer token presented on every call. Required — an unset token is a hard startup
|
||||
/// failure rather than an anonymous call, because the hub authenticates on this alone.
|
||||
@@ -69,8 +84,9 @@ public sealed class SecretsGrpcHubClientOptions
|
||||
/// operator actually has to edit rather than the default one.
|
||||
/// </param>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// The endpoint is missing or is not an absolute http/https URI, the bearer token is missing, or
|
||||
/// an interval is not positive.
|
||||
/// The endpoint is missing or is not an absolute http/https URI, a fallback endpoint is blank,
|
||||
/// not an absolute http/https URI, or duplicates another endpoint, the bearer token is missing,
|
||||
/// or an interval is not positive.
|
||||
/// </exception>
|
||||
public void Validate(string sectionPath = DefaultSectionPath)
|
||||
{
|
||||
@@ -92,6 +108,35 @@ public sealed class SecretsGrpcHubClientOptions
|
||||
$"({sectionPath}:Endpoint, was '{Endpoint}').");
|
||||
}
|
||||
|
||||
var seenEndpoints = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { Endpoint };
|
||||
|
||||
foreach (string fallback in FallbackEndpoints)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(fallback))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"gRPC secrets hub client fallback endpoints must not be blank " +
|
||||
$"({sectionPath}:FallbackEndpoints).");
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(fallback, UriKind.Absolute, out Uri? fallbackUri) ||
|
||||
(fallbackUri.Scheme != Uri.UriSchemeHttp && fallbackUri.Scheme != Uri.UriSchemeHttps))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"gRPC secrets hub client fallback endpoints must be absolute http or https " +
|
||||
$"URIs ({sectionPath}:FallbackEndpoints, entry '{fallback}').");
|
||||
}
|
||||
|
||||
if (!seenEndpoints.Add(fallback))
|
||||
{
|
||||
// A duplicated endpoint silently halves the failover the operator thinks they
|
||||
// configured: the rotation would try the same address twice and call it two tries.
|
||||
throw new InvalidOperationException(
|
||||
$"gRPC secrets hub client fallback endpoints must not repeat the Endpoint or " +
|
||||
$"each other ({sectionPath}:FallbackEndpoints, entry '{fallback}').");
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(BearerToken))
|
||||
{
|
||||
// Fail closed. An empty token would otherwise be presented as `Bearer `, which the hub
|
||||
|
||||
Reference in New Issue
Block a user