fix(secrets-grpc): hub-call cancellation surfaces as cancellation, not a failed sweep (scadaproj#45)

Secrets-019: the hub channels are created with ThrowOperationCanceledOnCancellation = true, so a
call cancelled through its token throws OperationCanceledException. Defensively, the failover
reader rethrows RpcException(Cancelled)-with-a-cancelled-token as cancellation instead of dialing
every fallback with the same cancelled token, and the sweep treats it as shutdown, not a failure
recorded in ISecretsHubSyncStatus. A server-side Cancelled status (token not cancelled) still fails
over.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 15:23:02 -05:00
parent 3aa7c728d5
commit fac3c5355c
4 changed files with 174 additions and 4 deletions
@@ -87,7 +87,8 @@ public static class SecretsGrpcHubClientExtensions
// would silently lose to that registration and this client would then dial someone else's
// endpoint with the hub credential attached.
services.TryAddKeyedSingleton(ChannelServiceKey, (sp, _) => GrpcChannel.ForAddress(
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>().Value.Endpoint));
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>().Value.Endpoint,
ChannelOptions()));
if (eager.FallbackEndpoints.Count == 0)
{
@@ -112,9 +113,10 @@ public static class SecretsGrpcHubClientExtensions
int index = i;
services.TryAddKeyedSingleton(
FallbackChannelServiceKey(index),
(sp, _) => GrpcChannel.ForAddress(sp
.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>()
.Value.FallbackEndpoints[index]));
(sp, _) => GrpcChannel.ForAddress(
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>()
.Value.FallbackEndpoints[index],
ChannelOptions()));
}
services.TryAddSingleton<ISecretsHubReader>(sp =>
@@ -173,4 +175,10 @@ public static class SecretsGrpcHubClientExtensions
/// <returns>The service key.</returns>
internal static string FallbackChannelServiceKey(int index) =>
$"{ChannelServiceKey}:fallback:{index}";
// Secrets-019: a call cancelled through its token must surface as OperationCanceledException,
// which is what the sweep treats as shutdown and the failover reader propagates — not as
// RpcException(Cancelled), the gRPC default, which the sweep recorded as a failed sync and the
// failover reader answered by dialing every other endpoint with the same cancelled token.
internal static GrpcChannelOptions ChannelOptions() => new() { ThrowOperationCanceledOnCancellation = true };
}
@@ -111,6 +111,13 @@ internal sealed partial class FailoverSecretsHubReader : ISecretsHubReader
Volatile.Write(ref _stickyIndex, index);
return result;
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
{
// The caller cancelled (host stopping) and the channel reported it gRPC-style. That
// is not "this endpoint did not answer": propagate as cancellation, never fail over
// with a token that is already cancelled (Secrets-019).
throw new OperationCanceledException(ex.Message, ex, ct);
}
catch (RpcException ex)
{
lastFailure = ex;
@@ -1,3 +1,4 @@
using Grpc.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -177,6 +178,12 @@ internal sealed partial class GrpcSecretSyncService : BackgroundService
{
// Host shutdown — not a failure.
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
{
// Host shutdown, reported gRPC-style by a channel not created through this library's
// registration (which sets ThrowOperationCanceledOnCancellation) — also not a failure
// (Secrets-019).
}
catch (Exception ex)
{
// Deliberately catch-all. Everything reachable from here — an unreachable hub, a