fix(secrets-grpc): hub-call cancellation surfaces as cancellation, not a failed sweep (scadaproj#45)
Secrets-019: the hub channels are created with ThrowOperationCanceledOnCancellation = true, so a call cancelled through its token throws OperationCanceledException. Defensively, the failover reader rethrows RpcException(Cancelled)-with-a-cancelled-token as cancellation instead of dialing every fallback with the same cancelled token, and the sweep treats it as shutdown, not a failure recorded in ISecretsHubSyncStatus. A server-side Cancelled status (token not cancelled) still fails over. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+12
-4
@@ -87,7 +87,8 @@ public static class SecretsGrpcHubClientExtensions
|
||||
// would silently lose to that registration and this client would then dial someone else's
|
||||
// endpoint with the hub credential attached.
|
||||
services.TryAddKeyedSingleton(ChannelServiceKey, (sp, _) => GrpcChannel.ForAddress(
|
||||
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>().Value.Endpoint));
|
||||
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>().Value.Endpoint,
|
||||
ChannelOptions()));
|
||||
|
||||
if (eager.FallbackEndpoints.Count == 0)
|
||||
{
|
||||
@@ -112,9 +113,10 @@ public static class SecretsGrpcHubClientExtensions
|
||||
int index = i;
|
||||
services.TryAddKeyedSingleton(
|
||||
FallbackChannelServiceKey(index),
|
||||
(sp, _) => GrpcChannel.ForAddress(sp
|
||||
.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>()
|
||||
.Value.FallbackEndpoints[index]));
|
||||
(sp, _) => GrpcChannel.ForAddress(
|
||||
sp.GetRequiredService<IOptions<SecretsGrpcHubClientOptions>>()
|
||||
.Value.FallbackEndpoints[index],
|
||||
ChannelOptions()));
|
||||
}
|
||||
|
||||
services.TryAddSingleton<ISecretsHubReader>(sp =>
|
||||
@@ -173,4 +175,10 @@ public static class SecretsGrpcHubClientExtensions
|
||||
/// <returns>The service key.</returns>
|
||||
internal static string FallbackChannelServiceKey(int index) =>
|
||||
$"{ChannelServiceKey}:fallback:{index}";
|
||||
|
||||
// Secrets-019: a call cancelled through its token must surface as OperationCanceledException,
|
||||
// which is what the sweep treats as shutdown and the failover reader propagates — not as
|
||||
// RpcException(Cancelled), the gRPC default, which the sweep recorded as a failed sync and the
|
||||
// failover reader answered by dialing every other endpoint with the same cancelled token.
|
||||
internal static GrpcChannelOptions ChannelOptions() => new() { ThrowOperationCanceledOnCancellation = true };
|
||||
}
|
||||
|
||||
@@ -111,6 +111,13 @@ internal sealed partial class FailoverSecretsHubReader : ISecretsHubReader
|
||||
Volatile.Write(ref _stickyIndex, index);
|
||||
return result;
|
||||
}
|
||||
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
|
||||
{
|
||||
// The caller cancelled (host stopping) and the channel reported it gRPC-style. That
|
||||
// is not "this endpoint did not answer": propagate as cancellation, never fail over
|
||||
// with a token that is already cancelled (Secrets-019).
|
||||
throw new OperationCanceledException(ex.Message, ex, ct);
|
||||
}
|
||||
catch (RpcException ex)
|
||||
{
|
||||
lastFailure = ex;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -177,6 +178,12 @@ internal sealed partial class GrpcSecretSyncService : BackgroundService
|
||||
{
|
||||
// Host shutdown — not a failure.
|
||||
}
|
||||
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
|
||||
{
|
||||
// Host shutdown, reported gRPC-style by a channel not created through this library's
|
||||
// registration (which sets ThrowOperationCanceledOnCancellation) — also not a failure
|
||||
// (Secrets-019).
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Deliberately catch-all. Everything reachable from here — an unreachable hub, a
|
||||
|
||||
Reference in New Issue
Block a user