Add Polly resilience policies

2026-04-27 15:37:56 -04:00
parent d431ff9660
commit bd4a09a35e
22 changed files with 611 additions and 21 deletions
@@ -0,0 +1,62 @@
+using Grpc.Core;
+using Microsoft.Extensions.Logging;
+using MxGateway.Contracts.Proto;
+using Polly;
+using Polly.Retry;
+
+namespace MxGateway.Client;
+
+internal static class MxGatewayClientRetryPolicy
+{
+    public static ResiliencePipeline Create(
+        MxGatewayClientRetryOptions options,
+        ILogger? logger)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        options.Validate();
+
+        return new ResiliencePipelineBuilder()
+            .AddRetry(new RetryStrategyOptions
+            {
+                MaxRetryAttempts = Math.Max(0, options.MaxAttempts - 1),
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = options.UseJitter,
+                Delay = options.Delay,
+                MaxDelay = options.MaxDelay,
+                ShouldHandle = new PredicateBuilder().Handle<Exception>(IsTransientGrpcFailure),
+                OnRetry = args =>
+                {
+                    logger?.LogDebug(
+                        args.Outcome.Exception,
+                        "Retrying MXAccess Gateway client call after transient gRPC failure. Attempt {Attempt}.",
+                        args.AttemptNumber + 1);
+                    return default;
+                },
+            })
+            .Build();
+    }
+
+    public static bool IsRetryableCommand(MxCommandKind kind)
+    {
+        return kind is MxCommandKind.Ping
+            or MxCommandKind.GetSessionState
+            or MxCommandKind.GetWorkerInfo;
+    }
+
+    private static bool IsTransientGrpcFailure(Exception exception)
+    {
+        return exception switch
+        {
+            RpcException rpcException => IsTransientStatus(rpcException.StatusCode),
+            MxGatewayException { InnerException: RpcException rpcException } => IsTransientStatus(rpcException.StatusCode),
+            _ => false,
+        };
+    }
+
+    private static bool IsTransientStatus(StatusCode statusCode)
+    {
+        return statusCode is StatusCode.Unavailable
+            or StatusCode.DeadlineExceeded
+            or StatusCode.ResourceExhausted;
+    }
+}