From fad1c11e447a819ac4c932f6868a4826eb201480 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:02:46 -0500 Subject: [PATCH] fix(client-dotnet): size the retry outer budget for all attempts (mxaccessgw#197) ExecuteSafeUnaryAsync capped the whole retry pipeline at one DefaultCallTimeout while each attempt carried the same deadline, so a DeadlineExceeded first attempt consumed the whole budget and no retry ever ran. The outer cap is now MaxAttempts x DefaultCallTimeout + (MaxAttempts-1) x MaxDelay in both MxGatewayClient and GalaxyRepositoryClient. Adds a slow-first-attempt test (fails without the fix) and a budget unit test. --- .../FakeGatewayTransport.cs | 15 +++- .../MxGatewayClientSessionTests.cs | 74 +++++++++++++++++++ .../GalaxyRepositoryClient.cs | 5 +- .../MxGatewayClient.cs | 5 +- .../MxGatewayClientRetryPolicy.cs | 23 ++++++ 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/FakeGatewayTransport.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/FakeGatewayTransport.cs index cb677fc0..7d027f96 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/FakeGatewayTransport.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/FakeGatewayTransport.cs @@ -126,18 +126,29 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx return Task.FromResult(CloseSessionReply); } + /// + /// Gets or sets an optional behaviour run (before any queued exception) on each InvokeAsync call; + /// receives the 1-based attempt number and the call options. + /// + public Func? InvokeBehavior { get; set; } + /// - public Task InvokeAsync( + public async Task InvokeAsync( MxCommandRequest request, CallOptions callOptions) { InvokeCalls.Add((request, callOptions)); + if (InvokeBehavior is not null) + { + await InvokeBehavior(InvokeCalls.Count, callOptions).ConfigureAwait(false); + } + if (InvokeExceptions.TryDequeue(out Exception? exception)) { throw exception; } - return Task.FromResult(_invokeReplies.Dequeue()); + return _invokeReplies.Dequeue(); } /// diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientSessionTests.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientSessionTests.cs index 9cf14e74..6b97c737 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientSessionTests.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientSessionTests.cs @@ -315,6 +315,80 @@ public sealed class MxGatewayClientSessionTests Assert.Equal(2, transport.InvokeCalls.Count); } + /// + /// Verifies that a first attempt which runs to its own call deadline and fails with DeadlineExceeded + /// does not exhaust the retry budget (mxaccessgw#197): the retry still runs and succeeds. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task InvokeAsync_RetriesAfterFirstAttemptRunsToItsDeadline() + { + FakeGatewayTransport transport = new(new MxGatewayClientOptions + { + Endpoint = new Uri("http://localhost:5000"), + ApiKey = "test-api-key", + DefaultCallTimeout = TimeSpan.FromMilliseconds(400), + Retry = new MxGatewayClientRetryOptions + { + MaxAttempts = 2, + Delay = TimeSpan.FromMilliseconds(10), + MaxDelay = TimeSpan.FromMilliseconds(20), + }, + }); + transport.AddInvokeReply(new MxCommandReply + { + SessionId = "session-fixture", + Kind = MxCommandKind.Ping, + ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, + }); + transport.InvokeBehavior = async (attempt, callOptions) => + { + if (attempt != 1) + { + return; + } + + // Simulate a gateway that never answers: block until this attempt's own deadline, + // then fail the way gRPC does. + TimeSpan remaining = callOptions.Deadline!.Value - DateTime.UtcNow; + if (remaining > TimeSpan.Zero) + { + await Task.Delay(remaining + TimeSpan.FromMilliseconds(20)); + } + + throw new RpcException(new Status(StatusCode.DeadlineExceeded, "deadline")); + }; + await using MxGatewayClient client = CreateClient(transport); + MxGatewaySession session = await client.OpenSessionAsync(); + + await session.InvokeAsync(new MxCommandRequest + { + SessionId = session.SessionId, + Command = new MxCommand { Kind = MxCommandKind.Ping, Ping = new PingCommand() }, + }); + + Assert.Equal(2, transport.InvokeCalls.Count); + } + + /// Verifies the overall retry budget spans every attempt plus back-off. + [Fact] + public void ComputeOverallBudget_CoversAllAttemptsAndBackoff() + { + MxGatewayClientRetryOptions retry = new() + { + MaxAttempts = 3, + Delay = TimeSpan.FromSeconds(1), + MaxDelay = TimeSpan.FromSeconds(2), + }; + + Assert.Equal( + TimeSpan.FromSeconds((3 * 30) + (2 * 2)), + MxGatewayClientRetryPolicy.ComputeOverallBudget(retry, TimeSpan.FromSeconds(30))); + Assert.Equal( + TimeSpan.FromSeconds(30), + MxGatewayClientRetryPolicy.ComputeOverallBudget(new MxGatewayClientRetryOptions { MaxAttempts = 1 }, TimeSpan.FromSeconds(30))); + } + /// Verifies that open session does not retry on transient RPC failure. /// A task that represents the asynchronous operation. [Fact] diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GalaxyRepositoryClient.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GalaxyRepositoryClient.cs index a259fe31..35937b80 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GalaxyRepositoryClient.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GalaxyRepositoryClient.cs @@ -518,7 +518,10 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable CancellationToken cancellationToken) { using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeout.CancelAfter(Options.DefaultCallTimeout); + // Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline) + // plus back-off; capping it at one DefaultCallTimeout starved retries of a + // DeadlineExceeded first attempt (mxaccessgw#197). + timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout)); return await _safeUnaryRetryPipeline.ExecuteAsync( async token => await call(token).ConfigureAwait(false), diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClient.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClient.cs index cb331250..cd0604f8 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClient.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClient.cs @@ -376,7 +376,10 @@ public sealed class MxGatewayClient : IAsyncDisposable CancellationToken cancellationToken) { using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeout.CancelAfter(Options.DefaultCallTimeout); + // Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline) + // plus back-off; capping it at one DefaultCallTimeout starved retries of a + // DeadlineExceeded first attempt (mxaccessgw#197). + timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout)); return await _safeUnaryRetryPipeline.ExecuteAsync( async token => await call(token).ConfigureAwait(false), diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClientRetryPolicy.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClientRetryPolicy.cs index 4d01bbb8..cf1c3dfd 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClientRetryPolicy.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayClientRetryPolicy.cs @@ -41,6 +41,29 @@ internal static class MxGatewayClientRetryPolicy .Build(); } + /// + /// Computes the overall time budget for a retried unary call: every attempt gets its own + /// deadline, plus the worst-case back-off between attempts. + /// An outer cap equal to a single would let a + /// DeadlineExceeded first attempt consume the whole budget so no retry could ever run. + /// + /// Retry configuration. + /// The per-attempt call timeout. + /// The overall budget, or if it exceeds the timer range. + public static TimeSpan ComputeOverallBudget(MxGatewayClientRetryOptions options, TimeSpan callTimeout) + { + ArgumentNullException.ThrowIfNull(options); + + int attempts = Math.Max(1, options.MaxAttempts); + double totalMs = (callTimeout.TotalMilliseconds * attempts) + + (options.MaxDelay.TotalMilliseconds * (attempts - 1)); + + // CancellationTokenSource.CancelAfter accepts at most int.MaxValue milliseconds. + return totalMs >= int.MaxValue + ? Timeout.InfiniteTimeSpan + : TimeSpan.FromMilliseconds(totalMs); + } + /// Returns whether a command kind is eligible for automatic retry on transient failures. /// The command kind to check. /// if the command kind may be automatically retried; otherwise, .