fix(client-dotnet): size the retry outer budget for all attempts (mxaccessgw#197)

ExecuteSafeUnaryAsync capped the whole retry pipeline at one DefaultCallTimeout
while each attempt carried the same deadline, so a DeadlineExceeded first
attempt consumed the whole budget and no retry ever ran. The outer cap is now
MaxAttempts x DefaultCallTimeout + (MaxAttempts-1) x MaxDelay in both
MxGatewayClient and GalaxyRepositoryClient. Adds a slow-first-attempt test
(fails without the fix) and a budget unit test.
This commit is contained in:
Joseph Doherty
2026-11-09 12:02:46 -05:00
parent 20138b60cc
commit fad1c11e44
5 changed files with 118 additions and 4 deletions
@@ -126,18 +126,29 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
return Task.FromResult(CloseSessionReply); return Task.FromResult(CloseSessionReply);
} }
/// <summary>
/// Gets or sets an optional behaviour run (before any queued exception) on each InvokeAsync call;
/// receives the 1-based attempt number and the call options.
/// </summary>
public Func<int, CallOptions, Task>? InvokeBehavior { get; set; }
/// <inheritdoc /> /// <inheritdoc />
public Task<MxCommandReply> InvokeAsync( public async Task<MxCommandReply> InvokeAsync(
MxCommandRequest request, MxCommandRequest request,
CallOptions callOptions) CallOptions callOptions)
{ {
InvokeCalls.Add((request, callOptions)); InvokeCalls.Add((request, callOptions));
if (InvokeBehavior is not null)
{
await InvokeBehavior(InvokeCalls.Count, callOptions).ConfigureAwait(false);
}
if (InvokeExceptions.TryDequeue(out Exception? exception)) if (InvokeExceptions.TryDequeue(out Exception? exception))
{ {
throw exception; throw exception;
} }
return Task.FromResult(_invokeReplies.Dequeue()); return _invokeReplies.Dequeue();
} }
/// <inheritdoc /> /// <inheritdoc />
@@ -315,6 +315,80 @@ public sealed class MxGatewayClientSessionTests
Assert.Equal(2, transport.InvokeCalls.Count); Assert.Equal(2, transport.InvokeCalls.Count);
} }
/// <summary>
/// Verifies that a first attempt which runs to its own call deadline and fails with DeadlineExceeded
/// does not exhaust the retry budget (mxaccessgw#197): the retry still runs and succeeds.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task InvokeAsync_RetriesAfterFirstAttemptRunsToItsDeadline()
{
FakeGatewayTransport transport = new(new MxGatewayClientOptions
{
Endpoint = new Uri("http://localhost:5000"),
ApiKey = "test-api-key",
DefaultCallTimeout = TimeSpan.FromMilliseconds(400),
Retry = new MxGatewayClientRetryOptions
{
MaxAttempts = 2,
Delay = TimeSpan.FromMilliseconds(10),
MaxDelay = TimeSpan.FromMilliseconds(20),
},
});
transport.AddInvokeReply(new MxCommandReply
{
SessionId = "session-fixture",
Kind = MxCommandKind.Ping,
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
});
transport.InvokeBehavior = async (attempt, callOptions) =>
{
if (attempt != 1)
{
return;
}
// Simulate a gateway that never answers: block until this attempt's own deadline,
// then fail the way gRPC does.
TimeSpan remaining = callOptions.Deadline!.Value - DateTime.UtcNow;
if (remaining > TimeSpan.Zero)
{
await Task.Delay(remaining + TimeSpan.FromMilliseconds(20));
}
throw new RpcException(new Status(StatusCode.DeadlineExceeded, "deadline"));
};
await using MxGatewayClient client = CreateClient(transport);
MxGatewaySession session = await client.OpenSessionAsync();
await session.InvokeAsync(new MxCommandRequest
{
SessionId = session.SessionId,
Command = new MxCommand { Kind = MxCommandKind.Ping, Ping = new PingCommand() },
});
Assert.Equal(2, transport.InvokeCalls.Count);
}
/// <summary>Verifies the overall retry budget spans every attempt plus back-off.</summary>
[Fact]
public void ComputeOverallBudget_CoversAllAttemptsAndBackoff()
{
MxGatewayClientRetryOptions retry = new()
{
MaxAttempts = 3,
Delay = TimeSpan.FromSeconds(1),
MaxDelay = TimeSpan.FromSeconds(2),
};
Assert.Equal(
TimeSpan.FromSeconds((3 * 30) + (2 * 2)),
MxGatewayClientRetryPolicy.ComputeOverallBudget(retry, TimeSpan.FromSeconds(30)));
Assert.Equal(
TimeSpan.FromSeconds(30),
MxGatewayClientRetryPolicy.ComputeOverallBudget(new MxGatewayClientRetryOptions { MaxAttempts = 1 }, TimeSpan.FromSeconds(30)));
}
/// <summary>Verifies that open session does not retry on transient RPC failure.</summary> /// <summary>Verifies that open session does not retry on transient RPC failure.</summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
@@ -518,7 +518,10 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeout.CancelAfter(Options.DefaultCallTimeout); // Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline)
// plus back-off; capping it at one DefaultCallTimeout starved retries of a
// DeadlineExceeded first attempt (mxaccessgw#197).
timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout));
return await _safeUnaryRetryPipeline.ExecuteAsync( return await _safeUnaryRetryPipeline.ExecuteAsync(
async token => await call(token).ConfigureAwait(false), async token => await call(token).ConfigureAwait(false),
@@ -376,7 +376,10 @@ public sealed class MxGatewayClient : IAsyncDisposable
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeout.CancelAfter(Options.DefaultCallTimeout); // Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline)
// plus back-off; capping it at one DefaultCallTimeout starved retries of a
// DeadlineExceeded first attempt (mxaccessgw#197).
timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout));
return await _safeUnaryRetryPipeline.ExecuteAsync( return await _safeUnaryRetryPipeline.ExecuteAsync(
async token => await call(token).ConfigureAwait(false), async token => await call(token).ConfigureAwait(false),
@@ -41,6 +41,29 @@ internal static class MxGatewayClientRetryPolicy
.Build(); .Build();
} }
/// <summary>
/// Computes the overall time budget for a retried unary call: every attempt gets its own
/// <paramref name="callTimeout"/> deadline, plus the worst-case back-off between attempts.
/// An outer cap equal to a single <paramref name="callTimeout"/> would let a
/// <c>DeadlineExceeded</c> first attempt consume the whole budget so no retry could ever run.
/// </summary>
/// <param name="options">Retry configuration.</param>
/// <param name="callTimeout">The per-attempt call timeout.</param>
/// <returns>The overall budget, or <see cref="Timeout.InfiniteTimeSpan"/> if it exceeds the timer range.</returns>
public static TimeSpan ComputeOverallBudget(MxGatewayClientRetryOptions options, TimeSpan callTimeout)
{
ArgumentNullException.ThrowIfNull(options);
int attempts = Math.Max(1, options.MaxAttempts);
double totalMs = (callTimeout.TotalMilliseconds * attempts)
+ (options.MaxDelay.TotalMilliseconds * (attempts - 1));
// CancellationTokenSource.CancelAfter accepts at most int.MaxValue milliseconds.
return totalMs >= int.MaxValue
? Timeout.InfiniteTimeSpan
: TimeSpan.FromMilliseconds(totalMs);
}
/// <summary>Returns whether a command kind is eligible for automatic retry on transient failures.</summary> /// <summary>Returns whether a command kind is eligible for automatic retry on transient failures.</summary>
/// <param name="kind">The command kind to check.</param> /// <param name="kind">The command kind to check.</param>
/// <returns><see langword="true"/> if the command kind may be automatically retried; otherwise, <see langword="false"/>.</returns> /// <returns><see langword="true"/> if the command kind may be automatically retried; otherwise, <see langword="false"/>.</returns>