fix(client-dotnet): size the retry outer budget for all attempts (mxaccessgw#197)
ExecuteSafeUnaryAsync capped the whole retry pipeline at one DefaultCallTimeout while each attempt carried the same deadline, so a DeadlineExceeded first attempt consumed the whole budget and no retry ever ran. The outer cap is now MaxAttempts x DefaultCallTimeout + (MaxAttempts-1) x MaxDelay in both MxGatewayClient and GalaxyRepositoryClient. Adds a slow-first-attempt test (fails without the fix) and a budget unit test.
This commit is contained in:
@@ -126,18 +126,29 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
|||||||
return Task.FromResult(CloseSessionReply);
|
return Task.FromResult(CloseSessionReply);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets an optional behaviour run (before any queued exception) on each InvokeAsync call;
|
||||||
|
/// receives the 1-based attempt number and the call options.
|
||||||
|
/// </summary>
|
||||||
|
public Func<int, CallOptions, Task>? InvokeBehavior { get; set; }
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public Task<MxCommandReply> InvokeAsync(
|
public async Task<MxCommandReply> InvokeAsync(
|
||||||
MxCommandRequest request,
|
MxCommandRequest request,
|
||||||
CallOptions callOptions)
|
CallOptions callOptions)
|
||||||
{
|
{
|
||||||
InvokeCalls.Add((request, callOptions));
|
InvokeCalls.Add((request, callOptions));
|
||||||
|
if (InvokeBehavior is not null)
|
||||||
|
{
|
||||||
|
await InvokeBehavior(InvokeCalls.Count, callOptions).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
if (InvokeExceptions.TryDequeue(out Exception? exception))
|
if (InvokeExceptions.TryDequeue(out Exception? exception))
|
||||||
{
|
{
|
||||||
throw exception;
|
throw exception;
|
||||||
}
|
}
|
||||||
|
|
||||||
return Task.FromResult(_invokeReplies.Dequeue());
|
return _invokeReplies.Dequeue();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
|
|||||||
@@ -315,6 +315,80 @@ public sealed class MxGatewayClientSessionTests
|
|||||||
Assert.Equal(2, transport.InvokeCalls.Count);
|
Assert.Equal(2, transport.InvokeCalls.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies that a first attempt which runs to its own call deadline and fails with DeadlineExceeded
|
||||||
|
/// does not exhaust the retry budget (mxaccessgw#197): the retry still runs and succeeds.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task InvokeAsync_RetriesAfterFirstAttemptRunsToItsDeadline()
|
||||||
|
{
|
||||||
|
FakeGatewayTransport transport = new(new MxGatewayClientOptions
|
||||||
|
{
|
||||||
|
Endpoint = new Uri("http://localhost:5000"),
|
||||||
|
ApiKey = "test-api-key",
|
||||||
|
DefaultCallTimeout = TimeSpan.FromMilliseconds(400),
|
||||||
|
Retry = new MxGatewayClientRetryOptions
|
||||||
|
{
|
||||||
|
MaxAttempts = 2,
|
||||||
|
Delay = TimeSpan.FromMilliseconds(10),
|
||||||
|
MaxDelay = TimeSpan.FromMilliseconds(20),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
transport.AddInvokeReply(new MxCommandReply
|
||||||
|
{
|
||||||
|
SessionId = "session-fixture",
|
||||||
|
Kind = MxCommandKind.Ping,
|
||||||
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||||
|
});
|
||||||
|
transport.InvokeBehavior = async (attempt, callOptions) =>
|
||||||
|
{
|
||||||
|
if (attempt != 1)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulate a gateway that never answers: block until this attempt's own deadline,
|
||||||
|
// then fail the way gRPC does.
|
||||||
|
TimeSpan remaining = callOptions.Deadline!.Value - DateTime.UtcNow;
|
||||||
|
if (remaining > TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
await Task.Delay(remaining + TimeSpan.FromMilliseconds(20));
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RpcException(new Status(StatusCode.DeadlineExceeded, "deadline"));
|
||||||
|
};
|
||||||
|
await using MxGatewayClient client = CreateClient(transport);
|
||||||
|
MxGatewaySession session = await client.OpenSessionAsync();
|
||||||
|
|
||||||
|
await session.InvokeAsync(new MxCommandRequest
|
||||||
|
{
|
||||||
|
SessionId = session.SessionId,
|
||||||
|
Command = new MxCommand { Kind = MxCommandKind.Ping, Ping = new PingCommand() },
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(2, transport.InvokeCalls.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Verifies the overall retry budget spans every attempt plus back-off.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void ComputeOverallBudget_CoversAllAttemptsAndBackoff()
|
||||||
|
{
|
||||||
|
MxGatewayClientRetryOptions retry = new()
|
||||||
|
{
|
||||||
|
MaxAttempts = 3,
|
||||||
|
Delay = TimeSpan.FromSeconds(1),
|
||||||
|
MaxDelay = TimeSpan.FromSeconds(2),
|
||||||
|
};
|
||||||
|
|
||||||
|
Assert.Equal(
|
||||||
|
TimeSpan.FromSeconds((3 * 30) + (2 * 2)),
|
||||||
|
MxGatewayClientRetryPolicy.ComputeOverallBudget(retry, TimeSpan.FromSeconds(30)));
|
||||||
|
Assert.Equal(
|
||||||
|
TimeSpan.FromSeconds(30),
|
||||||
|
MxGatewayClientRetryPolicy.ComputeOverallBudget(new MxGatewayClientRetryOptions { MaxAttempts = 1 }, TimeSpan.FromSeconds(30)));
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that open session does not retry on transient RPC failure.</summary>
|
/// <summary>Verifies that open session does not retry on transient RPC failure.</summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
@@ -518,7 +518,10 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable
|
|||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
timeout.CancelAfter(Options.DefaultCallTimeout);
|
// Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline)
|
||||||
|
// plus back-off; capping it at one DefaultCallTimeout starved retries of a
|
||||||
|
// DeadlineExceeded first attempt (mxaccessgw#197).
|
||||||
|
timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout));
|
||||||
|
|
||||||
return await _safeUnaryRetryPipeline.ExecuteAsync(
|
return await _safeUnaryRetryPipeline.ExecuteAsync(
|
||||||
async token => await call(token).ConfigureAwait(false),
|
async token => await call(token).ConfigureAwait(false),
|
||||||
|
|||||||
@@ -376,7 +376,10 @@ public sealed class MxGatewayClient : IAsyncDisposable
|
|||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
timeout.CancelAfter(Options.DefaultCallTimeout);
|
// Outer budget covers every attempt (each carries its own DefaultCallTimeout deadline)
|
||||||
|
// plus back-off; capping it at one DefaultCallTimeout starved retries of a
|
||||||
|
// DeadlineExceeded first attempt (mxaccessgw#197).
|
||||||
|
timeout.CancelAfter(MxGatewayClientRetryPolicy.ComputeOverallBudget(Options.Retry, Options.DefaultCallTimeout));
|
||||||
|
|
||||||
return await _safeUnaryRetryPipeline.ExecuteAsync(
|
return await _safeUnaryRetryPipeline.ExecuteAsync(
|
||||||
async token => await call(token).ConfigureAwait(false),
|
async token => await call(token).ConfigureAwait(false),
|
||||||
|
|||||||
@@ -41,6 +41,29 @@ internal static class MxGatewayClientRetryPolicy
|
|||||||
.Build();
|
.Build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Computes the overall time budget for a retried unary call: every attempt gets its own
|
||||||
|
/// <paramref name="callTimeout"/> deadline, plus the worst-case back-off between attempts.
|
||||||
|
/// An outer cap equal to a single <paramref name="callTimeout"/> would let a
|
||||||
|
/// <c>DeadlineExceeded</c> first attempt consume the whole budget so no retry could ever run.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="options">Retry configuration.</param>
|
||||||
|
/// <param name="callTimeout">The per-attempt call timeout.</param>
|
||||||
|
/// <returns>The overall budget, or <see cref="Timeout.InfiniteTimeSpan"/> if it exceeds the timer range.</returns>
|
||||||
|
public static TimeSpan ComputeOverallBudget(MxGatewayClientRetryOptions options, TimeSpan callTimeout)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(options);
|
||||||
|
|
||||||
|
int attempts = Math.Max(1, options.MaxAttempts);
|
||||||
|
double totalMs = (callTimeout.TotalMilliseconds * attempts)
|
||||||
|
+ (options.MaxDelay.TotalMilliseconds * (attempts - 1));
|
||||||
|
|
||||||
|
// CancellationTokenSource.CancelAfter accepts at most int.MaxValue milliseconds.
|
||||||
|
return totalMs >= int.MaxValue
|
||||||
|
? Timeout.InfiniteTimeSpan
|
||||||
|
: TimeSpan.FromMilliseconds(totalMs);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Returns whether a command kind is eligible for automatic retry on transient failures.</summary>
|
/// <summary>Returns whether a command kind is eligible for automatic retry on transient failures.</summary>
|
||||||
/// <param name="kind">The command kind to check.</param>
|
/// <param name="kind">The command kind to check.</param>
|
||||||
/// <returns><see langword="true"/> if the command kind may be automatically retried; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true"/> if the command kind may be automatically retried; otherwise, <see langword="false"/>.</returns>
|
||||||
|
|||||||
Reference in New Issue
Block a user