fix(gateway): remove the worker startup-probe retry options that could never retry (#188)
WorkerProcessStartedProbe only checks HasExited and throws a non-retryable WorkerProcessLaunchException, so StartupProbeRetryAttempts and StartupProbeRetryDelayMilliseconds configured a Polly retry that no shipped probe could trigger. Real readiness is WorkerClient's WorkerReady handshake under the same StartupTimeoutSeconds budget, so the probe now runs once and the two options, their validation, the Polly pipeline (and the server's now unused Polly.Core reference) and their docs are removed. A stale value in configuration is ignored by the binder. Tests: WorkerProcessLauncherTests.LaunchAsync_WhenStartupProbeFails_DoesNotRetryTheProbe, WorkerOptions_HasNoStartupProbeRetryOptions (both fail on the old code).
This commit is contained in:
@@ -246,14 +246,6 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
options.StartupTimeoutSeconds,
|
||||
"MxGateway:Worker:StartupTimeoutSeconds must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.StartupProbeRetryAttempts,
|
||||
"MxGateway:Worker:StartupProbeRetryAttempts must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.StartupProbeRetryDelayMilliseconds,
|
||||
"MxGateway:Worker:StartupProbeRetryDelayMilliseconds must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.PipeConnectAttemptTimeoutMilliseconds,
|
||||
"MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.",
|
||||
|
||||
@@ -15,12 +15,6 @@ public sealed class WorkerOptions
|
||||
/// <summary>The maximum time in seconds for the worker to start.</summary>
|
||||
public int StartupTimeoutSeconds { get; init; } = 30;
|
||||
|
||||
/// <summary>The number of retry attempts for the startup probe.</summary>
|
||||
public int StartupProbeRetryAttempts { get; init; } = 3;
|
||||
|
||||
/// <summary>The delay in milliseconds between startup probe retries.</summary>
|
||||
public int StartupProbeRetryDelayMilliseconds { get; init; } = 250;
|
||||
|
||||
/// <summary>The timeout in milliseconds for connecting to the worker pipe.</summary>
|
||||
public int PipeConnectAttemptTimeoutMilliseconds { get; init; } = 2000;
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using Polly;
|
||||
using Polly.Retry;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
@@ -140,15 +138,11 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
||||
using CancellationTokenSource startupTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
startupTimeout.CancelAfter(TimeSpan.FromSeconds(_workerOptions.StartupTimeoutSeconds));
|
||||
|
||||
await CreateStartupProbePipeline(process)
|
||||
.ExecuteAsync(
|
||||
async token =>
|
||||
{
|
||||
await _startupProbe
|
||||
.WaitUntilReadyAsync(process, request, token)
|
||||
.ConfigureAwait(false);
|
||||
},
|
||||
startupTimeout.Token)
|
||||
// The probe runs once. Real readiness is the WorkerReady handshake WorkerClient performs
|
||||
// after launch, under the same StartupTimeoutSeconds budget; the probe only catches a
|
||||
// worker that exited immediately, which no retry can recover (#188).
|
||||
await _startupProbe
|
||||
.WaitUntilReadyAsync(process, request, startupTimeout.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return new WorkerProcessHandle(process, commandLine, startedAt);
|
||||
@@ -310,43 +304,6 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
||||
}
|
||||
}
|
||||
|
||||
private ResiliencePipeline CreateStartupProbePipeline(IWorkerProcess process)
|
||||
{
|
||||
RetryStrategyOptions retryOptions = new()
|
||||
{
|
||||
MaxRetryAttempts = Math.Max(0, _workerOptions.StartupProbeRetryAttempts - 1),
|
||||
BackoffType = DelayBackoffType.Exponential,
|
||||
UseJitter = true,
|
||||
Delay = TimeSpan.FromMilliseconds(_workerOptions.StartupProbeRetryDelayMilliseconds),
|
||||
MaxDelay = TimeSpan.FromSeconds(2),
|
||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(exception =>
|
||||
ShouldRetryStartupProbe(exception, process)),
|
||||
OnRetry = args =>
|
||||
{
|
||||
_metrics.RetryAttempted("worker_startup");
|
||||
_logger.LogDebug(
|
||||
args.Outcome.Exception,
|
||||
"Retrying worker startup probe after transient failure. Attempt {Attempt}.",
|
||||
args.AttemptNumber + 1);
|
||||
return default;
|
||||
},
|
||||
};
|
||||
|
||||
return new ResiliencePipelineBuilder()
|
||||
.AddRetry(retryOptions)
|
||||
.Build();
|
||||
}
|
||||
|
||||
private static bool ShouldRetryStartupProbe(Exception exception, IWorkerProcess process)
|
||||
{
|
||||
if (exception is OperationCanceledException or WorkerProcessLaunchException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return !process.HasExited;
|
||||
}
|
||||
|
||||
private static void ValidateRequest(WorkerProcessLaunchRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.SessionId))
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
<!-- Security pin: GHSA-846f-f764-76mx / GHSA-2fmv-rv2c-59x7. Microsoft.Data.SqlClient 6.0.2
|
||||
pulls the vulnerable System.Security.Cryptography.Pkcs 9.0.4; 10.0.13 patches it. -->
|
||||
<PackageReference Include="System.Security.Cryptography.Pkcs" Version="10.0.13" />
|
||||
<PackageReference Include="Polly.Core" Version="8.6.6" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user