fix(server): remove inert worker startup probe retry options (mxaccessgw#188)
WorkerProcessStartedProbe only checks HasExited, so a retry could never change the outcome. Remove StartupProbeRetryAttempts/StartupProbeRetryDelayMilliseconds, their validation, the launcher's Polly retry pipeline and the docs; the probe now runs once under StartupTimeoutSeconds. Stale config keys are ignored by the binder.
This commit is contained in:
@@ -246,14 +246,6 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
options.StartupTimeoutSeconds,
|
||||
"MxGateway:Worker:StartupTimeoutSeconds must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.StartupProbeRetryAttempts,
|
||||
"MxGateway:Worker:StartupProbeRetryAttempts must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.StartupProbeRetryDelayMilliseconds,
|
||||
"MxGateway:Worker:StartupProbeRetryDelayMilliseconds must be greater than zero.",
|
||||
builder);
|
||||
AddIfNotPositive(
|
||||
options.PipeConnectAttemptTimeoutMilliseconds,
|
||||
"MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.",
|
||||
|
||||
@@ -15,12 +15,6 @@ public sealed class WorkerOptions
|
||||
/// <summary>The maximum time in seconds for the worker to start.</summary>
|
||||
public int StartupTimeoutSeconds { get; init; } = 30;
|
||||
|
||||
/// <summary>The number of retry attempts for the startup probe.</summary>
|
||||
public int StartupProbeRetryAttempts { get; init; } = 3;
|
||||
|
||||
/// <summary>The delay in milliseconds between startup probe retries.</summary>
|
||||
public int StartupProbeRetryDelayMilliseconds { get; init; } = 250;
|
||||
|
||||
/// <summary>The timeout in milliseconds for connecting to the worker pipe.</summary>
|
||||
public int PipeConnectAttemptTimeoutMilliseconds { get; init; } = 2000;
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using Polly;
|
||||
using Polly.Retry;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
@@ -140,15 +138,10 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
||||
using CancellationTokenSource startupTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
startupTimeout.CancelAfter(TimeSpan.FromSeconds(_workerOptions.StartupTimeoutSeconds));
|
||||
|
||||
await CreateStartupProbePipeline(process)
|
||||
.ExecuteAsync(
|
||||
async token =>
|
||||
{
|
||||
await _startupProbe
|
||||
.WaitUntilReadyAsync(process, request, token)
|
||||
.ConfigureAwait(false);
|
||||
},
|
||||
startupTimeout.Token)
|
||||
// Single-shot probe: the default WorkerProcessStartedProbe only checks HasExited, which can
|
||||
// never succeed on a retry, so there is deliberately no retry layer here (mxaccessgw#188).
|
||||
await _startupProbe
|
||||
.WaitUntilReadyAsync(process, request, startupTimeout.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return new WorkerProcessHandle(process, commandLine, startedAt);
|
||||
@@ -310,43 +303,6 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
||||
}
|
||||
}
|
||||
|
||||
private ResiliencePipeline CreateStartupProbePipeline(IWorkerProcess process)
|
||||
{
|
||||
RetryStrategyOptions retryOptions = new()
|
||||
{
|
||||
MaxRetryAttempts = Math.Max(0, _workerOptions.StartupProbeRetryAttempts - 1),
|
||||
BackoffType = DelayBackoffType.Exponential,
|
||||
UseJitter = true,
|
||||
Delay = TimeSpan.FromMilliseconds(_workerOptions.StartupProbeRetryDelayMilliseconds),
|
||||
MaxDelay = TimeSpan.FromSeconds(2),
|
||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(exception =>
|
||||
ShouldRetryStartupProbe(exception, process)),
|
||||
OnRetry = args =>
|
||||
{
|
||||
_metrics.RetryAttempted("worker_startup");
|
||||
_logger.LogDebug(
|
||||
args.Outcome.Exception,
|
||||
"Retrying worker startup probe after transient failure. Attempt {Attempt}.",
|
||||
args.AttemptNumber + 1);
|
||||
return default;
|
||||
},
|
||||
};
|
||||
|
||||
return new ResiliencePipelineBuilder()
|
||||
.AddRetry(retryOptions)
|
||||
.Build();
|
||||
}
|
||||
|
||||
private static bool ShouldRetryStartupProbe(Exception exception, IWorkerProcess process)
|
||||
{
|
||||
if (exception is OperationCanceledException or WorkerProcessLaunchException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return !process.HasExited;
|
||||
}
|
||||
|
||||
private static void ValidateRequest(WorkerProcessLaunchRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.SessionId))
|
||||
|
||||
@@ -38,8 +38,6 @@ public sealed class GatewayOptionsTests
|
||||
Assert.Equal(@"src\ZB.MOM.WW.MxGateway.Worker\bin\x86\Release\ZB.MOM.WW.MxGateway.Worker.exe", options.Worker.ExecutablePath);
|
||||
Assert.Equal(WorkerArchitecture.X86, options.Worker.RequiredArchitecture);
|
||||
Assert.Equal(30, options.Worker.StartupTimeoutSeconds);
|
||||
Assert.Equal(3, options.Worker.StartupProbeRetryAttempts);
|
||||
Assert.Equal(250, options.Worker.StartupProbeRetryDelayMilliseconds);
|
||||
Assert.Equal(2000, options.Worker.PipeConnectAttemptTimeoutMilliseconds);
|
||||
Assert.Equal(1500, options.Worker.WriteCompletionWaitMilliseconds);
|
||||
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
|
||||
@@ -110,7 +108,6 @@ public sealed class GatewayOptionsTests
|
||||
/// <param name="expectedFailure">Expected validation error message.</param>
|
||||
[Theory]
|
||||
[InlineData("MxGateway:Worker:ExecutablePath", "worker.dll", "MxGateway:Worker:ExecutablePath must point to a .exe file.")]
|
||||
[InlineData("MxGateway:Worker:StartupProbeRetryAttempts", "0", "MxGateway:Worker:StartupProbeRetryAttempts must be greater than zero.")]
|
||||
[InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Worker:WriteCompletionWaitMilliseconds", "-1", "MxGateway:Worker:WriteCompletionWaitMilliseconds must be greater than or equal to zero.")]
|
||||
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
||||
|
||||
@@ -167,34 +167,6 @@ public sealed class WorkerProcessLauncherTests
|
||||
Assert.Equal(1, metrics.GetSnapshot().WorkerKills);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that transient startup probe failures are retried without respawning the worker process.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenStartupProbeFailsTransiently_RetriesWithoutRespawningWorker()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234);
|
||||
FakeWorkerProcessFactory processFactory = new(process);
|
||||
GatewayMetrics metrics = new();
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
processFactory,
|
||||
new TransientStartupProbe(failuresBeforeSuccess: 1),
|
||||
metrics,
|
||||
startupProbeRetryAttempts: 2,
|
||||
startupProbeRetryDelayMilliseconds: 1);
|
||||
|
||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
|
||||
|
||||
Assert.Same(process, handle.Process);
|
||||
Assert.Equal(1, processFactory.StartCount);
|
||||
Assert.False(process.KillCalled);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.RetryAttempts);
|
||||
Assert.Equal(1, snapshot.RetryAttemptsByArea["worker_startup"]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a startup probe timeout kills and disposes the worker process.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -290,8 +262,6 @@ public sealed class WorkerProcessLauncherTests
|
||||
IWorkerStartupProbe startupProbe,
|
||||
GatewayMetrics? metrics = null,
|
||||
int startupTimeoutSeconds = 30,
|
||||
int startupProbeRetryAttempts = 3,
|
||||
int startupProbeRetryDelayMilliseconds = 250,
|
||||
int eventQueueCapacity = 10000,
|
||||
string ackClientName = "MxGateway")
|
||||
{
|
||||
@@ -303,8 +273,6 @@ public sealed class WorkerProcessLauncherTests
|
||||
ExecutablePath = executablePath,
|
||||
RequiredArchitecture = WorkerArchitecture.X86,
|
||||
StartupTimeoutSeconds = startupTimeoutSeconds,
|
||||
StartupProbeRetryAttempts = startupProbeRetryAttempts,
|
||||
StartupProbeRetryDelayMilliseconds = startupProbeRetryDelayMilliseconds,
|
||||
EventQueueCapacity = eventQueueCapacity,
|
||||
},
|
||||
};
|
||||
@@ -383,26 +351,6 @@ public sealed class WorkerProcessLauncherTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake startup probe that fails a configurable number of times before succeeding.</summary>
|
||||
private sealed class TransientStartupProbe(int failuresBeforeSuccess) : IWorkerStartupProbe
|
||||
{
|
||||
private int _attempts;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task WaitUntilReadyAsync(
|
||||
IWorkerProcess process,
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Increment(ref _attempts) <= failuresBeforeSuccess)
|
||||
{
|
||||
throw new IOException("The worker pipe was not ready yet.");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake pipe reservation for testing pipe lifecycle.</summary>
|
||||
private sealed class FakePipeReservation : IDisposable
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user