fix(server): remove inert worker startup probe retry options (mxaccessgw#188)
WorkerProcessStartedProbe only checks HasExited, so a retry could never change the outcome. Remove StartupProbeRetryAttempts/StartupProbeRetryDelayMilliseconds, their validation, the launcher's Polly retry pipeline and the docs; the probe now runs once under StartupTimeoutSeconds. Stale config keys are ignored by the binder.
This commit is contained in:
@@ -22,8 +22,6 @@ paths, timeouts, queue sizes, enum values, or protocol values are invalid.
|
|||||||
"WorkingDirectory": null,
|
"WorkingDirectory": null,
|
||||||
"RequiredArchitecture": "X86",
|
"RequiredArchitecture": "X86",
|
||||||
"StartupTimeoutSeconds": 30,
|
"StartupTimeoutSeconds": 30,
|
||||||
"StartupProbeRetryAttempts": 3,
|
|
||||||
"StartupProbeRetryDelayMilliseconds": 250,
|
|
||||||
"PipeConnectAttemptTimeoutMilliseconds": 2000,
|
"PipeConnectAttemptTimeoutMilliseconds": 2000,
|
||||||
"ShutdownTimeoutSeconds": 10,
|
"ShutdownTimeoutSeconds": 10,
|
||||||
"HeartbeatIntervalSeconds": 5,
|
"HeartbeatIntervalSeconds": 5,
|
||||||
@@ -116,8 +114,6 @@ launch CWD (SEC-01, SEC-33).
|
|||||||
| `MxGateway:Worker:WorkingDirectory` | `null` | Optional working directory for the worker process. When set, it must be a valid filesystem path. |
|
| `MxGateway:Worker:WorkingDirectory` | `null` | Optional working directory for the worker process. When set, it must be a valid filesystem path. |
|
||||||
| `MxGateway:Worker:RequiredArchitecture` | `X86` | Required Portable Executable architecture for the worker. Supported values are `X86` and `X64`; MXAccess parity uses `X86`. |
|
| `MxGateway:Worker:RequiredArchitecture` | `X86` | Required Portable Executable architecture for the worker. Supported values are `X86` and `X64`; MXAccess parity uses `X86`. |
|
||||||
| `MxGateway:Worker:StartupTimeoutSeconds` | `30` | Total startup budget for process launch, startup probe, pipe connect, handshake, and worker readiness. |
|
| `MxGateway:Worker:StartupTimeoutSeconds` | `30` | Total startup budget for process launch, startup probe, pipe connect, handshake, and worker readiness. |
|
||||||
| `MxGateway:Worker:StartupProbeRetryAttempts` | `3` | Number of retry attempts for transient worker startup probe failures before pipe connection and handshake continue. |
|
|
||||||
| `MxGateway:Worker:StartupProbeRetryDelayMilliseconds` | `250` | Delay between transient startup probe retry attempts. |
|
|
||||||
| `MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds` | `2000` | Per-attempt timeout used by the worker named-pipe connect retry path. The overall pipe connection still stays under the startup budget. |
|
| `MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds` | `2000` | Per-attempt timeout used by the worker named-pipe connect retry path. The overall pipe connection still stays under the startup budget. |
|
||||||
| `MxGateway:Worker:WriteCompletionWaitMilliseconds` | `1500` | Bounded wait the worker holds a unary write reply (`Write`/`Write2`/`WriteSecured`/`WriteSecured2`; bulk writes excluded) for the matching MXAccess `OnWriteComplete` callback, so the reply's `statuses` carry the real commit outcome. `0` disables the wait (pure fire-and-forget replies). Must be `>= 0`. The gateway conveys the value to the worker via the `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS` environment variable. Consumers that time their own writes must budget above this wait: OtOpcUa's GalaxyDriver wraps gateway writes in a 2 s Tier A resilience timeout, so a deployment raising this option past ~2000 must raise that driver `ResilienceConfig` write timeout in step or slow-but-successful commits surface as consumer-side failures. |
|
| `MxGateway:Worker:WriteCompletionWaitMilliseconds` | `1500` | Bounded wait the worker holds a unary write reply (`Write`/`Write2`/`WriteSecured`/`WriteSecured2`; bulk writes excluded) for the matching MXAccess `OnWriteComplete` callback, so the reply's `statuses` carry the real commit outcome. `0` disables the wait (pure fire-and-forget replies). Must be `>= 0`. The gateway conveys the value to the worker via the `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS` environment variable. Consumers that time their own writes must budget above this wait: OtOpcUa's GalaxyDriver wraps gateway writes in a 2 s Tier A resilience timeout, so a deployment raising this option past ~2000 must raise that driver `ResilienceConfig` write timeout in step or slow-but-successful commits surface as consumer-side failures. |
|
||||||
| `MxGateway:Worker:EventQueueCapacity` | `10000` | Capacity, in events, of the worker's outbound MXAccess event queue. Must be between `1000` and `1000000`. This is burst headroom, not a throttle: the queue has no drop policy, so filling it records a `QueueOverflow` worker fault and faults the session. Raise it for sessions whose subscription set can outrun the drain loop (large advise sets, slow event consumers); the backing queue pre-allocates its slots, so the ceiling keeps a mistyped value from committing the 32-bit worker to an outsized allocation. The gateway conveys the value to the worker via the `MXGATEWAY_EVENT_QUEUE_CAPACITY` environment variable; a missing or unusable value leaves the worker on the 10000 default rather than failing the session. |
|
| `MxGateway:Worker:EventQueueCapacity` | `10000` | Capacity, in events, of the worker's outbound MXAccess event queue. Must be between `1000` and `1000000`. This is burst headroom, not a throttle: the queue has no drop policy, so filling it records a `QueueOverflow` worker fault and faults the session. Raise it for sessions whose subscription set can outrun the drain loop (large advise sets, slow event consumers); the backing queue pre-allocates its slots, so the ceiling keeps a mistyped value from committing the 32-bit worker to an outsized allocation. The gateway conveys the value to the worker via the `MXGATEWAY_EVENT_QUEUE_CAPACITY` environment variable; a missing or unusable value leaves the worker on the 10000 default rather than failing the session. |
|
||||||
@@ -126,7 +122,6 @@ launch CWD (SEC-01, SEC-33).
|
|||||||
| `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. |
|
| `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. |
|
||||||
| `MxGateway:Worker:MaxMessageBytes` | `16842752` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`, and additionally requires this to be at least `MxGateway:Protocol:MaxGrpcMessageBytes` plus a 65536-byte (64 KiB) envelope-overhead reserve **and at most that cap plus twice the reserve**. The default is the 16 MiB public gRPC cap plus one reserve. The gateway conveys the negotiated value to the worker in the handshake (`GatewayHello.max_frame_bytes`), so the worker frames to the same limit rather than a hard-coded default. The band is narrow by design, so changing either message-size knob alone no longer validates — move `MxGateway:Protocol:MaxGrpcMessageBytes` and this one together, keeping this one one to two reserves above it. |
|
| `MxGateway:Worker:MaxMessageBytes` | `16842752` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`, and additionally requires this to be at least `MxGateway:Protocol:MaxGrpcMessageBytes` plus a 65536-byte (64 KiB) envelope-overhead reserve **and at most that cap plus twice the reserve**. The default is the 16 MiB public gRPC cap plus one reserve. The gateway conveys the negotiated value to the worker in the handshake (`GatewayHello.max_frame_bytes`), so the worker frames to the same limit rather than a hard-coded default. The band is narrow by design, so changing either message-size knob alone no longer validates — move `MxGateway:Protocol:MaxGrpcMessageBytes` and this one together, keeping this one one to two reserves above it. |
|
||||||
|
|
||||||
`StartupProbeRetryAttempts`, `StartupProbeRetryDelayMilliseconds`,
|
|
||||||
`PipeConnectAttemptTimeoutMilliseconds`, timeout values, heartbeat values, and
|
`PipeConnectAttemptTimeoutMilliseconds`, timeout values, heartbeat values, and
|
||||||
`MaxMessageBytes` must be positive. `MaxMessageBytes` is intentionally bounded
|
`MaxMessageBytes` must be positive. `MaxMessageBytes` is intentionally bounded
|
||||||
to avoid accidental large allocations from malformed or oversized frames. It
|
to avoid accidental large allocations from malformed or oversized frames. It
|
||||||
|
|||||||
@@ -940,8 +940,6 @@ Suggested configuration shape:
|
|||||||
"WorkingDirectory": null,
|
"WorkingDirectory": null,
|
||||||
"RequiredArchitecture": "X86",
|
"RequiredArchitecture": "X86",
|
||||||
"StartupTimeoutSeconds": 30,
|
"StartupTimeoutSeconds": 30,
|
||||||
"StartupProbeRetryAttempts": 3,
|
|
||||||
"StartupProbeRetryDelayMilliseconds": 250,
|
|
||||||
"PipeConnectAttemptTimeoutMilliseconds": 2000,
|
"PipeConnectAttemptTimeoutMilliseconds": 2000,
|
||||||
"ShutdownTimeoutSeconds": 10,
|
"ShutdownTimeoutSeconds": 10,
|
||||||
"HeartbeatIntervalSeconds": 5,
|
"HeartbeatIntervalSeconds": 5,
|
||||||
|
|||||||
+3
-5
@@ -109,7 +109,7 @@ The scalar fields mirror the counters and gauges. The four dictionaries provide
|
|||||||
- `CommandFailuresByMethod` keys by gRPC method name.
|
- `CommandFailuresByMethod` keys by gRPC method name.
|
||||||
- `EventsByFamily` keys by event family (the `Family` enum on a worker event).
|
- `EventsByFamily` keys by event family (the `Family` enum on a worker event).
|
||||||
- `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound.
|
- `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound.
|
||||||
- `RetryAttemptsByArea` keys by the resilience `area` tag, e.g. `worker_startup`.
|
- `RetryAttemptsByArea` keys by the resilience `area` tag, (no area is currently recorded by the launcher).
|
||||||
|
|
||||||
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (PERF-02). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients.
|
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (PERF-02). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients.
|
||||||
|
|
||||||
@@ -152,15 +152,13 @@ _metrics.RemoveSessionEvents(session.SessionId);
|
|||||||
|
|
||||||
### Worker process launcher
|
### Worker process launcher
|
||||||
|
|
||||||
`Workers/WorkerProcessLauncher.cs` records process-level kills and startup retries:
|
`Workers/WorkerProcessLauncher.cs` records process-level kills:
|
||||||
|
|
||||||
```csharp
|
```csharp
|
||||||
_metrics.WorkerKilled(reason);
|
_metrics.WorkerKilled(reason);
|
||||||
...
|
|
||||||
_metrics.RetryAttempted("worker_startup");
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The `worker_startup` tag is hard-coded so the `RetryAttemptsByArea` snapshot reports launcher retries distinctly from other resilience areas.
|
The launcher no longer retries the startup probe (mxaccessgw#188), so it no longer records `RetryAttempted("worker_startup")`.
|
||||||
|
|
||||||
### Session worker client factory
|
### Session worker client factory
|
||||||
|
|
||||||
|
|||||||
@@ -54,12 +54,12 @@ The default probe only verifies that the worker did not exit immediately. The
|
|||||||
worker client replaces this probe when pipe connection, hello, and
|
worker client replaces this probe when pipe connection, hello, and
|
||||||
`WorkerReady` handling are implemented.
|
`WorkerReady` handling are implemented.
|
||||||
|
|
||||||
Startup probing uses a bounded Polly retry policy. The gateway starts the worker
|
The startup probe runs once, bounded by `WorkerOptions.StartupTimeoutSeconds`;
|
||||||
process once, then retries only transient startup-probe failures while the
|
it is not retried. The default probe only checks that the process is still
|
||||||
process remains alive. The policy is configured by
|
alive, which cannot change on a retry, so the former
|
||||||
`WorkerOptions.StartupProbeRetryAttempts` and
|
`StartupProbeRetryAttempts` / `StartupProbeRetryDelayMilliseconds` options were
|
||||||
`WorkerOptions.StartupProbeRetryDelayMilliseconds`; the retry counter is
|
removed (mxaccessgw#188). Stale keys in existing configuration are ignored by
|
||||||
recorded as `mxgateway.retries.attempted` with `area=worker_startup`.
|
the binder. The worker pipe connect and handshake have their own retry loop.
|
||||||
|
|
||||||
The worker uses the exported pipe-attempt value inside its own bounded Polly
|
The worker uses the exported pipe-attempt value inside its own bounded Polly
|
||||||
retry loop.
|
retry loop.
|
||||||
|
|||||||
@@ -246,14 +246,6 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
|||||||
options.StartupTimeoutSeconds,
|
options.StartupTimeoutSeconds,
|
||||||
"MxGateway:Worker:StartupTimeoutSeconds must be greater than zero.",
|
"MxGateway:Worker:StartupTimeoutSeconds must be greater than zero.",
|
||||||
builder);
|
builder);
|
||||||
AddIfNotPositive(
|
|
||||||
options.StartupProbeRetryAttempts,
|
|
||||||
"MxGateway:Worker:StartupProbeRetryAttempts must be greater than zero.",
|
|
||||||
builder);
|
|
||||||
AddIfNotPositive(
|
|
||||||
options.StartupProbeRetryDelayMilliseconds,
|
|
||||||
"MxGateway:Worker:StartupProbeRetryDelayMilliseconds must be greater than zero.",
|
|
||||||
builder);
|
|
||||||
AddIfNotPositive(
|
AddIfNotPositive(
|
||||||
options.PipeConnectAttemptTimeoutMilliseconds,
|
options.PipeConnectAttemptTimeoutMilliseconds,
|
||||||
"MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.",
|
"MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.",
|
||||||
|
|||||||
@@ -15,12 +15,6 @@ public sealed class WorkerOptions
|
|||||||
/// <summary>The maximum time in seconds for the worker to start.</summary>
|
/// <summary>The maximum time in seconds for the worker to start.</summary>
|
||||||
public int StartupTimeoutSeconds { get; init; } = 30;
|
public int StartupTimeoutSeconds { get; init; } = 30;
|
||||||
|
|
||||||
/// <summary>The number of retry attempts for the startup probe.</summary>
|
|
||||||
public int StartupProbeRetryAttempts { get; init; } = 3;
|
|
||||||
|
|
||||||
/// <summary>The delay in milliseconds between startup probe retries.</summary>
|
|
||||||
public int StartupProbeRetryDelayMilliseconds { get; init; } = 250;
|
|
||||||
|
|
||||||
/// <summary>The timeout in milliseconds for connecting to the worker pipe.</summary>
|
/// <summary>The timeout in milliseconds for connecting to the worker pipe.</summary>
|
||||||
public int PipeConnectAttemptTimeoutMilliseconds { get; init; } = 2000;
|
public int PipeConnectAttemptTimeoutMilliseconds { get; init; } = 2000;
|
||||||
|
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ using Microsoft.Extensions.Options;
|
|||||||
using ZB.MOM.WW.MxGateway.Contracts;
|
using ZB.MOM.WW.MxGateway.Contracts;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||||
using Polly;
|
|
||||||
using Polly.Retry;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
||||||
|
|
||||||
@@ -140,15 +138,10 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
|||||||
using CancellationTokenSource startupTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
using CancellationTokenSource startupTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
startupTimeout.CancelAfter(TimeSpan.FromSeconds(_workerOptions.StartupTimeoutSeconds));
|
startupTimeout.CancelAfter(TimeSpan.FromSeconds(_workerOptions.StartupTimeoutSeconds));
|
||||||
|
|
||||||
await CreateStartupProbePipeline(process)
|
// Single-shot probe: the default WorkerProcessStartedProbe only checks HasExited, which can
|
||||||
.ExecuteAsync(
|
// never succeed on a retry, so there is deliberately no retry layer here (mxaccessgw#188).
|
||||||
async token =>
|
|
||||||
{
|
|
||||||
await _startupProbe
|
await _startupProbe
|
||||||
.WaitUntilReadyAsync(process, request, token)
|
.WaitUntilReadyAsync(process, request, startupTimeout.Token)
|
||||||
.ConfigureAwait(false);
|
|
||||||
},
|
|
||||||
startupTimeout.Token)
|
|
||||||
.ConfigureAwait(false);
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
return new WorkerProcessHandle(process, commandLine, startedAt);
|
return new WorkerProcessHandle(process, commandLine, startedAt);
|
||||||
@@ -310,43 +303,6 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResiliencePipeline CreateStartupProbePipeline(IWorkerProcess process)
|
|
||||||
{
|
|
||||||
RetryStrategyOptions retryOptions = new()
|
|
||||||
{
|
|
||||||
MaxRetryAttempts = Math.Max(0, _workerOptions.StartupProbeRetryAttempts - 1),
|
|
||||||
BackoffType = DelayBackoffType.Exponential,
|
|
||||||
UseJitter = true,
|
|
||||||
Delay = TimeSpan.FromMilliseconds(_workerOptions.StartupProbeRetryDelayMilliseconds),
|
|
||||||
MaxDelay = TimeSpan.FromSeconds(2),
|
|
||||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(exception =>
|
|
||||||
ShouldRetryStartupProbe(exception, process)),
|
|
||||||
OnRetry = args =>
|
|
||||||
{
|
|
||||||
_metrics.RetryAttempted("worker_startup");
|
|
||||||
_logger.LogDebug(
|
|
||||||
args.Outcome.Exception,
|
|
||||||
"Retrying worker startup probe after transient failure. Attempt {Attempt}.",
|
|
||||||
args.AttemptNumber + 1);
|
|
||||||
return default;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
return new ResiliencePipelineBuilder()
|
|
||||||
.AddRetry(retryOptions)
|
|
||||||
.Build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool ShouldRetryStartupProbe(Exception exception, IWorkerProcess process)
|
|
||||||
{
|
|
||||||
if (exception is OperationCanceledException or WorkerProcessLaunchException)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return !process.HasExited;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void ValidateRequest(WorkerProcessLaunchRequest request)
|
private static void ValidateRequest(WorkerProcessLaunchRequest request)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(request.SessionId))
|
if (string.IsNullOrWhiteSpace(request.SessionId))
|
||||||
|
|||||||
@@ -38,8 +38,6 @@ public sealed class GatewayOptionsTests
|
|||||||
Assert.Equal(@"src\ZB.MOM.WW.MxGateway.Worker\bin\x86\Release\ZB.MOM.WW.MxGateway.Worker.exe", options.Worker.ExecutablePath);
|
Assert.Equal(@"src\ZB.MOM.WW.MxGateway.Worker\bin\x86\Release\ZB.MOM.WW.MxGateway.Worker.exe", options.Worker.ExecutablePath);
|
||||||
Assert.Equal(WorkerArchitecture.X86, options.Worker.RequiredArchitecture);
|
Assert.Equal(WorkerArchitecture.X86, options.Worker.RequiredArchitecture);
|
||||||
Assert.Equal(30, options.Worker.StartupTimeoutSeconds);
|
Assert.Equal(30, options.Worker.StartupTimeoutSeconds);
|
||||||
Assert.Equal(3, options.Worker.StartupProbeRetryAttempts);
|
|
||||||
Assert.Equal(250, options.Worker.StartupProbeRetryDelayMilliseconds);
|
|
||||||
Assert.Equal(2000, options.Worker.PipeConnectAttemptTimeoutMilliseconds);
|
Assert.Equal(2000, options.Worker.PipeConnectAttemptTimeoutMilliseconds);
|
||||||
Assert.Equal(1500, options.Worker.WriteCompletionWaitMilliseconds);
|
Assert.Equal(1500, options.Worker.WriteCompletionWaitMilliseconds);
|
||||||
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
|
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
|
||||||
@@ -110,7 +108,6 @@ public sealed class GatewayOptionsTests
|
|||||||
/// <param name="expectedFailure">Expected validation error message.</param>
|
/// <param name="expectedFailure">Expected validation error message.</param>
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("MxGateway:Worker:ExecutablePath", "worker.dll", "MxGateway:Worker:ExecutablePath must point to a .exe file.")]
|
[InlineData("MxGateway:Worker:ExecutablePath", "worker.dll", "MxGateway:Worker:ExecutablePath must point to a .exe file.")]
|
||||||
[InlineData("MxGateway:Worker:StartupProbeRetryAttempts", "0", "MxGateway:Worker:StartupProbeRetryAttempts must be greater than zero.")]
|
|
||||||
[InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
|
[InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
|
||||||
[InlineData("MxGateway:Worker:WriteCompletionWaitMilliseconds", "-1", "MxGateway:Worker:WriteCompletionWaitMilliseconds must be greater than or equal to zero.")]
|
[InlineData("MxGateway:Worker:WriteCompletionWaitMilliseconds", "-1", "MxGateway:Worker:WriteCompletionWaitMilliseconds must be greater than or equal to zero.")]
|
||||||
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
||||||
|
|||||||
@@ -167,34 +167,6 @@ public sealed class WorkerProcessLauncherTests
|
|||||||
Assert.Equal(1, metrics.GetSnapshot().WorkerKills);
|
Assert.Equal(1, metrics.GetSnapshot().WorkerKills);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that transient startup probe failures are retried without respawning the worker process.</summary>
|
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
||||||
[Fact]
|
|
||||||
public async Task LaunchAsync_WhenStartupProbeFailsTransiently_RetriesWithoutRespawningWorker()
|
|
||||||
{
|
|
||||||
using TestDirectory directory = TestDirectory.Create();
|
|
||||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
|
||||||
FakeWorkerProcess process = new(processId: 1234);
|
|
||||||
FakeWorkerProcessFactory processFactory = new(process);
|
|
||||||
GatewayMetrics metrics = new();
|
|
||||||
WorkerProcessLauncher launcher = CreateLauncher(
|
|
||||||
executablePath,
|
|
||||||
processFactory,
|
|
||||||
new TransientStartupProbe(failuresBeforeSuccess: 1),
|
|
||||||
metrics,
|
|
||||||
startupProbeRetryAttempts: 2,
|
|
||||||
startupProbeRetryDelayMilliseconds: 1);
|
|
||||||
|
|
||||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
|
|
||||||
|
|
||||||
Assert.Same(process, handle.Process);
|
|
||||||
Assert.Equal(1, processFactory.StartCount);
|
|
||||||
Assert.False(process.KillCalled);
|
|
||||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
||||||
Assert.Equal(1, snapshot.RetryAttempts);
|
|
||||||
Assert.Equal(1, snapshot.RetryAttemptsByArea["worker_startup"]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>Verifies that a startup probe timeout kills and disposes the worker process.</summary>
|
/// <summary>Verifies that a startup probe timeout kills and disposes the worker process.</summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -290,8 +262,6 @@ public sealed class WorkerProcessLauncherTests
|
|||||||
IWorkerStartupProbe startupProbe,
|
IWorkerStartupProbe startupProbe,
|
||||||
GatewayMetrics? metrics = null,
|
GatewayMetrics? metrics = null,
|
||||||
int startupTimeoutSeconds = 30,
|
int startupTimeoutSeconds = 30,
|
||||||
int startupProbeRetryAttempts = 3,
|
|
||||||
int startupProbeRetryDelayMilliseconds = 250,
|
|
||||||
int eventQueueCapacity = 10000,
|
int eventQueueCapacity = 10000,
|
||||||
string ackClientName = "MxGateway")
|
string ackClientName = "MxGateway")
|
||||||
{
|
{
|
||||||
@@ -303,8 +273,6 @@ public sealed class WorkerProcessLauncherTests
|
|||||||
ExecutablePath = executablePath,
|
ExecutablePath = executablePath,
|
||||||
RequiredArchitecture = WorkerArchitecture.X86,
|
RequiredArchitecture = WorkerArchitecture.X86,
|
||||||
StartupTimeoutSeconds = startupTimeoutSeconds,
|
StartupTimeoutSeconds = startupTimeoutSeconds,
|
||||||
StartupProbeRetryAttempts = startupProbeRetryAttempts,
|
|
||||||
StartupProbeRetryDelayMilliseconds = startupProbeRetryDelayMilliseconds,
|
|
||||||
EventQueueCapacity = eventQueueCapacity,
|
EventQueueCapacity = eventQueueCapacity,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -383,26 +351,6 @@ public sealed class WorkerProcessLauncherTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Fake startup probe that fails a configurable number of times before succeeding.</summary>
|
|
||||||
private sealed class TransientStartupProbe(int failuresBeforeSuccess) : IWorkerStartupProbe
|
|
||||||
{
|
|
||||||
private int _attempts;
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
public Task WaitUntilReadyAsync(
|
|
||||||
IWorkerProcess process,
|
|
||||||
WorkerProcessLaunchRequest request,
|
|
||||||
CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
if (Interlocked.Increment(ref _attempts) <= failuresBeforeSuccess)
|
|
||||||
{
|
|
||||||
throw new IOException("The worker pipe was not ready yet.");
|
|
||||||
}
|
|
||||||
|
|
||||||
return Task.CompletedTask;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>Fake pipe reservation for testing pipe lifecycle.</summary>
|
/// <summary>Fake pipe reservation for testing pipe lifecycle.</summary>
|
||||||
private sealed class FakePipeReservation : IDisposable
|
private sealed class FakePipeReservation : IDisposable
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user