perf(worker): launcher-configurable event queue capacity

This commit is contained in:
Joseph Doherty
2026-08-15 17:10:54 -04:00
parent f3e1de5f37
commit 13583322b5
10 changed files with 253 additions and 5 deletions
@@ -10,6 +10,12 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
private const int MinimumMaxMessageBytes = 1024;
private const int MaximumMaxMessageBytes = 256 * 1024 * 1024;
// Bounds on the worker's outbound event-queue capacity. The floor keeps enough headroom that a
// normal subscription burst cannot overflow the queue (an overflow faults the whole session);
// the ceiling keeps a mistyped value from committing the x86 worker to an unbounded backlog.
private const int MinimumWorkerEventQueueCapacity = 1000;
private const int MaximumWorkerEventQueueCapacity = 1_000_000;
// Whether the host is running in the Production environment. Drives the production-only
// hard-stops (dashboard login disabled, plaintext LDAP transport) that must abort startup
// rather than merely warn. Non-production hosts keep the permissive dev posture.
@@ -275,6 +281,12 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatIntervalSeconds.");
}
if (options.EventQueueCapacity is < MinimumWorkerEventQueueCapacity or > MaximumWorkerEventQueueCapacity)
{
builder.Add(
$"MxGateway:Worker:EventQueueCapacity must be between {MinimumWorkerEventQueueCapacity} and {MaximumWorkerEventQueueCapacity}.");
}
if (options.MaxMessageBytes is < MinimumMaxMessageBytes or > MaximumMaxMessageBytes)
{
builder.Add(
@@ -33,6 +33,18 @@ public sealed class WorkerOptions
/// </summary>
public int WriteCompletionWaitMilliseconds { get; init; } = 1500;
/// <summary>
/// Capacity of the worker's outbound MXAccess event queue, in events.
/// Default 10,000; must be between 1,000 and 1,000,000. This is
/// headroom, not a throttle: the queue has no drop policy, so a burst
/// that fills it faults the session with a <c>QueueOverflow</c> worker
/// fault. Raise it for sessions whose subscription set can outrun the
/// drain loop (large advise sets, slow event consumers). Conveyed to
/// the worker through the <c>MXGATEWAY_EVENT_QUEUE_CAPACITY</c>
/// environment variable.
/// </summary>
public int EventQueueCapacity { get; init; } = 10000;
/// <summary>The maximum time in seconds for graceful shutdown.</summary>
public int ShutdownTimeoutSeconds { get; init; } = 10;
@@ -44,6 +44,13 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
public const string WorkerMaxAlarmsPerFetchEnvironmentVariableName =
"MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH";
/// <summary>
/// Conveys MxGateway:Worker:EventQueueCapacity to the worker: the capacity
/// of the outbound MXAccess event queue, whose overflow faults the session.
/// </summary>
public const string WorkerEventQueueCapacityEnvironmentVariableName =
"MXGATEWAY_EVENT_QUEUE_CAPACITY";
private readonly IWorkerProcessFactory _processFactory;
private readonly IWorkerStartupProbe _startupProbe;
private readonly GatewayMetrics _metrics;
@@ -202,6 +209,8 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
_workerOptions.PipeConnectAttemptTimeoutMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerWriteCompletionWaitEnvironmentVariableName] =
_workerOptions.WriteCompletionWaitMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerEventQueueCapacityEnvironmentVariableName] =
_workerOptions.EventQueueCapacity.ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerAlarmPollIntervalEnvironmentVariableName] =
_alarmsOptions.PollIntervalMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerMaxAlarmsPerFetchEnvironmentVariableName] =