feat(gateway,worker): configurable worker heartbeat send interval and stuck ceiling (mxaccessgw#218)
The worker's heartbeat send interval (5 s) and its own STA-watchdog stuck ceiling (75 s) were compile-time defaults the gateway never reached, and nothing checked that the gateway's HeartbeatGraceSeconds exceeded the real send interval, so a misconfigured grace could kill healthy workers. - New MxGateway:Worker options WorkerHeartbeatSendIntervalSeconds (default 5) and WorkerHeartbeatStuckCeilingSeconds (default 75), exported by WorkerProcessLauncher as MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS and MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS, the same env-var path the other worker settings use. - The worker's production WorkerPipeSession reads them through WorkerPipeSessionOptions.FromEnvironment; a missing/invalid value, or a ceiling not above the worker's 15 s watchdog grace, keeps the default rather than failing the worker. - Startup validation: HeartbeatGraceSeconds > WorkerHeartbeatSendIntervalSeconds; send interval > 0; worker ceiling > 15 s; both at most 86400 s. - Docs: GatewayConfiguration, WorkerProcessLauncher (env table now also lists the previously missing MXGATEWAY_ALARM_ACK_CLIENT_NAME), worker design doc. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -43,6 +43,8 @@ public sealed class GatewayOptionsTests
|
||||
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
|
||||
Assert.Equal(5, options.Worker.ResolveHeartbeatCheckIntervalSeconds());
|
||||
Assert.Equal(75, options.Worker.HeartbeatStuckCeilingSeconds);
|
||||
Assert.Equal(5, options.Worker.WorkerHeartbeatSendIntervalSeconds);
|
||||
Assert.Equal(75, options.Worker.WorkerHeartbeatStuckCeilingSeconds);
|
||||
Assert.Equal(5000, options.Worker.EventChannelFullModeTimeoutMilliseconds);
|
||||
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
|
||||
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
|
||||
@@ -117,6 +119,12 @@ public sealed class GatewayOptionsTests
|
||||
[InlineData("MxGateway:Worker:HeartbeatCheckIntervalSeconds", "16", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatCheckIntervalSeconds.")]
|
||||
[InlineData("MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds", "0", "MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Worker:HeartbeatStuckCeilingSeconds", "14", "MxGateway:Worker:HeartbeatStuckCeilingSeconds must be greater than or equal to HeartbeatGraceSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "0", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "15", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most 86400.")]
|
||||
[InlineData("MxGateway:Worker:HeartbeatGraceSeconds", "5", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's 15-second watchdog grace.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most 86400.")]
|
||||
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
|
||||
|
||||
@@ -23,6 +23,8 @@ public sealed class WorkerProcessLauncherTests
|
||||
WorkerLaunchEnvironment.AlarmPollIntervalMilliseconds,
|
||||
WorkerLaunchEnvironment.MaxAlarmsPerFetch,
|
||||
WorkerLaunchEnvironment.AlarmAckClientName,
|
||||
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds,
|
||||
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds,
|
||||
];
|
||||
|
||||
/// <summary>Verifies that a valid worker executable starts with correct bootstrap arguments and nonce environment variable.</summary>
|
||||
@@ -80,6 +82,15 @@ public sealed class WorkerProcessLauncherTests
|
||||
"MxGateway",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerLaunchEnvironment.AlarmAckClientName]);
|
||||
// The worker's heartbeat send interval and its own watchdog ceiling (mxaccessgw#218).
|
||||
Assert.Equal(
|
||||
"5000",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
|
||||
Assert.Equal(
|
||||
"75000",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
|
||||
Assert.All(
|
||||
RequiredLaunchEnvironmentVariables,
|
||||
variable => Assert.True(
|
||||
@@ -139,6 +150,35 @@ public sealed class WorkerProcessLauncherTests
|
||||
WorkerLaunchEnvironment.EventQueueCapacity]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that configured worker heartbeat settings — not the shipped defaults — reach the
|
||||
/// worker in milliseconds (mxaccessgw#218), so they are tunable without a worker rebuild.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WithConfiguredWorkerHeartbeat_ExportsItToTheWorkerEnvironment()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234));
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
processFactory,
|
||||
new SucceedingStartupProbe(),
|
||||
workerHeartbeatSendIntervalSeconds: 3,
|
||||
workerHeartbeatStuckCeilingSeconds: 240);
|
||||
|
||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
|
||||
|
||||
Assert.NotNull(processFactory.LastStartInfo);
|
||||
Assert.Equal(
|
||||
"3000",
|
||||
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
|
||||
Assert.Equal(
|
||||
"240000",
|
||||
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -263,7 +303,9 @@ public sealed class WorkerProcessLauncherTests
|
||||
GatewayMetrics? metrics = null,
|
||||
int startupTimeoutSeconds = 30,
|
||||
int eventQueueCapacity = 10000,
|
||||
string ackClientName = "MxGateway")
|
||||
string ackClientName = "MxGateway",
|
||||
int workerHeartbeatSendIntervalSeconds = WorkerOptions.DefaultWorkerHeartbeatSendIntervalSeconds,
|
||||
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds)
|
||||
{
|
||||
GatewayOptions options = new()
|
||||
{
|
||||
@@ -274,6 +316,8 @@ public sealed class WorkerProcessLauncherTests
|
||||
RequiredArchitecture = WorkerArchitecture.X86,
|
||||
StartupTimeoutSeconds = startupTimeoutSeconds,
|
||||
EventQueueCapacity = eventQueueCapacity,
|
||||
WorkerHeartbeatSendIntervalSeconds = workerHeartbeatSendIntervalSeconds,
|
||||
WorkerHeartbeatStuckCeilingSeconds = workerHeartbeatStuckCeilingSeconds,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user