feat(gateway,worker): configurable worker heartbeat send interval and stuck ceiling (mxaccessgw#218)
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m32s
ci / java (push) Successful in 2m57s
ci / portable (push) Failing after 5m1s

The worker's heartbeat send interval (5 s) and its own STA-watchdog stuck
ceiling (75 s) were compile-time defaults the gateway never reached, and
nothing checked that the gateway's HeartbeatGraceSeconds exceeded the real send
interval, so a misconfigured grace could kill healthy workers.

- New MxGateway:Worker options WorkerHeartbeatSendIntervalSeconds (default 5)
  and WorkerHeartbeatStuckCeilingSeconds (default 75), exported by
  WorkerProcessLauncher as MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS and
  MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS, the same env-var path the other
  worker settings use.
- The worker's production WorkerPipeSession reads them through
  WorkerPipeSessionOptions.FromEnvironment; a missing/invalid value, or a
  ceiling not above the worker's 15 s watchdog grace, keeps the default rather
  than failing the worker.
- Startup validation: HeartbeatGraceSeconds > WorkerHeartbeatSendIntervalSeconds;
  send interval > 0; worker ceiling > 15 s; both at most 86400 s.
- Docs: GatewayConfiguration, WorkerProcessLauncher (env table now also lists
  the previously missing MXGATEWAY_ALARM_ACK_CLIENT_NAME), worker design doc.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:19:32 -05:00
parent 29d9510b4e
commit 2e52f9f5e8
13 changed files with 284 additions and 9 deletions
@@ -43,6 +43,8 @@ public sealed class GatewayOptionsTests
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
Assert.Equal(5, options.Worker.ResolveHeartbeatCheckIntervalSeconds());
Assert.Equal(75, options.Worker.HeartbeatStuckCeilingSeconds);
Assert.Equal(5, options.Worker.WorkerHeartbeatSendIntervalSeconds);
Assert.Equal(75, options.Worker.WorkerHeartbeatStuckCeilingSeconds);
Assert.Equal(5000, options.Worker.EventChannelFullModeTimeoutMilliseconds);
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
@@ -117,6 +119,12 @@ public sealed class GatewayOptionsTests
[InlineData("MxGateway:Worker:HeartbeatCheckIntervalSeconds", "16", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatCheckIntervalSeconds.")]
[InlineData("MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds", "0", "MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds must be greater than zero.")]
[InlineData("MxGateway:Worker:HeartbeatStuckCeilingSeconds", "14", "MxGateway:Worker:HeartbeatStuckCeilingSeconds must be greater than or equal to HeartbeatGraceSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "0", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be greater than zero.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "15", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most 86400.")]
[InlineData("MxGateway:Worker:HeartbeatGraceSeconds", "5", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's 15-second watchdog grace.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most 86400.")]
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
@@ -23,6 +23,8 @@ public sealed class WorkerProcessLauncherTests
WorkerLaunchEnvironment.AlarmPollIntervalMilliseconds,
WorkerLaunchEnvironment.MaxAlarmsPerFetch,
WorkerLaunchEnvironment.AlarmAckClientName,
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds,
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds,
];
/// <summary>Verifies that a valid worker executable starts with correct bootstrap arguments and nonce environment variable.</summary>
@@ -80,6 +82,15 @@ public sealed class WorkerProcessLauncherTests
"MxGateway",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.AlarmAckClientName]);
// The worker's heartbeat send interval and its own watchdog ceiling (mxaccessgw#218).
Assert.Equal(
"5000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
Assert.Equal(
"75000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
Assert.All(
RequiredLaunchEnvironmentVariables,
variable => Assert.True(
@@ -139,6 +150,35 @@ public sealed class WorkerProcessLauncherTests
WorkerLaunchEnvironment.EventQueueCapacity]);
}
/// <summary>
/// Verifies that configured worker heartbeat settings — not the shipped defaults — reach the
/// worker in milliseconds (mxaccessgw#218), so they are tunable without a worker rebuild.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task LaunchAsync_WithConfiguredWorkerHeartbeat_ExportsItToTheWorkerEnvironment()
{
using TestDirectory directory = TestDirectory.Create();
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234));
WorkerProcessLauncher launcher = CreateLauncher(
executablePath,
processFactory,
new SucceedingStartupProbe(),
workerHeartbeatSendIntervalSeconds: 3,
workerHeartbeatStuckCeilingSeconds: 240);
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
Assert.NotNull(processFactory.LastStartInfo);
Assert.Equal(
"3000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
Assert.Equal(
"240000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
}
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -263,7 +303,9 @@ public sealed class WorkerProcessLauncherTests
GatewayMetrics? metrics = null,
int startupTimeoutSeconds = 30,
int eventQueueCapacity = 10000,
string ackClientName = "MxGateway")
string ackClientName = "MxGateway",
int workerHeartbeatSendIntervalSeconds = WorkerOptions.DefaultWorkerHeartbeatSendIntervalSeconds,
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds)
{
GatewayOptions options = new()
{
@@ -274,6 +316,8 @@ public sealed class WorkerProcessLauncherTests
RequiredArchitecture = WorkerArchitecture.X86,
StartupTimeoutSeconds = startupTimeoutSeconds,
EventQueueCapacity = eventQueueCapacity,
WorkerHeartbeatSendIntervalSeconds = workerHeartbeatSendIntervalSeconds,
WorkerHeartbeatStuckCeilingSeconds = workerHeartbeatStuckCeilingSeconds,
},
};