feat(gateway,worker): configurable worker heartbeat send interval and stuck ceiling (mxaccessgw#218)
The worker's heartbeat send interval (5 s) and its own STA-watchdog stuck ceiling (75 s) were compile-time defaults the gateway never reached, and nothing checked that the gateway's HeartbeatGraceSeconds exceeded the real send interval, so a misconfigured grace could kill healthy workers. - New MxGateway:Worker options WorkerHeartbeatSendIntervalSeconds (default 5) and WorkerHeartbeatStuckCeilingSeconds (default 75), exported by WorkerProcessLauncher as MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS and MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS, the same env-var path the other worker settings use. - The worker's production WorkerPipeSession reads them through WorkerPipeSessionOptions.FromEnvironment; a missing/invalid value, or a ceiling not above the worker's 15 s watchdog grace, keeps the default rather than failing the worker. - Startup validation: HeartbeatGraceSeconds > WorkerHeartbeatSendIntervalSeconds; send interval > 0; worker ceiling > 15 s; both at most 86400 s. - Docs: GatewayConfiguration, WorkerProcessLauncher (env table now also lists the previously missing MXGATEWAY_ALARM_ACK_CLIENT_NAME), worker design doc. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -16,6 +16,9 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
private const int MinimumWorkerEventQueueCapacity = 1000;
|
||||
private const int MaximumWorkerEventQueueCapacity = 1_000_000;
|
||||
|
||||
// The worker heartbeat settings travel as int milliseconds; a day keeps them far inside that.
|
||||
private const int MaximumWorkerHeartbeatSeconds = 86_400;
|
||||
|
||||
// Whether the host is running in the Production environment. Drives the production-only
|
||||
// hard-stops (dashboard login disabled, plaintext LDAP transport) that must abort startup
|
||||
// rather than merely warn. Non-production hosts keep the permissive dev posture.
|
||||
@@ -280,6 +283,39 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatCheckIntervalSeconds.");
|
||||
}
|
||||
|
||||
AddIfNotPositive(
|
||||
options.WorkerHeartbeatSendIntervalSeconds,
|
||||
"MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be greater than zero.",
|
||||
builder);
|
||||
|
||||
// The gateway's grace is measured against heartbeats the worker sends on its own interval:
|
||||
// a grace at or below that interval declares a healthy worker dead in the gap between two
|
||||
// heartbeats (mxaccessgw#218).
|
||||
if (options.WorkerHeartbeatSendIntervalSeconds > 0
|
||||
&& options.HeartbeatGraceSeconds <= options.WorkerHeartbeatSendIntervalSeconds)
|
||||
{
|
||||
builder.Add(
|
||||
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.");
|
||||
}
|
||||
|
||||
if (options.WorkerHeartbeatSendIntervalSeconds > MaximumWorkerHeartbeatSeconds)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
|
||||
}
|
||||
|
||||
if (options.WorkerHeartbeatStuckCeilingSeconds > MaximumWorkerHeartbeatSeconds)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
|
||||
}
|
||||
|
||||
if (options.WorkerHeartbeatStuckCeilingSeconds <= WorkerOptions.WorkerWatchdogGraceSeconds)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's {WorkerOptions.WorkerWatchdogGraceSeconds}-second watchdog grace.");
|
||||
}
|
||||
|
||||
AddIfNotPositive(
|
||||
options.EventChannelFullModeTimeoutMilliseconds,
|
||||
"MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds must be greater than zero.",
|
||||
|
||||
Reference in New Issue
Block a user