feat(gateway,worker): configurable worker heartbeat send interval and stuck ceiling (mxaccessgw#218)
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m32s
ci / java (push) Successful in 2m57s
ci / portable (push) Failing after 5m1s

The worker's heartbeat send interval (5 s) and its own STA-watchdog stuck
ceiling (75 s) were compile-time defaults the gateway never reached, and
nothing checked that the gateway's HeartbeatGraceSeconds exceeded the real send
interval, so a misconfigured grace could kill healthy workers.

- New MxGateway:Worker options WorkerHeartbeatSendIntervalSeconds (default 5)
  and WorkerHeartbeatStuckCeilingSeconds (default 75), exported by
  WorkerProcessLauncher as MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS and
  MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS, the same env-var path the other
  worker settings use.
- The worker's production WorkerPipeSession reads them through
  WorkerPipeSessionOptions.FromEnvironment; a missing/invalid value, or a
  ceiling not above the worker's 15 s watchdog grace, keeps the default rather
  than failing the worker.
- Startup validation: HeartbeatGraceSeconds > WorkerHeartbeatSendIntervalSeconds;
  send interval > 0; worker ceiling > 15 s; both at most 86400 s.
- Docs: GatewayConfiguration, WorkerProcessLauncher (env table now also lists
  the previously missing MXGATEWAY_ALARM_ACK_CLIENT_NAME), worker design doc.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:19:32 -05:00
parent 29d9510b4e
commit 2e52f9f5e8
13 changed files with 284 additions and 9 deletions
@@ -28,4 +28,18 @@ public static class WorkerLaunchEnvironment
/// Historian records it as the acknowledgement's OperatorNode.
/// </summary>
public const string AlarmAckClientName = "MXGATEWAY_ALARM_ACK_CLIENT_NAME";
/// <summary>
/// Interval, in milliseconds, at which the worker sends heartbeats to the gateway. Set
/// from <c>MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds</c>; the gateway's
/// <c>HeartbeatGraceSeconds</c> must exceed it or healthy workers are declared dead.
/// </summary>
public const string HeartbeatSendIntervalMilliseconds = "MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS";
/// <summary>
/// Ceiling, in milliseconds, after which the worker's own STA watchdog faults
/// <c>StaHung</c> even while a command is in flight. Set from
/// <c>MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds</c>.
/// </summary>
public const string HeartbeatStuckCeilingMilliseconds = "MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS";
}
@@ -16,6 +16,9 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
private const int MinimumWorkerEventQueueCapacity = 1000;
private const int MaximumWorkerEventQueueCapacity = 1_000_000;
// The worker heartbeat settings travel as int milliseconds; a day keeps them far inside that.
private const int MaximumWorkerHeartbeatSeconds = 86_400;
// Whether the host is running in the Production environment. Drives the production-only
// hard-stops (dashboard login disabled, plaintext LDAP transport) that must abort startup
// rather than merely warn. Non-production hosts keep the permissive dev posture.
@@ -280,6 +283,39 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatCheckIntervalSeconds.");
}
AddIfNotPositive(
options.WorkerHeartbeatSendIntervalSeconds,
"MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be greater than zero.",
builder);
// The gateway's grace is measured against heartbeats the worker sends on its own interval:
// a grace at or below that interval declares a healthy worker dead in the gap between two
// heartbeats (mxaccessgw#218).
if (options.WorkerHeartbeatSendIntervalSeconds > 0
&& options.HeartbeatGraceSeconds <= options.WorkerHeartbeatSendIntervalSeconds)
{
builder.Add(
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.");
}
if (options.WorkerHeartbeatSendIntervalSeconds > MaximumWorkerHeartbeatSeconds)
{
builder.Add(
$"MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
}
if (options.WorkerHeartbeatStuckCeilingSeconds > MaximumWorkerHeartbeatSeconds)
{
builder.Add(
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
}
if (options.WorkerHeartbeatStuckCeilingSeconds <= WorkerOptions.WorkerWatchdogGraceSeconds)
{
builder.Add(
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's {WorkerOptions.WorkerWatchdogGraceSeconds}-second watchdog grace.");
}
AddIfNotPositive(
options.EventChannelFullModeTimeoutMilliseconds,
"MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds must be greater than zero.",
@@ -48,8 +48,8 @@ public sealed class WorkerOptions
/// <summary>
/// How often, in seconds, the gateway checks the age of the worker's last heartbeat
/// against <see cref="HeartbeatGraceSeconds"/>. This is the gateway-side <em>check</em>
/// cadence only: the worker sends heartbeats on its own fixed 5-second interval, which
/// this option does not reach. <see langword="null"/> falls back to the deprecated
/// cadence only: the worker's own send interval is
/// <see cref="WorkerHeartbeatSendIntervalSeconds"/>. <see langword="null"/> falls back to the deprecated
/// <see cref="HeartbeatIntervalSeconds"/> key, then to
/// <see cref="DefaultHeartbeatCheckIntervalSeconds"/>; read the resolved value through
/// <see cref="ResolveHeartbeatCheckIntervalSeconds"/>.
@@ -81,11 +81,44 @@ public sealed class WorkerOptions
/// is pending the gateway tolerates a stale heartbeat (the worker STA is busy in a COM
/// call), but once the oldest pending command is older than this the heartbeat watchdog
/// faults the worker anyway. Default 75; must be at least <see cref="HeartbeatGraceSeconds"/>.
/// The worker's own watchdog keeps its separate, fixed 75-second ceiling.
/// The worker's own watchdog has its separate ceiling,
/// <see cref="WorkerHeartbeatStuckCeilingSeconds"/>.
/// </summary>
public int HeartbeatStuckCeilingSeconds { get; init; } =
(int)Workers.WorkerClientOptions.DefaultHeartbeatStuckCeiling.TotalSeconds;
/// <summary>Default for <see cref="WorkerHeartbeatSendIntervalSeconds"/>.</summary>
public const int DefaultWorkerHeartbeatSendIntervalSeconds = 5;
/// <summary>Default for <see cref="WorkerHeartbeatStuckCeilingSeconds"/>.</summary>
public const int DefaultWorkerHeartbeatStuckCeilingSeconds = 75;
/// <summary>
/// The worker's own STA-watchdog grace, in seconds. Not configurable: the worker keeps its
/// compiled default (<c>WorkerPipeSessionOptions.DefaultHeartbeatGrace</c>). Mirrored here
/// only so validation can hold <see cref="WorkerHeartbeatStuckCeilingSeconds"/> above it,
/// which the worker requires.
/// </summary>
public const int WorkerWatchdogGraceSeconds = 15;
/// <summary>
/// Interval, in seconds, at which the worker sends heartbeats to the gateway. Default 5.
/// <see cref="HeartbeatGraceSeconds"/> must be greater than this, or the gateway declares
/// a healthy worker dead between two heartbeats. Conveyed to the worker through the
/// <c>MXGATEWAY_WORKER_HEARTBEAT_SEND_INTERVAL_MS</c> environment variable.
/// </summary>
public int WorkerHeartbeatSendIntervalSeconds { get; init; } = DefaultWorkerHeartbeatSendIntervalSeconds;
/// <summary>
/// Ceiling, in seconds, on the worker's own STA watchdog: once STA activity has been stale
/// this long the worker faults <c>StaHung</c> even while a command is in flight. Default 75;
/// must be greater than the worker's fixed 15-second watchdog grace
/// (<see cref="WorkerWatchdogGraceSeconds"/>). Raise it, together with
/// <see cref="HeartbeatStuckCeilingSeconds"/>, for deployments that run very long bulk
/// operations. Conveyed through <c>MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS</c>.
/// </summary>
public int WorkerHeartbeatStuckCeilingSeconds { get; init; } = DefaultWorkerHeartbeatStuckCeilingSeconds;
/// <summary>
/// Resolves the effective heartbeat check interval: <see cref="HeartbeatCheckIntervalSeconds"/>,
/// else the deprecated <see cref="HeartbeatIntervalSeconds"/>, else
@@ -217,6 +217,10 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
_alarmsOptions.MaxAlarmsPerFetch.ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerLaunchEnvironment.AlarmAckClientName] = _alarmsOptions.AckClientName;
startInfo.Environment[WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds] =
(_workerOptions.WorkerHeartbeatSendIntervalSeconds * 1000L).ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds] =
(_workerOptions.WorkerHeartbeatStuckCeilingSeconds * 1000L).ToString(System.Globalization.CultureInfo.InvariantCulture);
commandLine = new WorkerProcessCommandLine(executablePath, arguments);
@@ -52,6 +52,8 @@
"HeartbeatCheckIntervalSeconds": 5,
"HeartbeatGraceSeconds": 15,
"HeartbeatStuckCeilingSeconds": 75,
"WorkerHeartbeatSendIntervalSeconds": 5,
"WorkerHeartbeatStuckCeilingSeconds": 75,
"EventChannelFullModeTimeoutMilliseconds": 5000,
"MaxMessageBytes": 16842752
},
@@ -43,6 +43,8 @@ public sealed class GatewayOptionsTests
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
Assert.Equal(5, options.Worker.ResolveHeartbeatCheckIntervalSeconds());
Assert.Equal(75, options.Worker.HeartbeatStuckCeilingSeconds);
Assert.Equal(5, options.Worker.WorkerHeartbeatSendIntervalSeconds);
Assert.Equal(75, options.Worker.WorkerHeartbeatStuckCeilingSeconds);
Assert.Equal(5000, options.Worker.EventChannelFullModeTimeoutMilliseconds);
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
@@ -117,6 +119,12 @@ public sealed class GatewayOptionsTests
[InlineData("MxGateway:Worker:HeartbeatCheckIntervalSeconds", "16", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatCheckIntervalSeconds.")]
[InlineData("MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds", "0", "MxGateway:Worker:EventChannelFullModeTimeoutMilliseconds must be greater than zero.")]
[InlineData("MxGateway:Worker:HeartbeatStuckCeilingSeconds", "14", "MxGateway:Worker:HeartbeatStuckCeilingSeconds must be greater than or equal to HeartbeatGraceSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "0", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be greater than zero.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "15", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most 86400.")]
[InlineData("MxGateway:Worker:HeartbeatGraceSeconds", "5", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's 15-second watchdog grace.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most 86400.")]
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
@@ -23,6 +23,8 @@ public sealed class WorkerProcessLauncherTests
WorkerLaunchEnvironment.AlarmPollIntervalMilliseconds,
WorkerLaunchEnvironment.MaxAlarmsPerFetch,
WorkerLaunchEnvironment.AlarmAckClientName,
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds,
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds,
];
/// <summary>Verifies that a valid worker executable starts with correct bootstrap arguments and nonce environment variable.</summary>
@@ -80,6 +82,15 @@ public sealed class WorkerProcessLauncherTests
"MxGateway",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.AlarmAckClientName]);
// The worker's heartbeat send interval and its own watchdog ceiling (mxaccessgw#218).
Assert.Equal(
"5000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
Assert.Equal(
"75000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
Assert.All(
RequiredLaunchEnvironmentVariables,
variable => Assert.True(
@@ -139,6 +150,35 @@ public sealed class WorkerProcessLauncherTests
WorkerLaunchEnvironment.EventQueueCapacity]);
}
/// <summary>
/// Verifies that configured worker heartbeat settings — not the shipped defaults — reach the
/// worker in milliseconds (mxaccessgw#218), so they are tunable without a worker rebuild.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task LaunchAsync_WithConfiguredWorkerHeartbeat_ExportsItToTheWorkerEnvironment()
{
using TestDirectory directory = TestDirectory.Create();
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234));
WorkerProcessLauncher launcher = CreateLauncher(
executablePath,
processFactory,
new SucceedingStartupProbe(),
workerHeartbeatSendIntervalSeconds: 3,
workerHeartbeatStuckCeilingSeconds: 240);
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
Assert.NotNull(processFactory.LastStartInfo);
Assert.Equal(
"3000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds]);
Assert.Equal(
"240000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
}
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -263,7 +303,9 @@ public sealed class WorkerProcessLauncherTests
GatewayMetrics? metrics = null,
int startupTimeoutSeconds = 30,
int eventQueueCapacity = 10000,
string ackClientName = "MxGateway")
string ackClientName = "MxGateway",
int workerHeartbeatSendIntervalSeconds = WorkerOptions.DefaultWorkerHeartbeatSendIntervalSeconds,
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds)
{
GatewayOptions options = new()
{
@@ -274,6 +316,8 @@ public sealed class WorkerProcessLauncherTests
RequiredArchitecture = WorkerArchitecture.X86,
StartupTimeoutSeconds = startupTimeoutSeconds,
EventQueueCapacity = eventQueueCapacity,
WorkerHeartbeatSendIntervalSeconds = workerHeartbeatSendIntervalSeconds,
WorkerHeartbeatStuckCeilingSeconds = workerHeartbeatStuckCeilingSeconds,
},
};
@@ -0,0 +1,72 @@
using System;
using System.Collections.Generic;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
namespace ZB.MOM.WW.MxGateway.Worker.Tests.Ipc;
/// <summary>
/// Covers <see cref="WorkerPipeSessionOptions.FromEnvironment"/>: the gateway conveys the
/// worker's heartbeat send interval and watchdog stuck ceiling through the launch environment
/// (mxaccessgw#218), and a bad value must fall back rather than keep the worker from starting.
/// </summary>
public sealed class WorkerPipeSessionOptionsTests
{
/// <summary>Verifies that launcher-provided values replace the compiled defaults.</summary>
[Fact]
public void FromEnvironment_WithLauncherValues_UsesThem()
{
WorkerPipeSessionOptions options = WorkerPipeSessionOptions.FromEnvironment(Lookup(
(WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds, "3000"),
(WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, "240000")));
Assert.Equal(TimeSpan.FromSeconds(3), options.HeartbeatInterval);
Assert.Equal(TimeSpan.FromSeconds(240), options.HeartbeatStuckCeiling);
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatGrace, options.HeartbeatGrace);
options.Validate();
}
/// <summary>Verifies that an absent environment yields the compiled defaults.</summary>
[Fact]
public void FromEnvironment_WithNothingSet_KeepsDefaults()
{
WorkerPipeSessionOptions options = WorkerPipeSessionOptions.FromEnvironment(Lookup());
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatInterval, options.HeartbeatInterval);
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatStuckCeiling, options.HeartbeatStuckCeiling);
}
/// <summary>
/// Verifies that unusable values fall back to the defaults, including a ceiling that would
/// fail <see cref="WorkerPipeSessionOptions.Validate"/> by not exceeding the grace.
/// </summary>
/// <param name="interval">Send-interval environment value.</param>
/// <param name="ceiling">Stuck-ceiling environment value.</param>
[Theory]
[InlineData("0", "0")]
[InlineData("-5", "-5")]
[InlineData("abc", "1.5")]
[InlineData("", "15000")]
[InlineData("99999999999", "99999999999")]
public void FromEnvironment_WithUnusableValues_FallsBackToDefaults(string interval, string ceiling)
{
WorkerPipeSessionOptions options = WorkerPipeSessionOptions.FromEnvironment(Lookup(
(WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds, interval),
(WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, ceiling)));
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatInterval, options.HeartbeatInterval);
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatStuckCeiling, options.HeartbeatStuckCeiling);
options.Validate();
}
private static Func<string, string?> Lookup(params (string Name, string Value)[] values)
{
Dictionary<string, string> map = new(StringComparer.Ordinal);
foreach ((string name, string value) in values)
{
map[name] = value;
}
return name => map.TryGetValue(name, out string? value) ? value : null;
}
}
@@ -75,7 +75,7 @@ public sealed class WorkerPipeSession
new WorkerFrameWriter(stream, options),
options,
() => Process.GetCurrentProcess().Id,
new WorkerPipeSessionOptions(),
WorkerPipeSessionOptions.FromEnvironment(),
() => CreateDefaultRuntimeSession(WorkerCommandLogging.IsEnabled() ? logger : null),
logger,
stream)
@@ -1,4 +1,6 @@
using System;
using System.Globalization;
using ZB.MOM.WW.MxGateway.Contracts;
namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -26,6 +28,56 @@ public sealed class WorkerPipeSessionOptions
HeartbeatStuckCeiling = DefaultHeartbeatStuckCeiling;
}
/// <summary>
/// Builds the production options from the launcher-provided environment
/// (<see cref="WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds"/> and
/// <see cref="WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds"/>), which the
/// gateway sets from <c>MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds</c> and
/// <c>WorkerHeartbeatStuckCeilingSeconds</c>. A missing, unparseable, or non-positive value
/// keeps its default, and so does a ceiling that would fail <see cref="Validate"/> (not
/// above <see cref="HeartbeatGrace"/>): the gateway validates both at startup, and a
/// worker that refused to start over a stray environment value would turn a tuning
/// mistake into a session outage. The grace is not conveyed and keeps its default.
/// </summary>
/// <param name="getEnvironmentVariable">
/// Environment lookup; <see langword="null"/> reads the process environment.
/// </param>
/// <returns>Options that always pass <see cref="Validate"/>.</returns>
public static WorkerPipeSessionOptions FromEnvironment(Func<string, string?>? getEnvironmentVariable = null)
{
Func<string, string?> lookup = getEnvironmentVariable ?? Environment.GetEnvironmentVariable;
WorkerPipeSessionOptions options = new WorkerPipeSessionOptions();
if (TryReadPositiveMilliseconds(lookup, WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds, out TimeSpan interval))
{
options.HeartbeatInterval = interval;
}
if (TryReadPositiveMilliseconds(lookup, WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, out TimeSpan ceiling)
&& ceiling > options.HeartbeatGrace)
{
options.HeartbeatStuckCeiling = ceiling;
}
return options;
}
private static bool TryReadPositiveMilliseconds(
Func<string, string?> lookup,
string name,
out TimeSpan value)
{
value = TimeSpan.Zero;
if (!int.TryParse(lookup(name), NumberStyles.Integer, CultureInfo.InvariantCulture, out int milliseconds)
|| milliseconds <= 0)
{
return false;
}
value = TimeSpan.FromMilliseconds(milliseconds);
return true;
}
/// <summary>Gets or sets the heartbeat interval.</summary>
public TimeSpan HeartbeatInterval { get; set; }