feat(gateway,worker): configurable worker watchdog grace; heartbeat settings in effective config and Settings page (mxaccessgw#224)
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m24s
ci / java (push) Successful in 2m15s
ci / portable (push) Failing after 4m17s

- New MxGateway:Worker:WorkerHeartbeatGraceSeconds (default 15), exported as
  MXGATEWAY_WORKER_HEARTBEAT_GRACE_MS and read by WorkerPipeSessionOptions.FromEnvironment
  (a grace the ceiling does not exceed falls back to the default so the worker
  still starts). Validation now checks WorkerHeartbeatStuckCeilingSeconds against
  the configured grace; the copied WorkerWatchdogGraceSeconds constant is gone.
- EffectiveWorkerConfiguration and the dashboard Settings page now show the worker
  heartbeat send interval, watchdog grace and stuck ceiling (render test added).
- Docs updated.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:29:10 -05:00
parent 381e772766
commit 22964e192b
16 changed files with 141 additions and 19 deletions
@@ -42,4 +42,12 @@ public static class WorkerLaunchEnvironment
/// <c>MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds</c>.
/// </summary>
public const string HeartbeatStuckCeilingMilliseconds = "MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS";
/// <summary>
/// Grace, in milliseconds, the worker's own STA watchdog allows STA activity to go stale
/// before faulting <c>StaHung</c> (when no command is in flight). Set from
/// <c>MxGateway:Worker:WorkerHeartbeatGraceSeconds</c>; the worker's stuck ceiling must
/// exceed it.
/// </summary>
public const string HeartbeatGraceMilliseconds = "MXGATEWAY_WORKER_HEARTBEAT_GRACE_MS";
}
@@ -9,5 +9,8 @@ public sealed record EffectiveWorkerConfiguration(
int HeartbeatCheckIntervalSeconds,
int HeartbeatGraceSeconds,
int HeartbeatStuckCeilingSeconds,
int WorkerHeartbeatSendIntervalSeconds,
int WorkerHeartbeatGraceSeconds,
int WorkerHeartbeatStuckCeilingSeconds,
int EventChannelFullModeTimeoutMilliseconds,
int MaxMessageBytes);
@@ -41,6 +41,9 @@ public sealed class GatewayConfigurationProvider(IOptions<GatewayOptions> option
HeartbeatCheckIntervalSeconds: value.Worker.ResolveHeartbeatCheckIntervalSeconds(),
HeartbeatGraceSeconds: value.Worker.HeartbeatGraceSeconds,
HeartbeatStuckCeilingSeconds: value.Worker.HeartbeatStuckCeilingSeconds,
WorkerHeartbeatSendIntervalSeconds: value.Worker.WorkerHeartbeatSendIntervalSeconds,
WorkerHeartbeatGraceSeconds: value.Worker.WorkerHeartbeatGraceSeconds,
WorkerHeartbeatStuckCeilingSeconds: value.Worker.WorkerHeartbeatStuckCeilingSeconds,
EventChannelFullModeTimeoutMilliseconds: value.Worker.EventChannelFullModeTimeoutMilliseconds,
MaxMessageBytes: value.Worker.MaxMessageBytes),
Sessions: new EffectiveSessionConfiguration(
@@ -310,10 +310,23 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
}
if (options.WorkerHeartbeatStuckCeilingSeconds <= WorkerOptions.WorkerWatchdogGraceSeconds)
AddIfNotPositive(
options.WorkerHeartbeatGraceSeconds,
"MxGateway:Worker:WorkerHeartbeatGraceSeconds must be greater than zero.",
builder);
if (options.WorkerHeartbeatGraceSeconds > MaximumWorkerHeartbeatSeconds)
{
builder.Add(
$"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's {WorkerOptions.WorkerWatchdogGraceSeconds}-second watchdog grace.");
$"MxGateway:Worker:WorkerHeartbeatGraceSeconds must be at most {MaximumWorkerHeartbeatSeconds}.");
}
// The worker refuses a ceiling at or below its own grace, so hold it above the configured
// grace (not a copied constant) -- mxaccessgw#224.
if (options.WorkerHeartbeatStuckCeilingSeconds <= options.WorkerHeartbeatGraceSeconds)
{
builder.Add(
"MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than WorkerHeartbeatGraceSeconds.");
}
AddIfNotPositive(
@@ -93,14 +93,17 @@ public sealed class WorkerOptions
/// <summary>Default for <see cref="WorkerHeartbeatStuckCeilingSeconds"/>.</summary>
public const int DefaultWorkerHeartbeatStuckCeilingSeconds = 75;
/// <summary>
/// The worker's own STA-watchdog grace, in seconds. Not configurable: the worker keeps its
/// compiled default (<c>WorkerPipeSessionOptions.DefaultHeartbeatGrace</c>). Mirrored here
/// only so validation can hold <see cref="WorkerHeartbeatStuckCeilingSeconds"/> above it,
/// which the worker requires.
/// </summary>
public const int WorkerWatchdogGraceSeconds = 15;
/// <summary>Default for <see cref="WorkerHeartbeatGraceSeconds"/>.</summary>
public const int DefaultWorkerHeartbeatGraceSeconds = 15;
/// <summary>
/// The worker's own STA-watchdog grace, in seconds: how long STA activity may go stale
/// before the worker faults <c>StaHung</c> (no command in flight). Default 15.
/// <see cref="WorkerHeartbeatStuckCeilingSeconds"/> must be greater than this, which
/// validation checks against the configured value rather than a copied constant.
/// Conveyed through <c>MXGATEWAY_WORKER_HEARTBEAT_GRACE_MS</c>.
/// </summary>
public int WorkerHeartbeatGraceSeconds { get; init; } = DefaultWorkerHeartbeatGraceSeconds;
/// <summary>
/// Interval, in seconds, at which the worker sends heartbeats to the gateway. Default 5.
/// <see cref="HeartbeatGraceSeconds"/> must be greater than this, or the gateway declares
@@ -112,8 +115,8 @@ public sealed class WorkerOptions
/// <summary>
/// Ceiling, in seconds, on the worker's own STA watchdog: once STA activity has been stale
/// this long the worker faults <c>StaHung</c> even while a command is in flight. Default 75;
/// must be greater than the worker's fixed 15-second watchdog grace
/// (<see cref="WorkerWatchdogGraceSeconds"/>). Raise it, together with
/// must be greater than the worker's watchdog grace
/// (<see cref="WorkerHeartbeatGraceSeconds"/>). Raise it, together with
/// <see cref="HeartbeatStuckCeilingSeconds"/>, for deployments that run very long bulk
/// operations. Conveyed through <c>MXGATEWAY_WORKER_HEARTBEAT_STUCK_CEILING_MS</c>.
/// </summary>
@@ -93,6 +93,9 @@ else
<tr><th scope="row">Heartbeat grace</th><td>@Snapshot.Configuration.Worker.HeartbeatGraceSeconds seconds</td></tr>
<tr><th scope="row">Heartbeat check interval</th><td>@Snapshot.Configuration.Worker.HeartbeatCheckIntervalSeconds seconds</td></tr>
<tr><th scope="row">Heartbeat stuck ceiling</th><td>@Snapshot.Configuration.Worker.HeartbeatStuckCeilingSeconds seconds</td></tr>
<tr><th scope="row">Worker heartbeat send interval</th><td>@Snapshot.Configuration.Worker.WorkerHeartbeatSendIntervalSeconds seconds</td></tr>
<tr><th scope="row">Worker watchdog grace</th><td>@Snapshot.Configuration.Worker.WorkerHeartbeatGraceSeconds seconds</td></tr>
<tr><th scope="row">Worker watchdog stuck ceiling</th><td>@Snapshot.Configuration.Worker.WorkerHeartbeatStuckCeilingSeconds seconds</td></tr>
<tr><th scope="row">Event channel full-mode timeout</th><td>@Snapshot.Configuration.Worker.EventChannelFullModeTimeoutMilliseconds ms</td></tr>
<tr><th scope="row">Default command timeout</th><td>@Snapshot.Configuration.Sessions.DefaultCommandTimeoutSeconds seconds</td></tr>
<tr><th scope="row">Max sessions</th><td>@Snapshot.Configuration.Sessions.MaxSessions</td></tr>
@@ -219,6 +219,8 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
startInfo.Environment[WorkerLaunchEnvironment.AlarmAckClientName] = _alarmsOptions.AckClientName;
startInfo.Environment[WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds] =
(_workerOptions.WorkerHeartbeatSendIntervalSeconds * 1000L).ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerLaunchEnvironment.HeartbeatGraceMilliseconds] =
(_workerOptions.WorkerHeartbeatGraceSeconds * 1000L).ToString(System.Globalization.CultureInfo.InvariantCulture);
startInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds] =
(_workerOptions.WorkerHeartbeatStuckCeilingSeconds * 1000L).ToString(System.Globalization.CultureInfo.InvariantCulture);
@@ -53,6 +53,7 @@
"HeartbeatGraceSeconds": 15,
"HeartbeatStuckCeilingSeconds": 75,
"WorkerHeartbeatSendIntervalSeconds": 5,
"WorkerHeartbeatGraceSeconds": 15,
"WorkerHeartbeatStuckCeilingSeconds": 75,
"EventChannelFullModeTimeoutMilliseconds": 5000,
"MaxMessageBytes": 16842752
@@ -45,6 +45,7 @@ public sealed class GatewayOptionsTests
Assert.Equal(75, options.Worker.HeartbeatStuckCeilingSeconds);
Assert.Equal(5, options.Worker.WorkerHeartbeatSendIntervalSeconds);
Assert.Equal(75, options.Worker.WorkerHeartbeatStuckCeilingSeconds);
Assert.Equal(15, options.Worker.WorkerHeartbeatGraceSeconds);
Assert.Equal(5000, options.Worker.EventChannelFullModeTimeoutMilliseconds);
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
@@ -123,7 +124,10 @@ public sealed class GatewayOptionsTests
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "15", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most 86400.")]
[InlineData("MxGateway:Worker:HeartbeatGraceSeconds", "5", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's 15-second watchdog grace.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than WorkerHeartbeatGraceSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "75", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than WorkerHeartbeatGraceSeconds.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "0", "MxGateway:Worker:WorkerHeartbeatGraceSeconds must be greater than zero.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatGraceSeconds must be at most 86400.")]
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most 86400.")]
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
@@ -120,6 +120,31 @@ public sealed class SettingsPageTagVisibilityRenderTests
Assert.Contains("""<th scope="row">Replay buffer max bytes</th><td>987654321</td>""", html, StringComparison.Ordinal);
}
/// <summary>
/// The worker-side heartbeat send interval, watchdog grace and stuck ceiling (mxaccessgw#218,
/// #224) reach the page, each in its own labelled row, distinct from the gateway-side ones.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SettingsPage_RendersWorkerHeartbeatOptions()
{
string html = await RenderAsync(new GatewayOptions
{
Worker = new WorkerOptions
{
HeartbeatStuckCeilingSeconds = 100,
WorkerHeartbeatSendIntervalSeconds = 3,
WorkerHeartbeatGraceSeconds = 31,
WorkerHeartbeatStuckCeilingSeconds = 241,
},
});
Assert.Contains("""<th scope="row">Worker heartbeat send interval</th><td>3 seconds</td>""", html, StringComparison.Ordinal);
Assert.Contains("""<th scope="row">Worker watchdog grace</th><td>31 seconds</td>""", html, StringComparison.Ordinal);
Assert.Contains("""<th scope="row">Worker watchdog stuck ceiling</th><td>241 seconds</td>""", html, StringComparison.Ordinal);
Assert.Contains("""<th scope="row">Heartbeat stuck ceiling</th><td>100 seconds</td>""", html, StringComparison.Ordinal);
}
private static Task<string> RenderAsync(DashboardOptions dashboard) =>
RenderAsync(new GatewayOptions { Dashboard = dashboard });
@@ -24,6 +24,7 @@ public sealed class WorkerProcessLauncherTests
WorkerLaunchEnvironment.MaxAlarmsPerFetch,
WorkerLaunchEnvironment.AlarmAckClientName,
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds,
WorkerLaunchEnvironment.HeartbeatGraceMilliseconds,
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds,
];
@@ -91,6 +92,10 @@ public sealed class WorkerProcessLauncherTests
"75000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
Assert.Equal(
"15000",
processFactory.LastStartInfo.Environment[
WorkerLaunchEnvironment.HeartbeatGraceMilliseconds]);
Assert.All(
RequiredLaunchEnvironmentVariables,
variable => Assert.True(
@@ -166,7 +171,8 @@ public sealed class WorkerProcessLauncherTests
processFactory,
new SucceedingStartupProbe(),
workerHeartbeatSendIntervalSeconds: 3,
workerHeartbeatStuckCeilingSeconds: 240);
workerHeartbeatStuckCeilingSeconds: 240,
workerHeartbeatGraceSeconds: 30);
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
@@ -177,6 +183,9 @@ public sealed class WorkerProcessLauncherTests
Assert.Equal(
"240000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
Assert.Equal(
"30000",
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatGraceMilliseconds]);
}
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
@@ -305,7 +314,8 @@ public sealed class WorkerProcessLauncherTests
int eventQueueCapacity = 10000,
string ackClientName = "MxGateway",
int workerHeartbeatSendIntervalSeconds = WorkerOptions.DefaultWorkerHeartbeatSendIntervalSeconds,
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds)
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds,
int workerHeartbeatGraceSeconds = WorkerOptions.DefaultWorkerHeartbeatGraceSeconds)
{
GatewayOptions options = new()
{
@@ -318,6 +328,7 @@ public sealed class WorkerProcessLauncherTests
EventQueueCapacity = eventQueueCapacity,
WorkerHeartbeatSendIntervalSeconds = workerHeartbeatSendIntervalSeconds,
WorkerHeartbeatStuckCeilingSeconds = workerHeartbeatStuckCeilingSeconds,
WorkerHeartbeatGraceSeconds = workerHeartbeatGraceSeconds,
},
};
@@ -26,6 +26,31 @@ public sealed class WorkerPipeSessionOptionsTests
options.Validate();
}
/// <summary>Verifies that a launcher-provided grace replaces the compiled default (mxaccessgw#224).</summary>
[Fact]
public void FromEnvironment_WithGrace_UsesItAndStaysValid()
{
WorkerPipeSessionOptions options = WorkerPipeSessionOptions.FromEnvironment(Lookup(
(WorkerLaunchEnvironment.HeartbeatGraceMilliseconds, "30000"),
(WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, "120000")));
Assert.Equal(TimeSpan.FromSeconds(30), options.HeartbeatGrace);
Assert.Equal(TimeSpan.FromSeconds(120), options.HeartbeatStuckCeiling);
options.Validate();
}
/// <summary>Verifies that a grace the ceiling does not exceed is dropped rather than failing the worker.</summary>
[Fact]
public void FromEnvironment_WithGraceAtOrAboveCeiling_FallsBackToDefaultGrace()
{
WorkerPipeSessionOptions options = WorkerPipeSessionOptions.FromEnvironment(Lookup(
(WorkerLaunchEnvironment.HeartbeatGraceMilliseconds, "80000")));
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatGrace, options.HeartbeatGrace);
Assert.Equal(WorkerPipeSessionOptions.DefaultHeartbeatStuckCeiling, options.HeartbeatStuckCeiling);
options.Validate();
}
/// <summary>Verifies that an absent environment yields the compiled defaults.</summary>
[Fact]
public void FromEnvironment_WithNothingSet_KeepsDefaults()
@@ -37,7 +37,8 @@ public sealed class WorkerPipeSessionOptions
/// keeps its default, and so does a ceiling that would fail <see cref="Validate"/> (not
/// above <see cref="HeartbeatGrace"/>): the gateway validates both at startup, and a
/// worker that refused to start over a stray environment value would turn a tuning
/// mistake into a session outage. The grace is not conveyed and keeps its default.
/// mistake into a session outage. The watchdog grace is conveyed too
/// (<see cref="WorkerLaunchEnvironment.HeartbeatGraceMilliseconds"/>) and must stay below the ceiling.
/// </summary>
/// <param name="getEnvironmentVariable">
/// Environment lookup; <see langword="null"/> reads the process environment.
@@ -53,8 +54,25 @@ public sealed class WorkerPipeSessionOptions
options.HeartbeatInterval = interval;
}
if (TryReadPositiveMilliseconds(lookup, WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, out TimeSpan ceiling)
&& ceiling > options.HeartbeatGrace)
TimeSpan grace = options.HeartbeatGrace;
bool haveGrace = TryReadPositiveMilliseconds(lookup, WorkerLaunchEnvironment.HeartbeatGraceMilliseconds, out TimeSpan configuredGrace);
if (haveGrace)
{
grace = configuredGrace;
}
bool haveCeiling = TryReadPositiveMilliseconds(lookup, WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds, out TimeSpan ceiling);
TimeSpan effectiveCeiling = haveCeiling ? ceiling : options.HeartbeatStuckCeiling;
// Keep ceiling > grace so Validate() always passes. A grace the ceiling does not exceed
// is dropped (back to the default) rather than failing the worker; an unusable ceiling
// is dropped first when the default grace still leaves it valid.
if (effectiveCeiling > grace)
{
options.HeartbeatGrace = grace;
options.HeartbeatStuckCeiling = effectiveCeiling;
}
else if (haveCeiling && ceiling > options.HeartbeatGrace)
{
options.HeartbeatStuckCeiling = ceiling;
}