feat(gateway,worker): configurable worker watchdog grace; heartbeat settings in effective config and Settings page (mxaccessgw#224)
- New MxGateway:Worker:WorkerHeartbeatGraceSeconds (default 15), exported as MXGATEWAY_WORKER_HEARTBEAT_GRACE_MS and read by WorkerPipeSessionOptions.FromEnvironment (a grace the ceiling does not exceed falls back to the default so the worker still starts). Validation now checks WorkerHeartbeatStuckCeilingSeconds against the configured grace; the copied WorkerWatchdogGraceSeconds constant is gone. - EffectiveWorkerConfiguration and the dashboard Settings page now show the worker heartbeat send interval, watchdog grace and stuck ceiling (render test added). - Docs updated. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -45,6 +45,7 @@ public sealed class GatewayOptionsTests
|
||||
Assert.Equal(75, options.Worker.HeartbeatStuckCeilingSeconds);
|
||||
Assert.Equal(5, options.Worker.WorkerHeartbeatSendIntervalSeconds);
|
||||
Assert.Equal(75, options.Worker.WorkerHeartbeatStuckCeilingSeconds);
|
||||
Assert.Equal(15, options.Worker.WorkerHeartbeatGraceSeconds);
|
||||
Assert.Equal(5000, options.Worker.EventChannelFullModeTimeoutMilliseconds);
|
||||
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
|
||||
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
|
||||
@@ -123,7 +124,10 @@ public sealed class GatewayOptionsTests
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "15", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatSendIntervalSeconds must be at most 86400.")]
|
||||
[InlineData("MxGateway:Worker:HeartbeatGraceSeconds", "5", "MxGateway:Worker:HeartbeatGraceSeconds must be greater than WorkerHeartbeatSendIntervalSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than the worker's 15-second watchdog grace.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "15", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than WorkerHeartbeatGraceSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "75", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be greater than WorkerHeartbeatGraceSeconds.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "0", "MxGateway:Worker:WorkerHeartbeatGraceSeconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatGraceSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatGraceSeconds must be at most 86400.")]
|
||||
[InlineData("MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds", "86401", "MxGateway:Worker:WorkerHeartbeatStuckCeilingSeconds must be at most 86400.")]
|
||||
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
||||
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
|
||||
|
||||
@@ -120,6 +120,31 @@ public sealed class SettingsPageTagVisibilityRenderTests
|
||||
Assert.Contains("""<th scope="row">Replay buffer max bytes</th><td>987654321</td>""", html, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker-side heartbeat send interval, watchdog grace and stuck ceiling (mxaccessgw#218,
|
||||
/// #224) reach the page, each in its own labelled row, distinct from the gateway-side ones.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SettingsPage_RendersWorkerHeartbeatOptions()
|
||||
{
|
||||
string html = await RenderAsync(new GatewayOptions
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
HeartbeatStuckCeilingSeconds = 100,
|
||||
WorkerHeartbeatSendIntervalSeconds = 3,
|
||||
WorkerHeartbeatGraceSeconds = 31,
|
||||
WorkerHeartbeatStuckCeilingSeconds = 241,
|
||||
},
|
||||
});
|
||||
|
||||
Assert.Contains("""<th scope="row">Worker heartbeat send interval</th><td>3 seconds</td>""", html, StringComparison.Ordinal);
|
||||
Assert.Contains("""<th scope="row">Worker watchdog grace</th><td>31 seconds</td>""", html, StringComparison.Ordinal);
|
||||
Assert.Contains("""<th scope="row">Worker watchdog stuck ceiling</th><td>241 seconds</td>""", html, StringComparison.Ordinal);
|
||||
Assert.Contains("""<th scope="row">Heartbeat stuck ceiling</th><td>100 seconds</td>""", html, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static Task<string> RenderAsync(DashboardOptions dashboard) =>
|
||||
RenderAsync(new GatewayOptions { Dashboard = dashboard });
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ public sealed class WorkerProcessLauncherTests
|
||||
WorkerLaunchEnvironment.MaxAlarmsPerFetch,
|
||||
WorkerLaunchEnvironment.AlarmAckClientName,
|
||||
WorkerLaunchEnvironment.HeartbeatSendIntervalMilliseconds,
|
||||
WorkerLaunchEnvironment.HeartbeatGraceMilliseconds,
|
||||
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds,
|
||||
];
|
||||
|
||||
@@ -91,6 +92,10 @@ public sealed class WorkerProcessLauncherTests
|
||||
"75000",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
|
||||
Assert.Equal(
|
||||
"15000",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerLaunchEnvironment.HeartbeatGraceMilliseconds]);
|
||||
Assert.All(
|
||||
RequiredLaunchEnvironmentVariables,
|
||||
variable => Assert.True(
|
||||
@@ -166,7 +171,8 @@ public sealed class WorkerProcessLauncherTests
|
||||
processFactory,
|
||||
new SucceedingStartupProbe(),
|
||||
workerHeartbeatSendIntervalSeconds: 3,
|
||||
workerHeartbeatStuckCeilingSeconds: 240);
|
||||
workerHeartbeatStuckCeilingSeconds: 240,
|
||||
workerHeartbeatGraceSeconds: 30);
|
||||
|
||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
|
||||
|
||||
@@ -177,6 +183,9 @@ public sealed class WorkerProcessLauncherTests
|
||||
Assert.Equal(
|
||||
"240000",
|
||||
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatStuckCeilingMilliseconds]);
|
||||
Assert.Equal(
|
||||
"30000",
|
||||
processFactory.LastStartInfo.Environment[WorkerLaunchEnvironment.HeartbeatGraceMilliseconds]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
|
||||
@@ -305,7 +314,8 @@ public sealed class WorkerProcessLauncherTests
|
||||
int eventQueueCapacity = 10000,
|
||||
string ackClientName = "MxGateway",
|
||||
int workerHeartbeatSendIntervalSeconds = WorkerOptions.DefaultWorkerHeartbeatSendIntervalSeconds,
|
||||
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds)
|
||||
int workerHeartbeatStuckCeilingSeconds = WorkerOptions.DefaultWorkerHeartbeatStuckCeilingSeconds,
|
||||
int workerHeartbeatGraceSeconds = WorkerOptions.DefaultWorkerHeartbeatGraceSeconds)
|
||||
{
|
||||
GatewayOptions options = new()
|
||||
{
|
||||
@@ -318,6 +328,7 @@ public sealed class WorkerProcessLauncherTests
|
||||
EventQueueCapacity = eventQueueCapacity,
|
||||
WorkerHeartbeatSendIntervalSeconds = workerHeartbeatSendIntervalSeconds,
|
||||
WorkerHeartbeatStuckCeilingSeconds = workerHeartbeatStuckCeilingSeconds,
|
||||
WorkerHeartbeatGraceSeconds = workerHeartbeatGraceSeconds,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user