From 61c00d59a9352a48642f6e7ad7908aea7299fc1e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 6 Nov 2026 16:13:09 -0500 Subject: [PATCH] feat(gateway): readiness check for the worker executable (#206) Readiness covered only the auth store, so a gateway whose worker exe was missing or of the wrong architecture reported ready while every OpenSession failed. The new ready-tier check worker-executable applies the launcher's own rule (path resolution, .exe, presence, PE architecture), now shared as WorkerExecutableValidator.ResolveAndValidate, and reports only the launch error code (the endpoint is anonymous). The session check stays on the active tier, so zero sessions keeps readiness green. The bounded Galaxy SQL check is not in this change: its tier, failure status and probe cadence need a decision (see the issue). Tests: WorkerExecutableHealthCheckTests (healthy, missing, wrong architecture, and the ready-tier registration, which fails without the wiring). --- docs/Diagnostics.md | 3 +- .../WorkerExecutableHealthCheck.cs | 55 +++++++++ .../GatewayApplication.cs | 6 + .../Workers/WorkerExecutableValidator.cs | 44 ++++++++ .../Workers/WorkerProcessLauncher.cs | 35 +----- .../WorkerExecutableHealthCheckTests.cs | 106 ++++++++++++++++++ 6 files changed, 215 insertions(+), 34 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index b3252a7..540bdde 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -218,11 +218,12 @@ The order matters: putting the logging scope first ensures that authentication f ## Health Checks The shared `ZB.MOM.WW.Health` package maps three endpoints — `/healthz` (live), `/health/ready`, and -`/health/active` — and each registered check opts into a tier by tag. The gateway registers two: +`/health/active` — and each registered check opts into a tier by tag. The gateway registers three: | Check | Endpoint tier | Fails when | |---|---|---| | `auth-store` | `ready` | The SQLite auth store cannot be opened. Every gRPC call authenticates against it, so its reachability genuinely gates whether the process should receive traffic. | +| `worker-executable` | `ready` | The configured `MxGateway:Worker:ExecutablePath` is not launchable — invalid path, not a `.exe`, missing, or not a PE image of `RequiredArchitecture`. It applies the launcher's own rule (`WorkerExecutableValidator.ResolveAndValidate`), so the probe and `OpenSession` cannot disagree. Every session launches the worker, so a gateway without one cannot serve MXAccess. It reports only the launch error code as entry `data` (`error_code`), never the path, because `/health/ready` is anonymous. It does not depend on session count. | | `mxaccess-sessions` | `active` | Sessions exist and their workers have faulted. Reports `total` / `ready` / `faulted` / `starting` / `closing` as entry `data`. | **Zero sessions is Healthy, and the tier choice follows from that.** The gateway opens an MXAccess diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs new file mode 100644 index 0000000..da1a6e3 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Workers; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// +/// Readiness probe: verifies the configured worker executable is present and launchable (valid +/// path, .exe, on disk, PE image of the required architecture). Every OpenSession +/// launches it, so a gateway whose worker is missing cannot serve MXAccess at all and should not +/// receive traffic. This applies the same rule the launcher applies +/// (), so the probe and the launch cannot +/// disagree. +/// +/// +/// The result never carries the executable path: /health/ready is anonymous, so only the +/// launch error code is reported. +/// +public sealed class WorkerExecutableHealthCheck : IHealthCheck +{ + private readonly IOptions _options; + + /// Initializes a new instance of the class. + /// Gateway options carrying the worker executable settings. + public WorkerExecutableHealthCheck(IOptions options) => + _options = options ?? throw new ArgumentNullException(nameof(options)); + + /// Checks the worker executable the next session launch would use. + /// The health check context. + /// Token to cancel the asynchronous operation. + /// Healthy when the executable is launchable; otherwise Unhealthy with the launch error code. + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + try + { + WorkerExecutableValidator.ResolveAndValidate(_options.Value.Worker); + return Task.FromResult(HealthCheckResult.Healthy("Worker executable is present.")); + } + catch (WorkerProcessLaunchException exception) + { + return Task.FromResult(HealthCheckResult.Unhealthy( + $"Worker executable is not launchable ({exception.ErrorCode}).", + data: new Dictionary { ["error_code"] = exception.ErrorCode.ToString() })); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + return Task.FromResult(HealthCheckResult.Unhealthy( + "Worker executable could not be read.", + data: new Dictionary { ["error_code"] = exception.GetType().Name })); + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs index d80f859..e86a74b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs @@ -120,6 +120,12 @@ public static class GatewayApplication "auth-store", failureStatus: null, tags: new[] { ZbHealthTags.Ready }) + // Ready: every OpenSession launches the worker, so a missing worker exe means the + // gateway cannot serve MXAccess. It never depends on session count. + .AddTypeActivatedCheck( + "worker-executable", + failureStatus: null, + tags: new[] { ZbHealthTags.Ready }) // Active, not Ready: a gateway holding no sessions is legitimately ready to serve. // See SessionHealthCheck for why zero sessions is healthy. .AddTypeActivatedCheck( diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerExecutableValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerExecutableValidator.cs index 5a9a9af..815f77d 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerExecutableValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerExecutableValidator.cs @@ -13,6 +13,50 @@ internal static class WorkerExecutableValidator private const int MachineOffsetFromPeHeader = PeSignatureSize; private const int MinimumHeaderSize = 0x40; + /// + /// Resolves the configured worker executable to a full path and checks it can be launched: + /// a valid path, a .exe extension, present on disk, and a PE image of the required + /// architecture. Shared by the launcher and the readiness check so both apply one rule. + /// + /// Worker options carrying the path and required architecture. + /// The full path of the validated worker executable. + /// Thrown when the executable is not launchable. + public static string ResolveAndValidate(WorkerOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + string executablePath; + try + { + executablePath = Path.GetFullPath(options.ExecutablePath); + } + catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException) + { + throw new WorkerProcessLaunchException( + WorkerProcessLaunchErrorCode.InvalidExecutable, + "Worker executable path is not a valid filesystem path.", + exception); + } + + if (!string.Equals(Path.GetExtension(executablePath), ".exe", StringComparison.OrdinalIgnoreCase)) + { + throw new WorkerProcessLaunchException( + WorkerProcessLaunchErrorCode.InvalidExecutable, + "Worker executable path must point to a .exe file."); + } + + if (!File.Exists(executablePath)) + { + throw new WorkerProcessLaunchException( + WorkerProcessLaunchErrorCode.ExecutableNotFound, + "Worker executable does not exist."); + } + + Validate(executablePath, options.RequiredArchitecture); + + return executablePath; + } + /// Validates that a worker executable file has the required architecture. /// Full path to the worker executable file. /// Required CPU architecture (x86 or x64). diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs index 178d9d7..d25ae53 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs @@ -224,39 +224,8 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher return startInfo; } - private string ResolveExecutablePath() - { - string executablePath; - try - { - executablePath = Path.GetFullPath(_workerOptions.ExecutablePath); - } - catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException) - { - throw new WorkerProcessLaunchException( - WorkerProcessLaunchErrorCode.InvalidExecutable, - "Worker executable path is not a valid filesystem path.", - exception); - } - - if (!string.Equals(Path.GetExtension(executablePath), ".exe", StringComparison.OrdinalIgnoreCase)) - { - throw new WorkerProcessLaunchException( - WorkerProcessLaunchErrorCode.InvalidExecutable, - "Worker executable path must point to a .exe file."); - } - - if (!File.Exists(executablePath)) - { - throw new WorkerProcessLaunchException( - WorkerProcessLaunchErrorCode.ExecutableNotFound, - "Worker executable does not exist."); - } - - WorkerExecutableValidator.Validate(executablePath, _workerOptions.RequiredArchitecture); - - return executablePath; - } + private string ResolveExecutablePath() => + WorkerExecutableValidator.ResolveAndValidate(_workerOptions); private string ResolveWorkingDirectory(string executablePath) { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs new file mode 100644 index 0000000..db26b12 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs @@ -0,0 +1,106 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Health; +using ZB.MOM.WW.MxGateway.Server; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +/// Readiness coverage for the worker executable (#206). +public sealed class WorkerExecutableHealthCheckTests : IDisposable +{ + private readonly string _directory = Path.Combine(Path.GetTempPath(), $"mxgw-workercheck-{Guid.NewGuid():N}"); + + /// Initializes a new instance of the class. + public WorkerExecutableHealthCheckTests() => Directory.CreateDirectory(_directory); + + /// + public void Dispose() => Directory.Delete(_directory, recursive: true); + + /// A present x86 worker image is healthy. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenWorkerExecutableIsLaunchable() + { + string path = WritePeStub(machine: 0x014c); + + HealthCheckResult result = await CheckAsync(path, WorkerArchitecture.X86); + + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + /// A missing worker executable is unhealthy and reports only the error code, never the path. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenWorkerExecutableIsMissing() + { + string path = Path.Combine(_directory, "missing", "ZB.MOM.WW.MxGateway.Worker.exe"); + + HealthCheckResult result = await CheckAsync(path, WorkerArchitecture.X86); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("ExecutableNotFound", result.Data["error_code"]); + Assert.DoesNotContain(_directory, result.Description, StringComparison.OrdinalIgnoreCase); + } + + /// A worker image of the wrong architecture is unhealthy, as the launcher would reject it. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenWorkerExecutableHasWrongArchitecture() + { + string path = WritePeStub(machine: 0x8664); + + HealthCheckResult result = await CheckAsync(path, WorkerArchitecture.X86); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("InvalidExecutable", result.Data["error_code"]); + } + + /// + /// The gateway registers the check on the ready tier, and the session check stays on the active + /// tier so zero sessions keeps readiness green. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Build_RegistersWorkerExecutableCheckOnReadyTier() + { + await using WebApplication app = GatewayApplication.Build([]); + HealthCheckServiceOptions options = + app.Services.GetRequiredService>().Value; + + HealthCheckRegistration worker = Assert.Single(options.Registrations, r => r.Name == "worker-executable"); + Assert.Contains(ZbHealthTags.Ready, worker.Tags); + HealthCheckRegistration sessions = Assert.Single(options.Registrations, r => r.Name == "mxaccess-sessions"); + Assert.DoesNotContain(ZbHealthTags.Ready, sessions.Tags); + } + + private static Task CheckAsync(string path, WorkerArchitecture architecture) + { + WorkerExecutableHealthCheck check = new(Options.Create(new GatewayOptions + { + Worker = new WorkerOptions + { + ExecutablePath = path, + RequiredArchitecture = architecture, + }, + })); + return check.CheckHealthAsync(new HealthCheckContext()); + } + + private string WritePeStub(ushort machine) + { + string path = Path.Combine(_directory, "ZB.MOM.WW.MxGateway.Worker.exe"); + byte[] bytes = new byte[0x100]; + bytes[0] = (byte)'M'; + bytes[1] = (byte)'Z'; + BitConverter.GetBytes(0x80).CopyTo(bytes, 0x3c); + bytes[0x80] = (byte)'P'; + bytes[0x81] = (byte)'E'; + BitConverter.GetBytes(machine).CopyTo(bytes, 0x84); + File.WriteAllBytes(path, bytes); + return path; + } +}