From 8c21678cb4b0e2bbfed067100f356585348cbcd2 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:51:22 -0500 Subject: [PATCH] feat(health): worker-exe presence and bounded Galaxy SQL readiness checks (mxaccessgw#206) Adds WorkerExecutableHealthCheck (configured worker .exe exists) and GalaxySqlHealthCheck (IGalaxyRepository.TestConnectionAsync bounded by a 5 s TimeProvider-driven timeout). Both are tagged ready only, never live, so an outage takes the node out of rotation without triggering a restart; sessions stay on the active tier so zero sessions is still healthy for Overview. Registrations move into AddGatewayHealthChecks() so the tier tagging is unit-tested. LDAP is not added (sign-in already surfaces bind failures). docs/Diagnostics.md lists the two new checks. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- docs/Diagnostics.md | 2 + .../Diagnostics/GalaxySqlHealthCheck.cs | 66 ++++++++++++++ .../GatewayHealthCheckExtensions.cs | 42 +++++++++ .../WorkerExecutableHealthCheck.cs | 45 ++++++++++ .../GatewayApplication.cs | 12 +-- .../Diagnostics/GalaxySqlHealthCheckTests.cs | 90 +++++++++++++++++++ .../GatewayHealthCheckRegistrationTests.cs | 32 +++++++ .../WorkerExecutableHealthCheckTests.cs | 50 +++++++++++ 8 files changed, 328 insertions(+), 11 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GalaxySqlHealthCheck.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayHealthCheckExtensions.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GalaxySqlHealthCheckTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayHealthCheckRegistrationTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index b3252a7e..34ae743e 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -223,6 +223,8 @@ The shared `ZB.MOM.WW.Health` package maps three endpoints — `/healthz` (live) | Check | Endpoint tier | Fails when | |---|---|---| | `auth-store` | `ready` | The SQLite auth store cannot be opened. Every gRPC call authenticates against it, so its reachability genuinely gates whether the process should receive traffic. | +| `worker-executable` | `ready` | The configured `MxGateway:Worker:ExecutablePath` does not resolve to an existing file, so every session start would fail with `ExecutableNotFound`. | +| `galaxy-sql` | `ready` | The Galaxy Repository SQL ping fails or exceeds its 5 s bound. Never `live`: an outage takes the node out of rotation but a restart cannot fix it. | | `mxaccess-sessions` | `active` | Sessions exist and their workers have faulted. Reports `total` / `ready` / `faulted` / `starting` / `closing` as entry `data`. | **Zero sessions is Healthy, and the tier choice follows from that.** The gateway opens an MXAccess diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GalaxySqlHealthCheck.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GalaxySqlHealthCheck.cs new file mode 100644 index 00000000..f566b958 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GalaxySqlHealthCheck.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.GalaxyRepository; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// +/// Readiness probe: pings the Galaxy Repository SQL database, bounded by a hard timeout so a +/// hung SQL Server cannot stall the health endpoint. Registered on the ready tier only +/// (never live): a Galaxy outage must take the gateway out of rotation but must not make +/// an orchestrator restart a process that restarting cannot fix. +/// +public sealed class GalaxySqlHealthCheck : IHealthCheck +{ + /// Default upper bound for the SQL ping. + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(5); + + private readonly IGalaxyRepository _repository; + private readonly TimeSpan _timeout; + private readonly TimeProvider _timeProvider; + + /// Initializes a new instance of the class. + /// Galaxy Repository SQL surface. + /// Ping bound; when null. + /// Clock driving the bound; the system clock when null. + public GalaxySqlHealthCheck( + IGalaxyRepository repository, + TimeSpan? timeout = null, + TimeProvider? timeProvider = null) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _timeout = timeout ?? DefaultTimeout; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// Pings the Galaxy Repository database within the configured timeout. + /// The health check context. + /// Token to cancel the asynchronous operation. + /// Healthy when the ping succeeds; Unhealthy on failure or timeout. + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + using CancellationTokenSource timeoutSource = new(_timeout, _timeProvider); + using CancellationTokenSource linked = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token); + try + { + bool ok = await _repository.TestConnectionAsync(linked.Token).ConfigureAwait(false); + return ok + ? HealthCheckResult.Healthy("Galaxy Repository SQL is reachable.") + : HealthCheckResult.Unhealthy("Galaxy Repository SQL ping failed."); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) + { + return HealthCheckResult.Unhealthy($"Galaxy Repository SQL ping timed out after {_timeout.TotalSeconds:0.#}s."); + } + catch (Exception exception) + { + return HealthCheckResult.Unhealthy("Galaxy Repository SQL is unreachable.", exception); + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayHealthCheckExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayHealthCheckExtensions.cs new file mode 100644 index 00000000..1ffa6005 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayHealthCheckExtensions.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.DependencyInjection; +using ZB.MOM.WW.Health; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// Registers the gateway's health checks on the shared three-tier ZbHealthTags scheme. +public static class GatewayHealthCheckExtensions +{ + /// + /// Registers the auth-store, worker-executable and Galaxy SQL readiness checks and the + /// MXAccess-session active-tier check. Nothing is tagged : a + /// missing worker exe or an unreachable Galaxy SQL must take the gateway out of rotation but + /// is not something a process restart fixes. The Galaxy ping is bounded by + /// , and sessions stay on the active tier so + /// zero sessions remains healthy for the Overview dashboard. + /// + /// The service collection. + /// The same service collection. + public static IServiceCollection AddGatewayHealthChecks(this IServiceCollection services) + { + services.AddHealthChecks() + .AddTypeActivatedCheck( + "auth-store", + failureStatus: null, + tags: new[] { ZbHealthTags.Ready }) + .AddTypeActivatedCheck( + "worker-executable", + failureStatus: null, + tags: new[] { ZbHealthTags.Ready }) + .AddTypeActivatedCheck( + "galaxy-sql", + failureStatus: null, + tags: new[] { ZbHealthTags.Ready }) + // Active, not Ready: a gateway holding no sessions is legitimately ready to serve. + // See SessionHealthCheck for why zero sessions is healthy. + .AddTypeActivatedCheck( + "mxaccess-sessions", + failureStatus: null, + tags: new[] { ZbHealthTags.Active }); + return services; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs new file mode 100644 index 00000000..f177c1c4 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/WorkerExecutableHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// +/// Readiness probe: verifies the configured x86 MXAccess worker executable exists on disk. Without +/// it every session start fails with ExecutableNotFound, so a gateway missing the file +/// should not receive traffic. Registered on the ready tier only; the check touches no +/// sessions, so zero sessions stays healthy (the Overview dashboard polls the active tier). +/// +public sealed class WorkerExecutableHealthCheck : IHealthCheck +{ + private readonly IOptions _options; + + /// Initializes a new instance of the class. + /// Gateway options carrying Worker.ExecutablePath. + public WorkerExecutableHealthCheck(IOptions options) => + _options = options ?? throw new ArgumentNullException(nameof(options)); + + /// Checks that the configured worker executable path resolves to an existing file. + /// The health check context. + /// Token to cancel the asynchronous operation. + /// Healthy when the file exists; otherwise Unhealthy. + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + string configured = _options.Value.Worker.ExecutablePath; + string fullPath; + try + { + fullPath = Path.GetFullPath(configured); + } + catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException) + { + return Task.FromResult(HealthCheckResult.Unhealthy("Worker executable path is not a valid filesystem path.", exception)); + } + + return Task.FromResult(File.Exists(fullPath) + ? HealthCheckResult.Healthy("Worker executable is present.") + : HealthCheckResult.Unhealthy($"Worker executable does not exist: {fullPath}")); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs index d80f8590..bd7e5e1a 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs @@ -115,17 +115,7 @@ public static class GatewayApplication builder.Services.AddGatewayGrpcAuthorization(); builder.Services.AddGatewayMetricsAuthorization(); AddLoginRateLimiter(builder); - builder.Services.AddHealthChecks() - .AddTypeActivatedCheck( - "auth-store", - failureStatus: null, - tags: new[] { ZbHealthTags.Ready }) - // Active, not Ready: a gateway holding no sessions is legitimately ready to serve. - // See SessionHealthCheck for why zero sessions is healthy. - .AddTypeActivatedCheck( - "mxaccess-sessions", - failureStatus: null, - tags: new[] { ZbHealthTags.Active }); + builder.Services.AddGatewayHealthChecks(); builder.Services.AddSingleton(); builder.AddZbTelemetry(o => { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GalaxySqlHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GalaxySqlHealthCheckTests.cs new file mode 100644 index 00000000..676fe98e --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GalaxySqlHealthCheckTests.cs @@ -0,0 +1,90 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Time.Testing; +using ZB.MOM.WW.GalaxyRepository; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +public sealed class GalaxySqlHealthCheckTests +{ + /// A successful ping is healthy. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenPingSucceeds() + { + GalaxySqlHealthCheck check = new(new PingRepository((_, _) => Task.FromResult(true))); + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + /// A false ping and a thrown ping are both unhealthy. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenPingFailsOrThrows() + { + HealthCheckResult returnedFalse = await new GalaxySqlHealthCheck(new PingRepository((_, _) => Task.FromResult(false))) + .CheckHealthAsync(new HealthCheckContext()); + HealthCheckResult threw = await new GalaxySqlHealthCheck(new PingRepository((_, _) => throw new InvalidOperationException("boom"))) + .CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, returnedFalse.Status); + Assert.Equal(HealthStatus.Unhealthy, threw.Status); + Assert.IsType(threw.Exception); + } + + /// A hung ping is cut off by the timeout (fake clock) and reported unhealthy. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenPingHangsPastTimeout() + { + FakeTimeProvider clock = new(); + PingRepository hung = new(async (_, ct) => + { + await Task.Delay(Timeout.Infinite, ct); + return true; + }); + GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), clock); + + // The timeout timer is armed synchronously inside CheckHealthAsync before the first await. + Task pending = check.CheckHealthAsync(new HealthCheckContext()); + Assert.False(pending.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(5)); + HealthCheckResult result = await pending.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("timed out", result.Description); + } + + /// Caller cancellation propagates instead of being reported as a timeout. + /// A task that represents the asynchronous operation. + [Fact] + public async Task CallerCancellation_Propagates() + { + PingRepository hung = new(async (_, ct) => + { + await Task.Delay(Timeout.Infinite, ct); + return true; + }); + GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), new FakeTimeProvider()); + using CancellationTokenSource caller = new(); + + Task pending = check.CheckHealthAsync(new HealthCheckContext(), caller.Token); + await caller.CancelAsync(); + + await Assert.ThrowsAnyAsync(async () => await pending.WaitAsync(TimeSpan.FromSeconds(5))); + } + + private sealed class PingRepository(Func> ping) : IGalaxyRepository + { + public Task TestConnectionAsync(CancellationToken ct = default) => ping(0, ct); + + public Task GetLastDeployTimeAsync(CancellationToken ct = default) => throw new NotSupportedException(); + + public Task> GetHierarchyAsync(CancellationToken ct = default) => throw new NotSupportedException(); + + public Task> GetAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException(); + + public Task> GetAlarmAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException(); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayHealthCheckRegistrationTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayHealthCheckRegistrationTests.cs new file mode 100644 index 00000000..0bb280cb --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayHealthCheckRegistrationTests.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Health; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +public sealed class GatewayHealthCheckRegistrationTests +{ + private static Dictionary Registrations() + { + ServiceCollection services = new(); + services.AddGatewayHealthChecks(); + using ServiceProvider provider = services.BuildServiceProvider(); + return provider.GetRequiredService>().Value.Registrations + .ToDictionary(r => r.Name, StringComparer.Ordinal); + } + + /// The worker-exe and Galaxy SQL checks are readiness-only; sessions stay on the active tier. + [Fact] + public void Checks_AreOnTheExpectedTiers_AndNoneIsLive() + { + Dictionary registrations = Registrations(); + + Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["worker-executable"].Tags); + Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["galaxy-sql"].Tags); + Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["auth-store"].Tags); + Assert.Equal(new[] { ZbHealthTags.Active }, registrations["mxaccess-sessions"].Tags); + Assert.DoesNotContain(registrations.Values, r => r.Tags.Contains(ZbHealthTags.Live)); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs new file mode 100644 index 00000000..654a057c --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/WorkerExecutableHealthCheckTests.cs @@ -0,0 +1,50 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +public sealed class WorkerExecutableHealthCheckTests +{ + private static WorkerExecutableHealthCheck CheckFor(string path) => + new(Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = path } })); + + /// An existing worker executable file is healthy. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenExecutableExists() + { + string file = Path.Combine(Path.GetTempPath(), $"worker-{Guid.NewGuid():N}.exe"); + await File.WriteAllTextAsync(file, "x"); + try + { + HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext()); + Assert.Equal(HealthStatus.Healthy, result.Status); + } + finally + { + File.Delete(file); + } + } + + /// A missing worker executable is unhealthy and names the resolved path. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenExecutableMissing() + { + string file = Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.exe"); + HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext()); + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains(file, result.Description); + } + + /// A path that cannot be resolved is unhealthy rather than throwing. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenPathInvalid() + { + HealthCheckResult result = await CheckFor("bad\0path.exe").CheckHealthAsync(new HealthCheckContext()); + Assert.Equal(HealthStatus.Unhealthy, result.Status); + } +}