feat(health): worker-exe presence and bounded Galaxy SQL readiness checks (mxaccessgw#206)

Adds WorkerExecutableHealthCheck (configured worker .exe exists) and
GalaxySqlHealthCheck (IGalaxyRepository.TestConnectionAsync bounded by a 5 s
TimeProvider-driven timeout). Both are tagged ready only, never live, so an
outage takes the node out of rotation without triggering a restart; sessions
stay on the active tier so zero sessions is still healthy for Overview.

Registrations move into AddGatewayHealthChecks() so the tier tagging is
unit-tested. LDAP is not added (sign-in already surfaces bind failures).
docs/Diagnostics.md lists the two new checks.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:51:22 -05:00
parent b5d11e5869
commit 8c21678cb4
8 changed files with 328 additions and 11 deletions
@@ -0,0 +1,66 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.GalaxyRepository;
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
/// <summary>
/// Readiness probe: pings the Galaxy Repository SQL database, bounded by a hard timeout so a
/// hung SQL Server cannot stall the health endpoint. Registered on the <c>ready</c> tier only
/// (never <c>live</c>): a Galaxy outage must take the gateway out of rotation but must not make
/// an orchestrator restart a process that restarting cannot fix.
/// </summary>
public sealed class GalaxySqlHealthCheck : IHealthCheck
{
/// <summary>Default upper bound for the SQL ping.</summary>
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(5);
private readonly IGalaxyRepository _repository;
private readonly TimeSpan _timeout;
private readonly TimeProvider _timeProvider;
/// <summary>Initializes a new instance of the <see cref="GalaxySqlHealthCheck"/> class.</summary>
/// <param name="repository">Galaxy Repository SQL surface.</param>
/// <param name="timeout">Ping bound; <see cref="DefaultTimeout"/> when null.</param>
/// <param name="timeProvider">Clock driving the bound; the system clock when null.</param>
public GalaxySqlHealthCheck(
IGalaxyRepository repository,
TimeSpan? timeout = null,
TimeProvider? timeProvider = null)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_timeout = timeout ?? DefaultTimeout;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>Pings the Galaxy Repository database within the configured timeout.</summary>
/// <param name="context">The health check context.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>Healthy when the ping succeeds; Unhealthy on failure or timeout.</returns>
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
using CancellationTokenSource timeoutSource = new(_timeout, _timeProvider);
using CancellationTokenSource linked =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
bool ok = await _repository.TestConnectionAsync(linked.Token).ConfigureAwait(false);
return ok
? HealthCheckResult.Healthy("Galaxy Repository SQL is reachable.")
: HealthCheckResult.Unhealthy("Galaxy Repository SQL ping failed.");
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (OperationCanceledException)
{
return HealthCheckResult.Unhealthy($"Galaxy Repository SQL ping timed out after {_timeout.TotalSeconds:0.#}s.");
}
catch (Exception exception)
{
return HealthCheckResult.Unhealthy("Galaxy Repository SQL is unreachable.", exception);
}
}
}
@@ -0,0 +1,42 @@
using Microsoft.Extensions.DependencyInjection;
using ZB.MOM.WW.Health;
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
/// <summary>Registers the gateway's health checks on the shared three-tier <c>ZbHealthTags</c> scheme.</summary>
public static class GatewayHealthCheckExtensions
{
/// <summary>
/// Registers the auth-store, worker-executable and Galaxy SQL readiness checks and the
/// MXAccess-session active-tier check. Nothing is tagged <see cref="ZbHealthTags.Live"/>: a
/// missing worker exe or an unreachable Galaxy SQL must take the gateway out of rotation but
/// is not something a process restart fixes. The Galaxy ping is bounded by
/// <see cref="GalaxySqlHealthCheck.DefaultTimeout"/>, and sessions stay on the active tier so
/// zero sessions remains healthy for the Overview dashboard.
/// </summary>
/// <param name="services">The service collection.</param>
/// <returns>The same service collection.</returns>
public static IServiceCollection AddGatewayHealthChecks(this IServiceCollection services)
{
services.AddHealthChecks()
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
"auth-store",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
.AddTypeActivatedCheck<WorkerExecutableHealthCheck>(
"worker-executable",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
.AddTypeActivatedCheck<GalaxySqlHealthCheck>(
"galaxy-sql",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// Active, not Ready: a gateway holding no sessions is legitimately ready to serve.
// See SessionHealthCheck for why zero sessions is healthy.
.AddTypeActivatedCheck<SessionHealthCheck>(
"mxaccess-sessions",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
return services;
}
}
@@ -0,0 +1,45 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Server.Configuration;
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
/// <summary>
/// Readiness probe: verifies the configured x86 MXAccess worker executable exists on disk. Without
/// it every session start fails with <c>ExecutableNotFound</c>, so a gateway missing the file
/// should not receive traffic. Registered on the <c>ready</c> tier only; the check touches no
/// sessions, so zero sessions stays healthy (the Overview dashboard polls the active tier).
/// </summary>
public sealed class WorkerExecutableHealthCheck : IHealthCheck
{
private readonly IOptions<GatewayOptions> _options;
/// <summary>Initializes a new instance of the <see cref="WorkerExecutableHealthCheck"/> class.</summary>
/// <param name="options">Gateway options carrying <c>Worker.ExecutablePath</c>.</param>
public WorkerExecutableHealthCheck(IOptions<GatewayOptions> options) =>
_options = options ?? throw new ArgumentNullException(nameof(options));
/// <summary>Checks that the configured worker executable path resolves to an existing file.</summary>
/// <param name="context">The health check context.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>Healthy when the file exists; otherwise Unhealthy.</returns>
public Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
string configured = _options.Value.Worker.ExecutablePath;
string fullPath;
try
{
fullPath = Path.GetFullPath(configured);
}
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
{
return Task.FromResult(HealthCheckResult.Unhealthy("Worker executable path is not a valid filesystem path.", exception));
}
return Task.FromResult(File.Exists(fullPath)
? HealthCheckResult.Healthy("Worker executable is present.")
: HealthCheckResult.Unhealthy($"Worker executable does not exist: {fullPath}"));
}
}
@@ -115,17 +115,7 @@ public static class GatewayApplication
builder.Services.AddGatewayGrpcAuthorization();
builder.Services.AddGatewayMetricsAuthorization();
AddLoginRateLimiter(builder);
builder.Services.AddHealthChecks()
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
"auth-store",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// Active, not Ready: a gateway holding no sessions is legitimately ready to serve.
// See SessionHealthCheck for why zero sessions is healthy.
.AddTypeActivatedCheck<SessionHealthCheck>(
"mxaccess-sessions",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
builder.Services.AddGatewayHealthChecks();
builder.Services.AddSingleton<GatewayMetrics>();
builder.AddZbTelemetry(o =>
{
@@ -0,0 +1,90 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.GalaxyRepository;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class GalaxySqlHealthCheckTests
{
/// <summary>A successful ping is healthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Healthy_WhenPingSucceeds()
{
GalaxySqlHealthCheck check = new(new PingRepository((_, _) => Task.FromResult(true)));
HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Healthy, result.Status);
}
/// <summary>A false ping and a thrown ping are both unhealthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPingFailsOrThrows()
{
HealthCheckResult returnedFalse = await new GalaxySqlHealthCheck(new PingRepository((_, _) => Task.FromResult(false)))
.CheckHealthAsync(new HealthCheckContext());
HealthCheckResult threw = await new GalaxySqlHealthCheck(new PingRepository((_, _) => throw new InvalidOperationException("boom")))
.CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, returnedFalse.Status);
Assert.Equal(HealthStatus.Unhealthy, threw.Status);
Assert.IsType<InvalidOperationException>(threw.Exception);
}
/// <summary>A hung ping is cut off by the timeout (fake clock) and reported unhealthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPingHangsPastTimeout()
{
FakeTimeProvider clock = new();
PingRepository hung = new(async (_, ct) =>
{
await Task.Delay(Timeout.Infinite, ct);
return true;
});
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), clock);
// The timeout timer is armed synchronously inside CheckHealthAsync before the first await.
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext());
Assert.False(pending.IsCompleted);
clock.Advance(TimeSpan.FromSeconds(5));
HealthCheckResult result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains("timed out", result.Description);
}
/// <summary>Caller cancellation propagates instead of being reported as a timeout.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CallerCancellation_Propagates()
{
PingRepository hung = new(async (_, ct) =>
{
await Task.Delay(Timeout.Infinite, ct);
return true;
});
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), new FakeTimeProvider());
using CancellationTokenSource caller = new();
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext(), caller.Token);
await caller.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await pending.WaitAsync(TimeSpan.FromSeconds(5)));
}
private sealed class PingRepository(Func<int, CancellationToken, Task<bool>> ping) : IGalaxyRepository
{
public Task<bool> TestConnectionAsync(CancellationToken ct = default) => ping(0, ct);
public Task<DateTime?> GetLastDeployTimeAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyHierarchyRow>> GetHierarchyAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyAttributeRow>> GetAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyAlarmAttributeRow>> GetAlarmAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
}
}
@@ -0,0 +1,32 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class GatewayHealthCheckRegistrationTests
{
private static Dictionary<string, HealthCheckRegistration> Registrations()
{
ServiceCollection services = new();
services.AddGatewayHealthChecks();
using ServiceProvider provider = services.BuildServiceProvider();
return provider.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations
.ToDictionary(r => r.Name, StringComparer.Ordinal);
}
/// <summary>The worker-exe and Galaxy SQL checks are readiness-only; sessions stay on the active tier.</summary>
[Fact]
public void Checks_AreOnTheExpectedTiers_AndNoneIsLive()
{
Dictionary<string, HealthCheckRegistration> registrations = Registrations();
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["worker-executable"].Tags);
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["galaxy-sql"].Tags);
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["auth-store"].Tags);
Assert.Equal(new[] { ZbHealthTags.Active }, registrations["mxaccess-sessions"].Tags);
Assert.DoesNotContain(registrations.Values, r => r.Tags.Contains(ZbHealthTags.Live));
}
}
@@ -0,0 +1,50 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class WorkerExecutableHealthCheckTests
{
private static WorkerExecutableHealthCheck CheckFor(string path) =>
new(Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = path } }));
/// <summary>An existing worker executable file is healthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Healthy_WhenExecutableExists()
{
string file = Path.Combine(Path.GetTempPath(), $"worker-{Guid.NewGuid():N}.exe");
await File.WriteAllTextAsync(file, "x");
try
{
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Healthy, result.Status);
}
finally
{
File.Delete(file);
}
}
/// <summary>A missing worker executable is unhealthy and names the resolved path.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenExecutableMissing()
{
string file = Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.exe");
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains(file, result.Description);
}
/// <summary>A path that cannot be resolved is unhealthy rather than throwing.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPathInvalid()
{
HealthCheckResult result = await CheckFor("bad\0path.exe").CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
}
}