feat(health): worker-exe presence and bounded Galaxy SQL readiness checks (mxaccessgw#206)
Adds WorkerExecutableHealthCheck (configured worker .exe exists) and GalaxySqlHealthCheck (IGalaxyRepository.TestConnectionAsync bounded by a 5 s TimeProvider-driven timeout). Both are tagged ready only, never live, so an outage takes the node out of rotation without triggering a restart; sessions stay on the active tier so zero sessions is still healthy for Overview. Registrations move into AddGatewayHealthChecks() so the tier tagging is unit-tested. LDAP is not added (sign-in already surfaces bind failures). docs/Diagnostics.md lists the two new checks. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.GalaxyRepository;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
/// <summary>
|
||||
/// Readiness probe: pings the Galaxy Repository SQL database, bounded by a hard timeout so a
|
||||
/// hung SQL Server cannot stall the health endpoint. Registered on the <c>ready</c> tier only
|
||||
/// (never <c>live</c>): a Galaxy outage must take the gateway out of rotation but must not make
|
||||
/// an orchestrator restart a process that restarting cannot fix.
|
||||
/// </summary>
|
||||
public sealed class GalaxySqlHealthCheck : IHealthCheck
|
||||
{
|
||||
/// <summary>Default upper bound for the SQL ping.</summary>
|
||||
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly IGalaxyRepository _repository;
|
||||
private readonly TimeSpan _timeout;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="GalaxySqlHealthCheck"/> class.</summary>
|
||||
/// <param name="repository">Galaxy Repository SQL surface.</param>
|
||||
/// <param name="timeout">Ping bound; <see cref="DefaultTimeout"/> when null.</param>
|
||||
/// <param name="timeProvider">Clock driving the bound; the system clock when null.</param>
|
||||
public GalaxySqlHealthCheck(
|
||||
IGalaxyRepository repository,
|
||||
TimeSpan? timeout = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
_timeout = timeout ?? DefaultTimeout;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <summary>Pings the Galaxy Repository database within the configured timeout.</summary>
|
||||
/// <param name="context">The health check context.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>Healthy when the ping succeeds; Unhealthy on failure or timeout.</returns>
|
||||
public async Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
using CancellationTokenSource timeoutSource = new(_timeout, _timeProvider);
|
||||
using CancellationTokenSource linked =
|
||||
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
try
|
||||
{
|
||||
bool ok = await _repository.TestConnectionAsync(linked.Token).ConfigureAwait(false);
|
||||
return ok
|
||||
? HealthCheckResult.Healthy("Galaxy Repository SQL is reachable.")
|
||||
: HealthCheckResult.Unhealthy("Galaxy Repository SQL ping failed.");
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return HealthCheckResult.Unhealthy($"Galaxy Repository SQL ping timed out after {_timeout.TotalSeconds:0.#}s.");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return HealthCheckResult.Unhealthy("Galaxy Repository SQL is unreachable.", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.Health;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
/// <summary>Registers the gateway's health checks on the shared three-tier <c>ZbHealthTags</c> scheme.</summary>
|
||||
public static class GatewayHealthCheckExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers the auth-store, worker-executable and Galaxy SQL readiness checks and the
|
||||
/// MXAccess-session active-tier check. Nothing is tagged <see cref="ZbHealthTags.Live"/>: a
|
||||
/// missing worker exe or an unreachable Galaxy SQL must take the gateway out of rotation but
|
||||
/// is not something a process restart fixes. The Galaxy ping is bounded by
|
||||
/// <see cref="GalaxySqlHealthCheck.DefaultTimeout"/>, and sessions stay on the active tier so
|
||||
/// zero sessions remains healthy for the Overview dashboard.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same service collection.</returns>
|
||||
public static IServiceCollection AddGatewayHealthChecks(this IServiceCollection services)
|
||||
{
|
||||
services.AddHealthChecks()
|
||||
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
|
||||
"auth-store",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Ready })
|
||||
.AddTypeActivatedCheck<WorkerExecutableHealthCheck>(
|
||||
"worker-executable",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Ready })
|
||||
.AddTypeActivatedCheck<GalaxySqlHealthCheck>(
|
||||
"galaxy-sql",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Ready })
|
||||
// Active, not Ready: a gateway holding no sessions is legitimately ready to serve.
|
||||
// See SessionHealthCheck for why zero sessions is healthy.
|
||||
.AddTypeActivatedCheck<SessionHealthCheck>(
|
||||
"mxaccess-sessions",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Active });
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
/// <summary>
|
||||
/// Readiness probe: verifies the configured x86 MXAccess worker executable exists on disk. Without
|
||||
/// it every session start fails with <c>ExecutableNotFound</c>, so a gateway missing the file
|
||||
/// should not receive traffic. Registered on the <c>ready</c> tier only; the check touches no
|
||||
/// sessions, so zero sessions stays healthy (the Overview dashboard polls the active tier).
|
||||
/// </summary>
|
||||
public sealed class WorkerExecutableHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly IOptions<GatewayOptions> _options;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="WorkerExecutableHealthCheck"/> class.</summary>
|
||||
/// <param name="options">Gateway options carrying <c>Worker.ExecutablePath</c>.</param>
|
||||
public WorkerExecutableHealthCheck(IOptions<GatewayOptions> options) =>
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
|
||||
/// <summary>Checks that the configured worker executable path resolves to an existing file.</summary>
|
||||
/// <param name="context">The health check context.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>Healthy when the file exists; otherwise Unhealthy.</returns>
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string configured = _options.Value.Worker.ExecutablePath;
|
||||
string fullPath;
|
||||
try
|
||||
{
|
||||
fullPath = Path.GetFullPath(configured);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return Task.FromResult(HealthCheckResult.Unhealthy("Worker executable path is not a valid filesystem path.", exception));
|
||||
}
|
||||
|
||||
return Task.FromResult(File.Exists(fullPath)
|
||||
? HealthCheckResult.Healthy("Worker executable is present.")
|
||||
: HealthCheckResult.Unhealthy($"Worker executable does not exist: {fullPath}"));
|
||||
}
|
||||
}
|
||||
@@ -115,17 +115,7 @@ public static class GatewayApplication
|
||||
builder.Services.AddGatewayGrpcAuthorization();
|
||||
builder.Services.AddGatewayMetricsAuthorization();
|
||||
AddLoginRateLimiter(builder);
|
||||
builder.Services.AddHealthChecks()
|
||||
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
|
||||
"auth-store",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Ready })
|
||||
// Active, not Ready: a gateway holding no sessions is legitimately ready to serve.
|
||||
// See SessionHealthCheck for why zero sessions is healthy.
|
||||
.AddTypeActivatedCheck<SessionHealthCheck>(
|
||||
"mxaccess-sessions",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Active });
|
||||
builder.Services.AddGatewayHealthChecks();
|
||||
builder.Services.AddSingleton<GatewayMetrics>();
|
||||
builder.AddZbTelemetry(o =>
|
||||
{
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using ZB.MOM.WW.GalaxyRepository;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class GalaxySqlHealthCheckTests
|
||||
{
|
||||
/// <summary>A successful ping is healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Healthy_WhenPingSucceeds()
|
||||
{
|
||||
GalaxySqlHealthCheck check = new(new PingRepository((_, _) => Task.FromResult(true)));
|
||||
HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Healthy, result.Status);
|
||||
}
|
||||
|
||||
/// <summary>A false ping and a thrown ping are both unhealthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPingFailsOrThrows()
|
||||
{
|
||||
HealthCheckResult returnedFalse = await new GalaxySqlHealthCheck(new PingRepository((_, _) => Task.FromResult(false)))
|
||||
.CheckHealthAsync(new HealthCheckContext());
|
||||
HealthCheckResult threw = await new GalaxySqlHealthCheck(new PingRepository((_, _) => throw new InvalidOperationException("boom")))
|
||||
.CheckHealthAsync(new HealthCheckContext());
|
||||
|
||||
Assert.Equal(HealthStatus.Unhealthy, returnedFalse.Status);
|
||||
Assert.Equal(HealthStatus.Unhealthy, threw.Status);
|
||||
Assert.IsType<InvalidOperationException>(threw.Exception);
|
||||
}
|
||||
|
||||
/// <summary>A hung ping is cut off by the timeout (fake clock) and reported unhealthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPingHangsPastTimeout()
|
||||
{
|
||||
FakeTimeProvider clock = new();
|
||||
PingRepository hung = new(async (_, ct) =>
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
return true;
|
||||
});
|
||||
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), clock);
|
||||
|
||||
// The timeout timer is armed synchronously inside CheckHealthAsync before the first await.
|
||||
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.False(pending.IsCompleted);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
HealthCheckResult result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains("timed out", result.Description);
|
||||
}
|
||||
|
||||
/// <summary>Caller cancellation propagates instead of being reported as a timeout.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CallerCancellation_Propagates()
|
||||
{
|
||||
PingRepository hung = new(async (_, ct) =>
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
return true;
|
||||
});
|
||||
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), new FakeTimeProvider());
|
||||
using CancellationTokenSource caller = new();
|
||||
|
||||
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext(), caller.Token);
|
||||
await caller.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await pending.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
private sealed class PingRepository(Func<int, CancellationToken, Task<bool>> ping) : IGalaxyRepository
|
||||
{
|
||||
public Task<bool> TestConnectionAsync(CancellationToken ct = default) => ping(0, ct);
|
||||
|
||||
public Task<DateTime?> GetLastDeployTimeAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyHierarchyRow>> GetHierarchyAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyAttributeRow>> GetAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyAlarmAttributeRow>> GetAlarmAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class GatewayHealthCheckRegistrationTests
|
||||
{
|
||||
private static Dictionary<string, HealthCheckRegistration> Registrations()
|
||||
{
|
||||
ServiceCollection services = new();
|
||||
services.AddGatewayHealthChecks();
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
return provider.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations
|
||||
.ToDictionary(r => r.Name, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>The worker-exe and Galaxy SQL checks are readiness-only; sessions stay on the active tier.</summary>
|
||||
[Fact]
|
||||
public void Checks_AreOnTheExpectedTiers_AndNoneIsLive()
|
||||
{
|
||||
Dictionary<string, HealthCheckRegistration> registrations = Registrations();
|
||||
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["worker-executable"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["galaxy-sql"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["auth-store"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Active }, registrations["mxaccess-sessions"].Tags);
|
||||
Assert.DoesNotContain(registrations.Values, r => r.Tags.Contains(ZbHealthTags.Live));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class WorkerExecutableHealthCheckTests
|
||||
{
|
||||
private static WorkerExecutableHealthCheck CheckFor(string path) =>
|
||||
new(Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = path } }));
|
||||
|
||||
/// <summary>An existing worker executable file is healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Healthy_WhenExecutableExists()
|
||||
{
|
||||
string file = Path.Combine(Path.GetTempPath(), $"worker-{Guid.NewGuid():N}.exe");
|
||||
await File.WriteAllTextAsync(file, "x");
|
||||
try
|
||||
{
|
||||
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Healthy, result.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A missing worker executable is unhealthy and names the resolved path.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenExecutableMissing()
|
||||
{
|
||||
string file = Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.exe");
|
||||
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains(file, result.Description);
|
||||
}
|
||||
|
||||
/// <summary>A path that cannot be resolved is unhealthy rather than throwing.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPathInvalid()
|
||||
{
|
||||
HealthCheckResult result = await CheckFor("bad\0path.exe").CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user