feat(health): worker-exe presence and bounded Galaxy SQL readiness checks (mxaccessgw#206)

Adds WorkerExecutableHealthCheck (configured worker .exe exists) and
GalaxySqlHealthCheck (IGalaxyRepository.TestConnectionAsync bounded by a 5 s
TimeProvider-driven timeout). Both are tagged ready only, never live, so an
outage takes the node out of rotation without triggering a restart; sessions
stay on the active tier so zero sessions is still healthy for Overview.

Registrations move into AddGatewayHealthChecks() so the tier tagging is
unit-tested. LDAP is not added (sign-in already surfaces bind failures).
docs/Diagnostics.md lists the two new checks.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:51:22 -05:00
parent b5d11e5869
commit 8c21678cb4
8 changed files with 328 additions and 11 deletions
@@ -0,0 +1,90 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.GalaxyRepository;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class GalaxySqlHealthCheckTests
{
/// <summary>A successful ping is healthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Healthy_WhenPingSucceeds()
{
GalaxySqlHealthCheck check = new(new PingRepository((_, _) => Task.FromResult(true)));
HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Healthy, result.Status);
}
/// <summary>A false ping and a thrown ping are both unhealthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPingFailsOrThrows()
{
HealthCheckResult returnedFalse = await new GalaxySqlHealthCheck(new PingRepository((_, _) => Task.FromResult(false)))
.CheckHealthAsync(new HealthCheckContext());
HealthCheckResult threw = await new GalaxySqlHealthCheck(new PingRepository((_, _) => throw new InvalidOperationException("boom")))
.CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, returnedFalse.Status);
Assert.Equal(HealthStatus.Unhealthy, threw.Status);
Assert.IsType<InvalidOperationException>(threw.Exception);
}
/// <summary>A hung ping is cut off by the timeout (fake clock) and reported unhealthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPingHangsPastTimeout()
{
FakeTimeProvider clock = new();
PingRepository hung = new(async (_, ct) =>
{
await Task.Delay(Timeout.Infinite, ct);
return true;
});
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), clock);
// The timeout timer is armed synchronously inside CheckHealthAsync before the first await.
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext());
Assert.False(pending.IsCompleted);
clock.Advance(TimeSpan.FromSeconds(5));
HealthCheckResult result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains("timed out", result.Description);
}
/// <summary>Caller cancellation propagates instead of being reported as a timeout.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CallerCancellation_Propagates()
{
PingRepository hung = new(async (_, ct) =>
{
await Task.Delay(Timeout.Infinite, ct);
return true;
});
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), new FakeTimeProvider());
using CancellationTokenSource caller = new();
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext(), caller.Token);
await caller.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await pending.WaitAsync(TimeSpan.FromSeconds(5)));
}
private sealed class PingRepository(Func<int, CancellationToken, Task<bool>> ping) : IGalaxyRepository
{
public Task<bool> TestConnectionAsync(CancellationToken ct = default) => ping(0, ct);
public Task<DateTime?> GetLastDeployTimeAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyHierarchyRow>> GetHierarchyAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyAttributeRow>> GetAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
public Task<List<GalaxyAlarmAttributeRow>> GetAlarmAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
}
}
@@ -0,0 +1,32 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class GatewayHealthCheckRegistrationTests
{
private static Dictionary<string, HealthCheckRegistration> Registrations()
{
ServiceCollection services = new();
services.AddGatewayHealthChecks();
using ServiceProvider provider = services.BuildServiceProvider();
return provider.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations
.ToDictionary(r => r.Name, StringComparer.Ordinal);
}
/// <summary>The worker-exe and Galaxy SQL checks are readiness-only; sessions stay on the active tier.</summary>
[Fact]
public void Checks_AreOnTheExpectedTiers_AndNoneIsLive()
{
Dictionary<string, HealthCheckRegistration> registrations = Registrations();
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["worker-executable"].Tags);
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["galaxy-sql"].Tags);
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["auth-store"].Tags);
Assert.Equal(new[] { ZbHealthTags.Active }, registrations["mxaccess-sessions"].Tags);
Assert.DoesNotContain(registrations.Values, r => r.Tags.Contains(ZbHealthTags.Live));
}
}
@@ -0,0 +1,50 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
public sealed class WorkerExecutableHealthCheckTests
{
private static WorkerExecutableHealthCheck CheckFor(string path) =>
new(Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = path } }));
/// <summary>An existing worker executable file is healthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Healthy_WhenExecutableExists()
{
string file = Path.Combine(Path.GetTempPath(), $"worker-{Guid.NewGuid():N}.exe");
await File.WriteAllTextAsync(file, "x");
try
{
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Healthy, result.Status);
}
finally
{
File.Delete(file);
}
}
/// <summary>A missing worker executable is unhealthy and names the resolved path.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenExecutableMissing()
{
string file = Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.exe");
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains(file, result.Description);
}
/// <summary>A path that cannot be resolved is unhealthy rather than throwing.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Unhealthy_WhenPathInvalid()
{
HealthCheckResult result = await CheckFor("bad\0path.exe").CheckHealthAsync(new HealthCheckContext());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
}
}