feat(health): worker-exe presence and bounded Galaxy SQL readiness checks (mxaccessgw#206)
Adds WorkerExecutableHealthCheck (configured worker .exe exists) and GalaxySqlHealthCheck (IGalaxyRepository.TestConnectionAsync bounded by a 5 s TimeProvider-driven timeout). Both are tagged ready only, never live, so an outage takes the node out of rotation without triggering a restart; sessions stay on the active tier so zero sessions is still healthy for Overview. Registrations move into AddGatewayHealthChecks() so the tier tagging is unit-tested. LDAP is not added (sign-in already surfaces bind failures). docs/Diagnostics.md lists the two new checks. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using ZB.MOM.WW.GalaxyRepository;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class GalaxySqlHealthCheckTests
|
||||
{
|
||||
/// <summary>A successful ping is healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Healthy_WhenPingSucceeds()
|
||||
{
|
||||
GalaxySqlHealthCheck check = new(new PingRepository((_, _) => Task.FromResult(true)));
|
||||
HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Healthy, result.Status);
|
||||
}
|
||||
|
||||
/// <summary>A false ping and a thrown ping are both unhealthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPingFailsOrThrows()
|
||||
{
|
||||
HealthCheckResult returnedFalse = await new GalaxySqlHealthCheck(new PingRepository((_, _) => Task.FromResult(false)))
|
||||
.CheckHealthAsync(new HealthCheckContext());
|
||||
HealthCheckResult threw = await new GalaxySqlHealthCheck(new PingRepository((_, _) => throw new InvalidOperationException("boom")))
|
||||
.CheckHealthAsync(new HealthCheckContext());
|
||||
|
||||
Assert.Equal(HealthStatus.Unhealthy, returnedFalse.Status);
|
||||
Assert.Equal(HealthStatus.Unhealthy, threw.Status);
|
||||
Assert.IsType<InvalidOperationException>(threw.Exception);
|
||||
}
|
||||
|
||||
/// <summary>A hung ping is cut off by the timeout (fake clock) and reported unhealthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPingHangsPastTimeout()
|
||||
{
|
||||
FakeTimeProvider clock = new();
|
||||
PingRepository hung = new(async (_, ct) =>
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
return true;
|
||||
});
|
||||
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), clock);
|
||||
|
||||
// The timeout timer is armed synchronously inside CheckHealthAsync before the first await.
|
||||
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.False(pending.IsCompleted);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
HealthCheckResult result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains("timed out", result.Description);
|
||||
}
|
||||
|
||||
/// <summary>Caller cancellation propagates instead of being reported as a timeout.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CallerCancellation_Propagates()
|
||||
{
|
||||
PingRepository hung = new(async (_, ct) =>
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
return true;
|
||||
});
|
||||
GalaxySqlHealthCheck check = new(hung, TimeSpan.FromSeconds(5), new FakeTimeProvider());
|
||||
using CancellationTokenSource caller = new();
|
||||
|
||||
Task<HealthCheckResult> pending = check.CheckHealthAsync(new HealthCheckContext(), caller.Token);
|
||||
await caller.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await pending.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
private sealed class PingRepository(Func<int, CancellationToken, Task<bool>> ping) : IGalaxyRepository
|
||||
{
|
||||
public Task<bool> TestConnectionAsync(CancellationToken ct = default) => ping(0, ct);
|
||||
|
||||
public Task<DateTime?> GetLastDeployTimeAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyHierarchyRow>> GetHierarchyAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyAttributeRow>> GetAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
|
||||
public Task<List<GalaxyAlarmAttributeRow>> GetAlarmAttributesAsync(CancellationToken ct = default) => throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class GatewayHealthCheckRegistrationTests
|
||||
{
|
||||
private static Dictionary<string, HealthCheckRegistration> Registrations()
|
||||
{
|
||||
ServiceCollection services = new();
|
||||
services.AddGatewayHealthChecks();
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
return provider.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations
|
||||
.ToDictionary(r => r.Name, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>The worker-exe and Galaxy SQL checks are readiness-only; sessions stay on the active tier.</summary>
|
||||
[Fact]
|
||||
public void Checks_AreOnTheExpectedTiers_AndNoneIsLive()
|
||||
{
|
||||
Dictionary<string, HealthCheckRegistration> registrations = Registrations();
|
||||
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["worker-executable"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["galaxy-sql"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Ready }, registrations["auth-store"].Tags);
|
||||
Assert.Equal(new[] { ZbHealthTags.Active }, registrations["mxaccess-sessions"].Tags);
|
||||
Assert.DoesNotContain(registrations.Values, r => r.Tags.Contains(ZbHealthTags.Live));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics;
|
||||
|
||||
public sealed class WorkerExecutableHealthCheckTests
|
||||
{
|
||||
private static WorkerExecutableHealthCheck CheckFor(string path) =>
|
||||
new(Options.Create(new GatewayOptions { Worker = new WorkerOptions { ExecutablePath = path } }));
|
||||
|
||||
/// <summary>An existing worker executable file is healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Healthy_WhenExecutableExists()
|
||||
{
|
||||
string file = Path.Combine(Path.GetTempPath(), $"worker-{Guid.NewGuid():N}.exe");
|
||||
await File.WriteAllTextAsync(file, "x");
|
||||
try
|
||||
{
|
||||
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Healthy, result.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A missing worker executable is unhealthy and names the resolved path.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenExecutableMissing()
|
||||
{
|
||||
string file = Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.exe");
|
||||
HealthCheckResult result = await CheckFor(file).CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains(file, result.Description);
|
||||
}
|
||||
|
||||
/// <summary>A path that cannot be resolved is unhealthy rather than throwing.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Unhealthy_WhenPathInvalid()
|
||||
{
|
||||
HealthCheckResult result = await CheckFor("bad\0path.exe").CheckHealthAsync(new HealthCheckContext());
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user