diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index 7ea29d4..70c24d0 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -217,6 +217,30 @@ The order matters: putting the logging scope first ensures that authentication f - `DashboardRedactor.Redact` delegates to `RedactClientIdentity` for any value containing the `mxgw_` marker, then falls back to a marker-keyword check for fields like `password` or `token`. This keeps dashboard renders aligned with log redaction. - `ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs` covers each redaction branch, including the assertion that `WriteSecured` values stay redacted even when `valueLoggingEnabled` is true. +## Health Checks + +The shared `ZB.MOM.WW.Health` package maps three endpoints — `/healthz` (live), `/health/ready`, and +`/health/active` — and each registered check opts into a tier by tag. The gateway registers two: + +| Check | Endpoint tier | Fails when | +|---|---|---| +| `auth-store` | `ready` | The SQLite auth store cannot be opened. Every gRPC call authenticates against it, so its reachability genuinely gates whether the process should receive traffic. | +| `mxaccess-sessions` | `active` | Sessions exist and their workers have faulted. Reports `total` / `ready` / `faulted` / `starting` / `closing` as entry `data`. | + +**Zero sessions is Healthy, and the tier choice follows from that.** The gateway opens an MXAccess +session when a client asks for one and holds none otherwise, so an idle gateway is working normally, +not broken. A count threshold ("unhealthy below N") would sit red forever on a host nothing dials +yet, and a permanently red probe is one operators stop reading — which leaves them worse off than no +probe at all. `mxaccess-sessions` is therefore graded on whether the sessions that exist are usable: + +- nothing faulted → **Healthy** (including no sessions at all) +- some faulted, some still ready or starting → **Degraded** +- every session faulted → **Unhealthy** + +For the same reason it is tagged `active` rather than `ready`. Readiness decides whether the process +should be sent traffic, and a gateway with no sessions is ready to serve; failing readiness there +would pull a working gateway out of rotation over a condition its clients create. + ## Related Documentation - [Identifying A Deployed Build](./runbooks/IdentifyingADeployedBuild.md) — mapping a running binary back to a commit, and why the `InformationalVersion` stamp cannot be trusted on Windows builds from 2026-07-09 to 2026-08-10 diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs new file mode 100644 index 0000000..84d040a --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs @@ -0,0 +1,114 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Sessions; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// +/// Reports how many MXAccess sessions are healthy. Each session is one worker process holding one +/// MXAccess COM instance — a live connection into a Galaxy — so this is the "how many Galaxy +/// connections are healthy" probe, expressed in the vocabulary the code actually uses. +/// +/// +/// +/// Zero sessions is healthy, deliberately. The gateway is a server: it opens a session when +/// a client asks and holds none otherwise, so idle-with-no-clients is the normal steady state, not +/// a fault. A count-based rule ("unhealthy below N") would sit red forever on a host nothing dials +/// yet, and a probe that is permanently red is one people learn to ignore — which costs more than +/// having no probe. The status here is therefore false only when a session exists and its worker +/// has actually failed. +/// +/// +/// This is tagged active rather than ready for the same reason. Readiness gates +/// whether the process should receive traffic, and a gateway with no sessions is legitimately ready +/// to serve — unlike the auth store, which every call depends on (see +/// ). Failing readiness on session state would take a working +/// gateway out of rotation for a condition its own clients cause. +/// +/// +public sealed class SessionHealthCheck : IHealthCheck +{ + private readonly ISessionRegistry _sessionRegistry; + + /// Initializes a new instance of the class. + /// Registry holding the live sessions. + public SessionHealthCheck(ISessionRegistry sessionRegistry) => + _sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry)); + + /// Buckets the live sessions by state and grades the result. + /// The health check context. + /// Token to cancel the asynchronous operation. + /// + /// Healthy when nothing is faulted (including when no sessions are open), Degraded when some + /// sessions are faulted but others are still usable, and Unhealthy when every session is + /// faulted. + /// + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + int ready = 0; + int faulted = 0; + int starting = 0; + int closing = 0; + + foreach (GatewaySession session in _sessionRegistry.Snapshot()) + { + switch (session.State) + { + case SessionState.Ready: + ready++; + break; + case SessionState.Faulted: + faulted++; + break; + case SessionState.Closing: + case SessionState.Closed: + // Counted but excluded from the verdict: a session on its way out is an + // expected lifecycle stage, not a failure, and Snapshot() still returns + // Closed sessions until they are removed from the registry. + closing++; + break; + default: + // Creating / StartingWorker / WaitingForPipe / Handshaking / + // InitializingWorker — mid-startup, not yet usable but not wrong. + // Unspecified lands here too; it is the proto zero value and should not occur. + starting++; + break; + } + } + + int total = ready + faulted + starting + closing; + int usable = ready + starting; + + Dictionary data = new(StringComparer.Ordinal) + { + ["total"] = total, + ["ready"] = ready, + ["faulted"] = faulted, + ["starting"] = starting, + ["closing"] = closing, + }; + + HealthCheckResult result = (faulted, usable) switch + { + (0, _) => HealthCheckResult.Healthy(Describe(total, ready, faulted), data), + (_, 0) => HealthCheckResult.Unhealthy(Describe(total, ready, faulted), data: data), + _ => HealthCheckResult.Degraded(Describe(total, ready, faulted), data: data), + }; + + return Task.FromResult(result); + } + + private static string Describe(int total, int ready, int faulted) + { + if (total == 0) + { + return "No MXAccess sessions are open."; + } + + return faulted == 0 + ? $"{ready} of {total} MXAccess sessions ready." + : $"{ready} of {total} MXAccess sessions ready, {faulted} faulted."; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs index 0a38696..49144ee 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs @@ -106,7 +106,13 @@ public static class GatewayApplication .AddTypeActivatedCheck( "auth-store", failureStatus: null, - tags: new[] { ZbHealthTags.Ready }); + tags: new[] { ZbHealthTags.Ready }) + // Active, not Ready: a gateway holding no sessions is legitimately ready to serve. + // See SessionHealthCheck for why zero sessions is healthy. + .AddTypeActivatedCheck( + "mxaccess-sessions", + failureStatus: null, + tags: new[] { ZbHealthTags.Active }); builder.Services.AddSingleton(); builder.AddZbTelemetry(o => { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs new file mode 100644 index 0000000..edff0d0 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; +using ZB.MOM.WW.MxGateway.Server.Sessions; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +public sealed class SessionHealthCheckTests +{ + /// + /// An idle gateway is healthy. This is the load-bearing case: a gateway holding no sessions is + /// the normal steady state on a host nothing dials yet, and a probe that reports red there is + /// one operators learn to ignore. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenNoSessionsAreOpen() + { + var check = new SessionHealthCheck(new SessionRegistry()); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(0, result.Data["total"]); + Assert.Equal("No MXAccess sessions are open.", result.Description); + } + + /// Every session ready reports healthy, with the counts carried as entry data. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenAllSessionsReady() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Ready, SessionState.Ready)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(2, result.Data["total"]); + Assert.Equal(2, result.Data["ready"]); + Assert.Equal(0, result.Data["faulted"]); + } + + /// + /// A faulted session alongside a usable one is degraded, not unhealthy — the gateway is still + /// serving the sessions that work. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Degraded_WhenSomeFaultedAndSomeReady() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Ready, SessionState.Faulted)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal(1, result.Data["ready"]); + Assert.Equal(1, result.Data["faulted"]); + Assert.Contains("1 faulted", result.Description, StringComparison.Ordinal); + } + + /// + /// A session still starting counts as usable for grading, so a fault beside it is degraded + /// rather than unhealthy — the startup has not failed yet. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Degraded_WhenFaultedBesideAStartingSession() + { + var check = new SessionHealthCheck( + RegistryWith(SessionState.Faulted, SessionState.StartingWorker)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal(1, result.Data["starting"]); + } + + /// Every session faulted is the genuinely bad condition, and the only unhealthy one. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenEverySessionIsFaulted() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Faulted, SessionState.Faulted)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal(2, result.Data["faulted"]); + Assert.Equal(0, result.Data["ready"]); + } + + /// + /// Closed sessions linger in the registry until they are removed. They are counted separately + /// and excluded from the verdict, so a gateway whose sessions all closed cleanly is healthy — + /// not unhealthy for having zero ready ones. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenOnlyClosedSessionsRemain() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Closed, SessionState.Closed)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(2, result.Data["closing"]); + Assert.Equal(0, result.Data["ready"]); + } + + private static SessionRegistry RegistryWith(params SessionState[] states) + { + var registry = new SessionRegistry(); + for (int i = 0; i < states.Length; i++) + { + GatewaySession session = CreateSession($"session-{i}"); + session.TransitionTo(states[i]); + Assert.True(registry.TryAdd(session)); + } + + return registry; + } + + private static GatewaySession CreateSession(string sessionId) + { + return new GatewaySession( + sessionId, + "mxaccess", + $"mxaccess-gateway-1-{sessionId}", + "nonce", + clientIdentity: null, + clientSessionName: "test-session", + clientCorrelationId: "client-correlation", + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(5), + DateTimeOffset.UnixEpoch); + } +}