From c69a1c441ba2493056d96b96461f421ab6ae29bd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 11 Aug 2026 08:41:54 -0400 Subject: [PATCH] feat(diagnostics): report MXAccess session health on the active probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `mxaccess-sessions` health check reporting how many MXAccess sessions are healthy. Each session is one worker process holding one MXAccess COM instance — a live connection into a Galaxy — so this answers "how many Galaxy connections are healthy" in the vocabulary the code actually uses. Zero sessions is Healthy, deliberately, and the rest of the design follows from that. The gateway opens a session when a client asks and holds none otherwise, so an idle gateway is working normally. A count threshold ("unhealthy below N") would sit red forever on a host nothing dials yet, and a permanently red probe is one operators stop reading — which leaves them worse off than no probe. The check therefore grades on whether the sessions that exist are usable: nothing faulted is Healthy, some faulted beside a ready or starting one is Degraded, and every session faulted is Unhealthy. Counts ride along as entry data for the family Overview dashboard. Tagged `active` rather than `ready` for the same reason. Readiness decides whether the process should be sent traffic, and a gateway with no sessions is ready to serve — unlike the auth store, which every call depends on. Failing readiness here would pull a working gateway out of rotation over a condition its own clients create. Reads ISessionRegistry, which already exposes Snapshot(); ISessionManager stays the command surface and grows no enumerator. --- docs/Diagnostics.md | 24 +++ .../Diagnostics/SessionHealthCheck.cs | 114 +++++++++++++++ .../GatewayApplication.cs | 8 +- .../Diagnostics/SessionHealthCheckTests.cs | 138 ++++++++++++++++++ 4 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index 7ea29d4..70c24d0 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -217,6 +217,30 @@ The order matters: putting the logging scope first ensures that authentication f - `DashboardRedactor.Redact` delegates to `RedactClientIdentity` for any value containing the `mxgw_` marker, then falls back to a marker-keyword check for fields like `password` or `token`. This keeps dashboard renders aligned with log redaction. - `ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs` covers each redaction branch, including the assertion that `WriteSecured` values stay redacted even when `valueLoggingEnabled` is true. +## Health Checks + +The shared `ZB.MOM.WW.Health` package maps three endpoints — `/healthz` (live), `/health/ready`, and +`/health/active` — and each registered check opts into a tier by tag. The gateway registers two: + +| Check | Endpoint tier | Fails when | +|---|---|---| +| `auth-store` | `ready` | The SQLite auth store cannot be opened. Every gRPC call authenticates against it, so its reachability genuinely gates whether the process should receive traffic. | +| `mxaccess-sessions` | `active` | Sessions exist and their workers have faulted. Reports `total` / `ready` / `faulted` / `starting` / `closing` as entry `data`. | + +**Zero sessions is Healthy, and the tier choice follows from that.** The gateway opens an MXAccess +session when a client asks for one and holds none otherwise, so an idle gateway is working normally, +not broken. A count threshold ("unhealthy below N") would sit red forever on a host nothing dials +yet, and a permanently red probe is one operators stop reading — which leaves them worse off than no +probe at all. `mxaccess-sessions` is therefore graded on whether the sessions that exist are usable: + +- nothing faulted → **Healthy** (including no sessions at all) +- some faulted, some still ready or starting → **Degraded** +- every session faulted → **Unhealthy** + +For the same reason it is tagged `active` rather than `ready`. Readiness decides whether the process +should be sent traffic, and a gateway with no sessions is ready to serve; failing readiness there +would pull a working gateway out of rotation over a condition its clients create. + ## Related Documentation - [Identifying A Deployed Build](./runbooks/IdentifyingADeployedBuild.md) — mapping a running binary back to a commit, and why the `InformationalVersion` stamp cannot be trusted on Windows builds from 2026-07-09 to 2026-08-10 diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs new file mode 100644 index 0000000..84d040a --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/SessionHealthCheck.cs @@ -0,0 +1,114 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Sessions; + +namespace ZB.MOM.WW.MxGateway.Server.Diagnostics; + +/// +/// Reports how many MXAccess sessions are healthy. Each session is one worker process holding one +/// MXAccess COM instance — a live connection into a Galaxy — so this is the "how many Galaxy +/// connections are healthy" probe, expressed in the vocabulary the code actually uses. +/// +/// +/// +/// Zero sessions is healthy, deliberately. The gateway is a server: it opens a session when +/// a client asks and holds none otherwise, so idle-with-no-clients is the normal steady state, not +/// a fault. A count-based rule ("unhealthy below N") would sit red forever on a host nothing dials +/// yet, and a probe that is permanently red is one people learn to ignore — which costs more than +/// having no probe. The status here is therefore false only when a session exists and its worker +/// has actually failed. +/// +/// +/// This is tagged active rather than ready for the same reason. Readiness gates +/// whether the process should receive traffic, and a gateway with no sessions is legitimately ready +/// to serve — unlike the auth store, which every call depends on (see +/// ). Failing readiness on session state would take a working +/// gateway out of rotation for a condition its own clients cause. +/// +/// +public sealed class SessionHealthCheck : IHealthCheck +{ + private readonly ISessionRegistry _sessionRegistry; + + /// Initializes a new instance of the class. + /// Registry holding the live sessions. + public SessionHealthCheck(ISessionRegistry sessionRegistry) => + _sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry)); + + /// Buckets the live sessions by state and grades the result. + /// The health check context. + /// Token to cancel the asynchronous operation. + /// + /// Healthy when nothing is faulted (including when no sessions are open), Degraded when some + /// sessions are faulted but others are still usable, and Unhealthy when every session is + /// faulted. + /// + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + int ready = 0; + int faulted = 0; + int starting = 0; + int closing = 0; + + foreach (GatewaySession session in _sessionRegistry.Snapshot()) + { + switch (session.State) + { + case SessionState.Ready: + ready++; + break; + case SessionState.Faulted: + faulted++; + break; + case SessionState.Closing: + case SessionState.Closed: + // Counted but excluded from the verdict: a session on its way out is an + // expected lifecycle stage, not a failure, and Snapshot() still returns + // Closed sessions until they are removed from the registry. + closing++; + break; + default: + // Creating / StartingWorker / WaitingForPipe / Handshaking / + // InitializingWorker — mid-startup, not yet usable but not wrong. + // Unspecified lands here too; it is the proto zero value and should not occur. + starting++; + break; + } + } + + int total = ready + faulted + starting + closing; + int usable = ready + starting; + + Dictionary data = new(StringComparer.Ordinal) + { + ["total"] = total, + ["ready"] = ready, + ["faulted"] = faulted, + ["starting"] = starting, + ["closing"] = closing, + }; + + HealthCheckResult result = (faulted, usable) switch + { + (0, _) => HealthCheckResult.Healthy(Describe(total, ready, faulted), data), + (_, 0) => HealthCheckResult.Unhealthy(Describe(total, ready, faulted), data: data), + _ => HealthCheckResult.Degraded(Describe(total, ready, faulted), data: data), + }; + + return Task.FromResult(result); + } + + private static string Describe(int total, int ready, int faulted) + { + if (total == 0) + { + return "No MXAccess sessions are open."; + } + + return faulted == 0 + ? $"{ready} of {total} MXAccess sessions ready." + : $"{ready} of {total} MXAccess sessions ready, {faulted} faulted."; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs index 0a38696..49144ee 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs @@ -106,7 +106,13 @@ public static class GatewayApplication .AddTypeActivatedCheck( "auth-store", failureStatus: null, - tags: new[] { ZbHealthTags.Ready }); + tags: new[] { ZbHealthTags.Ready }) + // Active, not Ready: a gateway holding no sessions is legitimately ready to serve. + // See SessionHealthCheck for why zero sessions is healthy. + .AddTypeActivatedCheck( + "mxaccess-sessions", + failureStatus: null, + tags: new[] { ZbHealthTags.Active }); builder.Services.AddSingleton(); builder.AddZbTelemetry(o => { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs new file mode 100644 index 0000000..edff0d0 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/SessionHealthCheckTests.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Diagnostics; +using ZB.MOM.WW.MxGateway.Server.Sessions; + +namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; + +public sealed class SessionHealthCheckTests +{ + /// + /// An idle gateway is healthy. This is the load-bearing case: a gateway holding no sessions is + /// the normal steady state on a host nothing dials yet, and a probe that reports red there is + /// one operators learn to ignore. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenNoSessionsAreOpen() + { + var check = new SessionHealthCheck(new SessionRegistry()); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(0, result.Data["total"]); + Assert.Equal("No MXAccess sessions are open.", result.Description); + } + + /// Every session ready reports healthy, with the counts carried as entry data. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenAllSessionsReady() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Ready, SessionState.Ready)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(2, result.Data["total"]); + Assert.Equal(2, result.Data["ready"]); + Assert.Equal(0, result.Data["faulted"]); + } + + /// + /// A faulted session alongside a usable one is degraded, not unhealthy — the gateway is still + /// serving the sessions that work. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Degraded_WhenSomeFaultedAndSomeReady() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Ready, SessionState.Faulted)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal(1, result.Data["ready"]); + Assert.Equal(1, result.Data["faulted"]); + Assert.Contains("1 faulted", result.Description, StringComparison.Ordinal); + } + + /// + /// A session still starting counts as usable for grading, so a fault beside it is degraded + /// rather than unhealthy — the startup has not failed yet. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Degraded_WhenFaultedBesideAStartingSession() + { + var check = new SessionHealthCheck( + RegistryWith(SessionState.Faulted, SessionState.StartingWorker)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal(1, result.Data["starting"]); + } + + /// Every session faulted is the genuinely bad condition, and the only unhealthy one. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Unhealthy_WhenEverySessionIsFaulted() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Faulted, SessionState.Faulted)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal(2, result.Data["faulted"]); + Assert.Equal(0, result.Data["ready"]); + } + + /// + /// Closed sessions linger in the registry until they are removed. They are counted separately + /// and excluded from the verdict, so a gateway whose sessions all closed cleanly is healthy — + /// not unhealthy for having zero ready ones. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task Healthy_WhenOnlyClosedSessionsRemain() + { + var check = new SessionHealthCheck(RegistryWith(SessionState.Closed, SessionState.Closed)); + + HealthCheckResult result = await check.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(2, result.Data["closing"]); + Assert.Equal(0, result.Data["ready"]); + } + + private static SessionRegistry RegistryWith(params SessionState[] states) + { + var registry = new SessionRegistry(); + for (int i = 0; i < states.Length; i++) + { + GatewaySession session = CreateSession($"session-{i}"); + session.TransitionTo(states[i]); + Assert.True(registry.TryAdd(session)); + } + + return registry; + } + + private static GatewaySession CreateSession(string sessionId) + { + return new GatewaySession( + sessionId, + "mxaccess", + $"mxaccess-gateway-1-{sessionId}", + "nonce", + clientIdentity: null, + clientSessionName: "test-session", + clientCorrelationId: "client-correlation", + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(5), + DateTimeOffset.UnixEpoch); + } +}