fix(dashboard): derive gateway status from health checks; send config once per hub connection
SEC-27: DashboardSnapshot.GatewayStatus was hardcoded Healthy. It is now the aggregate HealthCheckService status, evaluated once per snapshot tick (2s bound, zero-wait gate) and cached for GetSnapshot; Unknown before the first evaluation, on failure, or without a health service. The startup-static EffectiveGatewayConfiguration is JsonIgnore'd on the snapshot, so /hubs/snapshot ticks carry only volatile state. The hub sends it once per connection (ConfigurationUpdated) and serves GetConfiguration(). Snapshot broadcasts are time-bounded (1s) so one slow client cannot delay ticks. StatusBadge maps Unhealthy to Bad.
This commit is contained in:
@@ -10,7 +10,7 @@
|
||||
"Ready" or "Healthy" or "Active" => StatusState.Ok,
|
||||
"Creating" or "StartingWorker" or "WaitingForPipe" or "InitializingWorker" or "Closing"
|
||||
or "Stale" or "Degraded" or "Expiring" => StatusState.Warn,
|
||||
"Faulted" or "Unavailable" or "Expired" => StatusState.Bad,
|
||||
"Faulted" or "Unavailable" or "Expired" or "Unhealthy" => StatusState.Bad,
|
||||
_ => StatusState.Idle,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,7 +1,33 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// One dashboard refresh: gateway status, sessions, workers, metrics, faults, API
|
||||
/// keys, the effective configuration, and the Galaxy summary.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="Configuration"/> is excluded from JSON serialization. It is
|
||||
/// startup-static, yet it is the largest part of the record, and the only wire this
|
||||
/// record crosses is the <c>/hubs/snapshot</c> tick sent every snapshot interval.
|
||||
/// Remote clients receive it once per connection instead, as
|
||||
/// <see cref="Hubs.DashboardSnapshotHub.ConfigurationMessage"/> (or on demand via
|
||||
/// <see cref="Hubs.DashboardSnapshotHub.GetConfiguration"/>). In-process pages read the
|
||||
/// property directly and are unaffected.
|
||||
/// </remarks>
|
||||
/// <param name="GeneratedAt">When the snapshot was built.</param>
|
||||
/// <param name="GatewayStartedAt">When the gateway process started.</param>
|
||||
/// <param name="GatewayUptime">Elapsed time since start.</param>
|
||||
/// <param name="GatewayStatus">Aggregate health-check status, or <c>Unknown</c>.</param>
|
||||
/// <param name="GatewayVersion">Server assembly version.</param>
|
||||
/// <param name="Sessions">Most recent sessions, newest first.</param>
|
||||
/// <param name="Workers">Workers that are not closed.</param>
|
||||
/// <param name="Metrics">Gateway counters and gauges.</param>
|
||||
/// <param name="Faults">Recent session and worker faults.</param>
|
||||
/// <param name="ApiKeys">API key summaries (no secrets).</param>
|
||||
/// <param name="Configuration">Effective configuration; not serialized on the hub tick.</param>
|
||||
/// <param name="Galaxy">Galaxy hierarchy cache summary.</param>
|
||||
public sealed record DashboardSnapshot(
|
||||
DateTimeOffset GeneratedAt,
|
||||
DateTimeOffset GatewayStartedAt,
|
||||
@@ -13,5 +39,5 @@ public sealed record DashboardSnapshot(
|
||||
IReadOnlyList<DashboardMetricSummary> Metrics,
|
||||
IReadOnlyList<DashboardFaultSummary> Faults,
|
||||
IReadOnlyList<DashboardApiKeySummary> ApiKeys,
|
||||
EffectiveGatewayConfiguration Configuration,
|
||||
[property: JsonIgnore] EffectiveGatewayConfiguration Configuration,
|
||||
DashboardGalaxySummary Galaxy);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -14,7 +15,16 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
{
|
||||
private const string HealthyStatus = "Healthy";
|
||||
/// <summary>
|
||||
/// Status shown before the first health evaluation completes, when the health
|
||||
/// evaluation fails, and when no <see cref="HealthCheckService"/> is registered.
|
||||
/// Deliberately not "Healthy": claiming health that was never checked is how the
|
||||
/// dashboard used to show a green gateway status throughout an outage.
|
||||
/// </summary>
|
||||
internal const string UnknownStatus = "Unknown";
|
||||
|
||||
/// <summary>Upper bound on one health evaluation, so a hung check cannot stall the snapshot tick.</summary>
|
||||
private static readonly TimeSpan GatewayStatusRefreshTimeout = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Minimum spacing between API key list reads. The list is a SQLite query whose
|
||||
@@ -39,7 +49,10 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
private readonly int _recentFaultLimit;
|
||||
private readonly int _recentSessionLimit;
|
||||
private readonly ILogger<DashboardSnapshotService> _logger;
|
||||
private readonly HealthCheckService? _healthCheckService;
|
||||
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
||||
private readonly SemaphoreSlim _gatewayStatusRefreshGate = new(1, 1);
|
||||
private string _gatewayStatus = UnknownStatus;
|
||||
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
||||
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
|
||||
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
|
||||
@@ -65,6 +78,11 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
/// <param name="options">Gateway configuration options.</param>
|
||||
/// <param name="timeProvider">Provider for current time; defaults to system time.</param>
|
||||
/// <param name="logger">Optional logger for the dashboard snapshot service.</param>
|
||||
/// <param name="healthCheckService">
|
||||
/// The gateway's registered health checks. <see cref="DashboardSnapshot.GatewayStatus"/>
|
||||
/// is the aggregate status of one evaluation per snapshot tick; without it the
|
||||
/// status stays <see cref="UnknownStatus"/>.
|
||||
/// </param>
|
||||
public DashboardSnapshotService(
|
||||
ISessionRegistry sessionRegistry,
|
||||
GatewayMetrics metrics,
|
||||
@@ -73,7 +91,8 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
IApiKeyAdminStore apiKeyAdminStore,
|
||||
IOptions<GatewayOptions> options,
|
||||
TimeProvider? timeProvider = null,
|
||||
ILogger<DashboardSnapshotService>? logger = null)
|
||||
ILogger<DashboardSnapshotService>? logger = null,
|
||||
HealthCheckService? healthCheckService = null)
|
||||
{
|
||||
_sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
@@ -88,6 +107,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
_recentFaultLimit = options.Value.Dashboard.RecentFaultLimit;
|
||||
_recentSessionLimit = options.Value.Dashboard.RecentSessionLimit;
|
||||
_logger = logger ?? NullLogger<DashboardSnapshotService>.Instance;
|
||||
_healthCheckService = healthCheckService;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -111,7 +131,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
GeneratedAt: generatedAt,
|
||||
GatewayStartedAt: _gatewayStartedAt,
|
||||
GatewayUptime: generatedAt - _gatewayStartedAt,
|
||||
GatewayStatus: HealthyStatus,
|
||||
GatewayStatus: Volatile.Read(ref _gatewayStatus),
|
||||
GatewayVersion: typeof(DashboardSnapshotService).Assembly.GetName().Version?.ToString() ?? "unknown",
|
||||
Sessions: sessionSummaries,
|
||||
Workers: workerSummaries,
|
||||
@@ -171,6 +191,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
}
|
||||
|
||||
await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await RefreshGatewayStatusAsync(cancellationToken).ConfigureAwait(false);
|
||||
yield return GetSnapshot();
|
||||
|
||||
using PeriodicTimer timer = new(_snapshotInterval, _timeProvider);
|
||||
@@ -192,10 +213,57 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
}
|
||||
|
||||
await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await RefreshGatewayStatusAsync(cancellationToken).ConfigureAwait(false);
|
||||
yield return GetSnapshot();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates the registered health checks once and caches the aggregate status
|
||||
/// for <see cref="GetSnapshot"/>, which is synchronous and also serves page seeds
|
||||
/// and hub connects between ticks. One evaluation per tick, not per
|
||||
/// <see cref="GetSnapshot"/> call: the checks include an auth-store query, and the
|
||||
/// tick is already the cadence at which the dashboard refreshes.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the evaluation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_healthCheckService is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Zero-timeout gate: a concurrent enumeration (the hub publisher and the page
|
||||
// feed each run one) skips rather than queueing a second evaluation.
|
||||
if (!await _gatewayStatusRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeout.CancelAfter(GatewayStatusRefreshTimeout);
|
||||
HealthReport report = await _healthCheckService.CheckHealthAsync(timeout.Token).ConfigureAwait(false);
|
||||
Volatile.Write(ref _gatewayStatus, report.Status.ToString());
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A health evaluation that cannot complete is not evidence of health.
|
||||
Volatile.Write(ref _gatewayStatus, UnknownStatus);
|
||||
_logger.LogDebug(exception, "Dashboard gateway health evaluation failed.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gatewayStatusRefreshGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static DashboardSessionSummary CreateSessionSummary(
|
||||
GatewaySession session,
|
||||
GatewayMetricsSnapshot metricsSnapshot)
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// SignalR hub that pushes a fresh <see cref="DashboardSnapshot"/> on every
|
||||
/// snapshot refresh. New connections receive the current snapshot
|
||||
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
|
||||
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
|
||||
/// snapshot refresh. New connections receive the effective configuration and the
|
||||
/// current snapshot immediately via <see cref="OnConnectedAsync"/>; subsequent
|
||||
/// refreshes are broadcast by <see cref="DashboardSnapshotPublisher"/> without the
|
||||
/// configuration, which is startup-static and serialized only once per connection.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Connections are counted into <see cref="DashboardSnapshotHubConnectionCounter"/>
|
||||
@@ -22,13 +24,28 @@ public sealed class DashboardSnapshotHub(
|
||||
/// <summary>Method name used to push snapshot updates to clients.</summary>
|
||||
public const string SnapshotMessage = "SnapshotUpdated";
|
||||
|
||||
/// <summary>
|
||||
/// Method name of the one-per-connection effective-configuration push. Snapshot
|
||||
/// ticks omit the configuration, so a client keeps the copy it got here.
|
||||
/// </summary>
|
||||
public const string ConfigurationMessage = "ConfigurationUpdated";
|
||||
|
||||
/// <summary>
|
||||
/// Returns the effective gateway configuration — the same object
|
||||
/// <see cref="ConfigurationMessage"/> carries — for a client that needs it again.
|
||||
/// </summary>
|
||||
/// <returns>The effective configuration.</returns>
|
||||
public EffectiveGatewayConfiguration GetConfiguration() => snapshotService.GetSnapshot().Configuration;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task OnConnectedAsync()
|
||||
{
|
||||
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||
// no later than the first snapshot this connection renders.
|
||||
connectionCounter.Increment();
|
||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
|
||||
DashboardSnapshot snapshot = snapshotService.GetSnapshot();
|
||||
await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false);
|
||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false);
|
||||
await base.OnConnectedAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
|
||||
@@ -26,11 +26,20 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// connected. While the count is zero the publisher does not advance the snapshot
|
||||
/// enumerator at all, so the producing iterator stays suspended and builds nothing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Each broadcast is bounded by a send timeout (one second by default, the default
|
||||
/// snapshot interval). A broadcast completes only once every connection accepted the
|
||||
/// frame, so awaiting it unbounded let one slow remote client delay every tick for
|
||||
/// everyone; a send that misses the bound is abandoned and the next tick goes out on
|
||||
/// time. The tick omits the effective configuration (see
|
||||
/// <see cref="DashboardSnapshot"/>), so each broadcast is the volatile state only.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1);
|
||||
private static readonly TimeSpan DefaultSendTimeout = TimeSpan.FromSeconds(1);
|
||||
|
||||
private readonly IDashboardSnapshotService _snapshotService;
|
||||
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
|
||||
@@ -38,6 +47,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
private readonly ILogger<DashboardSnapshotPublisher> _logger;
|
||||
private readonly TimeSpan _reconnectDelay;
|
||||
private readonly TimeSpan _idlePollInterval;
|
||||
private readonly TimeSpan _sendTimeout;
|
||||
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
|
||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||
@@ -64,13 +74,15 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
|
||||
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
|
||||
/// <param name="sendTimeout">Upper bound on one broadcast; defaults to one second.</param>
|
||||
internal DashboardSnapshotPublisher(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
IHubContext<DashboardSnapshotHub> hubContext,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||
ILogger<DashboardSnapshotPublisher> logger,
|
||||
TimeSpan reconnectDelay,
|
||||
TimeSpan idlePollInterval)
|
||||
TimeSpan idlePollInterval,
|
||||
TimeSpan? sendTimeout = null)
|
||||
{
|
||||
_snapshotService = snapshotService;
|
||||
_hubContext = hubContext;
|
||||
@@ -78,6 +90,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
_logger = logger;
|
||||
_reconnectDelay = reconnectDelay;
|
||||
_idlePollInterval = idlePollInterval;
|
||||
_sendTimeout = sendTimeout ?? DefaultSendTimeout;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -117,10 +130,19 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
|
||||
try
|
||||
{
|
||||
await _hubContext.Clients
|
||||
.All
|
||||
.SendAsync(DashboardSnapshotHub.SnapshotMessage, snapshot, stoppingToken)
|
||||
bool sent = await BoundedHubSend.SendAsync(
|
||||
_hubContext.Clients.All,
|
||||
DashboardSnapshotHub.SnapshotMessage,
|
||||
snapshot,
|
||||
_sendTimeout,
|
||||
stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
if (!sent)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Snapshot broadcast exceeded {Timeout} and was abandoned; the next tick proceeds.",
|
||||
_sendTimeout);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
|
||||
@@ -138,6 +138,35 @@ public sealed class DashboardSnapshotPublisherTests
|
||||
Assert.True(snapshotService.PullCount >= 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A broadcast that never completes — one remote client whose transport is wedged —
|
||||
/// is abandoned at the send timeout, so later ticks still go out instead of every
|
||||
/// viewer waiting on the slowest one.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut()
|
||||
{
|
||||
CountingSnapshotService snapshotService = new();
|
||||
RecordingHubContext hubContext = new();
|
||||
hubContext.Proxy.StallFirstSend = true;
|
||||
DashboardSnapshotPublisher publisher = new(
|
||||
snapshotService,
|
||||
hubContext,
|
||||
ConnectedCounter(),
|
||||
NullLogger<DashboardSnapshotPublisher>.Instance,
|
||||
TimeSpan.FromMilliseconds(50),
|
||||
IdlePollInterval,
|
||||
sendTimeout: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
|
||||
await WaitUntilAsync(() => hubContext.SendCount >= 3);
|
||||
|
||||
await cts.CancelAsync();
|
||||
await publisher.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>Creates a connection counter that already has one live viewer.</summary>
|
||||
/// <returns>A counter reporting a single connection.</returns>
|
||||
private static DashboardSnapshotHubConnectionCounter ConnectedCounter()
|
||||
@@ -271,6 +300,9 @@ public sealed class DashboardSnapshotPublisherTests
|
||||
/// <summary>Gets the group manager.</summary>
|
||||
public IGroupManager Groups { get; } = new NoopGroupManager();
|
||||
|
||||
/// <summary>Gets the recording proxy every client lookup resolves to.</summary>
|
||||
public RecordingClientProxy Proxy => _clients.AllProxy;
|
||||
|
||||
/// <summary>Gets the number of send calls recorded.</summary>
|
||||
public int SendCount => _clients.AllProxy.SendCount;
|
||||
}
|
||||
@@ -329,6 +361,12 @@ public sealed class DashboardSnapshotPublisherTests
|
||||
{
|
||||
private int _sendCount;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the first send never completes and
|
||||
/// ignores its cancellation token — a wedged client transport.
|
||||
/// </summary>
|
||||
public bool StallFirstSend { get; set; }
|
||||
|
||||
/// <summary>Gets the number of send calls recorded.</summary>
|
||||
public int SendCount => Volatile.Read(ref _sendCount);
|
||||
|
||||
@@ -339,8 +377,8 @@ public sealed class DashboardSnapshotPublisherTests
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Interlocked.Increment(ref _sendCount);
|
||||
return Task.CompletedTask;
|
||||
int count = Interlocked.Increment(ref _sendCount);
|
||||
return StallFirstSend && count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
|
||||
@@ -18,7 +20,8 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
public sealed class DashboardSnapshotServiceTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies snapshot returns empty collections and healthy status when registry is empty.
|
||||
/// Verifies snapshot returns empty collections when registry is empty, and an
|
||||
/// <c>Unknown</c> status when no health check service is available to evaluate.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_WhenRegistryEmpty_ReturnsEmptyOperationalState()
|
||||
@@ -32,10 +35,76 @@ public sealed class DashboardSnapshotServiceTests
|
||||
Assert.Empty(snapshot.Workers);
|
||||
Assert.Empty(snapshot.Faults);
|
||||
Assert.Contains(snapshot.Metrics, metric => metric.Name == "mxgateway.sessions.open" && metric.Value == 0);
|
||||
Assert.Equal("Healthy", snapshot.GatewayStatus);
|
||||
Assert.Equal(DashboardSnapshotService.UnknownStatus, snapshot.GatewayStatus);
|
||||
Assert.NotNull(snapshot.Configuration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SEC-27: the gateway status is the aggregate of the registered health checks,
|
||||
/// evaluated on the snapshot tick, not a hardcoded "Healthy".
|
||||
/// </summary>
|
||||
/// <param name="status">The aggregate status the health checks report.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(HealthStatus.Healthy)]
|
||||
[InlineData(HealthStatus.Degraded)]
|
||||
[InlineData(HealthStatus.Unhealthy)]
|
||||
public async Task WatchSnapshotsAsync_ReportsTheHealthCheckStatus(HealthStatus status)
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
StubHealthCheckService health = new(status);
|
||||
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health);
|
||||
using CancellationTokenSource cancellation = new();
|
||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||
.WatchSnapshotsAsync(cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
|
||||
Assert.Equal(status.ToString(), enumerator.Current.GatewayStatus);
|
||||
|
||||
// Cached for the synchronous seed path too, without a second evaluation.
|
||||
Assert.Equal(status.ToString(), service.GetSnapshot().GatewayStatus);
|
||||
Assert.Equal(1, health.CheckCount);
|
||||
}
|
||||
|
||||
/// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchSnapshotsAsync_WhenHealthEvaluationThrows_ReportsUnknown()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
StubHealthCheckService health = new(HealthStatus.Healthy) { Throw = true };
|
||||
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health);
|
||||
using CancellationTokenSource cancellation = new();
|
||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||
.WatchSnapshotsAsync(cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
|
||||
Assert.Equal(DashboardSnapshotService.UnknownStatus, enumerator.Current.GatewayStatus);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The startup-static configuration is not part of the serialized snapshot, so the
|
||||
/// hub tick carries only volatile state; the hub sends it once per connection.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_WhenSerialized_OmitsTheConfiguration()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics);
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
using JsonDocument json = JsonDocument.Parse(JsonSerializer.Serialize(snapshot, JsonSerializerOptions.Web));
|
||||
|
||||
Assert.NotNull(snapshot.Configuration);
|
||||
Assert.False(json.RootElement.TryGetProperty("configuration", out _));
|
||||
Assert.True(json.RootElement.TryGetProperty("gatewayStatus", out _));
|
||||
Assert.True(json.RootElement.TryGetProperty("sessions", out _));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot projects active, faulted, and closed session states with worker and metrics data.
|
||||
/// </summary>
|
||||
@@ -615,7 +684,8 @@ public sealed class DashboardSnapshotServiceTests
|
||||
GatewayOptions? options = null,
|
||||
IGalaxyHierarchyCache? galaxyHierarchyCache = null,
|
||||
IApiKeyAdminStore? apiKeyAdminStore = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
TimeProvider? timeProvider = null,
|
||||
HealthCheckService? healthCheckService = null)
|
||||
{
|
||||
GatewayOptions resolvedOptions = options ?? new GatewayOptions
|
||||
{
|
||||
@@ -633,7 +703,38 @@ public sealed class DashboardSnapshotServiceTests
|
||||
galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty),
|
||||
apiKeyAdminStore ?? new FakeApiKeyAdminStore(),
|
||||
Options.Create(resolvedOptions),
|
||||
timeProvider);
|
||||
timeProvider,
|
||||
healthCheckService: healthCheckService);
|
||||
}
|
||||
|
||||
private sealed class StubHealthCheckService(HealthStatus status) : HealthCheckService
|
||||
{
|
||||
private int _checkCount;
|
||||
|
||||
/// <summary>Gets or sets a value indicating whether the evaluation throws.</summary>
|
||||
public bool Throw { get; set; }
|
||||
|
||||
/// <summary>Gets the number of evaluations requested.</summary>
|
||||
public int CheckCount => Volatile.Read(ref _checkCount);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<HealthReport> CheckHealthAsync(
|
||||
Func<HealthCheckRegistration, bool>? predicate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Interlocked.Increment(ref _checkCount);
|
||||
if (Throw)
|
||||
{
|
||||
throw new InvalidOperationException("health evaluation failed");
|
||||
}
|
||||
|
||||
return Task.FromResult(new HealthReport(
|
||||
new Dictionary<string, HealthReportEntry>
|
||||
{
|
||||
["stub"] = new(status, null, TimeSpan.Zero, null, null),
|
||||
},
|
||||
TimeSpan.Zero));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache
|
||||
|
||||
Reference in New Issue
Block a user