fix(dashboard): derive gateway status from health checks; send config once per hub connection
SEC-27: DashboardSnapshot.GatewayStatus was hardcoded Healthy. It is now the aggregate HealthCheckService status, evaluated once per snapshot tick (2s bound, zero-wait gate) and cached for GetSnapshot; Unknown before the first evaluation, on failure, or without a health service. The startup-static EffectiveGatewayConfiguration is JsonIgnore'd on the snapshot, so /hubs/snapshot ticks carry only volatile state. The hub sends it once per connection (ConfigurationUpdated) and serves GetConfiguration(). Snapshot broadcasts are time-bounded (1s) so one slow client cannot delay ticks. StatusBadge maps Unhealthy to Bad.
This commit is contained in:
Joseph Doherty
@@ -1,4 +1,5 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -14,7 +15,16 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
{
|
||||
private const string HealthyStatus = "Healthy";
|
||||
/// <summary>
|
||||
/// Status shown before the first health evaluation completes, when the health
|
||||
/// evaluation fails, and when no <see cref="HealthCheckService"/> is registered.
|
||||
/// Deliberately not "Healthy": claiming health that was never checked is how the
|
||||
/// dashboard used to show a green gateway status throughout an outage.
|
||||
/// </summary>
|
||||
internal const string UnknownStatus = "Unknown";
|
||||
|
||||
/// <summary>Upper bound on one health evaluation, so a hung check cannot stall the snapshot tick.</summary>
|
||||
private static readonly TimeSpan GatewayStatusRefreshTimeout = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Minimum spacing between API key list reads. The list is a SQLite query whose
|
||||
@@ -39,7 +49,10 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
private readonly int _recentFaultLimit;
|
||||
private readonly int _recentSessionLimit;
|
||||
private readonly ILogger<DashboardSnapshotService> _logger;
|
||||
private readonly HealthCheckService? _healthCheckService;
|
||||
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
||||
private readonly SemaphoreSlim _gatewayStatusRefreshGate = new(1, 1);
|
||||
private string _gatewayStatus = UnknownStatus;
|
||||
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
||||
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
|
||||
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
|
||||
@@ -65,6 +78,11 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
/// <param name="options">Gateway configuration options.</param>
|
||||
/// <param name="timeProvider">Provider for current time; defaults to system time.</param>
|
||||
/// <param name="logger">Optional logger for the dashboard snapshot service.</param>
|
||||
/// <param name="healthCheckService">
|
||||
/// The gateway's registered health checks. <see cref="DashboardSnapshot.GatewayStatus"/>
|
||||
/// is the aggregate status of one evaluation per snapshot tick; without it the
|
||||
/// status stays <see cref="UnknownStatus"/>.
|
||||
/// </param>
|
||||
public DashboardSnapshotService(
|
||||
ISessionRegistry sessionRegistry,
|
||||
GatewayMetrics metrics,
|
||||
@@ -73,7 +91,8 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
IApiKeyAdminStore apiKeyAdminStore,
|
||||
IOptions<GatewayOptions> options,
|
||||
TimeProvider? timeProvider = null,
|
||||
ILogger<DashboardSnapshotService>? logger = null)
|
||||
ILogger<DashboardSnapshotService>? logger = null,
|
||||
HealthCheckService? healthCheckService = null)
|
||||
{
|
||||
_sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
@@ -88,6 +107,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
_recentFaultLimit = options.Value.Dashboard.RecentFaultLimit;
|
||||
_recentSessionLimit = options.Value.Dashboard.RecentSessionLimit;
|
||||
_logger = logger ?? NullLogger<DashboardSnapshotService>.Instance;
|
||||
_healthCheckService = healthCheckService;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -111,7 +131,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
GeneratedAt: generatedAt,
|
||||
GatewayStartedAt: _gatewayStartedAt,
|
||||
GatewayUptime: generatedAt - _gatewayStartedAt,
|
||||
GatewayStatus: HealthyStatus,
|
||||
GatewayStatus: Volatile.Read(ref _gatewayStatus),
|
||||
GatewayVersion: typeof(DashboardSnapshotService).Assembly.GetName().Version?.ToString() ?? "unknown",
|
||||
Sessions: sessionSummaries,
|
||||
Workers: workerSummaries,
|
||||
@@ -171,6 +191,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
}
|
||||
|
||||
await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await RefreshGatewayStatusAsync(cancellationToken).ConfigureAwait(false);
|
||||
yield return GetSnapshot();
|
||||
|
||||
using PeriodicTimer timer = new(_snapshotInterval, _timeProvider);
|
||||
@@ -192,10 +213,57 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
}
|
||||
|
||||
await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await RefreshGatewayStatusAsync(cancellationToken).ConfigureAwait(false);
|
||||
yield return GetSnapshot();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates the registered health checks once and caches the aggregate status
|
||||
/// for <see cref="GetSnapshot"/>, which is synchronous and also serves page seeds
|
||||
/// and hub connects between ticks. One evaluation per tick, not per
|
||||
/// <see cref="GetSnapshot"/> call: the checks include an auth-store query, and the
|
||||
/// tick is already the cadence at which the dashboard refreshes.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the evaluation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_healthCheckService is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Zero-timeout gate: a concurrent enumeration (the hub publisher and the page
|
||||
// feed each run one) skips rather than queueing a second evaluation.
|
||||
if (!await _gatewayStatusRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeout.CancelAfter(GatewayStatusRefreshTimeout);
|
||||
HealthReport report = await _healthCheckService.CheckHealthAsync(timeout.Token).ConfigureAwait(false);
|
||||
Volatile.Write(ref _gatewayStatus, report.Status.ToString());
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A health evaluation that cannot complete is not evidence of health.
|
||||
Volatile.Write(ref _gatewayStatus, UnknownStatus);
|
||||
_logger.LogDebug(exception, "Dashboard gateway health evaluation failed.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gatewayStatusRefreshGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static DashboardSessionSummary CreateSessionSummary(
|
||||
GatewaySession session,
|
||||
GatewayMetricsSnapshot metricsSnapshot)
|
||||
|
||||
Reference in New Issue
Block a user