mxaccessgw/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs

using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.GalaxyRepository;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;

namespace ZB.MOM.WW.MxGateway.Server.Dashboard;

public sealed class DashboardSnapshotService : IDashboardSnapshotService
{
    private const string HealthyStatus = "Healthy";

    private readonly ISessionRegistry _sessionRegistry;
    private readonly GatewayMetrics _metrics;
    private readonly IGatewayConfigurationProvider _configurationProvider;
    private readonly IGalaxyHierarchyCache _galaxyHierarchyCache;
    private readonly IApiKeyAdminStore _apiKeyAdminStore;
    private readonly TimeProvider _timeProvider;
    private readonly DateTimeOffset _gatewayStartedAt;
    private readonly TimeSpan _snapshotInterval;
    private readonly TimeSpan _apiKeySummaryRefreshTimeout = TimeSpan.FromSeconds(2);
    private readonly int _recentFaultLimit;
    private readonly int _recentSessionLimit;
    private readonly ILogger<DashboardSnapshotService> _logger;
    private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
    private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
    // Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
    // library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
    // sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
    // O(1) for Galaxy. The summary's cheap volatile fields (status, timestamps, last error, counts)
    // are NOT memoized: the library mutates them in place at the same sequence on steady-state ticks
    // and on refresh failure, so they are copied fresh on every snapshot.
    private GalaxyBreakdownCache? _galaxyBreakdownCache;

    /// <summary>Initializes a new instance of the DashboardSnapshotService class.</summary>
    /// <param name="sessionRegistry">Registry of active gateway sessions.</param>
    /// <param name="metrics">Gateway metrics collector.</param>
    /// <param name="configurationProvider">Gateway configuration provider.</param>
    /// <param name="galaxyHierarchyCache">Galaxy hierarchy cache.</param>
    /// <param name="apiKeyAdminStore">API key administration store.</param>
    /// <param name="options">Gateway configuration options.</param>
    /// <param name="timeProvider">Provider for current time; defaults to system time.</param>
    /// <param name="logger">Optional logger for the dashboard snapshot service.</param>
    public DashboardSnapshotService(
        ISessionRegistry sessionRegistry,
        GatewayMetrics metrics,
        IGatewayConfigurationProvider configurationProvider,
        IGalaxyHierarchyCache galaxyHierarchyCache,
        IApiKeyAdminStore apiKeyAdminStore,
        IOptions<GatewayOptions> options,
        TimeProvider? timeProvider = null,
        ILogger<DashboardSnapshotService>? logger = null)
    {
        _sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry));
        _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
        _configurationProvider = configurationProvider ?? throw new ArgumentNullException(nameof(configurationProvider));
        _galaxyHierarchyCache = galaxyHierarchyCache ?? throw new ArgumentNullException(nameof(galaxyHierarchyCache));
        _apiKeyAdminStore = apiKeyAdminStore ?? throw new ArgumentNullException(nameof(apiKeyAdminStore));
        ArgumentNullException.ThrowIfNull(options);

        _timeProvider = timeProvider ?? TimeProvider.System;
        _gatewayStartedAt = _timeProvider.GetUtcNow();
        _snapshotInterval = TimeSpan.FromMilliseconds(options.Value.Dashboard.SnapshotIntervalMilliseconds);
        _recentFaultLimit = options.Value.Dashboard.RecentFaultLimit;
        _recentSessionLimit = options.Value.Dashboard.RecentSessionLimit;
        _logger = logger ?? NullLogger<DashboardSnapshotService>.Instance;
    }

    /// <inheritdoc />
    public DashboardSnapshot GetSnapshot()
    {
        DateTimeOffset generatedAt = _timeProvider.GetUtcNow();
        IReadOnlyList<GatewaySession> sessions = _sessionRegistry.Snapshot()
            .OrderByDescending(session => session.OpenedAt)
            .ToArray();
        GatewayMetricsSnapshot metricsSnapshot = _metrics.GetSnapshot();
        IReadOnlyList<DashboardSessionSummary> sessionSummaries = sessions
            .Take(ResolveLimit(_recentSessionLimit))
            .Select(session => CreateSessionSummary(session, metricsSnapshot))
            .ToArray();
        IReadOnlyList<DashboardWorkerSummary> workerSummaries = sessions
            .Where(session => session.WorkerClient is { State: not WorkerClientState.Closed })
            .Select(CreateWorkerSummary)
            .ToArray();

        return new DashboardSnapshot(
            GeneratedAt: generatedAt,
            GatewayStartedAt: _gatewayStartedAt,
            GatewayUptime: generatedAt - _gatewayStartedAt,
            GatewayStatus: HealthyStatus,
            GatewayVersion: typeof(DashboardSnapshotService).Assembly.GetName().Version?.ToString() ?? "unknown",
            Sessions: sessionSummaries,
            Workers: workerSummaries,
            Metrics: CreateMetricSummaries(metricsSnapshot),
            Faults: CreateFaultSummaries(sessions, generatedAt),
            ApiKeys: Volatile.Read(ref _apiKeySummaries),
            Configuration: _configurationProvider.GetEffectiveConfiguration(),
            Galaxy: ResolveGalaxySummary());
    }

    private DashboardGalaxySummary ResolveGalaxySummary()
    {
        GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
        long sequence = entry.Sequence;

        // Lock-free reuse of the O(N) breakdown only: a matching sequence means the object set is
        // unchanged, so the previously computed breakdown is still correct. A racing recompute for a
        // new sequence is harmless — the computation is pure, so any winner stores identical content
        // for that sequence. The cheap volatile fields are always taken from the current entry below.
        GalaxyBreakdownCache? cached = Volatile.Read(ref _galaxyBreakdownCache);
        GalaxyObjectBreakdown breakdown;
        if (cached is not null && cached.Sequence == sequence)
        {
            breakdown = cached.Breakdown;
        }
        else
        {
            breakdown = DashboardGalaxySummaryProjector.ComputeBreakdown(entry);
            Volatile.Write(ref _galaxyBreakdownCache, new GalaxyBreakdownCache(sequence, breakdown));
        }

        return DashboardGalaxySummaryProjector.BuildSummary(entry, breakdown);
    }

    private sealed record GalaxyBreakdownCache(long Sequence, GalaxyObjectBreakdown Breakdown);

    /// <inheritdoc />
    public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
        [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        if (cancellationToken.IsCancellationRequested)
        {
            yield break;
        }

        await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
        yield return GetSnapshot();

        using PeriodicTimer timer = new(_snapshotInterval, _timeProvider);
        while (true)
        {
            bool hasNext;
            try
            {
                hasNext = await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false);
            }
            catch (OperationCanceledException)
            {
                yield break;
            }

            if (!hasNext)
            {
                yield break;
            }

            await RefreshApiKeySummariesAsync(cancellationToken).ConfigureAwait(false);
            yield return GetSnapshot();
        }
    }

    private static DashboardSessionSummary CreateSessionSummary(
        GatewaySession session,
        GatewayMetricsSnapshot metricsSnapshot)
    {
        IWorkerClient? workerClient = session.WorkerClient;
        metricsSnapshot.EventsBySession.TryGetValue(session.SessionId, out long eventsReceived);

        return new DashboardSessionSummary(
            SessionId: session.SessionId,
            BackendName: session.BackendName,
            State: session.State,
            ClientIdentity: DashboardRedactor.Redact(session.ClientIdentity),
            ClientSessionName: DashboardRedactor.Redact(session.ClientSessionName),
            ClientCorrelationId: DashboardRedactor.Redact(session.ClientCorrelationId),
            OpenedAt: session.OpenedAt,
            LastClientActivityAt: session.LastClientActivityAt,
            LeaseExpiresAt: session.LeaseExpiresAt,
            WorkerProcessId: workerClient?.ProcessId,
            WorkerState: workerClient?.State,
            LastWorkerHeartbeatAt: workerClient?.LastHeartbeatAt,
            EventsReceived: eventsReceived,
            LastFault: DashboardRedactor.Redact(session.FinalFault));
    }

    private static DashboardWorkerSummary CreateWorkerSummary(GatewaySession session)
    {
        IWorkerClient workerClient = session.WorkerClient!;

        return new DashboardWorkerSummary(
            SessionId: session.SessionId,
            ProcessId: workerClient.ProcessId,
            State: workerClient.State,
            LastHeartbeatAt: workerClient.LastHeartbeatAt,
            LastFault: DashboardRedactor.Redact(session.FinalFault));
    }

    private static IReadOnlyList<DashboardMetricSummary> CreateMetricSummaries(GatewayMetricsSnapshot snapshot)
    {
        List<DashboardMetricSummary> metrics =
        [
            new("mxgateway.sessions.open", snapshot.OpenSessions),
            new("mxgateway.workers.running", snapshot.WorkersRunning),
            new("mxgateway.events.worker_queue.depth", snapshot.WorkerEventQueueDepth),
            new("mxgateway.events.grpc_stream_queue.depth", snapshot.GrpcEventStreamQueueDepth),
            new("mxgateway.sessions.opened", snapshot.SessionsOpened),
            new("mxgateway.sessions.closed", snapshot.SessionsClosed),
            new("mxgateway.commands.started", snapshot.CommandsStarted),
            new("mxgateway.commands.succeeded", snapshot.CommandsSucceeded),
            new("mxgateway.commands.failed", snapshot.CommandsFailed),
            new("mxgateway.events.received", snapshot.EventsReceived),
            new("mxgateway.queues.overflows", snapshot.QueueOverflows),
            new("mxgateway.faults", snapshot.Faults),
            new("mxgateway.workers.killed", snapshot.WorkerKills),
            new("mxgateway.workers.exited", snapshot.WorkerExits),
            new("mxgateway.heartbeats.failed", snapshot.HeartbeatFailures),
            new("mxgateway.grpc.streams.disconnected", snapshot.StreamDisconnects),
            new("mxgateway.alarms.provider_switches", snapshot.AlarmProviderSwitchCount),
        ];

        metrics.AddRange(snapshot.CommandFailuresByMethod
            .OrderBy(entry => entry.Key, StringComparer.OrdinalIgnoreCase)
            .Select(entry => new DashboardMetricSummary("mxgateway.commands.failed", entry.Value, entry.Key)));
        metrics.AddRange(snapshot.EventsByFamily
            .OrderBy(entry => entry.Key, StringComparer.OrdinalIgnoreCase)
            .Select(entry => new DashboardMetricSummary("mxgateway.events.received", entry.Value, entry.Key)));

        return metrics;
    }

    private IReadOnlyList<DashboardFaultSummary> CreateFaultSummaries(
        IReadOnlyList<GatewaySession> sessions,
        DateTimeOffset generatedAt)
    {
        return sessions
            .Where(HasFault)
            .Take(ResolveLimit(_recentFaultLimit))
            .Select(session => new DashboardFaultSummary(
                Source: session.WorkerClient?.State == WorkerClientState.Faulted ? "Worker" : "Session",
                SessionId: session.SessionId,
                WorkerProcessId: session.WorkerProcessId,
                State: session.WorkerClient?.State == WorkerClientState.Faulted
                    ? WorkerClientState.Faulted.ToString()
                    : session.State.ToString(),
                Message: DashboardRedactor.Redact(session.FinalFault) ?? "Faulted",
                ObservedAt: generatedAt))
            .ToArray();
    }

    private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
    {
        if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
        {
            return;
        }

        try
        {
            using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            timeout.CancelAfter(_apiKeySummaryRefreshTimeout);
            IReadOnlyList<DashboardApiKeySummary> summaries = (await _apiKeyAdminStore.ListAsync(timeout.Token)
                    .ConfigureAwait(false))
                .Select(key => new DashboardApiKeySummary(
                    KeyId: key.KeyId,
                    DisplayName: key.DisplayName,
                    Scopes: key.Scopes,
                    Constraints: ApiKeyConstraintSerializer.Deserialize(key.ConstraintsJson),
                    CreatedUtc: key.CreatedUtc,
                    LastUsedUtc: key.LastUsedUtc,
                    RevokedUtc: key.RevokedUtc,
                    ExpiresUtc: key.ExpiresUtc))
                .ToArray();

            Volatile.Write(ref _apiKeySummaries, summaries);
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            throw;
        }
        catch (OperationCanceledException)
        {
            _logger.LogWarning(
                "Timed out refreshing dashboard API key summaries after {Timeout}.",
                _apiKeySummaryRefreshTimeout);
        }
        catch (Exception)
        {
            _logger.LogWarning("Failed to refresh dashboard API key summaries.");
        }
        finally
        {
            _apiKeySummaryRefreshGate.Release();
        }
    }

    private static bool HasFault(GatewaySession session)
    {
        return session.State == ZB.MOM.WW.MxGateway.Contracts.Proto.SessionState.Faulted
            || session.WorkerClient?.State == WorkerClientState.Faulted
            || !string.IsNullOrWhiteSpace(session.FinalFault);
    }

    private static int ResolveLimit(int configuredLimit)
    {
        return configuredLimit < 0 ? 0 : configuredLimit;
    }
}