ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Scenarios/ObservabilityProbes.cs

using System.Diagnostics;
using Akka.Actor;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;

namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;

/// <summary>
/// The two "does observability still work at scale?" probes the target-scale test protocol
/// names — health report delivery timing and debug view latency — run continuously
/// alongside the sustained load rather than after it, so both are measured against a
/// site that is actually busy.
/// </summary>
public sealed class ObservabilityProbes : IAsyncDisposable
{
    private readonly CancellationTokenSource _cts = new();
    private readonly List<Task> _tasks = new();

    /// <summary>
    /// End-to-end health report latency: <c>SiteHealthCollector.CollectReport</c> plus
    /// the transport hop plus <c>CentralHealthAggregator.ProcessReport</c>. The
    /// interesting term at scale is <c>CollectReport</c>, which materializes the
    /// per-connection dictionaries for a site carrying 37,500 subscriptions.
    /// </summary>
    public LatencyHistogram HealthReportLatency { get; } = new();

    /// <summary>
    /// Debug view snapshot round-trip: an <c>Ask</c> of <c>DebugSnapshotRequest</c> to
    /// a live Instance Actor. This lands the request in the mailbox of an actor that
    /// is concurrently ingesting tag updates, so the measured time includes real
    /// queueing behind production traffic — which is the whole point of measuring it
    /// under load.
    /// </summary>
    public LatencyHistogram DebugSnapshotLatency { get; } = new();

    /// <summary>Health reports successfully ingested by the central aggregator.</summary>
    public long HealthReportsDelivered => Interlocked.Read(ref _healthReports);

    /// <summary>Debug snapshots that completed within the ask timeout.</summary>
    public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots);

    /// <summary>Debug snapshot asks that timed out.</summary>
    public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts);

    private long _healthReports;
    private long _debugSnapshots;
    private long _debugTimeouts;

    /// <summary>
    /// Starts both probes.
    /// </summary>
    /// <param name="sites">Sites to probe.</param>
    /// <param name="aggregator">The real central aggregator receiving the reports.</param>
    /// <param name="reportInterval">Health report cadence (production default 30 s).</param>
    /// <param name="debugProbeInterval">How often to take a debug snapshot.</param>
    /// <returns>The running probes.</returns>
    public static ObservabilityProbes Start(
        IReadOnlyList<SiteRuntimeFixture> sites,
        CentralHealthAggregator aggregator,
        TimeSpan reportInterval,
        TimeSpan debugProbeInterval)
    {
        var probes = new ObservabilityProbes();
        probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token)));
        probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token)));
        return probes;
    }

    private async Task HealthLoopAsync(
        IReadOnlyList<SiteRuntimeFixture> sites,
        CentralHealthAggregator aggregator,
        TimeSpan interval,
        CancellationToken cancellationToken)
    {
        using var timer = new PeriodicTimer(interval);
        try
        {
            while (await timer.WaitForNextTickAsync(cancellationToken))
            {
                foreach (var site in sites)
                {
                    var watch = Stopwatch.StartNew();
                    var report = site.HealthCollector.CollectReport(site.SiteId);
                    aggregator.ProcessReport(report);
                    watch.Stop();

                    HealthReportLatency.Record(watch.Elapsed);
                    Interlocked.Increment(ref _healthReports);
                }
            }
        }
        catch (OperationCanceledException)
        {
            // Normal teardown.
        }
    }

    private async Task DebugLoopAsync(
        IReadOnlyList<SiteRuntimeFixture> sites,
        TimeSpan interval,
        CancellationToken cancellationToken)
    {
        var random = new Random(20260815);
        using var timer = new PeriodicTimer(interval);
        try
        {
            while (await timer.WaitForNextTickAsync(cancellationToken))
            {
                var site = sites[random.Next(sites.Count)];
                if (site.InstanceActors.Count == 0)
                    continue;

                var index = random.Next(site.InstanceActors.Count);
                var actor = site.InstanceActors[index];
                var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N"));

                var watch = Stopwatch.StartNew();
                try
                {
                    await actor.Ask<DebugViewSnapshot>(request, TimeSpan.FromSeconds(10), cancellationToken);
                    watch.Stop();
                    DebugSnapshotLatency.Record(watch.Elapsed);
                    Interlocked.Increment(ref _debugSnapshots);
                }
                catch (AskTimeoutException)
                {
                    Interlocked.Increment(ref _debugTimeouts);
                }
            }
        }
        catch (OperationCanceledException)
        {
            // Normal teardown.
        }
    }

    private int _disposed;

    /// <summary>Stops the probes and releases their resources.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async ValueTask DisposeAsync()
    {
        // Idempotent: the orchestrator stops the probes early (so the register row 50
        // scenarios do not compete with them) and again in its finally block.
        if (Interlocked.Exchange(ref _disposed, 1) != 0)
            return;

        await _cts.CancelAsync();
        try
        {
            await Task.WhenAll(_tasks);
        }
        catch (OperationCanceledException)
        {
            // Expected.
        }

        _cts.Dispose();
    }
}