using System.Diagnostics;
using Akka.Actor;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
///
/// The two "does observability still work at scale?" probes WP-4's test protocol
/// names — health report delivery timing and debug view latency — run continuously
/// alongside the sustained load rather than after it, so both are measured against a
/// site that is actually busy.
///
public sealed class ObservabilityProbes : IAsyncDisposable
{
private readonly CancellationTokenSource _cts = new();
private readonly List _tasks = new();
///
/// End-to-end health report latency: SiteHealthCollector.CollectReport plus
/// the transport hop plus CentralHealthAggregator.ProcessReport. The
/// interesting term at scale is CollectReport, which materializes the
/// per-connection dictionaries for a site carrying 37,500 subscriptions.
///
public LatencyHistogram HealthReportLatency { get; } = new();
///
/// Debug view snapshot round-trip: an Ask of DebugSnapshotRequest to
/// a live Instance Actor. This lands the request in the mailbox of an actor that
/// is concurrently ingesting tag updates, so the measured time includes real
/// queueing behind production traffic — which is the whole point of measuring it
/// under load.
///
public LatencyHistogram DebugSnapshotLatency { get; } = new();
/// Health reports successfully ingested by the central aggregator.
public long HealthReportsDelivered => Interlocked.Read(ref _healthReports);
/// Debug snapshots that completed within the ask timeout.
public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots);
/// Debug snapshot asks that timed out.
public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts);
private long _healthReports;
private long _debugSnapshots;
private long _debugTimeouts;
///
/// Starts both probes.
///
/// Sites to probe.
/// The real central aggregator receiving the reports.
/// Health report cadence (production default 30 s).
/// How often to take a debug snapshot.
/// The running probes.
public static ObservabilityProbes Start(
IReadOnlyList sites,
CentralHealthAggregator aggregator,
TimeSpan reportInterval,
TimeSpan debugProbeInterval)
{
var probes = new ObservabilityProbes();
probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token)));
probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token)));
return probes;
}
private async Task HealthLoopAsync(
IReadOnlyList sites,
CentralHealthAggregator aggregator,
TimeSpan interval,
CancellationToken cancellationToken)
{
using var timer = new PeriodicTimer(interval);
try
{
while (await timer.WaitForNextTickAsync(cancellationToken))
{
foreach (var site in sites)
{
var watch = Stopwatch.StartNew();
var report = site.HealthCollector.CollectReport(site.SiteId);
aggregator.ProcessReport(report);
watch.Stop();
HealthReportLatency.Record(watch.Elapsed);
Interlocked.Increment(ref _healthReports);
}
}
}
catch (OperationCanceledException)
{
// Normal teardown.
}
}
private async Task DebugLoopAsync(
IReadOnlyList sites,
TimeSpan interval,
CancellationToken cancellationToken)
{
var random = new Random(20260815);
using var timer = new PeriodicTimer(interval);
try
{
while (await timer.WaitForNextTickAsync(cancellationToken))
{
var site = sites[random.Next(sites.Count)];
if (site.InstanceActors.Count == 0)
continue;
var index = random.Next(site.InstanceActors.Count);
var actor = site.InstanceActors[index];
var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N"));
var watch = Stopwatch.StartNew();
try
{
await actor.Ask(request, TimeSpan.FromSeconds(10), cancellationToken);
watch.Stop();
DebugSnapshotLatency.Record(watch.Elapsed);
Interlocked.Increment(ref _debugSnapshots);
}
catch (AskTimeoutException)
{
Interlocked.Increment(ref _debugTimeouts);
}
}
}
catch (OperationCanceledException)
{
// Normal teardown.
}
}
private int _disposed;
///
public async ValueTask DisposeAsync()
{
// Idempotent: the orchestrator stops the probes early (so the register row 50
// scenarios do not compete with them) and again in its finally block.
if (Interlocked.Exchange(ref _disposed, 1) != 0)
return;
await _cts.CancelAsync();
try
{
await Task.WhenAll(_tasks);
}
catch (OperationCanceledException)
{
// Expected.
}
_cts.Dispose();
}
}