using System.Diagnostics; using Akka.Actor; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics; namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios; /// /// The two "does observability still work at scale?" probes WP-4's test protocol /// names — health report delivery timing and debug view latency — run continuously /// alongside the sustained load rather than after it, so both are measured against a /// site that is actually busy. /// public sealed class ObservabilityProbes : IAsyncDisposable { private readonly CancellationTokenSource _cts = new(); private readonly List _tasks = new(); /// /// End-to-end health report latency: SiteHealthCollector.CollectReport plus /// the transport hop plus CentralHealthAggregator.ProcessReport. The /// interesting term at scale is CollectReport, which materializes the /// per-connection dictionaries for a site carrying 37,500 subscriptions. /// public LatencyHistogram HealthReportLatency { get; } = new(); /// /// Debug view snapshot round-trip: an Ask of DebugSnapshotRequest to /// a live Instance Actor. This lands the request in the mailbox of an actor that /// is concurrently ingesting tag updates, so the measured time includes real /// queueing behind production traffic — which is the whole point of measuring it /// under load. /// public LatencyHistogram DebugSnapshotLatency { get; } = new(); /// Health reports successfully ingested by the central aggregator. public long HealthReportsDelivered => Interlocked.Read(ref _healthReports); /// Debug snapshots that completed within the ask timeout. public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots); /// Debug snapshot asks that timed out. public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts); private long _healthReports; private long _debugSnapshots; private long _debugTimeouts; /// /// Starts both probes. /// /// Sites to probe. /// The real central aggregator receiving the reports. /// Health report cadence (production default 30 s). /// How often to take a debug snapshot. /// The running probes. public static ObservabilityProbes Start( IReadOnlyList sites, CentralHealthAggregator aggregator, TimeSpan reportInterval, TimeSpan debugProbeInterval) { var probes = new ObservabilityProbes(); probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token))); probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token))); return probes; } private async Task HealthLoopAsync( IReadOnlyList sites, CentralHealthAggregator aggregator, TimeSpan interval, CancellationToken cancellationToken) { using var timer = new PeriodicTimer(interval); try { while (await timer.WaitForNextTickAsync(cancellationToken)) { foreach (var site in sites) { var watch = Stopwatch.StartNew(); var report = site.HealthCollector.CollectReport(site.SiteId); aggregator.ProcessReport(report); watch.Stop(); HealthReportLatency.Record(watch.Elapsed); Interlocked.Increment(ref _healthReports); } } } catch (OperationCanceledException) { // Normal teardown. } } private async Task DebugLoopAsync( IReadOnlyList sites, TimeSpan interval, CancellationToken cancellationToken) { var random = new Random(20260815); using var timer = new PeriodicTimer(interval); try { while (await timer.WaitForNextTickAsync(cancellationToken)) { var site = sites[random.Next(sites.Count)]; if (site.InstanceActors.Count == 0) continue; var index = random.Next(site.InstanceActors.Count); var actor = site.InstanceActors[index]; var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N")); var watch = Stopwatch.StartNew(); try { await actor.Ask(request, TimeSpan.FromSeconds(10), cancellationToken); watch.Stop(); DebugSnapshotLatency.Record(watch.Elapsed); Interlocked.Increment(ref _debugSnapshots); } catch (AskTimeoutException) { Interlocked.Increment(ref _debugTimeouts); } } } catch (OperationCanceledException) { // Normal teardown. } } private int _disposed; /// public async ValueTask DisposeAsync() { // Idempotent: the orchestrator stops the probes early (so the register row 50 // scenarios do not compete with them) and again in its finally block. if (Interlocked.Exchange(ref _disposed, 1) != 0) return; await _cts.CancelAsync(); try { await Task.WhenAll(_tasks); } catch (OperationCanceledException) { // Expected. } _cts.Dispose(); } }