test(loadharness): target-scale load harness for WP-4 / register #25 + row 50

Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
2026-08-15 02:23:04 -04:00
parent 986e6e7ad5
commit 20f6b0b969
17 changed files with 2419 additions and 0 deletions
@@ -0,0 +1,160 @@
+using System.Diagnostics;
+using Akka.Actor;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
+
+namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
+
+/// <summary>
+/// The two "does observability still work at scale?" probes WP-4's test protocol
+/// names — health report delivery timing and debug view latency — run continuously
+/// alongside the sustained load rather than after it, so both are measured against a
+/// site that is actually busy.
+/// </summary>
+public sealed class ObservabilityProbes : IAsyncDisposable
+{
+    private readonly CancellationTokenSource _cts = new();
+    private readonly List<Task> _tasks = new();
+
+    /// <summary>
+    /// End-to-end health report latency: <c>SiteHealthCollector.CollectReport</c> plus
+    /// the transport hop plus <c>CentralHealthAggregator.ProcessReport</c>. The
+    /// interesting term at scale is <c>CollectReport</c>, which materializes the
+    /// per-connection dictionaries for a site carrying 37,500 subscriptions.
+    /// </summary>
+    public LatencyHistogram HealthReportLatency { get; } = new();
+
+    /// <summary>
+    /// Debug view snapshot round-trip: an <c>Ask</c> of <c>DebugSnapshotRequest</c> to
+    /// a live Instance Actor. This lands the request in the mailbox of an actor that
+    /// is concurrently ingesting tag updates, so the measured time includes real
+    /// queueing behind production traffic — which is the whole point of measuring it
+    /// under load.
+    /// </summary>
+    public LatencyHistogram DebugSnapshotLatency { get; } = new();
+
+    /// <summary>Health reports successfully ingested by the central aggregator.</summary>
+    public long HealthReportsDelivered => Interlocked.Read(ref _healthReports);
+
+    /// <summary>Debug snapshots that completed within the ask timeout.</summary>
+    public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots);
+
+    /// <summary>Debug snapshot asks that timed out.</summary>
+    public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts);
+
+    private long _healthReports;
+    private long _debugSnapshots;
+    private long _debugTimeouts;
+
+    /// <summary>
+    /// Starts both probes.
+    /// </summary>
+    /// <param name="sites">Sites to probe.</param>
+    /// <param name="aggregator">The real central aggregator receiving the reports.</param>
+    /// <param name="reportInterval">Health report cadence (production default 30 s).</param>
+    /// <param name="debugProbeInterval">How often to take a debug snapshot.</param>
+    /// <returns>The running probes.</returns>
+    public static ObservabilityProbes Start(
+        IReadOnlyList<SiteRuntimeFixture> sites,
+        CentralHealthAggregator aggregator,
+        TimeSpan reportInterval,
+        TimeSpan debugProbeInterval)
+    {
+        var probes = new ObservabilityProbes();
+        probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token)));
+        probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token)));
+        return probes;
+    }
+
+    private async Task HealthLoopAsync(
+        IReadOnlyList<SiteRuntimeFixture> sites,
+        CentralHealthAggregator aggregator,
+        TimeSpan interval,
+        CancellationToken cancellationToken)
+    {
+        using var timer = new PeriodicTimer(interval);
+        try
+        {
+            while (await timer.WaitForNextTickAsync(cancellationToken))
+            {
+                foreach (var site in sites)
+                {
+                    var watch = Stopwatch.StartNew();
+                    var report = site.HealthCollector.CollectReport(site.SiteId);
+                    aggregator.ProcessReport(report);
+                    watch.Stop();
+
+                    HealthReportLatency.Record(watch.Elapsed);
+                    Interlocked.Increment(ref _healthReports);
+                }
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            // Normal teardown.
+        }
+    }
+
+    private async Task DebugLoopAsync(
+        IReadOnlyList<SiteRuntimeFixture> sites,
+        TimeSpan interval,
+        CancellationToken cancellationToken)
+    {
+        var random = new Random(20260815);
+        using var timer = new PeriodicTimer(interval);
+        try
+        {
+            while (await timer.WaitForNextTickAsync(cancellationToken))
+            {
+                var site = sites[random.Next(sites.Count)];
+                if (site.InstanceActors.Count == 0)
+                    continue;
+
+                var index = random.Next(site.InstanceActors.Count);
+                var actor = site.InstanceActors[index];
+                var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N"));
+
+                var watch = Stopwatch.StartNew();
+                try
+                {
+                    await actor.Ask<DebugViewSnapshot>(request, TimeSpan.FromSeconds(10), cancellationToken);
+                    watch.Stop();
+                    DebugSnapshotLatency.Record(watch.Elapsed);
+                    Interlocked.Increment(ref _debugSnapshots);
+                }
+                catch (AskTimeoutException)
+                {
+                    Interlocked.Increment(ref _debugTimeouts);
+                }
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            // Normal teardown.
+        }
+    }
+
+    private int _disposed;
+
+    /// <inheritdoc />
+    public async ValueTask DisposeAsync()
+    {
+        // Idempotent: the orchestrator stops the probes early (so the register row 50
+        // scenarios do not compete with them) and again in its finally block.
+        if (Interlocked.Exchange(ref _disposed, 1) != 0)
+            return;
+
+        await _cts.CancelAsync();
+        try
+        {
+            await Task.WhenAll(_tasks);
+        }
+        catch (OperationCanceledException)
+        {
+            // Expected.
+        }
+
+        _cts.Dispose();
+    }
+}