test(loadharness): target-scale load harness for WP-4 / register #25 + row 50
Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,136 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// Lock-free logarithmic latency histogram sized for tens of thousands of samples
|
||||
/// per second across many threads.
|
||||
///
|
||||
/// <para>
|
||||
/// Buckets are 16-per-octave over microseconds, i.e. bucket <c>i</c> covers
|
||||
/// <c>[2^(i/16), 2^((i+1)/16))</c> µs. That bounds relative bucket width at
|
||||
/// <c>2^(1/16) - 1 ≈ 4.4%</c>, so a reported percentile is within ~4.4% of the true
|
||||
/// value — ample for the millisecond-scale thresholds this harness asserts, and far
|
||||
/// cheaper than retaining 45 million raw samples.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Recording is a <see cref="Math.Log2(double)"/> plus one
|
||||
/// <see cref="Interlocked.Increment(ref long)"/>; there is no allocation on the hot path.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class LatencyHistogram
|
||||
{
|
||||
private const int SubBucketsPerOctave = 16;
|
||||
private const int BucketCount = 64 * SubBucketsPerOctave;
|
||||
|
||||
private readonly long[] _buckets = new long[BucketCount];
|
||||
private long _count;
|
||||
private long _totalMicroseconds;
|
||||
private long _maxMicroseconds;
|
||||
|
||||
/// <summary>Number of samples recorded.</summary>
|
||||
public long Count => Interlocked.Read(ref _count);
|
||||
|
||||
/// <summary>Largest sample seen, in microseconds (exact — not bucketed).</summary>
|
||||
public double MaxMs => Interlocked.Read(ref _maxMicroseconds) / 1000.0;
|
||||
|
||||
/// <summary>Arithmetic mean in milliseconds (exact — accumulated, not bucketed).</summary>
|
||||
public double MeanMs
|
||||
{
|
||||
get
|
||||
{
|
||||
var count = Interlocked.Read(ref _count);
|
||||
return count == 0 ? 0 : Interlocked.Read(ref _totalMicroseconds) / 1000.0 / count;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records one sample. Negative durations (clock skew across the emit/receive
|
||||
/// boundary) are clamped to zero rather than discarded, so the sample count stays
|
||||
/// an honest denominator.
|
||||
/// </summary>
|
||||
/// <param name="elapsed">The measured latency.</param>
|
||||
public void Record(TimeSpan elapsed)
|
||||
{
|
||||
var micros = (long)(elapsed.TotalMilliseconds * 1000.0);
|
||||
if (micros < 0) micros = 0;
|
||||
|
||||
Interlocked.Increment(ref _count);
|
||||
Interlocked.Add(ref _totalMicroseconds, micros);
|
||||
|
||||
long observedMax;
|
||||
while (micros > (observedMax = Interlocked.Read(ref _maxMicroseconds)))
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _maxMicroseconds, micros, observedMax) == observedMax)
|
||||
break;
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _buckets[BucketIndex(micros)]);
|
||||
}
|
||||
|
||||
private static int BucketIndex(long micros)
|
||||
{
|
||||
if (micros <= 0) return 0;
|
||||
var index = (int)(Math.Log2(micros) * SubBucketsPerOctave);
|
||||
if (index < 0) return 0;
|
||||
return index >= BucketCount ? BucketCount - 1 : index;
|
||||
}
|
||||
|
||||
/// <summary>Bucket midpoint in milliseconds, used when reconstructing a percentile.</summary>
|
||||
private static double BucketMidpointMs(int index)
|
||||
{
|
||||
var low = Math.Pow(2, (double)index / SubBucketsPerOctave);
|
||||
var high = Math.Pow(2, (double)(index + 1) / SubBucketsPerOctave);
|
||||
return (low + high) / 2.0 / 1000.0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the requested percentile in milliseconds, or 0 when no samples were recorded.
|
||||
/// </summary>
|
||||
/// <param name="percentile">Percentile in the range 0..100 (e.g. 99 for P99).</param>
|
||||
/// <returns>The percentile value in milliseconds.</returns>
|
||||
public double PercentileMs(double percentile)
|
||||
{
|
||||
var total = Interlocked.Read(ref _count);
|
||||
if (total == 0) return 0;
|
||||
|
||||
var target = (long)Math.Ceiling(total * percentile / 100.0);
|
||||
if (target < 1) target = 1;
|
||||
|
||||
long cumulative = 0;
|
||||
for (var i = 0; i < BucketCount; i++)
|
||||
{
|
||||
cumulative += Interlocked.Read(ref _buckets[i]);
|
||||
if (cumulative >= target)
|
||||
return BucketMidpointMs(i);
|
||||
}
|
||||
|
||||
return MaxMs;
|
||||
}
|
||||
|
||||
/// <summary>Materializes the standard percentile set plus mean/max/count for reporting.</summary>
|
||||
/// <returns>A snapshot record of this histogram.</returns>
|
||||
public LatencySnapshot Snapshot() => new(
|
||||
Count,
|
||||
MeanMs,
|
||||
PercentileMs(50),
|
||||
PercentileMs(95),
|
||||
PercentileMs(99),
|
||||
PercentileMs(99.9),
|
||||
MaxMs);
|
||||
}
|
||||
|
||||
/// <summary>Point-in-time summary of a <see cref="LatencyHistogram"/>. All times in milliseconds.</summary>
|
||||
/// <param name="Count">Samples recorded.</param>
|
||||
/// <param name="MeanMs">Arithmetic mean.</param>
|
||||
/// <param name="P50Ms">Median.</param>
|
||||
/// <param name="P95Ms">95th percentile.</param>
|
||||
/// <param name="P99Ms">99th percentile.</param>
|
||||
/// <param name="P999Ms">99.9th percentile.</param>
|
||||
/// <param name="MaxMs">Largest observed sample.</param>
|
||||
public sealed record LatencySnapshot(
|
||||
long Count,
|
||||
double MeanMs,
|
||||
double P50Ms,
|
||||
double P95Ms,
|
||||
double P99Ms,
|
||||
double P999Ms,
|
||||
double MaxMs);
|
||||
@@ -0,0 +1,202 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// One periodic observation of process resource usage.
|
||||
/// </summary>
|
||||
/// <param name="ElapsedSeconds">Seconds since sampling started.</param>
|
||||
/// <param name="WorkingSetMb">Process working set (RSS).</param>
|
||||
/// <param name="ManagedHeapMb"><see cref="GC.GetTotalMemory(bool)"/> without forcing a collection.</param>
|
||||
/// <param name="CpuPercent">Mean CPU utilization since the previous sample, as a percentage of ONE core.</param>
|
||||
/// <param name="ThreadCount">OS threads in the process.</param>
|
||||
/// <param name="Gen2Collections">Cumulative gen-2 collections.</param>
|
||||
public sealed record ResourceSample(
|
||||
double ElapsedSeconds,
|
||||
double WorkingSetMb,
|
||||
double ManagedHeapMb,
|
||||
double CpuPercent,
|
||||
int ThreadCount,
|
||||
int Gen2Collections);
|
||||
|
||||
/// <summary>
|
||||
/// Samples working set, managed heap, CPU and thread count on a fixed cadence for the
|
||||
/// life of a run. CPU is differential (processor time delta / wall delta) so a sample
|
||||
/// reflects the interval it covers rather than the whole process lifetime.
|
||||
/// </summary>
|
||||
public sealed class ResourceSampler : IAsyncDisposable
|
||||
{
|
||||
private readonly List<ResourceSample> _samples = new();
|
||||
private readonly object _lock = new();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly Task _loop;
|
||||
private readonly Stopwatch _wall = Stopwatch.StartNew();
|
||||
|
||||
private TimeSpan _lastCpu;
|
||||
private double _lastElapsedSeconds;
|
||||
|
||||
private ResourceSampler(TimeSpan interval)
|
||||
{
|
||||
_lastCpu = Process.GetCurrentProcess().TotalProcessorTime;
|
||||
_loop = Task.Run(() => SampleLoopAsync(interval, _cts.Token));
|
||||
}
|
||||
|
||||
/// <summary>Starts sampling at the given cadence.</summary>
|
||||
/// <param name="interval">Sampling interval.</param>
|
||||
/// <returns>The running sampler.</returns>
|
||||
public static ResourceSampler Start(TimeSpan interval) => new(interval);
|
||||
|
||||
/// <summary>All samples collected so far, oldest first.</summary>
|
||||
/// <returns>A snapshot copy of the sample list.</returns>
|
||||
public IReadOnlyList<ResourceSample> Snapshot()
|
||||
{
|
||||
lock (_lock) return _samples.ToList();
|
||||
}
|
||||
|
||||
private async Task SampleLoopAsync(TimeSpan interval, CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
Capture();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private void Capture()
|
||||
{
|
||||
using var process = Process.GetCurrentProcess();
|
||||
process.Refresh();
|
||||
|
||||
var elapsedSeconds = _wall.Elapsed.TotalSeconds;
|
||||
var cpu = process.TotalProcessorTime;
|
||||
var wallDelta = elapsedSeconds - _lastElapsedSeconds;
|
||||
var cpuPercent = wallDelta > 0
|
||||
? (cpu - _lastCpu).TotalSeconds / wallDelta * 100.0
|
||||
: 0.0;
|
||||
|
||||
_lastCpu = cpu;
|
||||
_lastElapsedSeconds = elapsedSeconds;
|
||||
|
||||
var sample = new ResourceSample(
|
||||
elapsedSeconds,
|
||||
process.WorkingSet64 / 1024.0 / 1024.0,
|
||||
GC.GetTotalMemory(forceFullCollection: false) / 1024.0 / 1024.0,
|
||||
cpuPercent,
|
||||
process.Threads.Count,
|
||||
GC.CollectionCount(2));
|
||||
|
||||
lock (_lock) _samples.Add(sample);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summarizes the samples falling inside a window, expressed as seconds since
|
||||
/// sampling started. Memory growth is reported both as an absolute delta and as a
|
||||
/// least-squares slope, because a run that sawtooths around a stable mean and a
|
||||
/// run that climbs monotonically can share the same endpoint delta.
|
||||
/// </summary>
|
||||
/// <param name="fromSeconds">Window start (inclusive), seconds since start.</param>
|
||||
/// <param name="toSeconds">Window end (inclusive), seconds since start.</param>
|
||||
/// <returns>The window summary, or null when fewer than two samples fall inside it.</returns>
|
||||
public ResourceWindowSummary? Summarize(double fromSeconds, double toSeconds)
|
||||
{
|
||||
var window = Snapshot()
|
||||
.Where(s => s.ElapsedSeconds >= fromSeconds && s.ElapsedSeconds <= toSeconds)
|
||||
.ToList();
|
||||
|
||||
if (window.Count < 2)
|
||||
return null;
|
||||
|
||||
var first = window[0];
|
||||
var last = window[^1];
|
||||
|
||||
return new ResourceWindowSummary(
|
||||
SampleCount: window.Count,
|
||||
DurationSeconds: last.ElapsedSeconds - first.ElapsedSeconds,
|
||||
WorkingSetStartMb: first.WorkingSetMb,
|
||||
WorkingSetEndMb: last.WorkingSetMb,
|
||||
WorkingSetPeakMb: window.Max(s => s.WorkingSetMb),
|
||||
WorkingSetSlopeMbPerMinute: Slope(window, s => s.WorkingSetMb) * 60.0,
|
||||
ManagedHeapStartMb: first.ManagedHeapMb,
|
||||
ManagedHeapEndMb: last.ManagedHeapMb,
|
||||
ManagedHeapPeakMb: window.Max(s => s.ManagedHeapMb),
|
||||
ManagedHeapSlopeMbPerMinute: Slope(window, s => s.ManagedHeapMb) * 60.0,
|
||||
MeanCpuPercentOfOneCore: window.Average(s => s.CpuPercent),
|
||||
PeakCpuPercentOfOneCore: window.Max(s => s.CpuPercent),
|
||||
MeanThreadCount: window.Average(s => s.ThreadCount),
|
||||
Gen2Collections: last.Gen2Collections - first.Gen2Collections);
|
||||
}
|
||||
|
||||
private static double Slope(IReadOnlyList<ResourceSample> samples, Func<ResourceSample, double> selector)
|
||||
{
|
||||
var n = samples.Count;
|
||||
var meanX = samples.Average(s => s.ElapsedSeconds);
|
||||
var meanY = samples.Average(selector);
|
||||
|
||||
double numerator = 0, denominator = 0;
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var dx = samples[i].ElapsedSeconds - meanX;
|
||||
numerator += dx * (selector(samples[i]) - meanY);
|
||||
denominator += dx * dx;
|
||||
}
|
||||
|
||||
return denominator == 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await _loop;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected.
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Aggregate resource behaviour over a measurement window.</summary>
|
||||
/// <param name="SampleCount">Samples in the window.</param>
|
||||
/// <param name="DurationSeconds">Window length.</param>
|
||||
/// <param name="WorkingSetStartMb">Working set at window start.</param>
|
||||
/// <param name="WorkingSetEndMb">Working set at window end.</param>
|
||||
/// <param name="WorkingSetPeakMb">Peak working set in the window.</param>
|
||||
/// <param name="WorkingSetSlopeMbPerMinute">Least-squares working-set growth rate.</param>
|
||||
/// <param name="ManagedHeapStartMb">Managed heap at window start.</param>
|
||||
/// <param name="ManagedHeapEndMb">Managed heap at window end.</param>
|
||||
/// <param name="ManagedHeapPeakMb">Peak managed heap in the window.</param>
|
||||
/// <param name="ManagedHeapSlopeMbPerMinute">Least-squares managed-heap growth rate.</param>
|
||||
/// <param name="MeanCpuPercentOfOneCore">Mean CPU as a percentage of one core (1400% = 14 cores saturated).</param>
|
||||
/// <param name="PeakCpuPercentOfOneCore">Peak single-sample CPU as a percentage of one core.</param>
|
||||
/// <param name="MeanThreadCount">Mean OS thread count.</param>
|
||||
/// <param name="Gen2Collections">Gen-2 collections during the window.</param>
|
||||
public sealed record ResourceWindowSummary(
|
||||
int SampleCount,
|
||||
double DurationSeconds,
|
||||
double WorkingSetStartMb,
|
||||
double WorkingSetEndMb,
|
||||
double WorkingSetPeakMb,
|
||||
double WorkingSetSlopeMbPerMinute,
|
||||
double ManagedHeapStartMb,
|
||||
double ManagedHeapEndMb,
|
||||
double ManagedHeapPeakMb,
|
||||
double ManagedHeapSlopeMbPerMinute,
|
||||
double MeanCpuPercentOfOneCore,
|
||||
double PeakCpuPercentOfOneCore,
|
||||
double MeanThreadCount,
|
||||
int Gen2Collections);
|
||||
Reference in New Issue
Block a user