test(loadharness)+docs: wire-path stream probes — R2 batching benchmark-proven at target scale

The WP-4 runs measured the stream path only up to the bounded send channel; the probe deliberately substituted the socket hop with a controllable reader, which is exactly where the R2 SiteStreamEventBatcher lives — so batching had never been in any benchmark's hot path. The harness now also attaches per-site wire probes: a real Kestrel h2c listener hosting the production SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with batching negotiated, batch sizes read from the production batch_size histogram via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON): 899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99 43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a 75-event burst never hits the 100 cap — zero drops, zero faults, offered load 100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire path carries traffic and observes frames.
2026-08-18 10:32:46 -04:00
parent d3be5f0527
commit e9e44a9ef0
14 changed files with 3400 additions and 8 deletions
@@ -0,0 +1,136 @@
+using System.Diagnostics.Metrics;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
+
+namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
+
+/// <summary>
+/// Captures the production <c>scadabridge.site.stream.batch_size</c> histogram via a
+/// <see cref="MeterListener"/>, so a wire-path run can report the actual frame/batch
+/// distribution the R2 batcher produced — not a client-side reconstruction. The
+/// instrument is recorded once per emitted gRPC frame (negotiated streams only), with
+/// the frame's event count as the value, which makes <c>count = frames</c> and
+/// <c>sum = events</c> exactly.
+///
+/// <para>
+/// The harness hosts every site's <see cref="ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer"/>
+/// in-process, so this listener sees every wire frame the run emits and nothing else —
+/// no other component in the harness records this instrument.
+/// </para>
+/// </summary>
+public sealed class BatchSizeAggregator : IDisposable
+{
+    /// <summary>
+    /// Sizes above this are clamped into the last bucket. The production cap is
+    /// <c>GrpcStreamBatchMaxEvents</c> = 100, so 512 leaves generous headroom for a
+    /// configured-up run without an unbounded array.
+    /// </summary>
+    public const int MaxTrackedBatchSize = 512;
+
+    private readonly MeterListener _listener;
+    private readonly long[] _framesBySize = new long[MaxTrackedBatchSize + 1];
+
+    /// <summary>Starts listening. Must be created BEFORE the first stream opens.</summary>
+    public BatchSizeAggregator()
+    {
+        _listener = new MeterListener
+        {
+            InstrumentPublished = (instrument, listener) =>
+            {
+                if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
+                    instrument.Name == "scadabridge.site.stream.batch_size")
+                {
+                    listener.EnableMeasurementEvents(instrument);
+                }
+            },
+        };
+        _listener.SetMeasurementEventCallback<int>((_, value, _, _) =>
+        {
+            var size = Math.Clamp(value, 1, MaxTrackedBatchSize);
+            Interlocked.Increment(ref _framesBySize[size]);
+        });
+        _listener.Start();
+    }
+
+    /// <summary>Copies the cumulative per-size frame counts.</summary>
+    /// <returns>Index = events-per-frame (clamped), value = frames emitted at that size.</returns>
+    public long[] Snapshot()
+    {
+        var copy = new long[_framesBySize.Length];
+        for (var i = 0; i < copy.Length; i++)
+            copy[i] = Interlocked.Read(ref _framesBySize[i]);
+        return copy;
+    }
+
+    /// <inheritdoc />
+    public void Dispose() => _listener.Dispose();
+}
+
+/// <summary>
+/// Batch-size distribution over one measurement window, computed from two
+/// <see cref="BatchSizeAggregator.Snapshot"/>s. Percentiles are exact (integer sizes,
+/// full counts — nothing is bucketed away).
+/// </summary>
+/// <param name="Frames">gRPC frames written in the window.</param>
+/// <param name="Events">Events those frames carried.</param>
+/// <param name="BatchedFrames">Frames that carried more than one event.</param>
+/// <param name="MeanEventsPerFrame">Events / Frames.</param>
+/// <param name="P50">Median events-per-frame.</param>
+/// <param name="P95">95th percentile events-per-frame.</param>
+/// <param name="P99">99th percentile events-per-frame.</param>
+/// <param name="Max">Largest frame observed in the window.</param>
+public sealed record BatchSizeWindow(
+    long Frames,
+    long Events,
+    long BatchedFrames,
+    double MeanEventsPerFrame,
+    int P50,
+    int P95,
+    int P99,
+    int Max)
+{
+    /// <summary>Computes the window distribution from cumulative start/end snapshots.</summary>
+    /// <param name="start">Snapshot at window start.</param>
+    /// <param name="end">Snapshot at window end.</param>
+    /// <returns>The distribution of frames emitted between the two snapshots.</returns>
+    public static BatchSizeWindow From(long[] start, long[] end)
+    {
+        long frames = 0, events = 0, batched = 0;
+        var max = 0;
+        var delta = new long[end.Length];
+        for (var size = 1; size < end.Length; size++)
+        {
+            var n = end[size] - start[size];
+            if (n <= 0) continue;
+            delta[size] = n;
+            frames += n;
+            events += n * size;
+            if (size > 1) batched += n;
+            max = size;
+        }
+
+        return new BatchSizeWindow(
+            frames, events, batched,
+            frames == 0 ? 0 : events / (double)frames,
+            Percentile(delta, frames, 50),
+            Percentile(delta, frames, 95),
+            Percentile(delta, frames, 99),
+            max);
+    }
+
+    private static int Percentile(long[] delta, long frames, double percentile)
+    {
+        if (frames == 0) return 0;
+        var target = (long)Math.Ceiling(frames * percentile / 100.0);
+        if (target < 1) target = 1;
+
+        long cumulative = 0;
+        for (var size = 1; size < delta.Length; size++)
+        {
+            cumulative += delta[size];
+            if (cumulative >= target)
+                return size;
+        }
+
+        return delta.Length - 1;
+    }
+}