dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

test(loadharness)+docs: wire-path stream probes — R2 batching benchmark-proven at target scale

Browse Source 
The WP-4 runs measured the stream path only up to the bounded send channel;
the probe deliberately substituted the socket hop with a controllable reader,
which is exactly where the R2 SiteStreamEventBatcher lives — so batching had
never been in any benchmark's hot path. The harness now also attaches per-site
wire probes: a real Kestrel h2c listener hosting the production
SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with
batching negotiated, batch sizes read from the production batch_size histogram
via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old
shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON):
899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99
43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a
75-event burst never hits the 100 cap — zero drops, zero faults, offered load
100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire
path carries traffic and observes frames.
This commit is contained in:
Joseph Doherty
2026-08-18 10:32:46 -04:00
parent d3be5f0527
commit e9e44a9ef0
14 changed files with 3400 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessRun.cs
+74 -2
View File
@@ -40,6 +40,10 @@ namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
/// <param name="StreamProbeDropped">Events evicted by live subscribers' bounded channels.</param>
/// <param name="StoreAndForwardDrain">Store-and-forward drain measurement (register row 50).</param>
/// <param name="SlowSubscriber">Slow-subscriber isolation measurement (register row 50).</param>
/// <param name="WireStream">
/// Wire-path stream measurement — real Kestrel + real <c>SiteStreamGrpcClient</c> with
/// R2 batching negotiated — or null when <see cref="HarnessConfig.WireProbesPerSite"/> is 0.
/// </param>
public sealed record HarnessRunResult(
HarnessConfig Config,
EnvironmentInfo Environment,
@@ -67,7 +71,30 @@ public sealed record HarnessRunResult(
long StreamProbeReceived,
long StreamProbeDropped,
StoreAndForwardDrainResult? StoreAndForwardDrain,
SlowSubscriberResult? SlowSubscriber);
SlowSubscriberResult? SlowSubscriber,
WireStreamResult? WireStream);
/// <summary>
/// Steady-state-window measurement of the wire-path stream probes: the production
/// gRPC server (Kestrel h2c) and client with the R2 batcher, protobuf serialization
/// and HTTP/2 flow control INSIDE the measured path. All figures are deltas over the
/// steady-state window only.
/// </summary>
/// <param name="ProbeCount">Wire subscribers attached across the fleet.</param>
/// <param name="Latency">End-to-end DCL-boundary → post-unpack client callback latency.</param>
/// <param name="EventsReceived">Domain events delivered by the client in the window.</param>
/// <param name="ServerDroppedEvents">Events evicted by the wire servers' bounded send channels in the window.</param>
/// <param name="StreamFaults">Stream faults reported by the client (expected 0).</param>
/// <param name="GracefulCompletions">Graceful server-side stream completions (expected 0 inside the 4 h lifetime).</param>
/// <param name="Batches">The frame/batch-size distribution the R2 batcher actually produced.</param>
public sealed record WireStreamResult(
int ProbeCount,
LatencySnapshot Latency,
long EventsReceived,
long ServerDroppedEvents,
long StreamFaults,
long GracefulCompletions,
BatchSizeWindow Batches);
/// <summary>Host and runtime facts recorded alongside the numbers.</summary>
/// <param name="MachineName">Host name.</param>
@@ -120,6 +147,7 @@ public static class HarnessRun
ObservabilityProbes? probes = null;
StoreAndForwardDrainResult? drainResult = null;
SlowSubscriberResult? slowResult = null;
BatchSizeAggregator? batchAggregator = null;
var aggregator = new CentralHealthAggregator(
Options.Create(new HealthMonitoringOptions
@@ -154,6 +182,18 @@ public static class HarnessRun
site.AttachStreamProbes(latency);
log($" {sites.Sum(s => s.Probes.Count)} live stream subscribers attached");
// Wire path (real gRPC): the aggregator must exist BEFORE the first stream
// opens or the MeterListener misses early frames.
var wireLatency = new LatencyHistogram();
if (config.WireProbesPerSite > 0)
{
batchAggregator = new BatchSizeAggregator();
foreach (var site in sites)
await site.AttachWireStreamProbesAsync(config.WireProbesPerSite, wireLatency, log);
log($" {sites.Sum(s => s.WireSubscribers.Count)} wire stream subscribers attached " +
"(real Kestrel + SiteStreamGrpcClient, batching negotiated)");
}
// Let Instance Actors complete their DCL subscribe round-trips before the
// driver starts; an emit before SubscribeBatchAsync has captured the
// callback would be silently discarded.
@@ -184,6 +224,18 @@ public static class HarnessRun
probe.RetargetLatency(steadyLatency);
}
// Wire path: same ramp/steady histogram separation, plus window-start
// baselines for the client/server counters and the batch distribution.
var steadyWireLatency = new LatencyHistogram();
var wireReceivedAtWindowStart = sites.Sum(s => s.WireSubscribers.Sum(p => p.ReceivedEvents));
var wireDroppedAtWindowStart = sites.Sum(s => s.WireHost?.Server.DroppedStreamEventCount ?? 0);
var batchesAtWindowStart = batchAggregator?.Snapshot();
foreach (var site in sites)
{
foreach (var wire in site.WireSubscribers)
wire.RetargetLatency(steadyWireLatency);
}
var windowWatch = Stopwatch.StartNew();
log($"Sustained measurement window: {config.SustainDuration.TotalMinutes:F1} min...");
await Task.Delay(config.SustainDuration, cancellationToken);
@@ -199,6 +251,24 @@ public static class HarnessRun
var probeReceived = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents)) - receivedAtWindowStart;
var probeDropped = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents)) - droppedAtWindowStart;
// Wire-path window deltas, captured BEFORE the row-50 scenarios below so
// their publishing cannot contaminate the wire figures.
WireStreamResult? wireResult = null;
if (batchAggregator is not null)
{
wireResult = new WireStreamResult(
ProbeCount: sites.Sum(s => s.WireSubscribers.Count),
Latency: steadyWireLatency.Snapshot(),
EventsReceived: sites.Sum(s => s.WireSubscribers.Sum(p => p.ReceivedEvents)) - wireReceivedAtWindowStart,
ServerDroppedEvents: sites.Sum(s => s.WireHost?.Server.DroppedStreamEventCount ?? 0) - wireDroppedAtWindowStart,
StreamFaults: sites.Sum(s => s.WireSubscribers.Sum(p => p.Faults)),
GracefulCompletions: sites.Sum(s => s.WireSubscribers.Sum(p => p.GracefulCompletions)),
Batches: BatchSizeWindow.From(batchesAtWindowStart!, batchAggregator.Snapshot()));
log($" wire path: {wireResult.EventsReceived:N0} events in {wireResult.Batches.Frames:N0} frames " +
$"(mean {wireResult.Batches.MeanEventsPerFrame:F1} ev/frame, max {wireResult.Batches.Max}), " +
$"P99 {wireResult.Latency.P99Ms:F2}ms");
}
// Register row 50 — measured after the sustained window so the numbers are
// not competing with the full tag load for CPU, and reported separately for
// the same reason.
@@ -250,12 +320,14 @@ public static class HarnessRun
StreamProbeReceived: probeReceived,
StreamProbeDropped: probeDropped,
StoreAndForwardDrain: drainResult,
SlowSubscriber: slowResult);
SlowSubscriber: slowResult,
WireStream: wireResult);
}
finally
{
if (driver != null) await driver.DisposeAsync();
if (probes != null) await probes.DisposeAsync();
batchAggregator?.Dispose();
await sampler.DisposeAsync();
log("Tearing down sites...");