test(loadharness)+docs: wire-path stream probes — R2 batching benchmark-proven at target scale

The WP-4 runs measured the stream path only up to the bounded send channel;
the probe deliberately substituted the socket hop with a controllable reader,
which is exactly where the R2 SiteStreamEventBatcher lives — so batching had
never been in any benchmark's hot path. The harness now also attaches per-site
wire probes: a real Kestrel h2c listener hosting the production
SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with
batching negotiated, batch sizes read from the production batch_size histogram
via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old
shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON):
899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99
43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a
75-event burst never hits the 100 cap — zero drops, zero faults, offered load
100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire
path carries traffic and observes frames.
This commit is contained in:
Joseph Doherty
2026-08-18 10:32:46 -04:00
parent d3be5f0527
commit e9e44a9ef0
14 changed files with 3400 additions and 8 deletions
@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
@@ -43,7 +44,11 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
private readonly ILocalDb _localDb;
private readonly List<IActorRef> _instanceActors = new();
private readonly List<StreamSubscriberProbe> _probes = new();
private readonly List<WireStreamSubscriber> _wireSubscribers = new();
private readonly string _dataDirectory;
private SiteWireStreamHost? _wireHost;
private SiteStreamGrpcClient? _wireClient;
private CancellationTokenSource? _wireCts;
/// <summary>Site identifier, e.g. <c>site-01</c>.</summary>
public string SiteId { get; }
@@ -72,6 +77,12 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
/// <summary>The live stream subscribers attached to this site.</summary>
public IReadOnlyList<StreamSubscriberProbe> Probes => _probes;
/// <summary>The wire-path (real gRPC) subscribers attached to this site, if any.</summary>
public IReadOnlyList<WireStreamSubscriber> WireSubscribers => _wireSubscribers;
/// <summary>The site's real gRPC listener, when wire probes are attached.</summary>
public SiteWireStreamHost? WireHost => _wireHost;
/// <summary>Instance Actors created on this site.</summary>
public IReadOnlyList<IActorRef> InstanceActors => _instanceActors;
@@ -344,12 +355,63 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
}
}
/// <summary>
/// Starts the site's real Kestrel/gRPC listener and attaches
/// <paramref name="count"/> subscribers through the production
/// <see cref="SiteStreamGrpcClient"/> — one channel per site with the streams
/// multiplexed on it, exactly the production shape (central keys one client per
/// <c>(siteId, endpoint)</c>). Instance selection uses the same stride as
/// <see cref="AttachStreamProbes"/>, so the wire probes watch the SAME instances
/// as the in-process probes and the two latency distributions are directly
/// comparable within one run.
/// </summary>
/// <param name="count">Wire subscribers to attach.</param>
/// <param name="latency">Histogram that receives end-to-end (over-the-wire) latencies.</param>
/// <param name="log">Progress sink for fault reporting.</param>
/// <returns>
/// A task that completes when the listener is up and every stream has been
/// initiated. Stream ESTABLISHMENT finishes asynchronously — comfortably inside
/// the subscribe-settle + settle windows that separate attach from measurement.
/// </returns>
public async Task AttachWireStreamProbesAsync(int count, LatencyHistogram latency, Action<string> log)
{
_wireHost = await SiteWireStreamHost.StartAsync(this);
_wireClient = new SiteStreamGrpcClient(_wireHost.Endpoint, NullLogger.Instance);
_wireCts = new CancellationTokenSource();
var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, count));
for (var p = 0; p < count; p++)
{
var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1);
var subscriber = WireStreamSubscriber.Attach(
_wireClient, InstanceName(instanceIndex), $"wire-{SiteId}-p{p:D2}",
latency, log, _wireCts.Token);
_wireSubscribers.Add(subscriber);
}
}
/// <summary>Total tag paths registered across this site's connections.</summary>
public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count);
/// <inheritdoc />
public async ValueTask DisposeAsync()
{
// Wire path first: cancel the client streams, wait for their loops to exit,
// then drop the channel and the listener — the same order production teardown
// takes (client goes away, server cancels, actor system last).
if (_wireCts is not null)
{
await _wireCts.CancelAsync();
foreach (var subscriber in _wireSubscribers)
await subscriber.WaitForShutdownAsync();
_wireCts.Dispose();
}
if (_wireClient is not null)
await _wireClient.DisposeAsync();
if (_wireHost is not null)
await _wireHost.DisposeAsync();
foreach (var probe in _probes)
await probe.DisposeAsync();