test(loadharness)+docs: wire-path stream probes — R2 batching benchmark-proven at target scale
The WP-4 runs measured the stream path only up to the bounded send channel; the probe deliberately substituted the socket hop with a controllable reader, which is exactly where the R2 SiteStreamEventBatcher lives — so batching had never been in any benchmark's hot path. The harness now also attaches per-site wire probes: a real Kestrel h2c listener hosting the production SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with batching negotiated, batch sizes read from the production batch_size histogram via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON): 899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99 43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a 75-event burst never hits the 100 cap — zero drops, zero faults, offered load 100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire path carries traffic and observes frames.
This commit is contained in:
@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
|
||||
@@ -43,7 +44,11 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
|
||||
private readonly ILocalDb _localDb;
|
||||
private readonly List<IActorRef> _instanceActors = new();
|
||||
private readonly List<StreamSubscriberProbe> _probes = new();
|
||||
private readonly List<WireStreamSubscriber> _wireSubscribers = new();
|
||||
private readonly string _dataDirectory;
|
||||
private SiteWireStreamHost? _wireHost;
|
||||
private SiteStreamGrpcClient? _wireClient;
|
||||
private CancellationTokenSource? _wireCts;
|
||||
|
||||
/// <summary>Site identifier, e.g. <c>site-01</c>.</summary>
|
||||
public string SiteId { get; }
|
||||
@@ -72,6 +77,12 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
|
||||
/// <summary>The live stream subscribers attached to this site.</summary>
|
||||
public IReadOnlyList<StreamSubscriberProbe> Probes => _probes;
|
||||
|
||||
/// <summary>The wire-path (real gRPC) subscribers attached to this site, if any.</summary>
|
||||
public IReadOnlyList<WireStreamSubscriber> WireSubscribers => _wireSubscribers;
|
||||
|
||||
/// <summary>The site's real gRPC listener, when wire probes are attached.</summary>
|
||||
public SiteWireStreamHost? WireHost => _wireHost;
|
||||
|
||||
/// <summary>Instance Actors created on this site.</summary>
|
||||
public IReadOnlyList<IActorRef> InstanceActors => _instanceActors;
|
||||
|
||||
@@ -344,12 +355,63 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the site's real Kestrel/gRPC listener and attaches
|
||||
/// <paramref name="count"/> subscribers through the production
|
||||
/// <see cref="SiteStreamGrpcClient"/> — one channel per site with the streams
|
||||
/// multiplexed on it, exactly the production shape (central keys one client per
|
||||
/// <c>(siteId, endpoint)</c>). Instance selection uses the same stride as
|
||||
/// <see cref="AttachStreamProbes"/>, so the wire probes watch the SAME instances
|
||||
/// as the in-process probes and the two latency distributions are directly
|
||||
/// comparable within one run.
|
||||
/// </summary>
|
||||
/// <param name="count">Wire subscribers to attach.</param>
|
||||
/// <param name="latency">Histogram that receives end-to-end (over-the-wire) latencies.</param>
|
||||
/// <param name="log">Progress sink for fault reporting.</param>
|
||||
/// <returns>
|
||||
/// A task that completes when the listener is up and every stream has been
|
||||
/// initiated. Stream ESTABLISHMENT finishes asynchronously — comfortably inside
|
||||
/// the subscribe-settle + settle windows that separate attach from measurement.
|
||||
/// </returns>
|
||||
public async Task AttachWireStreamProbesAsync(int count, LatencyHistogram latency, Action<string> log)
|
||||
{
|
||||
_wireHost = await SiteWireStreamHost.StartAsync(this);
|
||||
_wireClient = new SiteStreamGrpcClient(_wireHost.Endpoint, NullLogger.Instance);
|
||||
_wireCts = new CancellationTokenSource();
|
||||
|
||||
var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, count));
|
||||
for (var p = 0; p < count; p++)
|
||||
{
|
||||
var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1);
|
||||
var subscriber = WireStreamSubscriber.Attach(
|
||||
_wireClient, InstanceName(instanceIndex), $"wire-{SiteId}-p{p:D2}",
|
||||
latency, log, _wireCts.Token);
|
||||
_wireSubscribers.Add(subscriber);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total tag paths registered across this site's connections.</summary>
|
||||
public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// Wire path first: cancel the client streams, wait for their loops to exit,
|
||||
// then drop the channel and the listener — the same order production teardown
|
||||
// takes (client goes away, server cancels, actor system last).
|
||||
if (_wireCts is not null)
|
||||
{
|
||||
await _wireCts.CancelAsync();
|
||||
foreach (var subscriber in _wireSubscribers)
|
||||
await subscriber.WaitForShutdownAsync();
|
||||
_wireCts.Dispose();
|
||||
}
|
||||
|
||||
if (_wireClient is not null)
|
||||
await _wireClient.DisposeAsync();
|
||||
if (_wireHost is not null)
|
||||
await _wireHost.DisposeAsync();
|
||||
|
||||
foreach (var probe in _probes)
|
||||
await probe.DisposeAsync();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user