Joseph Doherty
e9e44a9ef0
The WP-4 runs measured the stream path only up to the bounded send channel; the probe deliberately substituted the socket hop with a controllable reader, which is exactly where the R2 SiteStreamEventBatcher lives — so batching had never been in any benchmark's hot path. The harness now also attaches per-site wire probes: a real Kestrel h2c listener hosting the production SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with batching negotiated, batch sizes read from the production batch_size histogram via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON): 899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99 43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a 75-event burst never hits the 100 cap — zero drops, zero faults, offered load 100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire path carries traffic and observes frames.
98 lines
5.0 KiB
C#
98 lines
5.0 KiB
C#
using ZB.MOM.WW.ScadaBridge.LoadHarness;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.TargetScale;
|
|
|
|
/// <summary>
|
|
/// Keeps the target-scale load harness (deferred-work register #25 / Phase-8 WP-4)
|
|
/// honest at CI scale.
|
|
///
|
|
/// <para>
|
|
/// The full protocol — 10 sites x 500 instances x 75 tags sustained for 20 minutes —
|
|
/// deliberately lives in the standalone <c>ZB.MOM.WW.ScadaBridge.LoadHarness</c>
|
|
/// executable, NOT here: perf tests in this project run as part of an ordinary
|
|
/// <c>dotnet test ZB.MOM.WW.ScadaBridge.slnx</c> (the <c>Category=Performance</c>
|
|
/// trait enables a filter, it does not exclude by default), and a 20-minute test
|
|
/// would be intolerable there. What this test protects is that the harness still
|
|
/// compiles, wires up, and produces coherent measurements — so #25's evidence can be
|
|
/// regenerated on demand rather than bit-rotting.
|
|
/// </para>
|
|
/// <para>
|
|
/// Results doc: <c>docs/plans/2026-08-15-target-scale-load-test-results.md</c>.
|
|
/// Design memo: <c>docs/plans/2026-08-15-target-scale-load-test-design.md</c>.
|
|
/// </para>
|
|
/// </summary>
|
|
public class TargetScaleHarnessSmokeTests
|
|
{
|
|
/// <summary>
|
|
/// Runs the harness at ~1/1000th of target scale for a few seconds and asserts the
|
|
/// pipeline is intact end to end: tag updates reach live stream subscribers, the
|
|
/// central health aggregator tracks every site, debug snapshots answer, the
|
|
/// store-and-forward buffer drains to empty, and a stalled subscriber does not cost
|
|
/// the healthy ones any events.
|
|
/// </summary>
|
|
/// <returns>A task representing the test run.</returns>
|
|
[Trait("Category", "Performance")]
|
|
[Fact]
|
|
public async Task Harness_AtSmokeScale_ProducesCoherentMeasurements()
|
|
{
|
|
var config = new HarnessConfig
|
|
{
|
|
Sites = 2,
|
|
InstancesPerSite = 10,
|
|
TagsPerInstance = 5,
|
|
TagUpdatePeriod = TimeSpan.FromSeconds(1),
|
|
SettleDuration = TimeSpan.FromSeconds(5),
|
|
SustainDuration = TimeSpan.FromSeconds(15),
|
|
SampleInterval = TimeSpan.FromSeconds(2),
|
|
// Shortened from the production 30 s only because the smoke window is 20 s.
|
|
HealthReportInterval = TimeSpan.FromSeconds(2),
|
|
DebugProbeInterval = TimeSpan.FromSeconds(2),
|
|
SubscribeSettleDuration = TimeSpan.FromSeconds(5),
|
|
StreamProbesPerSite = 3,
|
|
WireProbesPerSite = 2,
|
|
StoreAndForwardDrainMessages = 200,
|
|
SlowSubscriberEvents = 2_000,
|
|
ResultsPath = Path.Combine(Path.GetTempPath(), $"loadharness-smoke-{Guid.NewGuid():N}.json"),
|
|
};
|
|
|
|
using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(10));
|
|
var result = await HarnessRun.ExecuteAsync(config, _ => { }, cancellation.Token);
|
|
|
|
// Tag updates flowed all the way through DCL -> InstanceActor -> site stream ->
|
|
// StreamRelayActor -> bounded channel -> subscriber.
|
|
Assert.True(result.TagUpdateLatency.Count > 0,
|
|
"No tag update latency samples — the DCL -> stream -> subscriber path did not carry traffic.");
|
|
Assert.True(result.StreamProbeReceived > 0, "Live stream subscribers received nothing.");
|
|
Assert.Equal(0, result.DriverSkippedNoCallback);
|
|
|
|
// The wire path (real Kestrel + real SiteStreamGrpcClient, R2 batching
|
|
// negotiated) carried traffic end to end: events crossed a real HTTP/2
|
|
// socket, the batch-size histogram observed real frames, and nothing faulted.
|
|
Assert.NotNull(result.WireStream);
|
|
Assert.True(result.WireStream!.EventsReceived > 0,
|
|
"Wire subscribers received nothing — the Kestrel/gRPC hop did not carry traffic.");
|
|
Assert.True(result.WireStream.Latency.Count > 0,
|
|
"No wire latency samples — the over-the-wire measurement path is broken.");
|
|
Assert.True(result.WireStream.Batches.Frames > 0,
|
|
"The batch-size MeterListener observed no frames — the R2 histogram path is broken.");
|
|
Assert.Equal(0, result.WireStream.StreamFaults);
|
|
|
|
// Observability held up.
|
|
Assert.Equal(config.Sites, result.SitesTrackedByAggregator);
|
|
Assert.True(result.DebugSnapshotsCompleted > 0, "No debug snapshot completed.");
|
|
Assert.Equal(0, result.DebugSnapshotTimeouts);
|
|
|
|
// Store-and-forward drained completely (register row 50, first half).
|
|
Assert.NotNull(result.StoreAndForwardDrain);
|
|
Assert.Equal(0, result.StoreAndForwardDrain!.ResidualDepth);
|
|
Assert.True(result.StoreAndForwardDrain.DrainPerSecond > 0);
|
|
|
|
// A stalled subscriber costs the healthy ones nothing (register row 50, second
|
|
// half). This is the design's isolation claim, asserted rather than assumed.
|
|
Assert.NotNull(result.SlowSubscriber);
|
|
Assert.Equal(1.0, result.SlowSubscriber!.HealthyMinDeliveryRatio, precision: 2);
|
|
Assert.True(result.SlowSubscriber.SlowDeliveryRatio < 1.0,
|
|
"The deliberately stalled subscriber kept up, so the probe proved nothing.");
|
|
}
|
|
}
|