test(loadharness): target-scale load harness for WP-4 / register #25 + row 50
Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,154 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>Result of one store-and-forward drain measurement.</summary>
|
||||
/// <param name="SiteId">Site the buffer belonged to.</param>
|
||||
/// <param name="MessageCount">Messages buffered before the drain began.</param>
|
||||
/// <param name="EnqueueSeconds">Wall time to buffer them (concurrent, many origin instances).</param>
|
||||
/// <param name="EnqueuePerSecond">Buffering throughput.</param>
|
||||
/// <param name="TimeToFirstDeliverySeconds">
|
||||
/// Wall time from the first sweep to the first successful delivery. With
|
||||
/// <c>attemptImmediateDelivery: false</c> the engine stamps <c>LastAttemptAt</c>, so the
|
||||
/// row is not due until one <c>DefaultRetryInterval</c> (30 s) has passed — this is the
|
||||
/// configured retry latency, not drain slowness, and is reported separately for that reason.
|
||||
/// </param>
|
||||
/// <param name="DrainSeconds">Wall time from the first sweep to an empty buffer (includes the retry wait).</param>
|
||||
/// <param name="DrainPerSecond">
|
||||
/// Drain throughput measured from the FIRST delivery to an empty buffer — the engine's
|
||||
/// actual capacity, and the headline number for register row 50.
|
||||
/// </param>
|
||||
/// <param name="ResidualDepth">Buffer depth left when the measurement stopped (0 = fully drained).</param>
|
||||
/// <param name="Progress">Delivered-count samples during the drain, so a steady rate can be told from a stall-then-burst.</param>
|
||||
public sealed record StoreAndForwardDrainResult(
|
||||
string SiteId,
|
||||
int MessageCount,
|
||||
double EnqueueSeconds,
|
||||
double EnqueuePerSecond,
|
||||
double TimeToFirstDeliverySeconds,
|
||||
double DrainSeconds,
|
||||
double DrainPerSecond,
|
||||
int ResidualDepth,
|
||||
IReadOnlyList<DrainProgressSample> Progress);
|
||||
|
||||
/// <summary>One observation during the drain.</summary>
|
||||
/// <param name="ElapsedSeconds">Seconds since the drain began.</param>
|
||||
/// <param name="Delivered">Cumulative successful deliveries.</param>
|
||||
/// <param name="Depth">Remaining buffer depth.</param>
|
||||
public sealed record DrainProgressSample(double ElapsedSeconds, long Delivered, int Depth);
|
||||
|
||||
/// <summary>
|
||||
/// Measures store-and-forward buffering and drain throughput (deferred-work register
|
||||
/// row 50, first half) using the real <c>StoreAndForwardService</c>, the real
|
||||
/// <c>StoreAndForwardStorage</c> and the real SQLite file — only the delivery target
|
||||
/// is a counting stub, because what is being measured is the site-local buffer's
|
||||
/// throughput, not a remote endpoint's.
|
||||
///
|
||||
/// <para>
|
||||
/// Phase 1 buffers <paramref name="messageCount"/> messages with
|
||||
/// <c>attemptImmediateDelivery: false</c>, spread across many origin instance names
|
||||
/// and issued from many concurrent tasks — the "concurrent buffering from multiple
|
||||
/// instances" WP-4 asks about (<c>[xc-7]</c>). Phase 2 registers a delivery handler
|
||||
/// that always succeeds and drives sweeps to completion, timing the drain.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The sweep is driven explicitly rather than waiting on the 10 s
|
||||
/// <c>RetryTimerInterval</c> so the number reported is the engine's drain capacity,
|
||||
/// not its polling cadence. The per-sweep batch is <c>SweepBatchLimit</c> (500) with
|
||||
/// <c>SweepTargetParallelism</c> (4) lanes, both at their production defaults.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class StoreAndForwardDrainScenario
|
||||
{
|
||||
/// <summary>Runs the drain measurement against one site's real S&F engine.</summary>
|
||||
/// <param name="site">The site whose store-and-forward engine is exercised.</param>
|
||||
/// <param name="messageCount">Messages to buffer.</param>
|
||||
/// <param name="concurrency">Concurrent enqueue tasks (distinct origin instances).</param>
|
||||
/// <param name="cancellationToken">Cancels the measurement.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<StoreAndForwardDrainResult> RunAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int messageCount,
|
||||
int concurrency,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var service = site.StoreAndForward;
|
||||
var payload = $"{{\"site\":\"{site.SiteId}\",\"body\":\"{new string('x', 256)}\"}}";
|
||||
|
||||
// Phase 1 — concurrent buffering from many instances, no delivery attempted.
|
||||
var enqueueWatch = Stopwatch.StartNew();
|
||||
var perTask = messageCount / concurrency;
|
||||
var enqueueTasks = new List<Task>(concurrency);
|
||||
for (var t = 0; t < concurrency; t++)
|
||||
{
|
||||
var taskIndex = t;
|
||||
enqueueTasks.Add(Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < perTask; i++)
|
||||
{
|
||||
await service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
target: $"load-target-{taskIndex % 4}",
|
||||
payloadJson: payload,
|
||||
originInstanceName: site.InstanceName(taskIndex),
|
||||
attemptImmediateDelivery: false);
|
||||
}
|
||||
}, cancellationToken));
|
||||
}
|
||||
|
||||
await Task.WhenAll(enqueueTasks);
|
||||
enqueueWatch.Stop();
|
||||
var buffered = perTask * concurrency;
|
||||
|
||||
// Phase 2 — a delivery target that always succeeds; time the drain to empty.
|
||||
var delivered = 0L;
|
||||
service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ =>
|
||||
{
|
||||
Interlocked.Increment(ref delivered);
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
// Sweeps are driven explicitly rather than waiting on the 10 s RetryTimerInterval:
|
||||
// the number wanted is the engine's drain CAPACITY, not its polling cadence. One
|
||||
// sweep moves at most SweepBatchLimit (500) messages, so a large backlog needs
|
||||
// many, and the progress series below is what distinguishes a genuinely slow
|
||||
// drain from an artefact of this polling loop.
|
||||
var drainWatch = Stopwatch.StartNew();
|
||||
var deadline = DateTimeOffset.UtcNow.AddMinutes(10);
|
||||
var progress = new List<DrainProgressSample>();
|
||||
int depth;
|
||||
while (true)
|
||||
{
|
||||
service.TriggerSweep();
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
|
||||
|
||||
var depths = await service.GetBufferDepthAsync();
|
||||
depth = depths.Values.Sum();
|
||||
progress.Add(new DrainProgressSample(
|
||||
drainWatch.Elapsed.TotalSeconds, Interlocked.Read(ref delivered), depth));
|
||||
|
||||
if (depth == 0 || DateTimeOffset.UtcNow > deadline)
|
||||
break;
|
||||
}
|
||||
|
||||
drainWatch.Stop();
|
||||
|
||||
// Split the retry wait from the drain: the first sample with a non-zero delivered
|
||||
// count marks the moment the backlog actually became due.
|
||||
var firstDelivery = progress.FirstOrDefault(s => s.Delivered > 0);
|
||||
var timeToFirstDelivery = firstDelivery?.ElapsedSeconds ?? drainWatch.Elapsed.TotalSeconds;
|
||||
var activeDrainSeconds = Math.Max(0.001, drainWatch.Elapsed.TotalSeconds - timeToFirstDelivery);
|
||||
|
||||
return new StoreAndForwardDrainResult(
|
||||
SiteId: site.SiteId,
|
||||
MessageCount: buffered,
|
||||
EnqueueSeconds: enqueueWatch.Elapsed.TotalSeconds,
|
||||
EnqueuePerSecond: buffered / Math.Max(0.001, enqueueWatch.Elapsed.TotalSeconds),
|
||||
TimeToFirstDeliverySeconds: timeToFirstDelivery,
|
||||
DrainSeconds: drainWatch.Elapsed.TotalSeconds,
|
||||
DrainPerSecond: Interlocked.Read(ref delivered) / activeDrainSeconds,
|
||||
ResidualDepth: depth,
|
||||
Progress: progress);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user