test(loadharness): target-scale load harness for WP-4 / register #25 + row 50
Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,144 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// Scale and duration knobs for a load-harness run. Defaults are the Phase-8 WP-4
|
||||
/// target scale (10 sites x 500 instances x 75 tags = 375,000 subscriptions).
|
||||
/// Every value is overridable from the command line so the same binary serves both
|
||||
/// the full-scale protocol run and the scaled-down CI smoke.
|
||||
/// </summary>
|
||||
public sealed record HarnessConfig
|
||||
{
|
||||
/// <summary>Number of simulated sites (WP-4 acceptance criterion <c>[2.5-1]</c>).</summary>
|
||||
public int Sites { get; init; } = 10;
|
||||
|
||||
/// <summary>Instance Actors per site (WP-4 <c>[2.5-2]</c>).</summary>
|
||||
public int InstancesPerSite { get; init; } = 500;
|
||||
|
||||
/// <summary>Data-sourced attributes ("live tags") per instance (WP-4 <c>[2.5-3]</c>).</summary>
|
||||
public int TagsPerInstance { get; init; } = 75;
|
||||
|
||||
/// <summary>
|
||||
/// Nominal per-tag update period. The driver emits each tag once per period, so
|
||||
/// the site-wide event rate is <c>InstancesPerSite * TagsPerInstance / period</c>.
|
||||
/// The 10 s default puts the target-scale fleet at 37,500 tag updates/second.
|
||||
/// </summary>
|
||||
public TimeSpan TagUpdatePeriod { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Length of the steady-state measurement window, measured AFTER the ramp
|
||||
/// completes and after the post-ramp settle. Memory-growth and CPU figures are
|
||||
/// computed over exactly this window.
|
||||
/// </summary>
|
||||
public TimeSpan SustainDuration { get; init; } = TimeSpan.FromMinutes(20);
|
||||
|
||||
/// <summary>
|
||||
/// Quiet period between the last instance starting and the start of the
|
||||
/// measurement window — lets startup allocations settle so the memory-growth
|
||||
/// slope reflects steady state, not the ramp.
|
||||
/// </summary>
|
||||
public TimeSpan SettleDuration { get; init; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Resource-sampling cadence (working set, GC heap, CPU, thread count).</summary>
|
||||
public TimeSpan SampleInterval { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Health report cadence. Defaults to the production
|
||||
/// <c>HealthMonitoringOptions.ReportInterval</c> (30 s) so the timing measured is
|
||||
/// the one that ships; only the CI smoke shortens it, because a 20-second smoke
|
||||
/// window would otherwise never see a single tick.
|
||||
/// </summary>
|
||||
public TimeSpan HealthReportInterval { get; init; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>How often to take a debug view snapshot of a random live instance.</summary>
|
||||
public TimeSpan DebugProbeInterval { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Pause between the last Instance Actor starting and the tag driver starting, to
|
||||
/// let every instance complete its DCL subscribe round-trip. The adapter callback
|
||||
/// is captured at subscribe time, so an emit before that lands is silently
|
||||
/// discarded — this window is what keeps <c>SkippedNoCallback</c> at zero.
|
||||
/// </summary>
|
||||
public TimeSpan SubscribeSettleDuration { get; init; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Instances per site that carry a live stream subscriber (a real
|
||||
/// <c>StreamRelayActor</c> + bounded DropOldest channel, i.e. the production
|
||||
/// Debug View / central shape). Every subscriber's stream graph sees the FULL
|
||||
/// site event flow and filters it, so this is the fan-out multiplier.
|
||||
/// </summary>
|
||||
public int StreamProbesPerSite { get; init; } = 10;
|
||||
|
||||
/// <summary>Store-and-forward messages enqueued for the drain-rate measurement (register row 50).</summary>
|
||||
public int StoreAndForwardDrainMessages { get; init; } = 20_000;
|
||||
|
||||
/// <summary>Events published at the slow-subscriber isolation probe (register row 50).</summary>
|
||||
public int SlowSubscriberEvents { get; init; } = 200_000;
|
||||
|
||||
/// <summary>Directory for the site SQLite files. A temp directory is used when null.</summary>
|
||||
public string? DataDirectory { get; init; }
|
||||
|
||||
/// <summary>Path the JSON metrics document is written to.</summary>
|
||||
public string ResultsPath { get; init; } = "loadharness-results.json";
|
||||
|
||||
/// <summary>Total live tag subscriptions across the fleet.</summary>
|
||||
public int TotalSubscriptions => Sites * InstancesPerSite * TagsPerInstance;
|
||||
|
||||
/// <summary>Nominal fleet-wide tag updates per second implied by the scale and update period.</summary>
|
||||
public double NominalUpdatesPerSecond => TotalSubscriptions / TagUpdatePeriod.TotalSeconds;
|
||||
|
||||
/// <summary>
|
||||
/// Parses <c>--key value</c> / <c>--key=value</c> arguments over the defaults.
|
||||
/// Unknown keys throw so a typo in a 20-minute run's command line fails fast
|
||||
/// rather than silently measuring the wrong scale.
|
||||
/// </summary>
|
||||
/// <param name="args">Raw command-line arguments.</param>
|
||||
/// <returns>The parsed configuration.</returns>
|
||||
public static HarnessConfig Parse(string[] args)
|
||||
{
|
||||
var config = new HarnessConfig();
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
if (!arg.StartsWith("--", StringComparison.Ordinal))
|
||||
throw new ArgumentException($"Unexpected argument '{arg}' (expected --key value).");
|
||||
|
||||
string key;
|
||||
string value;
|
||||
var eq = arg.IndexOf('=', StringComparison.Ordinal);
|
||||
if (eq >= 0)
|
||||
{
|
||||
key = arg[2..eq];
|
||||
value = arg[(eq + 1)..];
|
||||
}
|
||||
else
|
||||
{
|
||||
key = arg[2..];
|
||||
if (i + 1 >= args.Length)
|
||||
throw new ArgumentException($"Option '--{key}' requires a value.");
|
||||
value = args[++i];
|
||||
}
|
||||
|
||||
config = key switch
|
||||
{
|
||||
"sites" => config with { Sites = int.Parse(value) },
|
||||
"instances-per-site" => config with { InstancesPerSite = int.Parse(value) },
|
||||
"tags-per-instance" => config with { TagsPerInstance = int.Parse(value) },
|
||||
"tag-update-period-seconds" => config with { TagUpdatePeriod = TimeSpan.FromSeconds(double.Parse(value)) },
|
||||
"sustain-minutes" => config with { SustainDuration = TimeSpan.FromMinutes(double.Parse(value)) },
|
||||
"settle-minutes" => config with { SettleDuration = TimeSpan.FromMinutes(double.Parse(value)) },
|
||||
"subscribe-settle-seconds" => config with { SubscribeSettleDuration = TimeSpan.FromSeconds(double.Parse(value)) },
|
||||
"sample-seconds" => config with { SampleInterval = TimeSpan.FromSeconds(double.Parse(value)) },
|
||||
"health-interval-seconds" => config with { HealthReportInterval = TimeSpan.FromSeconds(double.Parse(value)) },
|
||||
"debug-probe-interval-seconds" => config with { DebugProbeInterval = TimeSpan.FromSeconds(double.Parse(value)) },
|
||||
"stream-probes-per-site" => config with { StreamProbesPerSite = int.Parse(value) },
|
||||
"sf-drain-messages" => config with { StoreAndForwardDrainMessages = int.Parse(value) },
|
||||
"slow-subscriber-events" => config with { SlowSubscriberEvents = int.Parse(value) },
|
||||
"data-dir" => config with { DataDirectory = value },
|
||||
"results" => config with { ResultsPath = value },
|
||||
_ => throw new ArgumentException($"Unknown option '--{key}'."),
|
||||
};
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>Everything one harness run measured, ready for serialization.</summary>
|
||||
/// <param name="Config">The configuration the run executed under.</param>
|
||||
/// <param name="Environment">Host/runtime description.</param>
|
||||
/// <param name="StartedUtc">Run start.</param>
|
||||
/// <param name="TotalSeconds">Total wall time including ramp and teardown.</param>
|
||||
/// <param name="SiteRampSeconds">Wall time to build all site fixtures (before instances).</param>
|
||||
/// <param name="InstanceRampSeconds">Wall time to create every Instance Actor across all sites.</param>
|
||||
/// <param name="SlowestSiteInstanceRampSeconds">Slowest single site's instance ramp — the "deploy 500 instances to a site" figure.</param>
|
||||
/// <param name="TagUpdateLatency">End-to-end DCL-boundary to stream-subscriber latency.</param>
|
||||
/// <param name="EmittedTagUpdates">Tag updates offered during the whole run.</param>
|
||||
/// <param name="SteadyStateEmittedTagUpdates">Tag updates offered during the measurement window only.</param>
|
||||
/// <param name="AchievedUpdatesPerSecond">Offered load actually achieved in the measurement window.</param>
|
||||
/// <param name="NominalUpdatesPerSecond">Offered load the configuration called for.</param>
|
||||
/// <param name="DriverLagSeconds">Cumulative driver slice overrun (harness-bound load shortfall).</param>
|
||||
/// <param name="DriverSkippedNoCallback">Emissions skipped before subscriptions existed.</param>
|
||||
/// <param name="SteadyStateResources">Resource behaviour over the measurement window.</param>
|
||||
/// <param name="WholeRunResources">Resource behaviour over the whole run.</param>
|
||||
/// <param name="HealthReportLatency">Health report collect+ingest latency.</param>
|
||||
/// <param name="HealthReportsDelivered">Health reports ingested by the central aggregator.</param>
|
||||
/// <param name="SitesTrackedByAggregator">Sites the central aggregator ended up tracking.</param>
|
||||
/// <param name="DebugSnapshotLatency">Debug view snapshot round-trip latency under load.</param>
|
||||
/// <param name="DebugSnapshotsCompleted">Debug snapshots that completed.</param>
|
||||
/// <param name="DebugSnapshotTimeouts">Debug snapshots that timed out.</param>
|
||||
/// <param name="StreamProbeReceived">Events delivered to live stream subscribers.</param>
|
||||
/// <param name="StreamProbeDropped">Events evicted by live subscribers' bounded channels.</param>
|
||||
/// <param name="StoreAndForwardDrain">Store-and-forward drain measurement (register row 50).</param>
|
||||
/// <param name="SlowSubscriber">Slow-subscriber isolation measurement (register row 50).</param>
|
||||
public sealed record HarnessRunResult(
|
||||
HarnessConfig Config,
|
||||
EnvironmentInfo Environment,
|
||||
DateTimeOffset StartedUtc,
|
||||
double TotalSeconds,
|
||||
double SiteRampSeconds,
|
||||
double InstanceRampSeconds,
|
||||
double SlowestSiteInstanceRampSeconds,
|
||||
LatencySnapshot TagUpdateLatency,
|
||||
long EmittedTagUpdates,
|
||||
long SteadyStateEmittedTagUpdates,
|
||||
double AchievedUpdatesPerSecond,
|
||||
double NominalUpdatesPerSecond,
|
||||
double DriverLagSeconds,
|
||||
long DriverSkippedNoCallback,
|
||||
ResourceWindowSummary? SteadyStateResources,
|
||||
ResourceWindowSummary? WholeRunResources,
|
||||
LatencySnapshot HealthReportLatency,
|
||||
long HealthReportsDelivered,
|
||||
int SitesTrackedByAggregator,
|
||||
LatencySnapshot DebugSnapshotLatency,
|
||||
long DebugSnapshotsCompleted,
|
||||
long DebugSnapshotTimeouts,
|
||||
long StreamProbeReceived,
|
||||
long StreamProbeDropped,
|
||||
StoreAndForwardDrainResult? StoreAndForwardDrain,
|
||||
SlowSubscriberResult? SlowSubscriber);
|
||||
|
||||
/// <summary>Host and runtime facts recorded alongside the numbers.</summary>
|
||||
/// <param name="MachineName">Host name.</param>
|
||||
/// <param name="OsDescription">Operating system description.</param>
|
||||
/// <param name="ProcessorCount">Logical processors visible to the process.</param>
|
||||
/// <param name="RuntimeVersion">.NET runtime version.</param>
|
||||
/// <param name="ServerGc">Whether server GC is active.</param>
|
||||
public sealed record EnvironmentInfo(
|
||||
string MachineName,
|
||||
string OsDescription,
|
||||
int ProcessorCount,
|
||||
string RuntimeVersion,
|
||||
bool ServerGc);
|
||||
|
||||
/// <summary>
|
||||
/// Orchestrates a full run: build sites, ramp instances, attach subscribers, drive
|
||||
/// tag updates for the sustained window while sampling resources and probing
|
||||
/// observability, then run the two register-row-50 scenarios on a dedicated site.
|
||||
/// </summary>
|
||||
public static class HarnessRun
|
||||
{
|
||||
/// <summary>Data connections each site spreads its tags across.</summary>
|
||||
public const int ConnectionsPerSite = 5;
|
||||
|
||||
/// <summary>Subscribers attached in the slow-subscriber isolation probe.</summary>
|
||||
public const int SlowSubscriberProbeCount = 5;
|
||||
|
||||
/// <summary>Concurrent enqueue tasks in the store-and-forward drain probe.</summary>
|
||||
public const int StoreAndForwardConcurrency = 25;
|
||||
|
||||
/// <summary>Executes a run end to end.</summary>
|
||||
/// <param name="config">Scale and duration configuration.</param>
|
||||
/// <param name="log">Progress sink (stdout in the console app).</param>
|
||||
/// <param name="cancellationToken">Cancels the run.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<HarnessRunResult> ExecuteAsync(
|
||||
HarnessConfig config, Action<string> log, CancellationToken cancellationToken)
|
||||
{
|
||||
var startedUtc = DateTimeOffset.UtcNow;
|
||||
var totalWatch = Stopwatch.StartNew();
|
||||
var dataRoot = config.DataDirectory
|
||||
?? Path.Combine(Path.GetTempPath(), $"scadabridge-loadharness-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(dataRoot);
|
||||
|
||||
var sampler = ResourceSampler.Start(config.SampleInterval);
|
||||
var sites = new List<SiteRuntimeFixture>(config.Sites);
|
||||
var latency = new LatencyHistogram();
|
||||
|
||||
TagUpdateDriver? driver = null;
|
||||
ObservabilityProbes? probes = null;
|
||||
StoreAndForwardDrainResult? drainResult = null;
|
||||
SlowSubscriberResult? slowResult = null;
|
||||
|
||||
var aggregator = new CentralHealthAggregator(
|
||||
Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
ReportInterval = config.HealthReportInterval,
|
||||
OfflineTimeout = config.HealthReportInterval * 2,
|
||||
}),
|
||||
NullLogger<CentralHealthAggregator>.Instance);
|
||||
|
||||
try
|
||||
{
|
||||
log($"Building {config.Sites} sites ({config.InstancesPerSite} instances x " +
|
||||
$"{config.TagsPerInstance} tags each = {config.TotalSubscriptions:N0} subscriptions)...");
|
||||
|
||||
var siteWatch = Stopwatch.StartNew();
|
||||
for (var s = 0; s < config.Sites; s++)
|
||||
sites.Add(await SiteRuntimeFixture.CreateAsync(s, config, dataRoot, ConnectionsPerSite));
|
||||
siteWatch.Stop();
|
||||
log($" sites built in {siteWatch.Elapsed.TotalSeconds:F1}s");
|
||||
|
||||
// Instance ramp — sites in parallel (they are independent actor systems,
|
||||
// exactly as 10 real sites would be), each site internally staggered at the
|
||||
// production StartupBatchSize/StartupBatchDelayMs pacing.
|
||||
var rampWatch = Stopwatch.StartNew();
|
||||
await Task.WhenAll(sites.Select(site => site.StartInstancesAsync(cancellationToken)));
|
||||
rampWatch.Stop();
|
||||
var slowestSiteRamp = sites.Max(s => s.InstanceRampDuration.TotalSeconds);
|
||||
log($" {config.Sites * config.InstancesPerSite:N0} instance actors created in " +
|
||||
$"{rampWatch.Elapsed.TotalSeconds:F1}s (slowest site {slowestSiteRamp:F1}s)");
|
||||
|
||||
foreach (var site in sites)
|
||||
site.AttachStreamProbes(latency);
|
||||
log($" {sites.Sum(s => s.Probes.Count)} live stream subscribers attached");
|
||||
|
||||
// Let Instance Actors complete their DCL subscribe round-trips before the
|
||||
// driver starts; an emit before SubscribeBatchAsync has captured the
|
||||
// callback would be silently discarded.
|
||||
await Task.Delay(config.SubscribeSettleDuration, cancellationToken);
|
||||
|
||||
driver = TagUpdateDriver.Start(sites, config);
|
||||
log($" tag driver started, nominal {config.NominalUpdatesPerSecond:N0} updates/s");
|
||||
|
||||
probes = ObservabilityProbes.Start(
|
||||
sites, aggregator, config.HealthReportInterval, config.DebugProbeInterval);
|
||||
|
||||
log($"Settling for {config.SettleDuration.TotalMinutes:F1} min...");
|
||||
await Task.Delay(config.SettleDuration, cancellationToken);
|
||||
|
||||
// Switch to a fresh latency histogram so ramp-window outliers do not
|
||||
// contaminate the steady-state percentiles. Everything reported as "steady
|
||||
// state" is measured strictly after this point; the subscriptions
|
||||
// themselves are left untouched.
|
||||
var steadyStateStartSeconds = sampler.Snapshot().LastOrDefault()?.ElapsedSeconds ?? 0;
|
||||
var emittedAtWindowStart = driver.EmittedCount;
|
||||
var receivedAtWindowStart = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents));
|
||||
var droppedAtWindowStart = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents));
|
||||
|
||||
var steadyLatency = new LatencyHistogram();
|
||||
foreach (var site in sites)
|
||||
{
|
||||
foreach (var probe in site.Probes)
|
||||
probe.RetargetLatency(steadyLatency);
|
||||
}
|
||||
|
||||
var windowWatch = Stopwatch.StartNew();
|
||||
log($"Sustained measurement window: {config.SustainDuration.TotalMinutes:F1} min...");
|
||||
await Task.Delay(config.SustainDuration, cancellationToken);
|
||||
windowWatch.Stop();
|
||||
|
||||
var steadyStateEndSeconds = sampler.Snapshot().LastOrDefault()?.ElapsedSeconds ?? 0;
|
||||
var emittedInWindow = driver.EmittedCount - emittedAtWindowStart;
|
||||
|
||||
var steadyResources = sampler.Summarize(steadyStateStartSeconds, steadyStateEndSeconds);
|
||||
log($" window complete: {emittedInWindow:N0} updates offered, " +
|
||||
$"{emittedInWindow / windowWatch.Elapsed.TotalSeconds:N0}/s achieved");
|
||||
|
||||
var probeReceived = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents)) - receivedAtWindowStart;
|
||||
var probeDropped = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents)) - droppedAtWindowStart;
|
||||
|
||||
// Register row 50 — measured after the sustained window so the numbers are
|
||||
// not competing with the full tag load for CPU, and reported separately for
|
||||
// the same reason.
|
||||
log("Stopping tag driver for the register row 50 scenarios...");
|
||||
var driverLagSeconds = driver.EmitLagSeconds;
|
||||
var driverSkipped = driver.SkippedNoCallback;
|
||||
await driver.DisposeAsync();
|
||||
driver = null;
|
||||
await probes.DisposeAsync();
|
||||
|
||||
log($"Store-and-forward drain: {config.StoreAndForwardDrainMessages:N0} messages...");
|
||||
drainResult = await StoreAndForwardDrainScenario.RunAsync(
|
||||
sites[0], config.StoreAndForwardDrainMessages, StoreAndForwardConcurrency, cancellationToken);
|
||||
log($" buffered {drainResult.EnqueuePerSecond:N0}/s, drained {drainResult.DrainPerSecond:N0}/s");
|
||||
|
||||
log($"Slow-subscriber isolation: {SlowSubscriberProbeCount} subscribers, " +
|
||||
$"{config.SlowSubscriberEvents:N0} events...");
|
||||
slowResult = await SlowSubscriberScenario.RunAsync(
|
||||
sites[1 % sites.Count], SlowSubscriberProbeCount, config.SlowSubscriberEvents, cancellationToken);
|
||||
log($" healthy min delivery {slowResult.HealthyMinDeliveryRatio:P2}, " +
|
||||
$"stalled {slowResult.SlowDeliveryRatio:P2}");
|
||||
|
||||
totalWatch.Stop();
|
||||
|
||||
return new HarnessRunResult(
|
||||
Config: config,
|
||||
Environment: CaptureEnvironment(),
|
||||
StartedUtc: startedUtc,
|
||||
TotalSeconds: totalWatch.Elapsed.TotalSeconds,
|
||||
SiteRampSeconds: siteWatch.Elapsed.TotalSeconds,
|
||||
InstanceRampSeconds: rampWatch.Elapsed.TotalSeconds,
|
||||
SlowestSiteInstanceRampSeconds: slowestSiteRamp,
|
||||
TagUpdateLatency: steadyLatency.Snapshot(),
|
||||
EmittedTagUpdates: emittedAtWindowStart + emittedInWindow,
|
||||
SteadyStateEmittedTagUpdates: emittedInWindow,
|
||||
AchievedUpdatesPerSecond: emittedInWindow / windowWatch.Elapsed.TotalSeconds,
|
||||
NominalUpdatesPerSecond: config.NominalUpdatesPerSecond,
|
||||
DriverLagSeconds: driverLagSeconds,
|
||||
DriverSkippedNoCallback: driverSkipped,
|
||||
SteadyStateResources: steadyResources,
|
||||
WholeRunResources: sampler.Summarize(0, double.MaxValue),
|
||||
HealthReportLatency: probes.HealthReportLatency.Snapshot(),
|
||||
HealthReportsDelivered: probes.HealthReportsDelivered,
|
||||
SitesTrackedByAggregator: aggregator.GetAllSiteStates().Count,
|
||||
DebugSnapshotLatency: probes.DebugSnapshotLatency.Snapshot(),
|
||||
DebugSnapshotsCompleted: probes.DebugSnapshotsCompleted,
|
||||
DebugSnapshotTimeouts: probes.DebugSnapshotTimeouts,
|
||||
StreamProbeReceived: probeReceived,
|
||||
StreamProbeDropped: probeDropped,
|
||||
StoreAndForwardDrain: drainResult,
|
||||
SlowSubscriber: slowResult);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (driver != null) await driver.DisposeAsync();
|
||||
if (probes != null) await probes.DisposeAsync();
|
||||
await sampler.DisposeAsync();
|
||||
|
||||
log("Tearing down sites...");
|
||||
foreach (var site in sites)
|
||||
await site.DisposeAsync();
|
||||
|
||||
try
|
||||
{
|
||||
if (config.DataDirectory == null && Directory.Exists(dataRoot))
|
||||
Directory.Delete(dataRoot, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Temp cleanup only.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static EnvironmentInfo CaptureEnvironment() => new(
|
||||
System.Environment.MachineName,
|
||||
System.Runtime.InteropServices.RuntimeInformation.OSDescription,
|
||||
System.Environment.ProcessorCount,
|
||||
System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription,
|
||||
System.Runtime.GCSettings.IsServerGC);
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// Lock-free logarithmic latency histogram sized for tens of thousands of samples
|
||||
/// per second across many threads.
|
||||
///
|
||||
/// <para>
|
||||
/// Buckets are 16-per-octave over microseconds, i.e. bucket <c>i</c> covers
|
||||
/// <c>[2^(i/16), 2^((i+1)/16))</c> µs. That bounds relative bucket width at
|
||||
/// <c>2^(1/16) - 1 ≈ 4.4%</c>, so a reported percentile is within ~4.4% of the true
|
||||
/// value — ample for the millisecond-scale thresholds this harness asserts, and far
|
||||
/// cheaper than retaining 45 million raw samples.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Recording is a <see cref="Math.Log2(double)"/> plus one
|
||||
/// <see cref="Interlocked.Increment(ref long)"/>; there is no allocation on the hot path.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class LatencyHistogram
|
||||
{
|
||||
private const int SubBucketsPerOctave = 16;
|
||||
private const int BucketCount = 64 * SubBucketsPerOctave;
|
||||
|
||||
private readonly long[] _buckets = new long[BucketCount];
|
||||
private long _count;
|
||||
private long _totalMicroseconds;
|
||||
private long _maxMicroseconds;
|
||||
|
||||
/// <summary>Number of samples recorded.</summary>
|
||||
public long Count => Interlocked.Read(ref _count);
|
||||
|
||||
/// <summary>Largest sample seen, in microseconds (exact — not bucketed).</summary>
|
||||
public double MaxMs => Interlocked.Read(ref _maxMicroseconds) / 1000.0;
|
||||
|
||||
/// <summary>Arithmetic mean in milliseconds (exact — accumulated, not bucketed).</summary>
|
||||
public double MeanMs
|
||||
{
|
||||
get
|
||||
{
|
||||
var count = Interlocked.Read(ref _count);
|
||||
return count == 0 ? 0 : Interlocked.Read(ref _totalMicroseconds) / 1000.0 / count;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records one sample. Negative durations (clock skew across the emit/receive
|
||||
/// boundary) are clamped to zero rather than discarded, so the sample count stays
|
||||
/// an honest denominator.
|
||||
/// </summary>
|
||||
/// <param name="elapsed">The measured latency.</param>
|
||||
public void Record(TimeSpan elapsed)
|
||||
{
|
||||
var micros = (long)(elapsed.TotalMilliseconds * 1000.0);
|
||||
if (micros < 0) micros = 0;
|
||||
|
||||
Interlocked.Increment(ref _count);
|
||||
Interlocked.Add(ref _totalMicroseconds, micros);
|
||||
|
||||
long observedMax;
|
||||
while (micros > (observedMax = Interlocked.Read(ref _maxMicroseconds)))
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _maxMicroseconds, micros, observedMax) == observedMax)
|
||||
break;
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _buckets[BucketIndex(micros)]);
|
||||
}
|
||||
|
||||
private static int BucketIndex(long micros)
|
||||
{
|
||||
if (micros <= 0) return 0;
|
||||
var index = (int)(Math.Log2(micros) * SubBucketsPerOctave);
|
||||
if (index < 0) return 0;
|
||||
return index >= BucketCount ? BucketCount - 1 : index;
|
||||
}
|
||||
|
||||
/// <summary>Bucket midpoint in milliseconds, used when reconstructing a percentile.</summary>
|
||||
private static double BucketMidpointMs(int index)
|
||||
{
|
||||
var low = Math.Pow(2, (double)index / SubBucketsPerOctave);
|
||||
var high = Math.Pow(2, (double)(index + 1) / SubBucketsPerOctave);
|
||||
return (low + high) / 2.0 / 1000.0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the requested percentile in milliseconds, or 0 when no samples were recorded.
|
||||
/// </summary>
|
||||
/// <param name="percentile">Percentile in the range 0..100 (e.g. 99 for P99).</param>
|
||||
/// <returns>The percentile value in milliseconds.</returns>
|
||||
public double PercentileMs(double percentile)
|
||||
{
|
||||
var total = Interlocked.Read(ref _count);
|
||||
if (total == 0) return 0;
|
||||
|
||||
var target = (long)Math.Ceiling(total * percentile / 100.0);
|
||||
if (target < 1) target = 1;
|
||||
|
||||
long cumulative = 0;
|
||||
for (var i = 0; i < BucketCount; i++)
|
||||
{
|
||||
cumulative += Interlocked.Read(ref _buckets[i]);
|
||||
if (cumulative >= target)
|
||||
return BucketMidpointMs(i);
|
||||
}
|
||||
|
||||
return MaxMs;
|
||||
}
|
||||
|
||||
/// <summary>Materializes the standard percentile set plus mean/max/count for reporting.</summary>
|
||||
/// <returns>A snapshot record of this histogram.</returns>
|
||||
public LatencySnapshot Snapshot() => new(
|
||||
Count,
|
||||
MeanMs,
|
||||
PercentileMs(50),
|
||||
PercentileMs(95),
|
||||
PercentileMs(99),
|
||||
PercentileMs(99.9),
|
||||
MaxMs);
|
||||
}
|
||||
|
||||
/// <summary>Point-in-time summary of a <see cref="LatencyHistogram"/>. All times in milliseconds.</summary>
|
||||
/// <param name="Count">Samples recorded.</param>
|
||||
/// <param name="MeanMs">Arithmetic mean.</param>
|
||||
/// <param name="P50Ms">Median.</param>
|
||||
/// <param name="P95Ms">95th percentile.</param>
|
||||
/// <param name="P99Ms">99th percentile.</param>
|
||||
/// <param name="P999Ms">99.9th percentile.</param>
|
||||
/// <param name="MaxMs">Largest observed sample.</param>
|
||||
public sealed record LatencySnapshot(
|
||||
long Count,
|
||||
double MeanMs,
|
||||
double P50Ms,
|
||||
double P95Ms,
|
||||
double P99Ms,
|
||||
double P999Ms,
|
||||
double MaxMs);
|
||||
@@ -0,0 +1,202 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// One periodic observation of process resource usage.
|
||||
/// </summary>
|
||||
/// <param name="ElapsedSeconds">Seconds since sampling started.</param>
|
||||
/// <param name="WorkingSetMb">Process working set (RSS).</param>
|
||||
/// <param name="ManagedHeapMb"><see cref="GC.GetTotalMemory(bool)"/> without forcing a collection.</param>
|
||||
/// <param name="CpuPercent">Mean CPU utilization since the previous sample, as a percentage of ONE core.</param>
|
||||
/// <param name="ThreadCount">OS threads in the process.</param>
|
||||
/// <param name="Gen2Collections">Cumulative gen-2 collections.</param>
|
||||
public sealed record ResourceSample(
|
||||
double ElapsedSeconds,
|
||||
double WorkingSetMb,
|
||||
double ManagedHeapMb,
|
||||
double CpuPercent,
|
||||
int ThreadCount,
|
||||
int Gen2Collections);
|
||||
|
||||
/// <summary>
|
||||
/// Samples working set, managed heap, CPU and thread count on a fixed cadence for the
|
||||
/// life of a run. CPU is differential (processor time delta / wall delta) so a sample
|
||||
/// reflects the interval it covers rather than the whole process lifetime.
|
||||
/// </summary>
|
||||
public sealed class ResourceSampler : IAsyncDisposable
|
||||
{
|
||||
private readonly List<ResourceSample> _samples = new();
|
||||
private readonly object _lock = new();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly Task _loop;
|
||||
private readonly Stopwatch _wall = Stopwatch.StartNew();
|
||||
|
||||
private TimeSpan _lastCpu;
|
||||
private double _lastElapsedSeconds;
|
||||
|
||||
private ResourceSampler(TimeSpan interval)
|
||||
{
|
||||
_lastCpu = Process.GetCurrentProcess().TotalProcessorTime;
|
||||
_loop = Task.Run(() => SampleLoopAsync(interval, _cts.Token));
|
||||
}
|
||||
|
||||
/// <summary>Starts sampling at the given cadence.</summary>
|
||||
/// <param name="interval">Sampling interval.</param>
|
||||
/// <returns>The running sampler.</returns>
|
||||
public static ResourceSampler Start(TimeSpan interval) => new(interval);
|
||||
|
||||
/// <summary>All samples collected so far, oldest first.</summary>
|
||||
/// <returns>A snapshot copy of the sample list.</returns>
|
||||
public IReadOnlyList<ResourceSample> Snapshot()
|
||||
{
|
||||
lock (_lock) return _samples.ToList();
|
||||
}
|
||||
|
||||
private async Task SampleLoopAsync(TimeSpan interval, CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
Capture();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private void Capture()
|
||||
{
|
||||
using var process = Process.GetCurrentProcess();
|
||||
process.Refresh();
|
||||
|
||||
var elapsedSeconds = _wall.Elapsed.TotalSeconds;
|
||||
var cpu = process.TotalProcessorTime;
|
||||
var wallDelta = elapsedSeconds - _lastElapsedSeconds;
|
||||
var cpuPercent = wallDelta > 0
|
||||
? (cpu - _lastCpu).TotalSeconds / wallDelta * 100.0
|
||||
: 0.0;
|
||||
|
||||
_lastCpu = cpu;
|
||||
_lastElapsedSeconds = elapsedSeconds;
|
||||
|
||||
var sample = new ResourceSample(
|
||||
elapsedSeconds,
|
||||
process.WorkingSet64 / 1024.0 / 1024.0,
|
||||
GC.GetTotalMemory(forceFullCollection: false) / 1024.0 / 1024.0,
|
||||
cpuPercent,
|
||||
process.Threads.Count,
|
||||
GC.CollectionCount(2));
|
||||
|
||||
lock (_lock) _samples.Add(sample);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summarizes the samples falling inside a window, expressed as seconds since
|
||||
/// sampling started. Memory growth is reported both as an absolute delta and as a
|
||||
/// least-squares slope, because a run that sawtooths around a stable mean and a
|
||||
/// run that climbs monotonically can share the same endpoint delta.
|
||||
/// </summary>
|
||||
/// <param name="fromSeconds">Window start (inclusive), seconds since start.</param>
|
||||
/// <param name="toSeconds">Window end (inclusive), seconds since start.</param>
|
||||
/// <returns>The window summary, or null when fewer than two samples fall inside it.</returns>
|
||||
public ResourceWindowSummary? Summarize(double fromSeconds, double toSeconds)
|
||||
{
|
||||
var window = Snapshot()
|
||||
.Where(s => s.ElapsedSeconds >= fromSeconds && s.ElapsedSeconds <= toSeconds)
|
||||
.ToList();
|
||||
|
||||
if (window.Count < 2)
|
||||
return null;
|
||||
|
||||
var first = window[0];
|
||||
var last = window[^1];
|
||||
|
||||
return new ResourceWindowSummary(
|
||||
SampleCount: window.Count,
|
||||
DurationSeconds: last.ElapsedSeconds - first.ElapsedSeconds,
|
||||
WorkingSetStartMb: first.WorkingSetMb,
|
||||
WorkingSetEndMb: last.WorkingSetMb,
|
||||
WorkingSetPeakMb: window.Max(s => s.WorkingSetMb),
|
||||
WorkingSetSlopeMbPerMinute: Slope(window, s => s.WorkingSetMb) * 60.0,
|
||||
ManagedHeapStartMb: first.ManagedHeapMb,
|
||||
ManagedHeapEndMb: last.ManagedHeapMb,
|
||||
ManagedHeapPeakMb: window.Max(s => s.ManagedHeapMb),
|
||||
ManagedHeapSlopeMbPerMinute: Slope(window, s => s.ManagedHeapMb) * 60.0,
|
||||
MeanCpuPercentOfOneCore: window.Average(s => s.CpuPercent),
|
||||
PeakCpuPercentOfOneCore: window.Max(s => s.CpuPercent),
|
||||
MeanThreadCount: window.Average(s => s.ThreadCount),
|
||||
Gen2Collections: last.Gen2Collections - first.Gen2Collections);
|
||||
}
|
||||
|
||||
private static double Slope(IReadOnlyList<ResourceSample> samples, Func<ResourceSample, double> selector)
|
||||
{
|
||||
var n = samples.Count;
|
||||
var meanX = samples.Average(s => s.ElapsedSeconds);
|
||||
var meanY = samples.Average(selector);
|
||||
|
||||
double numerator = 0, denominator = 0;
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var dx = samples[i].ElapsedSeconds - meanX;
|
||||
numerator += dx * (selector(samples[i]) - meanY);
|
||||
denominator += dx * dx;
|
||||
}
|
||||
|
||||
return denominator == 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await _loop;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected.
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Aggregate resource behaviour over a measurement window.</summary>
|
||||
/// <param name="SampleCount">Samples in the window.</param>
|
||||
/// <param name="DurationSeconds">Window length.</param>
|
||||
/// <param name="WorkingSetStartMb">Working set at window start.</param>
|
||||
/// <param name="WorkingSetEndMb">Working set at window end.</param>
|
||||
/// <param name="WorkingSetPeakMb">Peak working set in the window.</param>
|
||||
/// <param name="WorkingSetSlopeMbPerMinute">Least-squares working-set growth rate.</param>
|
||||
/// <param name="ManagedHeapStartMb">Managed heap at window start.</param>
|
||||
/// <param name="ManagedHeapEndMb">Managed heap at window end.</param>
|
||||
/// <param name="ManagedHeapPeakMb">Peak managed heap in the window.</param>
|
||||
/// <param name="ManagedHeapSlopeMbPerMinute">Least-squares managed-heap growth rate.</param>
|
||||
/// <param name="MeanCpuPercentOfOneCore">Mean CPU as a percentage of one core (1400% = 14 cores saturated).</param>
|
||||
/// <param name="PeakCpuPercentOfOneCore">Peak single-sample CPU as a percentage of one core.</param>
|
||||
/// <param name="MeanThreadCount">Mean OS thread count.</param>
|
||||
/// <param name="Gen2Collections">Gen-2 collections during the window.</param>
|
||||
public sealed record ResourceWindowSummary(
|
||||
int SampleCount,
|
||||
double DurationSeconds,
|
||||
double WorkingSetStartMb,
|
||||
double WorkingSetEndMb,
|
||||
double WorkingSetPeakMb,
|
||||
double WorkingSetSlopeMbPerMinute,
|
||||
double ManagedHeapStartMb,
|
||||
double ManagedHeapEndMb,
|
||||
double ManagedHeapPeakMb,
|
||||
double ManagedHeapSlopeMbPerMinute,
|
||||
double MeanCpuPercentOfOneCore,
|
||||
double PeakCpuPercentOfOneCore,
|
||||
double MeanThreadCount,
|
||||
int Gen2Collections);
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Threading.Channels;
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
|
||||
|
||||
/// <summary>
|
||||
/// A live site-stream subscriber assembled from the SAME parts
|
||||
/// <c>SiteStreamGrpcServer.RunSubscriptionStreamAsync</c> uses:
|
||||
///
|
||||
/// <list type="number">
|
||||
/// <item><description><c>SiteStreamManager.Subscribe</c> — materializes the
|
||||
/// per-subscriber graph (<c>Where</c> instance filter → <c>Buffer(StreamBufferSize,
|
||||
/// DropHead)</c> → <c>Sink.ForEach(Tell)</c>).</description></item>
|
||||
/// <item><description>A real <see cref="StreamRelayActor"/>, which converts the Akka
|
||||
/// record to the protobuf <c>SiteStreamEvent</c> and <c>TryWrite</c>s it.</description></item>
|
||||
/// <item><description>A bounded <c>DropOldest</c> <see cref="Channel"/> of the
|
||||
/// production capacity (<c>GrpcInstanceStreamChannelCapacity</c> = 1000) with the
|
||||
/// eviction counter wired to <see cref="DroppedEvents"/>.</description></item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>
|
||||
/// The single substitution is the final hop: instead of
|
||||
/// <c>responseStream.WriteAsync</c> pushing onto a socket, a reader task drains the
|
||||
/// channel. That is deliberate — it is precisely the hop whose slowness register row
|
||||
/// 50 asks about, and a controllable reader is the only way to hold it still.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class StreamSubscriberProbe : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Production per-instance stream channel capacity (<c>GrpcInstanceStreamChannelCapacity</c>).</summary>
|
||||
public const int ProductionChannelCapacity = 1000;
|
||||
|
||||
private readonly SiteStreamManager _manager;
|
||||
private readonly string _subscriptionId;
|
||||
private readonly IActorRef _relayActor;
|
||||
private readonly ActorSystem _system;
|
||||
private readonly Channel<SiteStreamEvent> _channel;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly Task _readerTask;
|
||||
private volatile LatencyHistogram? _latency;
|
||||
|
||||
private readonly DropCounter _dropCounter;
|
||||
private long _received;
|
||||
private long _readerDelayMicroseconds;
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>Human-readable probe name (also the relay actor's name suffix).</summary>
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary>Events evicted by the bounded channel's DropOldest policy.</summary>
|
||||
public long DroppedEvents => _dropCounter.Value;
|
||||
|
||||
/// <summary>Events successfully drained by the reader (i.e. "sent to the client").</summary>
|
||||
public long ReceivedEvents => Interlocked.Read(ref _received);
|
||||
|
||||
/// <summary>
|
||||
/// Repoints the latency histogram this probe records into, without tearing the
|
||||
/// subscription down. Used to separate ramp-window samples from steady-state ones:
|
||||
/// re-attaching probes instead would open a zero-subscriber gap (during which
|
||||
/// <c>PublishAttributeValueChanged</c> short-circuits) and risk reusing an actor
|
||||
/// name whose previous incarnation has not finished terminating.
|
||||
/// </summary>
|
||||
/// <param name="latency">The histogram to record into from now on, or null to stop recording.</param>
|
||||
public void RetargetLatency(LatencyHistogram? latency) => _latency = latency;
|
||||
|
||||
/// <summary>
|
||||
/// Artificial per-event reader delay, in microseconds. Zero is a healthy
|
||||
/// subscriber; a large value models a stalled WAN link or a wedged client.
|
||||
/// </summary>
|
||||
public long ReaderDelayMicroseconds
|
||||
{
|
||||
get => Interlocked.Read(ref _readerDelayMicroseconds);
|
||||
set => Interlocked.Exchange(ref _readerDelayMicroseconds, value);
|
||||
}
|
||||
|
||||
private StreamSubscriberProbe(
|
||||
ActorSystem system,
|
||||
SiteStreamManager manager,
|
||||
string name,
|
||||
Channel<SiteStreamEvent> channel,
|
||||
IActorRef relayActor,
|
||||
string subscriptionId,
|
||||
LatencyHistogram? latency,
|
||||
DropCounter dropCounter)
|
||||
{
|
||||
_system = system;
|
||||
_manager = manager;
|
||||
Name = name;
|
||||
_channel = channel;
|
||||
_relayActor = relayActor;
|
||||
_subscriptionId = subscriptionId;
|
||||
_latency = latency;
|
||||
_dropCounter = dropCounter;
|
||||
_readerTask = Task.Run(() => ReadLoopAsync(_cts.Token));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and attaches a probe subscribed to one instance's events.
|
||||
/// </summary>
|
||||
/// <param name="system">The site actor system.</param>
|
||||
/// <param name="manager">The site stream manager to subscribe against.</param>
|
||||
/// <param name="instanceUniqueName">Instance whose events this probe receives.</param>
|
||||
/// <param name="name">Probe name, used for the relay actor's path.</param>
|
||||
/// <param name="latency">Optional histogram fed with end-to-end event latency.</param>
|
||||
/// <returns>The attached probe.</returns>
|
||||
public static StreamSubscriberProbe Attach(
|
||||
ActorSystem system,
|
||||
SiteStreamManager manager,
|
||||
string instanceUniqueName,
|
||||
string name,
|
||||
LatencyHistogram? latency)
|
||||
{
|
||||
var dropCounter = new DropCounter();
|
||||
var channel = Channel.CreateBounded<SiteStreamEvent>(
|
||||
new BoundedChannelOptions(ProductionChannelCapacity)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
},
|
||||
_ => dropCounter.Increment());
|
||||
|
||||
var relayActor = system.ActorOf(
|
||||
Props.Create(typeof(StreamRelayActor), name, channel.Writer),
|
||||
$"stream-relay-{name}");
|
||||
|
||||
var subscriptionId = manager.Subscribe(instanceUniqueName, relayActor);
|
||||
|
||||
return new StreamSubscriberProbe(
|
||||
system, manager, name, channel, relayActor, subscriptionId, latency, dropCounter);
|
||||
}
|
||||
|
||||
private async Task ReadLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (var evt in _channel.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
Interlocked.Increment(ref _received);
|
||||
|
||||
var latency = _latency;
|
||||
if (latency != null && evt.AttributeChanged != null)
|
||||
{
|
||||
// The emit instant travels verbatim: the driver stamps it on
|
||||
// TagValueUpdate.Timestamp, DataConnectionActor forwards it,
|
||||
// InstanceActor copies it onto AttributeValueChanged.Timestamp, and
|
||||
// StreamRelayActor maps it onto the proto Timestamp. So this is a
|
||||
// true end-to-end DCL-boundary → subscriber measurement.
|
||||
var emitted = evt.AttributeChanged.Timestamp.ToDateTimeOffset();
|
||||
latency.Record(DateTimeOffset.UtcNow - emitted);
|
||||
}
|
||||
|
||||
var delay = Interlocked.Read(ref _readerDelayMicroseconds);
|
||||
if (delay > 0)
|
||||
await Task.Delay(TimeSpan.FromMicroseconds(delay), cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Detaches the subscription and stops the relay actor and reader.</summary>
|
||||
/// <returns>A task that completes when the probe is torn down.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
_manager.Unsubscribe(_subscriptionId);
|
||||
_channel.Writer.TryComplete();
|
||||
await _cts.CancelAsync();
|
||||
|
||||
try
|
||||
{
|
||||
await _readerTask;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected.
|
||||
}
|
||||
|
||||
_system.Stop(_relayActor);
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe counter for a bounded channel's <c>itemDropped</c> callback. A tiny
|
||||
/// class rather than a captured local so the probe and the channel share exactly one
|
||||
/// counter instance without a second closure.
|
||||
/// </summary>
|
||||
public sealed class DropCounter
|
||||
{
|
||||
private long _value;
|
||||
|
||||
/// <summary>Current count.</summary>
|
||||
public long Value => Interlocked.Read(ref _value);
|
||||
|
||||
/// <summary>Increments the counter.</summary>
|
||||
public void Increment() => Interlocked.Increment(ref _value);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System.Text.Json;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
// Target-scale load harness (Phase-8 WP-4 / deferred-work register #25 + row 50).
|
||||
//
|
||||
// Full-scale protocol run (the WP-4 numbers):
|
||||
// dotnet run -c Release --project tests/ZB.MOM.WW.ScadaBridge.LoadHarness -- \
|
||||
// --sustain-minutes 20 --results loadharness-results.json
|
||||
//
|
||||
// Full 1-hour version, unchanged in every other respect:
|
||||
// ... -- --sustain-minutes 60 --results loadharness-results-1h.json
|
||||
//
|
||||
// Scaled-down smoke (what the CI [Fact] runs):
|
||||
// ... -- --sites 2 --instances-per-site 10 --tags-per-instance 5 \
|
||||
// --settle-minutes 0.1 --sustain-minutes 0.2 --sample-seconds 2 \
|
||||
// --sf-drain-messages 200 --slow-subscriber-events 2000
|
||||
|
||||
var config = HarnessConfig.Parse(args);
|
||||
|
||||
Console.WriteLine("ScadaBridge target-scale load harness");
|
||||
Console.WriteLine($" sites {config.Sites}");
|
||||
Console.WriteLine($" instances/site {config.InstancesPerSite}");
|
||||
Console.WriteLine($" tags/instance {config.TagsPerInstance}");
|
||||
Console.WriteLine($" total subscriptions {config.TotalSubscriptions:N0}");
|
||||
Console.WriteLine($" nominal update rate {config.NominalUpdatesPerSecond:N0}/s");
|
||||
Console.WriteLine($" settle / sustain {config.SettleDuration.TotalMinutes:F1} / {config.SustainDuration.TotalMinutes:F1} min");
|
||||
Console.WriteLine();
|
||||
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
Console.CancelKeyPress += (_, e) =>
|
||||
{
|
||||
e.Cancel = true;
|
||||
cancellation.Cancel();
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await HarnessRun.ExecuteAsync(config, Console.WriteLine, cancellation.Token);
|
||||
|
||||
var json = JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true });
|
||||
await File.WriteAllTextAsync(config.ResultsPath, json, cancellation.Token);
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(ResultsFormatter.Format(result));
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"JSON metrics written to {Path.GetFullPath(config.ResultsPath)}");
|
||||
return 0;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Console.Error.WriteLine("Run cancelled.");
|
||||
return 130;
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// Renders a <see cref="HarnessRunResult"/> as the markdown table that goes into the
|
||||
/// results document, so the published numbers and the JSON come from one source.
|
||||
/// </summary>
|
||||
public static class ResultsFormatter
|
||||
{
|
||||
/// <summary>Formats a run result for console output and the results doc.</summary>
|
||||
/// <param name="result">The run to format.</param>
|
||||
/// <returns>A markdown-ish plain-text report.</returns>
|
||||
public static string Format(HarnessRunResult result)
|
||||
{
|
||||
var culture = CultureInfo.InvariantCulture;
|
||||
var builder = new StringBuilder();
|
||||
|
||||
builder.AppendLine("=== MEASURED ===");
|
||||
builder.AppendLine(culture, $"host {result.Environment.MachineName} / {result.Environment.OsDescription}");
|
||||
builder.AppendLine(culture, $"cpus / runtime / gc {result.Environment.ProcessorCount} / {result.Environment.RuntimeVersion} / serverGC={result.Environment.ServerGc}");
|
||||
builder.AppendLine(culture, $"scale {result.Config.Sites} sites x {result.Config.InstancesPerSite} instances x {result.Config.TagsPerInstance} tags = {result.Config.TotalSubscriptions:N0} subscriptions");
|
||||
builder.AppendLine(culture, $"total run {result.TotalSeconds:F1}s");
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- deployment / ramp --");
|
||||
builder.AppendLine(culture, $"site fixtures built {result.SiteRampSeconds:F1}s");
|
||||
builder.AppendLine(culture, $"all instance actors {result.InstanceRampSeconds:F1}s");
|
||||
builder.AppendLine(culture, $"slowest single site {result.SlowestSiteInstanceRampSeconds:F1}s ({result.Config.InstancesPerSite} instances)");
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- offered load (steady-state window) --");
|
||||
builder.AppendLine(culture, $"nominal {result.NominalUpdatesPerSecond:N0} updates/s");
|
||||
builder.AppendLine(culture, $"achieved {result.AchievedUpdatesPerSecond:N0} updates/s ({result.AchievedUpdatesPerSecond / result.NominalUpdatesPerSecond:P1} of nominal)");
|
||||
builder.AppendLine(culture, $"updates offered {result.SteadyStateEmittedTagUpdates:N0}");
|
||||
builder.AppendLine(culture, $"driver slice overrun {result.DriverLagSeconds:F1}s cumulative");
|
||||
builder.AppendLine(culture, $"skipped (no callback) {result.DriverSkippedNoCallback:N0}");
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- tag update latency (DCL boundary -> stream subscriber) --");
|
||||
AppendLatency(builder, culture, result.TagUpdateLatency);
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- live stream subscribers (steady-state window) --");
|
||||
builder.AppendLine(culture, $"events delivered {result.StreamProbeReceived:N0}");
|
||||
builder.AppendLine(culture, $"events dropped {result.StreamProbeDropped:N0}");
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- health report delivery --");
|
||||
AppendLatency(builder, culture, result.HealthReportLatency);
|
||||
builder.AppendLine(culture, $"reports ingested {result.HealthReportsDelivered:N0}");
|
||||
builder.AppendLine(culture, $"sites tracked centrally {result.SitesTrackedByAggregator}");
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("-- debug view snapshot (under load) --");
|
||||
AppendLatency(builder, culture, result.DebugSnapshotLatency);
|
||||
builder.AppendLine(culture, $"completed / timed out {result.DebugSnapshotsCompleted:N0} / {result.DebugSnapshotTimeouts:N0}");
|
||||
builder.AppendLine();
|
||||
|
||||
if (result.SteadyStateResources is { } steady)
|
||||
{
|
||||
builder.AppendLine("-- resources (steady-state window) --");
|
||||
builder.AppendLine(culture, $"window {steady.DurationSeconds:F0}s over {steady.SampleCount} samples");
|
||||
builder.AppendLine(culture, $"working set {steady.WorkingSetStartMb:F0} -> {steady.WorkingSetEndMb:F0} MB (peak {steady.WorkingSetPeakMb:F0} MB)");
|
||||
builder.AppendLine(culture, $"working set slope {steady.WorkingSetSlopeMbPerMinute:F2} MB/min");
|
||||
builder.AppendLine(culture, $"managed heap {steady.ManagedHeapStartMb:F0} -> {steady.ManagedHeapEndMb:F0} MB (peak {steady.ManagedHeapPeakMb:F0} MB)");
|
||||
builder.AppendLine(culture, $"managed heap slope {steady.ManagedHeapSlopeMbPerMinute:F2} MB/min");
|
||||
builder.AppendLine(culture, $"cpu mean / peak {steady.MeanCpuPercentOfOneCore:F0}% / {steady.PeakCpuPercentOfOneCore:F0}% of one core ({steady.MeanCpuPercentOfOneCore / result.Environment.ProcessorCount:F1}% of the box)");
|
||||
builder.AppendLine(culture, $"threads / gen2 GCs {steady.MeanThreadCount:F0} / {steady.Gen2Collections}");
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
if (result.StoreAndForwardDrain is { } drain)
|
||||
{
|
||||
builder.AppendLine("-- store-and-forward (register row 50) --");
|
||||
builder.AppendLine(culture, $"messages {drain.MessageCount:N0}");
|
||||
builder.AppendLine(culture, $"concurrent buffering {drain.EnqueuePerSecond:N0} msg/s ({drain.EnqueueSeconds:F1}s)");
|
||||
builder.AppendLine(culture, $"retry wait before drain {drain.TimeToFirstDeliverySeconds:F1}s (DefaultRetryInterval)");
|
||||
builder.AppendLine(culture, $"drain throughput {drain.DrainPerSecond:N0} msg/s (active drain {drain.DrainSeconds - drain.TimeToFirstDeliverySeconds:F2}s)");
|
||||
builder.AppendLine(culture, $"residual depth {drain.ResidualDepth}");
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
if (result.SlowSubscriber is { } slow)
|
||||
{
|
||||
builder.AppendLine("-- slow-subscriber isolation (register row 50) --");
|
||||
builder.AppendLine(culture, $"published {slow.PublishedEvents:N0} at {slow.PublishPerSecond:N0}/s");
|
||||
builder.AppendLine(culture, $"healthy min delivery {slow.HealthyMinDeliveryRatio:P2}");
|
||||
builder.AppendLine(culture, $"stalled delivery {slow.SlowDeliveryRatio:P2}");
|
||||
foreach (var outcome in slow.Outcomes)
|
||||
{
|
||||
builder.AppendLine(culture,
|
||||
$" {outcome.Name,-28} {(outcome.IsSlow ? "STALLED" : "healthy"),-8} " +
|
||||
$"recv {outcome.Received,8:N0} dropped {outcome.Dropped,8:N0} ratio {outcome.DeliveryRatio:P2}");
|
||||
}
|
||||
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static void AppendLatency(StringBuilder builder, CultureInfo culture, Metrics.LatencySnapshot snapshot)
|
||||
{
|
||||
builder.AppendLine(culture,
|
||||
$"samples {snapshot.Count:N0} mean {snapshot.MeanMs:F2}ms p50 {snapshot.P50Ms:F2}ms " +
|
||||
$"p95 {snapshot.P95Ms:F2}ms p99 {snapshot.P99Ms:F2}ms p99.9 {snapshot.P999Ms:F2}ms max {snapshot.MaxMs:F2}ms");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>
|
||||
/// The two "does observability still work at scale?" probes WP-4's test protocol
|
||||
/// names — health report delivery timing and debug view latency — run continuously
|
||||
/// alongside the sustained load rather than after it, so both are measured against a
|
||||
/// site that is actually busy.
|
||||
/// </summary>
|
||||
public sealed class ObservabilityProbes : IAsyncDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly List<Task> _tasks = new();
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end health report latency: <c>SiteHealthCollector.CollectReport</c> plus
|
||||
/// the transport hop plus <c>CentralHealthAggregator.ProcessReport</c>. The
|
||||
/// interesting term at scale is <c>CollectReport</c>, which materializes the
|
||||
/// per-connection dictionaries for a site carrying 37,500 subscriptions.
|
||||
/// </summary>
|
||||
public LatencyHistogram HealthReportLatency { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Debug view snapshot round-trip: an <c>Ask</c> of <c>DebugSnapshotRequest</c> to
|
||||
/// a live Instance Actor. This lands the request in the mailbox of an actor that
|
||||
/// is concurrently ingesting tag updates, so the measured time includes real
|
||||
/// queueing behind production traffic — which is the whole point of measuring it
|
||||
/// under load.
|
||||
/// </summary>
|
||||
public LatencyHistogram DebugSnapshotLatency { get; } = new();
|
||||
|
||||
/// <summary>Health reports successfully ingested by the central aggregator.</summary>
|
||||
public long HealthReportsDelivered => Interlocked.Read(ref _healthReports);
|
||||
|
||||
/// <summary>Debug snapshots that completed within the ask timeout.</summary>
|
||||
public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots);
|
||||
|
||||
/// <summary>Debug snapshot asks that timed out.</summary>
|
||||
public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts);
|
||||
|
||||
private long _healthReports;
|
||||
private long _debugSnapshots;
|
||||
private long _debugTimeouts;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both probes.
|
||||
/// </summary>
|
||||
/// <param name="sites">Sites to probe.</param>
|
||||
/// <param name="aggregator">The real central aggregator receiving the reports.</param>
|
||||
/// <param name="reportInterval">Health report cadence (production default 30 s).</param>
|
||||
/// <param name="debugProbeInterval">How often to take a debug snapshot.</param>
|
||||
/// <returns>The running probes.</returns>
|
||||
public static ObservabilityProbes Start(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
CentralHealthAggregator aggregator,
|
||||
TimeSpan reportInterval,
|
||||
TimeSpan debugProbeInterval)
|
||||
{
|
||||
var probes = new ObservabilityProbes();
|
||||
probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token)));
|
||||
probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token)));
|
||||
return probes;
|
||||
}
|
||||
|
||||
private async Task HealthLoopAsync(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
CentralHealthAggregator aggregator,
|
||||
TimeSpan interval,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
{
|
||||
foreach (var site in sites)
|
||||
{
|
||||
var watch = Stopwatch.StartNew();
|
||||
var report = site.HealthCollector.CollectReport(site.SiteId);
|
||||
aggregator.ProcessReport(report);
|
||||
watch.Stop();
|
||||
|
||||
HealthReportLatency.Record(watch.Elapsed);
|
||||
Interlocked.Increment(ref _healthReports);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DebugLoopAsync(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
TimeSpan interval,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var random = new Random(20260815);
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
{
|
||||
var site = sites[random.Next(sites.Count)];
|
||||
if (site.InstanceActors.Count == 0)
|
||||
continue;
|
||||
|
||||
var index = random.Next(site.InstanceActors.Count);
|
||||
var actor = site.InstanceActors[index];
|
||||
var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N"));
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await actor.Ask<DebugViewSnapshot>(request, TimeSpan.FromSeconds(10), cancellationToken);
|
||||
watch.Stop();
|
||||
DebugSnapshotLatency.Record(watch.Elapsed);
|
||||
Interlocked.Increment(ref _debugSnapshots);
|
||||
}
|
||||
catch (AskTimeoutException)
|
||||
{
|
||||
Interlocked.Increment(ref _debugTimeouts);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// Idempotent: the orchestrator stops the probes early (so the register row 50
|
||||
// scenarios do not compete with them) and again in its finally block.
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_tasks);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected.
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>Per-subscriber outcome from the slow-subscriber isolation probe.</summary>
|
||||
/// <param name="Name">Probe name.</param>
|
||||
/// <param name="IsSlow">Whether this probe's reader was deliberately stalled.</param>
|
||||
/// <param name="Received">Events the reader drained.</param>
|
||||
/// <param name="Dropped">Events evicted by this probe's bounded DropOldest channel.</param>
|
||||
/// <param name="DeliveryRatio">Received / published, before accounting for the site-stream buffer.</param>
|
||||
public sealed record SubscriberOutcome(
|
||||
string Name,
|
||||
bool IsSlow,
|
||||
long Received,
|
||||
long Dropped,
|
||||
double DeliveryRatio);
|
||||
|
||||
/// <summary>Result of the slow-subscriber isolation measurement.</summary>
|
||||
/// <param name="PublishedEvents">Events published to the site stream during the probe.</param>
|
||||
/// <param name="PublishSeconds">Wall time the publisher took.</param>
|
||||
/// <param name="PublishPerSecond">Publish throughput observed by the producer.</param>
|
||||
/// <param name="Outcomes">Per-subscriber outcomes.</param>
|
||||
/// <param name="HealthyMinDeliveryRatio">Worst delivery ratio among the healthy subscribers.</param>
|
||||
/// <param name="SlowDeliveryRatio">Delivery ratio of the stalled subscriber.</param>
|
||||
public sealed record SlowSubscriberResult(
|
||||
int PublishedEvents,
|
||||
double PublishSeconds,
|
||||
double PublishPerSecond,
|
||||
IReadOnlyList<SubscriberOutcome> Outcomes,
|
||||
double HealthyMinDeliveryRatio,
|
||||
double SlowDeliveryRatio);
|
||||
|
||||
/// <summary>
|
||||
/// Register row 50, second half: what does a slow or stalled gRPC subscriber do to
|
||||
/// per-subscriber buffering when several subscribers are attached?
|
||||
///
|
||||
/// <para>
|
||||
/// Several probes are attached to the SAME instance so every one of them is offered
|
||||
/// exactly the same event sequence — otherwise a difference in delivery could be a
|
||||
/// difference in offered load rather than a backpressure effect. One probe's reader
|
||||
/// is then stalled (a large per-event delay, standing in for a wedged client or a
|
||||
/// dead WAN link) while the rest read as fast as they can. Events are published
|
||||
/// through the real <c>SiteStreamManager</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The question the numbers answer: does the stalled subscriber's backlog propagate
|
||||
/// upstream — evicting events for the healthy subscribers or slowing the publisher —
|
||||
/// or is it confined to its own <c>Buffer(DropHead)</c> stage and its own bounded
|
||||
/// <c>DropOldest</c> channel? The design intends the latter; this measures it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SlowSubscriberScenario
|
||||
{
|
||||
/// <summary>Per-event reader delay applied to the stalled subscriber.</summary>
|
||||
public const int SlowReaderDelayMicroseconds = 50_000;
|
||||
|
||||
/// <summary>
|
||||
/// Publish rate for the probe. Deliberately paced rather than a tight burst: the
|
||||
/// publish source is a single <c>Source.ActorRef(StreamBufferSize, DropHead)</c>
|
||||
/// SHARED by every attribute subscriber, so an unpaced burst saturates that shared
|
||||
/// stage and every subscriber loses events for a reason that has nothing to do
|
||||
/// with the slow one. Pacing below the shared stage's capacity is what isolates
|
||||
/// the variable under test.
|
||||
/// </summary>
|
||||
public const int PublishRatePerSecond = 2_000;
|
||||
|
||||
/// <summary>Runs the isolation probe on a dedicated site.</summary>
|
||||
/// <param name="site">Site whose stream manager is used.</param>
|
||||
/// <param name="subscriberCount">Total subscribers to attach (one of them is stalled).</param>
|
||||
/// <param name="eventCount">Events to publish.</param>
|
||||
/// <param name="cancellationToken">Cancels the measurement.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<SlowSubscriberResult> RunAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int subscriberCount,
|
||||
int eventCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var instanceName = site.InstanceName(0);
|
||||
var probes = new List<StreamSubscriberProbe>(subscriberCount);
|
||||
|
||||
try
|
||||
{
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
var probe = StreamSubscriberProbe.Attach(
|
||||
site.System, site.StreamManager, instanceName,
|
||||
$"{site.SiteId}-slowprobe-{i:D2}", latency: null);
|
||||
|
||||
// Probe 0 is the pathological one.
|
||||
if (i == 0)
|
||||
probe.ReaderDelayMicroseconds = SlowReaderDelayMicroseconds;
|
||||
|
||||
probes.Add(probe);
|
||||
}
|
||||
|
||||
// Let every subscription's stream graph finish materializing before the burst.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
const int sliceMilliseconds = 50;
|
||||
var perSlice = Math.Max(1, PublishRatePerSecond * sliceMilliseconds / 1000);
|
||||
var published = 0;
|
||||
while (published < eventCount && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var sliceStart = Stopwatch.GetTimestamp();
|
||||
var end = Math.Min(published + perSlice, eventCount);
|
||||
for (var i = published; i < end; i++)
|
||||
{
|
||||
site.StreamManager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
instanceName, "Tag000", "Tag000", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
published = end;
|
||||
|
||||
var elapsedMs = (Stopwatch.GetTimestamp() - sliceStart) * 1000.0 / Stopwatch.Frequency;
|
||||
if (elapsedMs < sliceMilliseconds)
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(sliceMilliseconds - elapsedMs), cancellationToken);
|
||||
}
|
||||
|
||||
watch.Stop();
|
||||
|
||||
// Give the healthy readers time to finish; the stalled one will not.
|
||||
await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken);
|
||||
|
||||
var outcomes = probes
|
||||
.Select((p, i) => new SubscriberOutcome(
|
||||
p.Name,
|
||||
IsSlow: i == 0,
|
||||
p.ReceivedEvents,
|
||||
p.DroppedEvents,
|
||||
p.ReceivedEvents / (double)eventCount))
|
||||
.ToList();
|
||||
|
||||
var healthy = outcomes.Where(o => !o.IsSlow).ToList();
|
||||
|
||||
return new SlowSubscriberResult(
|
||||
PublishedEvents: eventCount,
|
||||
PublishSeconds: watch.Elapsed.TotalSeconds,
|
||||
PublishPerSecond: eventCount / Math.Max(0.001, watch.Elapsed.TotalSeconds),
|
||||
Outcomes: outcomes,
|
||||
HealthyMinDeliveryRatio: healthy.Count == 0 ? 0 : healthy.Min(o => o.DeliveryRatio),
|
||||
SlowDeliveryRatio: outcomes[0].DeliveryRatio);
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var probe in probes)
|
||||
await probe.DisposeAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>Result of one store-and-forward drain measurement.</summary>
|
||||
/// <param name="SiteId">Site the buffer belonged to.</param>
|
||||
/// <param name="MessageCount">Messages buffered before the drain began.</param>
|
||||
/// <param name="EnqueueSeconds">Wall time to buffer them (concurrent, many origin instances).</param>
|
||||
/// <param name="EnqueuePerSecond">Buffering throughput.</param>
|
||||
/// <param name="TimeToFirstDeliverySeconds">
|
||||
/// Wall time from the first sweep to the first successful delivery. With
|
||||
/// <c>attemptImmediateDelivery: false</c> the engine stamps <c>LastAttemptAt</c>, so the
|
||||
/// row is not due until one <c>DefaultRetryInterval</c> (30 s) has passed — this is the
|
||||
/// configured retry latency, not drain slowness, and is reported separately for that reason.
|
||||
/// </param>
|
||||
/// <param name="DrainSeconds">Wall time from the first sweep to an empty buffer (includes the retry wait).</param>
|
||||
/// <param name="DrainPerSecond">
|
||||
/// Drain throughput measured from the FIRST delivery to an empty buffer — the engine's
|
||||
/// actual capacity, and the headline number for register row 50.
|
||||
/// </param>
|
||||
/// <param name="ResidualDepth">Buffer depth left when the measurement stopped (0 = fully drained).</param>
|
||||
/// <param name="Progress">Delivered-count samples during the drain, so a steady rate can be told from a stall-then-burst.</param>
|
||||
public sealed record StoreAndForwardDrainResult(
|
||||
string SiteId,
|
||||
int MessageCount,
|
||||
double EnqueueSeconds,
|
||||
double EnqueuePerSecond,
|
||||
double TimeToFirstDeliverySeconds,
|
||||
double DrainSeconds,
|
||||
double DrainPerSecond,
|
||||
int ResidualDepth,
|
||||
IReadOnlyList<DrainProgressSample> Progress);
|
||||
|
||||
/// <summary>One observation during the drain.</summary>
|
||||
/// <param name="ElapsedSeconds">Seconds since the drain began.</param>
|
||||
/// <param name="Delivered">Cumulative successful deliveries.</param>
|
||||
/// <param name="Depth">Remaining buffer depth.</param>
|
||||
public sealed record DrainProgressSample(double ElapsedSeconds, long Delivered, int Depth);
|
||||
|
||||
/// <summary>
|
||||
/// Measures store-and-forward buffering and drain throughput (deferred-work register
|
||||
/// row 50, first half) using the real <c>StoreAndForwardService</c>, the real
|
||||
/// <c>StoreAndForwardStorage</c> and the real SQLite file — only the delivery target
|
||||
/// is a counting stub, because what is being measured is the site-local buffer's
|
||||
/// throughput, not a remote endpoint's.
|
||||
///
|
||||
/// <para>
|
||||
/// Phase 1 buffers <paramref name="messageCount"/> messages with
|
||||
/// <c>attemptImmediateDelivery: false</c>, spread across many origin instance names
|
||||
/// and issued from many concurrent tasks — the "concurrent buffering from multiple
|
||||
/// instances" WP-4 asks about (<c>[xc-7]</c>). Phase 2 registers a delivery handler
|
||||
/// that always succeeds and drives sweeps to completion, timing the drain.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The sweep is driven explicitly rather than waiting on the 10 s
|
||||
/// <c>RetryTimerInterval</c> so the number reported is the engine's drain capacity,
|
||||
/// not its polling cadence. The per-sweep batch is <c>SweepBatchLimit</c> (500) with
|
||||
/// <c>SweepTargetParallelism</c> (4) lanes, both at their production defaults.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class StoreAndForwardDrainScenario
|
||||
{
|
||||
/// <summary>Runs the drain measurement against one site's real S&F engine.</summary>
|
||||
/// <param name="site">The site whose store-and-forward engine is exercised.</param>
|
||||
/// <param name="messageCount">Messages to buffer.</param>
|
||||
/// <param name="concurrency">Concurrent enqueue tasks (distinct origin instances).</param>
|
||||
/// <param name="cancellationToken">Cancels the measurement.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<StoreAndForwardDrainResult> RunAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int messageCount,
|
||||
int concurrency,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var service = site.StoreAndForward;
|
||||
var payload = $"{{\"site\":\"{site.SiteId}\",\"body\":\"{new string('x', 256)}\"}}";
|
||||
|
||||
// Phase 1 — concurrent buffering from many instances, no delivery attempted.
|
||||
var enqueueWatch = Stopwatch.StartNew();
|
||||
var perTask = messageCount / concurrency;
|
||||
var enqueueTasks = new List<Task>(concurrency);
|
||||
for (var t = 0; t < concurrency; t++)
|
||||
{
|
||||
var taskIndex = t;
|
||||
enqueueTasks.Add(Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < perTask; i++)
|
||||
{
|
||||
await service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
target: $"load-target-{taskIndex % 4}",
|
||||
payloadJson: payload,
|
||||
originInstanceName: site.InstanceName(taskIndex),
|
||||
attemptImmediateDelivery: false);
|
||||
}
|
||||
}, cancellationToken));
|
||||
}
|
||||
|
||||
await Task.WhenAll(enqueueTasks);
|
||||
enqueueWatch.Stop();
|
||||
var buffered = perTask * concurrency;
|
||||
|
||||
// Phase 2 — a delivery target that always succeeds; time the drain to empty.
|
||||
var delivered = 0L;
|
||||
service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ =>
|
||||
{
|
||||
Interlocked.Increment(ref delivered);
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
// Sweeps are driven explicitly rather than waiting on the 10 s RetryTimerInterval:
|
||||
// the number wanted is the engine's drain CAPACITY, not its polling cadence. One
|
||||
// sweep moves at most SweepBatchLimit (500) messages, so a large backlog needs
|
||||
// many, and the progress series below is what distinguishes a genuinely slow
|
||||
// drain from an artefact of this polling loop.
|
||||
var drainWatch = Stopwatch.StartNew();
|
||||
var deadline = DateTimeOffset.UtcNow.AddMinutes(10);
|
||||
var progress = new List<DrainProgressSample>();
|
||||
int depth;
|
||||
while (true)
|
||||
{
|
||||
service.TriggerSweep();
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
|
||||
|
||||
var depths = await service.GetBufferDepthAsync();
|
||||
depth = depths.Values.Sum();
|
||||
progress.Add(new DrainProgressSample(
|
||||
drainWatch.Elapsed.TotalSeconds, Interlocked.Read(ref delivered), depth));
|
||||
|
||||
if (depth == 0 || DateTimeOffset.UtcNow > deadline)
|
||||
break;
|
||||
}
|
||||
|
||||
drainWatch.Stop();
|
||||
|
||||
// Split the retry wait from the drain: the first sample with a non-zero delivered
|
||||
// count marks the moment the backlog actually became due.
|
||||
var firstDelivery = progress.FirstOrDefault(s => s.Delivered > 0);
|
||||
var timeToFirstDelivery = firstDelivery?.ElapsedSeconds ?? drainWatch.Elapsed.TotalSeconds;
|
||||
var activeDrainSeconds = Math.Max(0.001, drainWatch.Elapsed.TotalSeconds - timeToFirstDelivery);
|
||||
|
||||
return new StoreAndForwardDrainResult(
|
||||
SiteId: site.SiteId,
|
||||
MessageCount: buffered,
|
||||
EnqueueSeconds: enqueueWatch.Elapsed.TotalSeconds,
|
||||
EnqueuePerSecond: buffered / Math.Max(0.001, enqueueWatch.Elapsed.TotalSeconds),
|
||||
TimeToFirstDeliverySeconds: timeToFirstDelivery,
|
||||
DrainSeconds: drainWatch.Elapsed.TotalSeconds,
|
||||
DrainPerSecond: Interlocked.Read(ref delivered) / activeDrainSeconds,
|
||||
ResidualDepth: depth,
|
||||
Progress: progress);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// In-process stand-in for an OPC UA / MxGateway server, registered on the REAL
|
||||
/// <c>DataConnectionFactory</c> under the <see cref="ProtocolName"/> protocol so the
|
||||
/// whole Data Connection Layer above it — <c>DataConnectionManagerActor</c>,
|
||||
/// <c>DataConnectionActor</c>, its <c>_instancesByTag</c> fan-out and the
|
||||
/// <c>TagValueUpdate</c> hand-off to Instance Actors — runs unmodified.
|
||||
///
|
||||
/// <para>
|
||||
/// It implements <see cref="IBatchSubscribableConnection"/> because the batch path is
|
||||
/// the one a real site takes at this scale; the per-tag fallback would make
|
||||
/// subscribe setup, not steady-state throughput, the thing being measured.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why faking here does not invalidate the measurement.</b> Everything this class
|
||||
/// replaces is on the far side of the process boundary: socket I/O, the OPC UA SDK's
|
||||
/// own session/subscription machinery, and the device. The system under test — the
|
||||
/// actor hierarchy, the site stream, store-and-forward, health, and the streaming
|
||||
/// relay — begins at the <see cref="SubscriptionCallback"/> invocation, which is
|
||||
/// exactly where the real adapter hands off. Driving 375,000 genuine monitored items
|
||||
/// would measure the OPC UA stack, not ScadaBridge.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SimulatedDataConnection : IDataConnection, IBatchSubscribableConnection
|
||||
{
|
||||
/// <summary>Protocol discriminator this adapter registers under on the factory.</summary>
|
||||
public const string ProtocolName = "LoadSim";
|
||||
|
||||
private SubscriptionCallback? _callback;
|
||||
private int _subscriptionCounter;
|
||||
|
||||
/// <summary>
|
||||
/// The data connection name this adapter was created for, taken from the connection
|
||||
/// details. The DCL factory creates adapters as its manager actor processes the
|
||||
/// CreateConnectionCommands, so creation order is not connection order — the driver
|
||||
/// resolves an adapter by name rather than by index.
|
||||
/// </summary>
|
||||
public string ConnectionName { get; private set; } = string.Empty;
|
||||
|
||||
/// <summary>Key under which the connection name travels in the connection details.</summary>
|
||||
public const string ConnectionNameKey = "connectionName";
|
||||
|
||||
/// <inheritdoc />
|
||||
public ConnectionHealth Status { get; private set; } = ConnectionHealth.Disconnected;
|
||||
|
||||
/// <inheritdoc />
|
||||
public event Action? Disconnected;
|
||||
|
||||
/// <summary>
|
||||
/// The callback captured at subscribe time. The tag driver invokes this to inject
|
||||
/// a value change, mirroring what the OPC UA SDK's notification thread does.
|
||||
/// Null until the site's Instance Actors have subscribed.
|
||||
/// </summary>
|
||||
public SubscriptionCallback? ValueCallback => _callback;
|
||||
|
||||
/// <summary>Number of tag paths this connection has accepted subscriptions for.</summary>
|
||||
public int SubscribedTagCount => _subscriptionCounter;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ConnectAsync(IDictionary<string, string> connectionDetails, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (connectionDetails.TryGetValue(ConnectionNameKey, out var name))
|
||||
ConnectionName = name;
|
||||
|
||||
Status = ConnectionHealth.Connected;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task DisconnectAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
Status = ConnectionHealth.Disconnected;
|
||||
Disconnected?.Invoke();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<string> SubscribeAsync(string tagPath, SubscriptionCallback callback, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_callback = callback;
|
||||
return Task.FromResult($"sub-{Interlocked.Increment(ref _subscriptionCounter)}");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<TagSubscribeResult>> SubscribeBatchAsync(
|
||||
IReadOnlyList<string> tagPaths,
|
||||
SubscriptionCallback callback,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
_callback = callback;
|
||||
var results = new List<TagSubscribeResult>(tagPaths.Count);
|
||||
foreach (var path in tagPaths)
|
||||
{
|
||||
results.Add(new TagSubscribeResult(
|
||||
path, true, $"sub-{Interlocked.Increment(ref _subscriptionCounter)}", null));
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<TagSubscribeResult>>(results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeAsync(string subscriptionId, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeBatchAsync(IReadOnlyList<string> subscriptionIds, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<ReadResult> ReadAsync(string tagPath, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ReadResult(true, new TagValue(0d, QualityCode.Good, DateTimeOffset.UtcNow), null));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyDictionary<string, ReadResult>> ReadBatchAsync(
|
||||
IEnumerable<string> tagPaths, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var results = new Dictionary<string, ReadResult>();
|
||||
foreach (var path in tagPaths)
|
||||
results[path] = new ReadResult(true, new TagValue(0d, QualityCode.Good, now), null);
|
||||
|
||||
return Task.FromResult<IReadOnlyDictionary<string, ReadResult>>(results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WriteResult> WriteAsync(string tagPath, object? value, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new WriteResult(true, null));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyDictionary<string, WriteResult>> WriteBatchAsync(
|
||||
IDictionary<string, object?> values, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new Dictionary<string, WriteResult>();
|
||||
foreach (var key in values.Keys)
|
||||
results[key] = new WriteResult(true, null);
|
||||
|
||||
return Task.FromResult<IReadOnlyDictionary<string, WriteResult>>(results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<bool> WriteBatchAndWaitAsync(
|
||||
IDictionary<string, object?> values, string flagPath, object? flagValue,
|
||||
string responsePath, object? responseValue, TimeSpan timeout,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// One simulated site: its own <see cref="ActorSystem"/>, its own LocalDb SQLite
|
||||
/// file, a real Data Connection Layer over <see cref="SimulatedDataConnection"/>
|
||||
/// adapters, real Instance Actors, a real <see cref="SiteStreamManager"/>, real
|
||||
/// store-and-forward, and a real <see cref="SiteHealthCollector"/>.
|
||||
///
|
||||
/// <para>
|
||||
/// The sites are separate, non-clustered ActorSystems rather than 10 real two-node
|
||||
/// Akka clusters. Cluster membership, singleton placement and failover timing are
|
||||
/// already measured on a real two-node rig by
|
||||
/// <c>PerformanceTests/Failover/FailoverTimingTests.cs</c>; what WP-4 asks about is
|
||||
/// the load-bearing hierarchy under each singleton, which is what this builds.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SiteRuntimeFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly HarnessConfig _config;
|
||||
private readonly List<SimulatedDataConnection> _connections;
|
||||
private readonly ServiceProvider _localDbProvider;
|
||||
private readonly ILocalDb _localDb;
|
||||
private readonly List<IActorRef> _instanceActors = new();
|
||||
private readonly List<StreamSubscriberProbe> _probes = new();
|
||||
private readonly string _dataDirectory;
|
||||
|
||||
/// <summary>Site identifier, e.g. <c>site-01</c>.</summary>
|
||||
public string SiteId { get; }
|
||||
|
||||
/// <summary>This site's actor system.</summary>
|
||||
public ActorSystem System { get; }
|
||||
|
||||
/// <summary>The real site-wide broadcast stream.</summary>
|
||||
public SiteStreamManager StreamManager { get; }
|
||||
|
||||
/// <summary>The real site health collector feeding the 30 s report.</summary>
|
||||
public SiteHealthCollector HealthCollector { get; }
|
||||
|
||||
/// <summary>The real store-and-forward engine for this site.</summary>
|
||||
public StoreAndForwardService StoreAndForward { get; }
|
||||
|
||||
/// <summary>The real DCL manager actor.</summary>
|
||||
public IActorRef DataConnectionManager { get; }
|
||||
|
||||
/// <summary>The simulated adapters, one per data connection, in creation order.</summary>
|
||||
public IReadOnlyList<SimulatedDataConnection> Connections
|
||||
{
|
||||
get { lock (_connections) return _connections.ToList(); }
|
||||
}
|
||||
|
||||
/// <summary>The live stream subscribers attached to this site.</summary>
|
||||
public IReadOnlyList<StreamSubscriberProbe> Probes => _probes;
|
||||
|
||||
/// <summary>Instance Actors created on this site.</summary>
|
||||
public IReadOnlyList<IActorRef> InstanceActors => _instanceActors;
|
||||
|
||||
/// <summary>Tag paths per connection index, in the order they were assigned.</summary>
|
||||
public IReadOnlyList<List<string>> TagPathsByConnection { get; }
|
||||
|
||||
/// <summary>Wall-clock time the instance ramp took, measured by <see cref="StartInstancesAsync"/>.</summary>
|
||||
public TimeSpan InstanceRampDuration { get; private set; }
|
||||
|
||||
private SiteRuntimeFixture(
|
||||
string siteId,
|
||||
HarnessConfig config,
|
||||
string dataDirectory,
|
||||
ServiceProvider localDbProvider,
|
||||
ILocalDb localDb,
|
||||
ActorSystem system,
|
||||
SiteStreamManager streamManager,
|
||||
SiteHealthCollector healthCollector,
|
||||
StoreAndForwardService storeAndForward,
|
||||
IActorRef dataConnectionManager,
|
||||
SiteStorageService storage,
|
||||
ScriptCompilationService compilationService,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions siteOptions,
|
||||
List<List<string>> tagPathsByConnection,
|
||||
List<SimulatedDataConnection> connections)
|
||||
{
|
||||
_connections = connections;
|
||||
SiteId = siteId;
|
||||
_config = config;
|
||||
_dataDirectory = dataDirectory;
|
||||
_localDbProvider = localDbProvider;
|
||||
_localDb = localDb;
|
||||
System = system;
|
||||
StreamManager = streamManager;
|
||||
HealthCollector = healthCollector;
|
||||
StoreAndForward = storeAndForward;
|
||||
DataConnectionManager = dataConnectionManager;
|
||||
Storage = storage;
|
||||
CompilationService = compilationService;
|
||||
SharedScriptLibrary = sharedScriptLibrary;
|
||||
SiteOptions = siteOptions;
|
||||
TagPathsByConnection = tagPathsByConnection;
|
||||
}
|
||||
|
||||
private SiteStorageService Storage { get; }
|
||||
private ScriptCompilationService CompilationService { get; }
|
||||
private SharedScriptLibrary SharedScriptLibrary { get; }
|
||||
private SiteRuntimeOptions SiteOptions { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Builds a site: LocalDb file, actor system, DCL with its simulated connections,
|
||||
/// stream manager, health collector and store-and-forward engine. Instance Actors
|
||||
/// are created separately by <see cref="StartInstancesAsync"/> so the deployment
|
||||
/// ramp can be timed on its own.
|
||||
/// </summary>
|
||||
/// <param name="siteIndex">Zero-based site index.</param>
|
||||
/// <param name="config">Harness configuration.</param>
|
||||
/// <param name="rootDataDirectory">Directory under which this site's SQLite files live.</param>
|
||||
/// <param name="connectionsPerSite">Number of data connections to spread the site's tags across.</param>
|
||||
/// <returns>The started fixture.</returns>
|
||||
public static async Task<SiteRuntimeFixture> CreateAsync(
|
||||
int siteIndex, HarnessConfig config, string rootDataDirectory, int connectionsPerSite)
|
||||
{
|
||||
var siteId = $"site-{siteIndex + 1:D2}";
|
||||
var dataDirectory = Path.Combine(rootDataDirectory, siteId);
|
||||
Directory.CreateDirectory(dataDirectory);
|
||||
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = Path.Combine(dataDirectory, "site-localdb.db"),
|
||||
})
|
||||
.Build();
|
||||
|
||||
var localDbProvider = new ServiceCollection()
|
||||
.AddZbLocalDb(configuration)
|
||||
.BuildServiceProvider();
|
||||
var localDb = localDbProvider.GetRequiredService<ILocalDb>();
|
||||
|
||||
var storage = new SiteStorageService(localDb, NullLogger<SiteStorageService>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var compilationService = new ScriptCompilationService(NullLogger<ScriptCompilationService>.Instance);
|
||||
var sharedScriptLibrary = new SharedScriptLibrary(compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
|
||||
// Production defaults throughout — the point of the run is to measure the
|
||||
// shipped configuration, so nothing here is tuned for the harness.
|
||||
var siteOptions = new SiteRuntimeOptions();
|
||||
|
||||
// WARNING-level logging. At 37,500 updates/s per site, Akka's INFO output would
|
||||
// itself become a measured load; and the InstanceActorInitialized dead letters are
|
||||
// a harness artifact (see StartInstancesAsync) rather than a real condition.
|
||||
var system = ActorSystem.Create($"loadharness-{siteId}", Akka.Configuration.ConfigurationFactory.ParseString(
|
||||
"akka.loglevel = WARNING\nakka.stdout-loglevel = WARNING\nakka.log-dead-letters = 0\nakka.log-dead-letters-during-shutdown = off"));
|
||||
|
||||
var streamManager = new SiteStreamManager(siteOptions, NullLogger<SiteStreamManager>.Instance);
|
||||
streamManager.Initialize(system);
|
||||
|
||||
var healthCollector = new SiteHealthCollector();
|
||||
healthCollector.SetActiveNode(true);
|
||||
healthCollector.SetNodeHostname($"{siteId}-node-a");
|
||||
|
||||
var sfStorage = new StoreAndForwardStorage(localDb, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
var storeAndForward = new StoreAndForwardService(
|
||||
sfStorage,
|
||||
new StoreAndForwardOptions(),
|
||||
NullLogger<StoreAndForwardService>.Instance,
|
||||
siteId: siteId);
|
||||
await storeAndForward.StartAsync();
|
||||
|
||||
// Real DCL, with the simulated adapter registered on the real factory via its
|
||||
// documented RegisterAdapter extension point.
|
||||
var loggerFactory = NullLoggerFactory.Instance;
|
||||
var factory = new DataConnectionFactory(loggerFactory);
|
||||
var connections = new List<SimulatedDataConnection>();
|
||||
factory.RegisterAdapter(SimulatedDataConnection.ProtocolName, _ =>
|
||||
{
|
||||
var connection = new SimulatedDataConnection();
|
||||
lock (connections) connections.Add(connection);
|
||||
return connection;
|
||||
});
|
||||
|
||||
var dclManager = system.ActorOf(
|
||||
Props.Create(() => new DataConnectionManagerActor(
|
||||
factory, new DataConnectionOptions(), healthCollector, null, null)),
|
||||
"data-connection-manager");
|
||||
|
||||
for (var c = 0; c < connectionsPerSite; c++)
|
||||
{
|
||||
dclManager.Tell(new CreateConnectionCommand(
|
||||
ConnectionName: ConnectionName(c),
|
||||
ProtocolType: SimulatedDataConnection.ProtocolName,
|
||||
PrimaryConnectionDetails: new Dictionary<string, string>
|
||||
{
|
||||
["endpoint"] = $"sim://{siteId}/{c}",
|
||||
[SimulatedDataConnection.ConnectionNameKey] = ConnectionName(c),
|
||||
}));
|
||||
}
|
||||
|
||||
var tagPathsByConnection = new List<List<string>>();
|
||||
for (var c = 0; c < connectionsPerSite; c++)
|
||||
tagPathsByConnection.Add(new List<string>());
|
||||
|
||||
var fixture = new SiteRuntimeFixture(
|
||||
siteId, config, dataDirectory, localDbProvider, localDb, system, streamManager,
|
||||
healthCollector, storeAndForward, dclManager, storage, compilationService,
|
||||
sharedScriptLibrary, siteOptions, tagPathsByConnection, connections);
|
||||
|
||||
return fixture;
|
||||
}
|
||||
|
||||
/// <summary>Deterministic connection name for a connection index.</summary>
|
||||
/// <param name="connectionIndex">Zero-based connection index.</param>
|
||||
/// <returns>The connection name used in configs and DCL commands.</returns>
|
||||
public static string ConnectionName(int connectionIndex) => $"sim-conn-{connectionIndex:D2}";
|
||||
|
||||
/// <summary>
|
||||
/// Creates this site's Instance Actors in production-shaped staggered batches
|
||||
/// (<see cref="SiteRuntimeOptions.StartupBatchSize"/> /
|
||||
/// <see cref="SiteRuntimeOptions.StartupBatchDelayMs"/>) and records how long the
|
||||
/// ramp took. This is the harness's stand-in for "deployment of 500 instances to
|
||||
/// a site" — it exercises the same per-instance construction, config
|
||||
/// deserialization, override load and DCL subscribe that a real deploy triggers.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancels the ramp.</param>
|
||||
/// <returns>A task that completes when every instance actor exists.</returns>
|
||||
public async Task StartInstancesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var started = Stopwatch.StartNew();
|
||||
var connectionCount = TagPathsByConnection.Count;
|
||||
|
||||
for (var i = 0; i < _config.InstancesPerSite; i++)
|
||||
{
|
||||
var instanceName = InstanceName(i);
|
||||
var connectionIndex = i % connectionCount;
|
||||
var connectionName = ConnectionName(connectionIndex);
|
||||
|
||||
var attributes = new List<ResolvedAttribute>(_config.TagsPerInstance);
|
||||
for (var t = 0; t < _config.TagsPerInstance; t++)
|
||||
{
|
||||
var tagPath = $"{instanceName}.Tag{t:D3}";
|
||||
TagPathsByConnection[connectionIndex].Add(tagPath);
|
||||
attributes.Add(new ResolvedAttribute
|
||||
{
|
||||
CanonicalName = $"Tag{t:D3}",
|
||||
DataType = "Double",
|
||||
DataSourceReference = tagPath,
|
||||
BoundDataConnectionId = connectionIndex + 1,
|
||||
BoundDataConnectionName = connectionName,
|
||||
BoundDataConnectionProtocol = SimulatedDataConnection.ProtocolName,
|
||||
});
|
||||
}
|
||||
|
||||
var configuration = new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
TemplateId = 1,
|
||||
SiteId = 1,
|
||||
Attributes = attributes,
|
||||
Connections = new Dictionary<string, ConnectionConfig>
|
||||
{
|
||||
[connectionName] = new()
|
||||
{
|
||||
Protocol = SimulatedDataConnection.ProtocolName,
|
||||
ConfigurationJson = "{}",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var configJson = JsonSerializer.Serialize(configuration);
|
||||
var actor = System.ActorOf(
|
||||
Props.Create(() => new InstanceActor(
|
||||
instanceName, configJson, Storage, CompilationService, SharedScriptLibrary,
|
||||
StreamManager, SiteOptions, NullLogger<InstanceActor>.Instance,
|
||||
DataConnectionManager, HealthCollector, null, null)),
|
||||
instanceName);
|
||||
_instanceActors.Add(actor);
|
||||
|
||||
// Production staggered-startup pacing (SiteRuntimeOptions defaults):
|
||||
// batches of StartupBatchSize separated by StartupBatchDelayMs, which is
|
||||
// exactly what DeploymentManagerActor does on a real site start.
|
||||
if ((i + 1) % SiteOptions.StartupBatchSize == 0)
|
||||
await Task.Delay(SiteOptions.StartupBatchDelayMs, cancellationToken);
|
||||
}
|
||||
|
||||
started.Stop();
|
||||
InstanceRampDuration = started.Elapsed;
|
||||
|
||||
HealthCollector.SetInstanceCounts(
|
||||
_config.InstancesPerSite, _config.InstancesPerSite, 0);
|
||||
for (var c = 0; c < connectionCount; c++)
|
||||
{
|
||||
HealthCollector.UpdateTagResolution(
|
||||
ConnectionName(c), TagPathsByConnection[c].Count, TagPathsByConnection[c].Count);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Deterministic instance unique name for an instance index.</summary>
|
||||
/// <param name="instanceIndex">Zero-based instance index.</param>
|
||||
/// <returns>The instance unique name.</returns>
|
||||
public string InstanceName(int instanceIndex) => $"{SiteId}-inst-{instanceIndex:D4}";
|
||||
|
||||
/// <summary>
|
||||
/// Attaches <see cref="HarnessConfig.StreamProbesPerSite"/> live stream subscribers,
|
||||
/// each built from the production pieces the gRPC server uses: a real
|
||||
/// <see cref="StreamRelayActor"/> writing into a bounded <c>DropOldest</c> channel
|
||||
/// of the production capacity, subscribed through the real
|
||||
/// <see cref="SiteStreamManager.Subscribe"/>. Only the socket writer is replaced —
|
||||
/// by a reader task under the harness's control, which is what makes the
|
||||
/// slow-subscriber scenario possible at all.
|
||||
///
|
||||
/// <para>
|
||||
/// Attaching at least one subscriber is also load-bearing:
|
||||
/// <c>PublishAttributeValueChanged</c> short-circuits at zero subscribers, so an
|
||||
/// unsubscribed site would publish nothing and measure nothing.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="latency">Histogram that receives end-to-end tag update latencies.</param>
|
||||
public void AttachStreamProbes(LatencyHistogram latency)
|
||||
{
|
||||
var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, _config.StreamProbesPerSite));
|
||||
for (var p = 0; p < _config.StreamProbesPerSite; p++)
|
||||
{
|
||||
var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1);
|
||||
var instanceName = InstanceName(instanceIndex);
|
||||
var probe = StreamSubscriberProbe.Attach(
|
||||
System, StreamManager, instanceName, $"{SiteId}-probe-{p:D2}", latency);
|
||||
_probes.Add(probe);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total tag paths registered across this site's connections.</summary>
|
||||
public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (var probe in _probes)
|
||||
await probe.DisposeAsync();
|
||||
|
||||
await StoreAndForward.StopAsync();
|
||||
await System.Terminate();
|
||||
await _localDbProvider.DisposeAsync();
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_dataDirectory))
|
||||
Directory.Delete(_dataDirectory, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort cleanup of a temp directory; a lingering WAL handle is not
|
||||
// a harness failure and must not mask the measured result.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// Drives simulated tag value changes into a site's connections at a fixed nominal
|
||||
/// rate, standing in for an OPC UA server's notification thread.
|
||||
///
|
||||
/// <para>
|
||||
/// One driver task per data connection, because that mirrors production: each
|
||||
/// <c>DataConnectionActor</c> has exactly one adapter feeding it, and every update
|
||||
/// for that connection funnels through that actor's single mailbox. Sharding the
|
||||
/// emitters differently would hide the per-connection-actor serialization point,
|
||||
/// which is one of the things WP-4 needs to characterize.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The emitter walks the connection's tag list in slices sized so that one full pass
|
||||
/// takes <see cref="HarnessConfig.TagUpdatePeriod"/>, then sleeps out the remainder
|
||||
/// of each slice's budget. If a slice overruns its budget the driver does NOT try to
|
||||
/// catch up — it records the shortfall in <see cref="EmitLagSeconds"/> so the results
|
||||
/// can say honestly whether the offered load was actually delivered.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class TagUpdateDriver : IAsyncDisposable
|
||||
{
|
||||
private const int SlicesPerPeriod = 20;
|
||||
|
||||
private readonly List<Task> _tasks = new();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private long _emitted;
|
||||
private long _skippedNoCallback;
|
||||
private long _lagTicks;
|
||||
|
||||
/// <summary>Total tag value changes handed to adapter callbacks.</summary>
|
||||
public long EmittedCount => Interlocked.Read(ref _emitted);
|
||||
|
||||
/// <summary>
|
||||
/// Emissions skipped because no Instance Actor had subscribed to that connection
|
||||
/// yet (the adapter callback is captured at subscribe time). Non-zero only during
|
||||
/// the ramp; a non-zero value in the steady window would mean lost offered load.
|
||||
/// </summary>
|
||||
public long SkippedNoCallback => Interlocked.Read(ref _skippedNoCallback);
|
||||
|
||||
/// <summary>
|
||||
/// Cumulative seconds by which emit slices overran their time budget, summed
|
||||
/// across driver tasks. Large values mean the harness itself could not offer the
|
||||
/// nominal rate and the measured throughput is driver-bound, not system-bound.
|
||||
/// </summary>
|
||||
public double EmitLagSeconds => Interlocked.Read(ref _lagTicks) / (double)Stopwatch.Frequency;
|
||||
|
||||
/// <summary>
|
||||
/// Starts one emitter task per connection across every site.
|
||||
/// </summary>
|
||||
/// <param name="sites">The sites to drive.</param>
|
||||
/// <param name="config">Harness configuration supplying the update period.</param>
|
||||
/// <returns>The running driver.</returns>
|
||||
public static TagUpdateDriver Start(IReadOnlyList<SiteRuntimeFixture> sites, HarnessConfig config)
|
||||
{
|
||||
var driver = new TagUpdateDriver();
|
||||
foreach (var site in sites)
|
||||
{
|
||||
for (var c = 0; c < site.TagPathsByConnection.Count; c++)
|
||||
{
|
||||
var connectionIndex = c;
|
||||
var tagPaths = site.TagPathsByConnection[connectionIndex].ToArray();
|
||||
driver._tasks.Add(Task.Run(() => driver.EmitLoopAsync(
|
||||
site, connectionIndex, tagPaths, config.TagUpdatePeriod, driver._cts.Token)));
|
||||
}
|
||||
}
|
||||
|
||||
return driver;
|
||||
}
|
||||
|
||||
private async Task EmitLoopAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int connectionIndex,
|
||||
string[] tagPaths,
|
||||
TimeSpan period,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (tagPaths.Length == 0)
|
||||
return;
|
||||
|
||||
var sliceBudget = period / SlicesPerPeriod;
|
||||
var sliceSize = Math.Max(1, (int)Math.Ceiling(tagPaths.Length / (double)SlicesPerPeriod));
|
||||
var cursor = 0;
|
||||
var sequence = 0d;
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var sliceStart = Stopwatch.GetTimestamp();
|
||||
var callback = ResolveCallback(site, connectionIndex);
|
||||
|
||||
if (callback == null)
|
||||
{
|
||||
Interlocked.Add(ref _skippedNoCallback, sliceSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
var end = Math.Min(cursor + sliceSize, tagPaths.Length);
|
||||
for (var i = cursor; i < end; i++)
|
||||
{
|
||||
// Stamped HERE: this instant rides TagValueUpdate.Timestamp all the
|
||||
// way to the subscriber, so the probe's subtraction is a genuine
|
||||
// end-to-end latency and not a re-stamped approximation.
|
||||
callback(tagPaths[i], new TagValue(sequence, QualityCode.Good, DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
Interlocked.Add(ref _emitted, end - cursor);
|
||||
cursor = end;
|
||||
}
|
||||
|
||||
if (cursor >= tagPaths.Length)
|
||||
{
|
||||
cursor = 0;
|
||||
sequence += 1d;
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetTimestamp() - sliceStart;
|
||||
var budgetTicks = (long)(sliceBudget.TotalSeconds * Stopwatch.Frequency);
|
||||
if (elapsed < budgetTicks)
|
||||
{
|
||||
var remaining = TimeSpan.FromSeconds((budgetTicks - elapsed) / (double)Stopwatch.Frequency);
|
||||
try
|
||||
{
|
||||
await Task.Delay(remaining, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Interlocked.Add(ref _lagTicks, elapsed - budgetTicks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the live adapter callback for a connection, by NAME. The DCL factory
|
||||
/// appends adapters as its manager actor processes CreateConnectionCommands, so
|
||||
/// list position does not track connection index.
|
||||
/// </summary>
|
||||
private static SubscriptionCallback? ResolveCallback(SiteRuntimeFixture site, int connectionIndex)
|
||||
{
|
||||
var name = SiteRuntimeFixture.ConnectionName(connectionIndex);
|
||||
// Last match wins: a reconnect would create a fresh adapter for the same name,
|
||||
// and only the newest one holds the live subscription callback.
|
||||
return site.Connections.LastOrDefault(c => c.ConnectionName == name)?.ValueCallback;
|
||||
}
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_tasks);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected on shutdown.
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<!--
|
||||
Target-scale load harness (deferred-work register #25 / Phase-8 WP-4, absorbing row 50).
|
||||
|
||||
Deliberately an Exe, not an xunit project: the full-scale protocol is a
|
||||
~20-minute sustained run at 10 sites x 500 instances x 75 tags, which must NOT
|
||||
execute as part of `dotnet test ZB.MOM.WW.ScadaBridge.slnx`. The PerformanceTests
|
||||
project carries a tiny scaled-down smoke [Fact] that references this assembly so
|
||||
the harness stays compiled and exercised by CI without the runtime cost.
|
||||
|
||||
Design memo: docs/plans/2026-08-15-target-scale-load-test-design.md
|
||||
Results: docs/plans/2026-08-15-target-scale-load-test-results.md
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<OutputType>Exe</OutputType>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsPackable>false</IsPackable>
|
||||
<RootNamespace>ZB.MOM.WW.ScadaBridge.LoadHarness</RootNamespace>
|
||||
<!-- Server GC: the harness holds ~5,000 actors and 375,000 attribute slots; the
|
||||
workstation collector's single heap is not representative of a site node,
|
||||
which runs under the ASP.NET Core host defaults (server GC on). -->
|
||||
<ServerGarbageCollection>true</ServerGarbageCollection>
|
||||
<ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ZB.MOM.WW.ScadaBridge.HealthMonitoring.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ZB.MOM.WW.ScadaBridge.StoreAndForward.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.AuditLog/ZB.MOM.WW.ScadaBridge.AuditLog.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.TargetScale;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the target-scale load harness (deferred-work register #25 / Phase-8 WP-4)
|
||||
/// honest at CI scale.
|
||||
///
|
||||
/// <para>
|
||||
/// The full protocol — 10 sites x 500 instances x 75 tags sustained for 20 minutes —
|
||||
/// deliberately lives in the standalone <c>ZB.MOM.WW.ScadaBridge.LoadHarness</c>
|
||||
/// executable, NOT here: perf tests in this project run as part of an ordinary
|
||||
/// <c>dotnet test ZB.MOM.WW.ScadaBridge.slnx</c> (the <c>Category=Performance</c>
|
||||
/// trait enables a filter, it does not exclude by default), and a 20-minute test
|
||||
/// would be intolerable there. What this test protects is that the harness still
|
||||
/// compiles, wires up, and produces coherent measurements — so #25's evidence can be
|
||||
/// regenerated on demand rather than bit-rotting.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Results doc: <c>docs/plans/2026-08-15-target-scale-load-test-results.md</c>.
|
||||
/// Design memo: <c>docs/plans/2026-08-15-target-scale-load-test-design.md</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class TargetScaleHarnessSmokeTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Runs the harness at ~1/1000th of target scale for a few seconds and asserts the
|
||||
/// pipeline is intact end to end: tag updates reach live stream subscribers, the
|
||||
/// central health aggregator tracks every site, debug snapshots answer, the
|
||||
/// store-and-forward buffer drains to empty, and a stalled subscriber does not cost
|
||||
/// the healthy ones any events.
|
||||
/// </summary>
|
||||
/// <returns>A task representing the test run.</returns>
|
||||
[Trait("Category", "Performance")]
|
||||
[Fact]
|
||||
public async Task Harness_AtSmokeScale_ProducesCoherentMeasurements()
|
||||
{
|
||||
var config = new HarnessConfig
|
||||
{
|
||||
Sites = 2,
|
||||
InstancesPerSite = 10,
|
||||
TagsPerInstance = 5,
|
||||
TagUpdatePeriod = TimeSpan.FromSeconds(1),
|
||||
SettleDuration = TimeSpan.FromSeconds(5),
|
||||
SustainDuration = TimeSpan.FromSeconds(15),
|
||||
SampleInterval = TimeSpan.FromSeconds(2),
|
||||
// Shortened from the production 30 s only because the smoke window is 20 s.
|
||||
HealthReportInterval = TimeSpan.FromSeconds(2),
|
||||
DebugProbeInterval = TimeSpan.FromSeconds(2),
|
||||
SubscribeSettleDuration = TimeSpan.FromSeconds(5),
|
||||
StreamProbesPerSite = 3,
|
||||
StoreAndForwardDrainMessages = 200,
|
||||
SlowSubscriberEvents = 2_000,
|
||||
ResultsPath = Path.Combine(Path.GetTempPath(), $"loadharness-smoke-{Guid.NewGuid():N}.json"),
|
||||
};
|
||||
|
||||
using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(10));
|
||||
var result = await HarnessRun.ExecuteAsync(config, _ => { }, cancellation.Token);
|
||||
|
||||
// Tag updates flowed all the way through DCL -> InstanceActor -> site stream ->
|
||||
// StreamRelayActor -> bounded channel -> subscriber.
|
||||
Assert.True(result.TagUpdateLatency.Count > 0,
|
||||
"No tag update latency samples — the DCL -> stream -> subscriber path did not carry traffic.");
|
||||
Assert.True(result.StreamProbeReceived > 0, "Live stream subscribers received nothing.");
|
||||
Assert.Equal(0, result.DriverSkippedNoCallback);
|
||||
|
||||
// Observability held up.
|
||||
Assert.Equal(config.Sites, result.SitesTrackedByAggregator);
|
||||
Assert.True(result.DebugSnapshotsCompleted > 0, "No debug snapshot completed.");
|
||||
Assert.Equal(0, result.DebugSnapshotTimeouts);
|
||||
|
||||
// Store-and-forward drained completely (register row 50, first half).
|
||||
Assert.NotNull(result.StoreAndForwardDrain);
|
||||
Assert.Equal(0, result.StoreAndForwardDrain!.ResidualDepth);
|
||||
Assert.True(result.StoreAndForwardDrain.DrainPerSecond > 0);
|
||||
|
||||
// A stalled subscriber costs the healthy ones nothing (register row 50, second
|
||||
// half). This is the design's isolation claim, asserted rather than assumed.
|
||||
Assert.NotNull(result.SlowSubscriber);
|
||||
Assert.Equal(1.0, result.SlowSubscriber!.HealthyMinDeliveryRatio, precision: 2);
|
||||
Assert.True(result.SlowSubscriber.SlowDeliveryRatio < 1.0,
|
||||
"The deliberately stalled subscriber kept up, so the probe proved nothing.");
|
||||
}
|
||||
}
|
||||
+3
@@ -31,6 +31,9 @@
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
|
||||
<!-- Two-node failover rig (TwoNodeClusterFixture) — brings Host + Akka transitively. -->
|
||||
<ProjectReference Include="../../tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/ZB.MOM.WW.ScadaBridge.IntegrationTests.csproj" />
|
||||
<!-- Target-scale load harness (register #25). Referenced so TargetScaleHarnessSmokeTests
|
||||
can run it at CI scale; the full-scale protocol run is the standalone executable. -->
|
||||
<ProjectReference Include="../../tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ZB.MOM.WW.ScadaBridge.LoadHarness.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user