using System.Diagnostics;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
/// Everything one harness run measured, ready for serialization.
/// The configuration the run executed under.
/// Host/runtime description.
/// Run start.
/// Total wall time including ramp and teardown.
/// Wall time to build all site fixtures (before instances).
/// Wall time to create every Instance Actor across all sites.
/// Slowest single site's instance ramp — the "deploy 500 instances to a site" figure.
/// End-to-end DCL-boundary to stream-subscriber latency.
/// Tag updates offered during the whole run.
/// Tag updates offered during the measurement window only.
/// Offered load actually achieved in the measurement window.
/// Offered load the configuration called for.
/// Cumulative driver slice overrun (harness-bound load shortfall).
/// Emissions skipped before subscriptions existed.
/// Resource behaviour over the measurement window.
/// Resource behaviour over the whole run.
///
/// Every raw resource sample. Retained because the summary alone cannot distinguish a
/// slow leak from a heap that simply sawtooths — especially when the run records zero
/// gen-2 collections, where a positive least-squares slope may be nothing more than
/// gen-2 garbage that was never collected.
///
/// Health report collect+ingest latency.
/// Health reports ingested by the central aggregator.
/// Sites the central aggregator ended up tracking.
/// Debug view snapshot round-trip latency under load.
/// Debug snapshots that completed.
/// Debug snapshots that timed out.
/// Events delivered to live stream subscribers.
/// Events evicted by live subscribers' bounded channels.
/// Store-and-forward drain measurement (register row 50).
/// Slow-subscriber isolation measurement (register row 50).
public sealed record HarnessRunResult(
HarnessConfig Config,
EnvironmentInfo Environment,
DateTimeOffset StartedUtc,
double TotalSeconds,
double SiteRampSeconds,
double InstanceRampSeconds,
double SlowestSiteInstanceRampSeconds,
LatencySnapshot TagUpdateLatency,
long EmittedTagUpdates,
long SteadyStateEmittedTagUpdates,
double AchievedUpdatesPerSecond,
double NominalUpdatesPerSecond,
double DriverLagSeconds,
long DriverSkippedNoCallback,
ResourceWindowSummary? SteadyStateResources,
ResourceWindowSummary? WholeRunResources,
IReadOnlyList ResourceSamples,
LatencySnapshot HealthReportLatency,
long HealthReportsDelivered,
int SitesTrackedByAggregator,
LatencySnapshot DebugSnapshotLatency,
long DebugSnapshotsCompleted,
long DebugSnapshotTimeouts,
long StreamProbeReceived,
long StreamProbeDropped,
StoreAndForwardDrainResult? StoreAndForwardDrain,
SlowSubscriberResult? SlowSubscriber);
/// Host and runtime facts recorded alongside the numbers.
/// Host name.
/// Operating system description.
/// Logical processors visible to the process.
/// .NET runtime version.
/// Whether server GC is active.
public sealed record EnvironmentInfo(
string MachineName,
string OsDescription,
int ProcessorCount,
string RuntimeVersion,
bool ServerGc);
///
/// Orchestrates a full run: build sites, ramp instances, attach subscribers, drive
/// tag updates for the sustained window while sampling resources and probing
/// observability, then run the two register-row-50 scenarios on a dedicated site.
///
public static class HarnessRun
{
/// Data connections each site spreads its tags across.
public const int ConnectionsPerSite = 5;
/// Subscribers attached in the slow-subscriber isolation probe.
public const int SlowSubscriberProbeCount = 5;
/// Concurrent enqueue tasks in the store-and-forward drain probe.
public const int StoreAndForwardConcurrency = 25;
/// Executes a run end to end.
/// Scale and duration configuration.
/// Progress sink (stdout in the console app).
/// Cancels the run.
/// The measured result.
public static async Task ExecuteAsync(
HarnessConfig config, Action log, CancellationToken cancellationToken)
{
var startedUtc = DateTimeOffset.UtcNow;
var totalWatch = Stopwatch.StartNew();
var dataRoot = config.DataDirectory
?? Path.Combine(Path.GetTempPath(), $"scadabridge-loadharness-{Guid.NewGuid():N}");
Directory.CreateDirectory(dataRoot);
var sampler = ResourceSampler.Start(config.SampleInterval);
var sites = new List(config.Sites);
var latency = new LatencyHistogram();
TagUpdateDriver? driver = null;
ObservabilityProbes? probes = null;
StoreAndForwardDrainResult? drainResult = null;
SlowSubscriberResult? slowResult = null;
var aggregator = new CentralHealthAggregator(
Options.Create(new HealthMonitoringOptions
{
ReportInterval = config.HealthReportInterval,
OfflineTimeout = config.HealthReportInterval * 2,
}),
NullLogger.Instance);
try
{
log($"Building {config.Sites} sites ({config.InstancesPerSite} instances x " +
$"{config.TagsPerInstance} tags each = {config.TotalSubscriptions:N0} subscriptions)...");
var siteWatch = Stopwatch.StartNew();
for (var s = 0; s < config.Sites; s++)
sites.Add(await SiteRuntimeFixture.CreateAsync(s, config, dataRoot, ConnectionsPerSite));
siteWatch.Stop();
log($" sites built in {siteWatch.Elapsed.TotalSeconds:F1}s");
// Instance ramp — sites in parallel (they are independent actor systems,
// exactly as 10 real sites would be), each site internally staggered at the
// production StartupBatchSize/StartupBatchDelayMs pacing.
var rampWatch = Stopwatch.StartNew();
await Task.WhenAll(sites.Select(site => site.StartInstancesAsync(cancellationToken)));
rampWatch.Stop();
var slowestSiteRamp = sites.Max(s => s.InstanceRampDuration.TotalSeconds);
log($" {config.Sites * config.InstancesPerSite:N0} instance actors created in " +
$"{rampWatch.Elapsed.TotalSeconds:F1}s (slowest site {slowestSiteRamp:F1}s)");
foreach (var site in sites)
site.AttachStreamProbes(latency);
log($" {sites.Sum(s => s.Probes.Count)} live stream subscribers attached");
// Let Instance Actors complete their DCL subscribe round-trips before the
// driver starts; an emit before SubscribeBatchAsync has captured the
// callback would be silently discarded.
await Task.Delay(config.SubscribeSettleDuration, cancellationToken);
driver = TagUpdateDriver.Start(sites, config);
log($" tag driver started, nominal {config.NominalUpdatesPerSecond:N0} updates/s");
probes = ObservabilityProbes.Start(
sites, aggregator, config.HealthReportInterval, config.DebugProbeInterval);
log($"Settling for {config.SettleDuration.TotalMinutes:F1} min...");
await Task.Delay(config.SettleDuration, cancellationToken);
// Switch to a fresh latency histogram so ramp-window outliers do not
// contaminate the steady-state percentiles. Everything reported as "steady
// state" is measured strictly after this point; the subscriptions
// themselves are left untouched.
var steadyStateStartSeconds = sampler.Snapshot().LastOrDefault()?.ElapsedSeconds ?? 0;
var emittedAtWindowStart = driver.EmittedCount;
var receivedAtWindowStart = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents));
var droppedAtWindowStart = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents));
var steadyLatency = new LatencyHistogram();
foreach (var site in sites)
{
foreach (var probe in site.Probes)
probe.RetargetLatency(steadyLatency);
}
var windowWatch = Stopwatch.StartNew();
log($"Sustained measurement window: {config.SustainDuration.TotalMinutes:F1} min...");
await Task.Delay(config.SustainDuration, cancellationToken);
windowWatch.Stop();
var steadyStateEndSeconds = sampler.Snapshot().LastOrDefault()?.ElapsedSeconds ?? 0;
var emittedInWindow = driver.EmittedCount - emittedAtWindowStart;
var steadyResources = sampler.Summarize(steadyStateStartSeconds, steadyStateEndSeconds);
log($" window complete: {emittedInWindow:N0} updates offered, " +
$"{emittedInWindow / windowWatch.Elapsed.TotalSeconds:N0}/s achieved");
var probeReceived = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents)) - receivedAtWindowStart;
var probeDropped = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents)) - droppedAtWindowStart;
// Register row 50 — measured after the sustained window so the numbers are
// not competing with the full tag load for CPU, and reported separately for
// the same reason.
log("Stopping tag driver for the register row 50 scenarios...");
var driverLagSeconds = driver.EmitLagSeconds;
var driverSkipped = driver.SkippedNoCallback;
await driver.DisposeAsync();
driver = null;
await probes.DisposeAsync();
log($"Store-and-forward drain: {config.StoreAndForwardDrainMessages:N0} messages...");
drainResult = await StoreAndForwardDrainScenario.RunAsync(
sites[0], config.StoreAndForwardDrainMessages, StoreAndForwardConcurrency, cancellationToken);
log($" buffered {drainResult.EnqueuePerSecond:N0}/s, drained {drainResult.DrainPerSecond:N0}/s");
log($"Slow-subscriber isolation: {SlowSubscriberProbeCount} subscribers, " +
$"{config.SlowSubscriberEvents:N0} events...");
slowResult = await SlowSubscriberScenario.RunAsync(
sites[1 % sites.Count], SlowSubscriberProbeCount, config.SlowSubscriberEvents, cancellationToken);
log($" healthy min delivery {slowResult.HealthyMinDeliveryRatio:P2}, " +
$"stalled {slowResult.SlowDeliveryRatio:P2}");
totalWatch.Stop();
return new HarnessRunResult(
Config: config,
Environment: CaptureEnvironment(),
StartedUtc: startedUtc,
TotalSeconds: totalWatch.Elapsed.TotalSeconds,
SiteRampSeconds: siteWatch.Elapsed.TotalSeconds,
InstanceRampSeconds: rampWatch.Elapsed.TotalSeconds,
SlowestSiteInstanceRampSeconds: slowestSiteRamp,
TagUpdateLatency: steadyLatency.Snapshot(),
EmittedTagUpdates: emittedAtWindowStart + emittedInWindow,
SteadyStateEmittedTagUpdates: emittedInWindow,
AchievedUpdatesPerSecond: emittedInWindow / windowWatch.Elapsed.TotalSeconds,
NominalUpdatesPerSecond: config.NominalUpdatesPerSecond,
DriverLagSeconds: driverLagSeconds,
DriverSkippedNoCallback: driverSkipped,
SteadyStateResources: steadyResources,
WholeRunResources: sampler.Summarize(0, double.MaxValue),
ResourceSamples: sampler.Snapshot(),
HealthReportLatency: probes.HealthReportLatency.Snapshot(),
HealthReportsDelivered: probes.HealthReportsDelivered,
SitesTrackedByAggregator: aggregator.GetAllSiteStates().Count,
DebugSnapshotLatency: probes.DebugSnapshotLatency.Snapshot(),
DebugSnapshotsCompleted: probes.DebugSnapshotsCompleted,
DebugSnapshotTimeouts: probes.DebugSnapshotTimeouts,
StreamProbeReceived: probeReceived,
StreamProbeDropped: probeDropped,
StoreAndForwardDrain: drainResult,
SlowSubscriber: slowResult);
}
finally
{
if (driver != null) await driver.DisposeAsync();
if (probes != null) await probes.DisposeAsync();
await sampler.DisposeAsync();
log("Tearing down sites...");
foreach (var site in sites)
await site.DisposeAsync();
try
{
if (config.DataDirectory == null && Directory.Exists(dataRoot))
Directory.Delete(dataRoot, recursive: true);
}
catch (IOException)
{
// Temp cleanup only.
}
}
}
private static EnvironmentInfo CaptureEnvironment() => new(
System.Environment.MachineName,
System.Runtime.InteropServices.RuntimeInformation.OSDescription,
System.Environment.ProcessorCount,
System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription,
System.Runtime.GCSettings.IsServerGC);
}