test(loadharness): target-scale load harness for WP-4 / register #25 + row 50
Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,371 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
|
||||
|
||||
/// <summary>
|
||||
/// One simulated site: its own <see cref="ActorSystem"/>, its own LocalDb SQLite
|
||||
/// file, a real Data Connection Layer over <see cref="SimulatedDataConnection"/>
|
||||
/// adapters, real Instance Actors, a real <see cref="SiteStreamManager"/>, real
|
||||
/// store-and-forward, and a real <see cref="SiteHealthCollector"/>.
|
||||
///
|
||||
/// <para>
|
||||
/// The sites are separate, non-clustered ActorSystems rather than 10 real two-node
|
||||
/// Akka clusters. Cluster membership, singleton placement and failover timing are
|
||||
/// already measured on a real two-node rig by
|
||||
/// <c>PerformanceTests/Failover/FailoverTimingTests.cs</c>; what WP-4 asks about is
|
||||
/// the load-bearing hierarchy under each singleton, which is what this builds.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SiteRuntimeFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly HarnessConfig _config;
|
||||
private readonly List<SimulatedDataConnection> _connections;
|
||||
private readonly ServiceProvider _localDbProvider;
|
||||
private readonly ILocalDb _localDb;
|
||||
private readonly List<IActorRef> _instanceActors = new();
|
||||
private readonly List<StreamSubscriberProbe> _probes = new();
|
||||
private readonly string _dataDirectory;
|
||||
|
||||
/// <summary>Site identifier, e.g. <c>site-01</c>.</summary>
|
||||
public string SiteId { get; }
|
||||
|
||||
/// <summary>This site's actor system.</summary>
|
||||
public ActorSystem System { get; }
|
||||
|
||||
/// <summary>The real site-wide broadcast stream.</summary>
|
||||
public SiteStreamManager StreamManager { get; }
|
||||
|
||||
/// <summary>The real site health collector feeding the 30 s report.</summary>
|
||||
public SiteHealthCollector HealthCollector { get; }
|
||||
|
||||
/// <summary>The real store-and-forward engine for this site.</summary>
|
||||
public StoreAndForwardService StoreAndForward { get; }
|
||||
|
||||
/// <summary>The real DCL manager actor.</summary>
|
||||
public IActorRef DataConnectionManager { get; }
|
||||
|
||||
/// <summary>The simulated adapters, one per data connection, in creation order.</summary>
|
||||
public IReadOnlyList<SimulatedDataConnection> Connections
|
||||
{
|
||||
get { lock (_connections) return _connections.ToList(); }
|
||||
}
|
||||
|
||||
/// <summary>The live stream subscribers attached to this site.</summary>
|
||||
public IReadOnlyList<StreamSubscriberProbe> Probes => _probes;
|
||||
|
||||
/// <summary>Instance Actors created on this site.</summary>
|
||||
public IReadOnlyList<IActorRef> InstanceActors => _instanceActors;
|
||||
|
||||
/// <summary>Tag paths per connection index, in the order they were assigned.</summary>
|
||||
public IReadOnlyList<List<string>> TagPathsByConnection { get; }
|
||||
|
||||
/// <summary>Wall-clock time the instance ramp took, measured by <see cref="StartInstancesAsync"/>.</summary>
|
||||
public TimeSpan InstanceRampDuration { get; private set; }
|
||||
|
||||
private SiteRuntimeFixture(
|
||||
string siteId,
|
||||
HarnessConfig config,
|
||||
string dataDirectory,
|
||||
ServiceProvider localDbProvider,
|
||||
ILocalDb localDb,
|
||||
ActorSystem system,
|
||||
SiteStreamManager streamManager,
|
||||
SiteHealthCollector healthCollector,
|
||||
StoreAndForwardService storeAndForward,
|
||||
IActorRef dataConnectionManager,
|
||||
SiteStorageService storage,
|
||||
ScriptCompilationService compilationService,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions siteOptions,
|
||||
List<List<string>> tagPathsByConnection,
|
||||
List<SimulatedDataConnection> connections)
|
||||
{
|
||||
_connections = connections;
|
||||
SiteId = siteId;
|
||||
_config = config;
|
||||
_dataDirectory = dataDirectory;
|
||||
_localDbProvider = localDbProvider;
|
||||
_localDb = localDb;
|
||||
System = system;
|
||||
StreamManager = streamManager;
|
||||
HealthCollector = healthCollector;
|
||||
StoreAndForward = storeAndForward;
|
||||
DataConnectionManager = dataConnectionManager;
|
||||
Storage = storage;
|
||||
CompilationService = compilationService;
|
||||
SharedScriptLibrary = sharedScriptLibrary;
|
||||
SiteOptions = siteOptions;
|
||||
TagPathsByConnection = tagPathsByConnection;
|
||||
}
|
||||
|
||||
private SiteStorageService Storage { get; }
|
||||
private ScriptCompilationService CompilationService { get; }
|
||||
private SharedScriptLibrary SharedScriptLibrary { get; }
|
||||
private SiteRuntimeOptions SiteOptions { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Builds a site: LocalDb file, actor system, DCL with its simulated connections,
|
||||
/// stream manager, health collector and store-and-forward engine. Instance Actors
|
||||
/// are created separately by <see cref="StartInstancesAsync"/> so the deployment
|
||||
/// ramp can be timed on its own.
|
||||
/// </summary>
|
||||
/// <param name="siteIndex">Zero-based site index.</param>
|
||||
/// <param name="config">Harness configuration.</param>
|
||||
/// <param name="rootDataDirectory">Directory under which this site's SQLite files live.</param>
|
||||
/// <param name="connectionsPerSite">Number of data connections to spread the site's tags across.</param>
|
||||
/// <returns>The started fixture.</returns>
|
||||
public static async Task<SiteRuntimeFixture> CreateAsync(
|
||||
int siteIndex, HarnessConfig config, string rootDataDirectory, int connectionsPerSite)
|
||||
{
|
||||
var siteId = $"site-{siteIndex + 1:D2}";
|
||||
var dataDirectory = Path.Combine(rootDataDirectory, siteId);
|
||||
Directory.CreateDirectory(dataDirectory);
|
||||
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = Path.Combine(dataDirectory, "site-localdb.db"),
|
||||
})
|
||||
.Build();
|
||||
|
||||
var localDbProvider = new ServiceCollection()
|
||||
.AddZbLocalDb(configuration)
|
||||
.BuildServiceProvider();
|
||||
var localDb = localDbProvider.GetRequiredService<ILocalDb>();
|
||||
|
||||
var storage = new SiteStorageService(localDb, NullLogger<SiteStorageService>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var compilationService = new ScriptCompilationService(NullLogger<ScriptCompilationService>.Instance);
|
||||
var sharedScriptLibrary = new SharedScriptLibrary(compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
|
||||
// Production defaults throughout — the point of the run is to measure the
|
||||
// shipped configuration, so nothing here is tuned for the harness.
|
||||
var siteOptions = new SiteRuntimeOptions();
|
||||
|
||||
// WARNING-level logging. At 37,500 updates/s per site, Akka's INFO output would
|
||||
// itself become a measured load; and the InstanceActorInitialized dead letters are
|
||||
// a harness artifact (see StartInstancesAsync) rather than a real condition.
|
||||
var system = ActorSystem.Create($"loadharness-{siteId}", Akka.Configuration.ConfigurationFactory.ParseString(
|
||||
"akka.loglevel = WARNING\nakka.stdout-loglevel = WARNING\nakka.log-dead-letters = 0\nakka.log-dead-letters-during-shutdown = off"));
|
||||
|
||||
var streamManager = new SiteStreamManager(siteOptions, NullLogger<SiteStreamManager>.Instance);
|
||||
streamManager.Initialize(system);
|
||||
|
||||
var healthCollector = new SiteHealthCollector();
|
||||
healthCollector.SetActiveNode(true);
|
||||
healthCollector.SetNodeHostname($"{siteId}-node-a");
|
||||
|
||||
var sfStorage = new StoreAndForwardStorage(localDb, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
var storeAndForward = new StoreAndForwardService(
|
||||
sfStorage,
|
||||
new StoreAndForwardOptions(),
|
||||
NullLogger<StoreAndForwardService>.Instance,
|
||||
siteId: siteId);
|
||||
await storeAndForward.StartAsync();
|
||||
|
||||
// Real DCL, with the simulated adapter registered on the real factory via its
|
||||
// documented RegisterAdapter extension point.
|
||||
var loggerFactory = NullLoggerFactory.Instance;
|
||||
var factory = new DataConnectionFactory(loggerFactory);
|
||||
var connections = new List<SimulatedDataConnection>();
|
||||
factory.RegisterAdapter(SimulatedDataConnection.ProtocolName, _ =>
|
||||
{
|
||||
var connection = new SimulatedDataConnection();
|
||||
lock (connections) connections.Add(connection);
|
||||
return connection;
|
||||
});
|
||||
|
||||
var dclManager = system.ActorOf(
|
||||
Props.Create(() => new DataConnectionManagerActor(
|
||||
factory, new DataConnectionOptions(), healthCollector, null, null)),
|
||||
"data-connection-manager");
|
||||
|
||||
for (var c = 0; c < connectionsPerSite; c++)
|
||||
{
|
||||
dclManager.Tell(new CreateConnectionCommand(
|
||||
ConnectionName: ConnectionName(c),
|
||||
ProtocolType: SimulatedDataConnection.ProtocolName,
|
||||
PrimaryConnectionDetails: new Dictionary<string, string>
|
||||
{
|
||||
["endpoint"] = $"sim://{siteId}/{c}",
|
||||
[SimulatedDataConnection.ConnectionNameKey] = ConnectionName(c),
|
||||
}));
|
||||
}
|
||||
|
||||
var tagPathsByConnection = new List<List<string>>();
|
||||
for (var c = 0; c < connectionsPerSite; c++)
|
||||
tagPathsByConnection.Add(new List<string>());
|
||||
|
||||
var fixture = new SiteRuntimeFixture(
|
||||
siteId, config, dataDirectory, localDbProvider, localDb, system, streamManager,
|
||||
healthCollector, storeAndForward, dclManager, storage, compilationService,
|
||||
sharedScriptLibrary, siteOptions, tagPathsByConnection, connections);
|
||||
|
||||
return fixture;
|
||||
}
|
||||
|
||||
/// <summary>Deterministic connection name for a connection index.</summary>
|
||||
/// <param name="connectionIndex">Zero-based connection index.</param>
|
||||
/// <returns>The connection name used in configs and DCL commands.</returns>
|
||||
public static string ConnectionName(int connectionIndex) => $"sim-conn-{connectionIndex:D2}";
|
||||
|
||||
/// <summary>
|
||||
/// Creates this site's Instance Actors in production-shaped staggered batches
|
||||
/// (<see cref="SiteRuntimeOptions.StartupBatchSize"/> /
|
||||
/// <see cref="SiteRuntimeOptions.StartupBatchDelayMs"/>) and records how long the
|
||||
/// ramp took. This is the harness's stand-in for "deployment of 500 instances to
|
||||
/// a site" — it exercises the same per-instance construction, config
|
||||
/// deserialization, override load and DCL subscribe that a real deploy triggers.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancels the ramp.</param>
|
||||
/// <returns>A task that completes when every instance actor exists.</returns>
|
||||
public async Task StartInstancesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var started = Stopwatch.StartNew();
|
||||
var connectionCount = TagPathsByConnection.Count;
|
||||
|
||||
for (var i = 0; i < _config.InstancesPerSite; i++)
|
||||
{
|
||||
var instanceName = InstanceName(i);
|
||||
var connectionIndex = i % connectionCount;
|
||||
var connectionName = ConnectionName(connectionIndex);
|
||||
|
||||
var attributes = new List<ResolvedAttribute>(_config.TagsPerInstance);
|
||||
for (var t = 0; t < _config.TagsPerInstance; t++)
|
||||
{
|
||||
var tagPath = $"{instanceName}.Tag{t:D3}";
|
||||
TagPathsByConnection[connectionIndex].Add(tagPath);
|
||||
attributes.Add(new ResolvedAttribute
|
||||
{
|
||||
CanonicalName = $"Tag{t:D3}",
|
||||
DataType = "Double",
|
||||
DataSourceReference = tagPath,
|
||||
BoundDataConnectionId = connectionIndex + 1,
|
||||
BoundDataConnectionName = connectionName,
|
||||
BoundDataConnectionProtocol = SimulatedDataConnection.ProtocolName,
|
||||
});
|
||||
}
|
||||
|
||||
var configuration = new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
TemplateId = 1,
|
||||
SiteId = 1,
|
||||
Attributes = attributes,
|
||||
Connections = new Dictionary<string, ConnectionConfig>
|
||||
{
|
||||
[connectionName] = new()
|
||||
{
|
||||
Protocol = SimulatedDataConnection.ProtocolName,
|
||||
ConfigurationJson = "{}",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var configJson = JsonSerializer.Serialize(configuration);
|
||||
var actor = System.ActorOf(
|
||||
Props.Create(() => new InstanceActor(
|
||||
instanceName, configJson, Storage, CompilationService, SharedScriptLibrary,
|
||||
StreamManager, SiteOptions, NullLogger<InstanceActor>.Instance,
|
||||
DataConnectionManager, HealthCollector, null, null)),
|
||||
instanceName);
|
||||
_instanceActors.Add(actor);
|
||||
|
||||
// Production staggered-startup pacing (SiteRuntimeOptions defaults):
|
||||
// batches of StartupBatchSize separated by StartupBatchDelayMs, which is
|
||||
// exactly what DeploymentManagerActor does on a real site start.
|
||||
if ((i + 1) % SiteOptions.StartupBatchSize == 0)
|
||||
await Task.Delay(SiteOptions.StartupBatchDelayMs, cancellationToken);
|
||||
}
|
||||
|
||||
started.Stop();
|
||||
InstanceRampDuration = started.Elapsed;
|
||||
|
||||
HealthCollector.SetInstanceCounts(
|
||||
_config.InstancesPerSite, _config.InstancesPerSite, 0);
|
||||
for (var c = 0; c < connectionCount; c++)
|
||||
{
|
||||
HealthCollector.UpdateTagResolution(
|
||||
ConnectionName(c), TagPathsByConnection[c].Count, TagPathsByConnection[c].Count);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Deterministic instance unique name for an instance index.</summary>
|
||||
/// <param name="instanceIndex">Zero-based instance index.</param>
|
||||
/// <returns>The instance unique name.</returns>
|
||||
public string InstanceName(int instanceIndex) => $"{SiteId}-inst-{instanceIndex:D4}";
|
||||
|
||||
/// <summary>
|
||||
/// Attaches <see cref="HarnessConfig.StreamProbesPerSite"/> live stream subscribers,
|
||||
/// each built from the production pieces the gRPC server uses: a real
|
||||
/// <see cref="StreamRelayActor"/> writing into a bounded <c>DropOldest</c> channel
|
||||
/// of the production capacity, subscribed through the real
|
||||
/// <see cref="SiteStreamManager.Subscribe"/>. Only the socket writer is replaced —
|
||||
/// by a reader task under the harness's control, which is what makes the
|
||||
/// slow-subscriber scenario possible at all.
|
||||
///
|
||||
/// <para>
|
||||
/// Attaching at least one subscriber is also load-bearing:
|
||||
/// <c>PublishAttributeValueChanged</c> short-circuits at zero subscribers, so an
|
||||
/// unsubscribed site would publish nothing and measure nothing.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="latency">Histogram that receives end-to-end tag update latencies.</param>
|
||||
public void AttachStreamProbes(LatencyHistogram latency)
|
||||
{
|
||||
var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, _config.StreamProbesPerSite));
|
||||
for (var p = 0; p < _config.StreamProbesPerSite; p++)
|
||||
{
|
||||
var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1);
|
||||
var instanceName = InstanceName(instanceIndex);
|
||||
var probe = StreamSubscriberProbe.Attach(
|
||||
System, StreamManager, instanceName, $"{SiteId}-probe-{p:D2}", latency);
|
||||
_probes.Add(probe);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total tag paths registered across this site's connections.</summary>
|
||||
public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (var probe in _probes)
|
||||
await probe.DisposeAsync();
|
||||
|
||||
await StoreAndForward.StopAsync();
|
||||
await System.Terminate();
|
||||
await _localDbProvider.DisposeAsync();
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_dataDirectory))
|
||||
Directory.Delete(_dataDirectory, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort cleanup of a temp directory; a lingering WAL handle is not
|
||||
// a harness failure and must not mask the measured result.
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user