using System.Diagnostics;
using System.Text.Json;
using Akka.Actor;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
namespace ZB.MOM.WW.ScadaBridge.LoadHarness;
///
/// One simulated site: its own , its own LocalDb SQLite
/// file, a real Data Connection Layer over
/// adapters, real Instance Actors, a real , real
/// store-and-forward, and a real .
///
///
/// The sites are separate, non-clustered ActorSystems rather than 10 real two-node
/// Akka clusters. Cluster membership, singleton placement and failover timing are
/// already measured on a real two-node rig by
/// PerformanceTests/Failover/FailoverTimingTests.cs; what WP-4 asks about is
/// the load-bearing hierarchy under each singleton, which is what this builds.
///
///
public sealed class SiteRuntimeFixture : IAsyncDisposable
{
private readonly HarnessConfig _config;
private readonly List _connections;
private readonly ServiceProvider _localDbProvider;
private readonly ILocalDb _localDb;
private readonly List _instanceActors = new();
private readonly List _probes = new();
private readonly string _dataDirectory;
/// Site identifier, e.g. site-01.
public string SiteId { get; }
/// This site's actor system.
public ActorSystem System { get; }
/// The real site-wide broadcast stream.
public SiteStreamManager StreamManager { get; }
/// The real site health collector feeding the 30 s report.
public SiteHealthCollector HealthCollector { get; }
/// The real store-and-forward engine for this site.
public StoreAndForwardService StoreAndForward { get; }
/// The real DCL manager actor.
public IActorRef DataConnectionManager { get; }
/// The simulated adapters, one per data connection, in creation order.
public IReadOnlyList Connections
{
get { lock (_connections) return _connections.ToList(); }
}
/// The live stream subscribers attached to this site.
public IReadOnlyList Probes => _probes;
/// Instance Actors created on this site.
public IReadOnlyList InstanceActors => _instanceActors;
/// Tag paths per connection index, in the order they were assigned.
public IReadOnlyList> TagPathsByConnection { get; }
/// Wall-clock time the instance ramp took, measured by .
public TimeSpan InstanceRampDuration { get; private set; }
private SiteRuntimeFixture(
string siteId,
HarnessConfig config,
string dataDirectory,
ServiceProvider localDbProvider,
ILocalDb localDb,
ActorSystem system,
SiteStreamManager streamManager,
SiteHealthCollector healthCollector,
StoreAndForwardService storeAndForward,
IActorRef dataConnectionManager,
SiteStorageService storage,
ScriptCompilationService compilationService,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions siteOptions,
List> tagPathsByConnection,
List connections)
{
_connections = connections;
SiteId = siteId;
_config = config;
_dataDirectory = dataDirectory;
_localDbProvider = localDbProvider;
_localDb = localDb;
System = system;
StreamManager = streamManager;
HealthCollector = healthCollector;
StoreAndForward = storeAndForward;
DataConnectionManager = dataConnectionManager;
Storage = storage;
CompilationService = compilationService;
SharedScriptLibrary = sharedScriptLibrary;
SiteOptions = siteOptions;
TagPathsByConnection = tagPathsByConnection;
}
private SiteStorageService Storage { get; }
private ScriptCompilationService CompilationService { get; }
private SharedScriptLibrary SharedScriptLibrary { get; }
private SiteRuntimeOptions SiteOptions { get; }
///
/// Builds a site: LocalDb file, actor system, DCL with its simulated connections,
/// stream manager, health collector and store-and-forward engine. Instance Actors
/// are created separately by so the deployment
/// ramp can be timed on its own.
///
/// Zero-based site index.
/// Harness configuration.
/// Directory under which this site's SQLite files live.
/// Number of data connections to spread the site's tags across.
/// The started fixture.
public static async Task CreateAsync(
int siteIndex, HarnessConfig config, string rootDataDirectory, int connectionsPerSite)
{
var siteId = $"site-{siteIndex + 1:D2}";
var dataDirectory = Path.Combine(rootDataDirectory, siteId);
Directory.CreateDirectory(dataDirectory);
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary
{
["LocalDb:Path"] = Path.Combine(dataDirectory, "site-localdb.db"),
})
.Build();
var localDbProvider = new ServiceCollection()
.AddZbLocalDb(configuration)
.BuildServiceProvider();
var localDb = localDbProvider.GetRequiredService();
var storage = new SiteStorageService(localDb, NullLogger.Instance);
await storage.InitializeAsync();
var compilationService = new ScriptCompilationService(NullLogger.Instance);
var sharedScriptLibrary = new SharedScriptLibrary(compilationService, NullLogger.Instance);
// Production defaults throughout — the point of the run is to measure the
// shipped configuration, so nothing here is tuned for the harness.
var siteOptions = new SiteRuntimeOptions();
// WARNING-level logging. At 37,500 updates/s per site, Akka's INFO output would
// itself become a measured load; and the InstanceActorInitialized dead letters are
// a harness artifact (see StartInstancesAsync) rather than a real condition.
var system = ActorSystem.Create($"loadharness-{siteId}", Akka.Configuration.ConfigurationFactory.ParseString(
"akka.loglevel = WARNING\nakka.stdout-loglevel = WARNING\nakka.log-dead-letters = 0\nakka.log-dead-letters-during-shutdown = off"));
var streamManager = new SiteStreamManager(siteOptions, NullLogger.Instance);
streamManager.Initialize(system);
var healthCollector = new SiteHealthCollector();
healthCollector.SetActiveNode(true);
healthCollector.SetNodeHostname($"{siteId}-node-a");
var sfStorage = new StoreAndForwardStorage(localDb, NullLogger.Instance);
var storeAndForward = new StoreAndForwardService(
sfStorage,
new StoreAndForwardOptions(),
NullLogger.Instance,
siteId: siteId);
await storeAndForward.StartAsync();
// Real DCL, with the simulated adapter registered on the real factory via its
// documented RegisterAdapter extension point.
var loggerFactory = NullLoggerFactory.Instance;
var factory = new DataConnectionFactory(loggerFactory);
var connections = new List();
factory.RegisterAdapter(SimulatedDataConnection.ProtocolName, _ =>
{
var connection = new SimulatedDataConnection();
lock (connections) connections.Add(connection);
return connection;
});
var dclManager = system.ActorOf(
Props.Create(() => new DataConnectionManagerActor(
factory, new DataConnectionOptions(), healthCollector, null, null)),
"data-connection-manager");
for (var c = 0; c < connectionsPerSite; c++)
{
dclManager.Tell(new CreateConnectionCommand(
ConnectionName: ConnectionName(c),
ProtocolType: SimulatedDataConnection.ProtocolName,
PrimaryConnectionDetails: new Dictionary
{
["endpoint"] = $"sim://{siteId}/{c}",
[SimulatedDataConnection.ConnectionNameKey] = ConnectionName(c),
}));
}
var tagPathsByConnection = new List>();
for (var c = 0; c < connectionsPerSite; c++)
tagPathsByConnection.Add(new List());
var fixture = new SiteRuntimeFixture(
siteId, config, dataDirectory, localDbProvider, localDb, system, streamManager,
healthCollector, storeAndForward, dclManager, storage, compilationService,
sharedScriptLibrary, siteOptions, tagPathsByConnection, connections);
return fixture;
}
/// Deterministic connection name for a connection index.
/// Zero-based connection index.
/// The connection name used in configs and DCL commands.
public static string ConnectionName(int connectionIndex) => $"sim-conn-{connectionIndex:D2}";
///
/// Creates this site's Instance Actors in production-shaped staggered batches
/// ( /
/// ) and records how long the
/// ramp took. This is the harness's stand-in for "deployment of 500 instances to
/// a site" — it exercises the same per-instance construction, config
/// deserialization, override load and DCL subscribe that a real deploy triggers.
///
/// Cancels the ramp.
/// A task that completes when every instance actor exists.
public async Task StartInstancesAsync(CancellationToken cancellationToken)
{
var started = Stopwatch.StartNew();
var connectionCount = TagPathsByConnection.Count;
for (var i = 0; i < _config.InstancesPerSite; i++)
{
var instanceName = InstanceName(i);
var connectionIndex = i % connectionCount;
var connectionName = ConnectionName(connectionIndex);
var attributes = new List(_config.TagsPerInstance);
for (var t = 0; t < _config.TagsPerInstance; t++)
{
var tagPath = $"{instanceName}.Tag{t:D3}";
TagPathsByConnection[connectionIndex].Add(tagPath);
attributes.Add(new ResolvedAttribute
{
CanonicalName = $"Tag{t:D3}",
DataType = "Double",
DataSourceReference = tagPath,
BoundDataConnectionId = connectionIndex + 1,
BoundDataConnectionName = connectionName,
BoundDataConnectionProtocol = SimulatedDataConnection.ProtocolName,
});
}
var configuration = new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
TemplateId = 1,
SiteId = 1,
Attributes = attributes,
Connections = new Dictionary
{
[connectionName] = new()
{
Protocol = SimulatedDataConnection.ProtocolName,
ConfigurationJson = "{}",
},
},
};
var configJson = JsonSerializer.Serialize(configuration);
var actor = System.ActorOf(
Props.Create(() => new InstanceActor(
instanceName, configJson, Storage, CompilationService, SharedScriptLibrary,
StreamManager, SiteOptions, NullLogger.Instance,
DataConnectionManager, HealthCollector, null, null)),
instanceName);
_instanceActors.Add(actor);
// Production staggered-startup pacing (SiteRuntimeOptions defaults):
// batches of StartupBatchSize separated by StartupBatchDelayMs, which is
// exactly what DeploymentManagerActor does on a real site start.
if ((i + 1) % SiteOptions.StartupBatchSize == 0)
await Task.Delay(SiteOptions.StartupBatchDelayMs, cancellationToken);
}
started.Stop();
InstanceRampDuration = started.Elapsed;
HealthCollector.SetInstanceCounts(
_config.InstancesPerSite, _config.InstancesPerSite, 0);
for (var c = 0; c < connectionCount; c++)
{
HealthCollector.UpdateTagResolution(
ConnectionName(c), TagPathsByConnection[c].Count, TagPathsByConnection[c].Count);
}
}
/// Deterministic instance unique name for an instance index.
/// Zero-based instance index.
/// The instance unique name.
public string InstanceName(int instanceIndex) => $"{SiteId}-inst-{instanceIndex:D4}";
///
/// Attaches live stream subscribers,
/// each built from the production pieces the gRPC server uses: a real
/// writing into a bounded DropOldest channel
/// of the production capacity, subscribed through the real
/// . Only the socket writer is replaced —
/// by a reader task under the harness's control, which is what makes the
/// slow-subscriber scenario possible at all.
///
///
/// Attaching at least one subscriber is also load-bearing:
/// PublishAttributeValueChanged short-circuits at zero subscribers, so an
/// unsubscribed site would publish nothing and measure nothing.
///
///
/// Histogram that receives end-to-end tag update latencies.
public void AttachStreamProbes(LatencyHistogram latency)
{
var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, _config.StreamProbesPerSite));
for (var p = 0; p < _config.StreamProbesPerSite; p++)
{
var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1);
var instanceName = InstanceName(instanceIndex);
var probe = StreamSubscriberProbe.Attach(
System, StreamManager, instanceName, $"{SiteId}-probe-{p:D2}", latency);
_probes.Add(probe);
}
}
/// Total tag paths registered across this site's connections.
public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count);
///
public async ValueTask DisposeAsync()
{
foreach (var probe in _probes)
await probe.DisposeAsync();
await StoreAndForward.StopAsync();
await System.Terminate();
await _localDbProvider.DisposeAsync();
try
{
if (Directory.Exists(_dataDirectory))
Directory.Delete(_dataDirectory, recursive: true);
}
catch (IOException)
{
// Best-effort cleanup of a temp directory; a lingering WAL handle is not
// a harness failure and must not mask the measured result.
}
}
}