test(perf): measure phase-8 WP-4 residual criteria xc-3/4/5/8 (#49)
New ResidualBench tool (eventlog, flatten, audit, audit-contention) and docker/measure-ui-responsiveness.sh. All four residuals pass; results in docs/plans/2026-11-10-wp4-residual-scope-results.md. Findings filed as #122, #123 and #124 (fixed in the following commit).
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ResidualBench;
|
||||
|
||||
/// <summary>
|
||||
/// [xc-5] Template flattening and validation complete within acceptable time for large
|
||||
/// templates. Runs the production <see cref="FlatteningService"/>, <see cref="ValidationService"/>
|
||||
/// (the deploy gate, including the Roslyn trust check + compile of every script) and
|
||||
/// <see cref="RevisionHashService"/> in-process over synthetic template graphs of three sizes.
|
||||
/// The configuration-database read that <c>FlatteningPipeline</c> does first is not in frame.
|
||||
/// </summary>
|
||||
internal static class FlattenBench
|
||||
{
|
||||
private sealed record Shape(
|
||||
string Name, int Depth, int AttrsPerLevel, int AlarmsPerLevel, int ScriptsPerLevel,
|
||||
int Modules, int ModuleAttrs, int ModuleAlarms, int ModuleScripts, int SubModuleAttrs);
|
||||
|
||||
private static readonly Shape[] Shapes =
|
||||
[
|
||||
new("typical", Depth: 2, AttrsPerLevel: 40, AlarmsPerLevel: 5, ScriptsPerLevel: 3,
|
||||
Modules: 2, ModuleAttrs: 10, ModuleAlarms: 1, ModuleScripts: 1, SubModuleAttrs: 0),
|
||||
new("large", Depth: 5, AttrsPerLevel: 100, AlarmsPerLevel: 10, ScriptsPerLevel: 6,
|
||||
Modules: 10, ModuleAttrs: 50, ModuleAlarms: 5, ModuleScripts: 3, SubModuleAttrs: 10),
|
||||
new("extra-large", Depth: 8, AttrsPerLevel: 250, AlarmsPerLevel: 25, ScriptsPerLevel: 15,
|
||||
Modules: 20, ModuleAttrs: 100, ModuleAlarms: 10, ModuleScripts: 5, SubModuleAttrs: 20),
|
||||
];
|
||||
|
||||
public static int Run(BenchArgs args)
|
||||
{
|
||||
var instances = args.GetInt("instances", 500);
|
||||
var repeats = args.GetInt("repeats", 20);
|
||||
args.RejectUnknown();
|
||||
|
||||
var flattener = new FlatteningService();
|
||||
var validator = new ValidationService();
|
||||
var hasher = new RevisionHashService();
|
||||
|
||||
// Warm the JIT and Roslyn once on a throwaway graph so the first shape doesn't pay it.
|
||||
{
|
||||
var warm = Build(Shapes[0], salt: "jit-warmup");
|
||||
var w = flattener.Flatten(warm.Instance, warm.Chain, warm.Compositions, warm.ComposedChains, warm.Connections).Value;
|
||||
validator.Validate(w);
|
||||
hasher.ComputeHash(w);
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("| shape | chain depth | flattened attrs / alarms / scripts | flatten P50 ms | validate, cold compile ms | validate, warm cache P50 ms | validate, compile off P50 ms | hash P50 ms | errors |");
|
||||
Console.WriteLine("|---|---|---|---|---|---|---|---|---|");
|
||||
foreach (var shape in Shapes)
|
||||
{
|
||||
var g = Build(shape, salt: shape.Name);
|
||||
FlattenedConfiguration config = null!;
|
||||
var flatten = new Samples();
|
||||
for (var i = 0; i < repeats; i++)
|
||||
flatten.Add(Samples.Time(() => config = Flatten(flattener, g)));
|
||||
|
||||
ScriptCompileVerdictCache.Clear();
|
||||
ValidationResult result = null!;
|
||||
var cold = Samples.Time(() => result = validator.Validate(config));
|
||||
|
||||
var warm = new Samples();
|
||||
for (var i = 0; i < repeats; i++) warm.Add(Samples.Time(() => validator.Validate(config)));
|
||||
var noCompile = new Samples();
|
||||
for (var i = 0; i < repeats; i++) noCompile.Add(Samples.Time(() => validator.Validate(config, validateScriptCompilation: false)));
|
||||
var hash = new Samples();
|
||||
for (var i = 0; i < repeats; i++) hash.Add(Samples.Time(() => hasher.ComputeHash(config)));
|
||||
|
||||
Console.WriteLine($"| {shape.Name} | {shape.Depth} | {config.Attributes.Count:N0} / {config.Alarms.Count} / {config.Scripts.Count} | " +
|
||||
$"{flatten.Percentile(50):F2} | {cold:F0} | {warm.Percentile(50):F2} | {noCompile.Percentile(50):F2} | {hash.Percentile(50):F2} | {result.Errors.Count} |");
|
||||
if (result.Errors.Count > 0)
|
||||
Console.WriteLine($" first errors: {result.SummarizeErrors(3)}");
|
||||
}
|
||||
|
||||
// Bulk-deploy CPU: N instances of one template (scripts shared, so after the first
|
||||
// instance every compile is a verdict-cache hit — the production shape).
|
||||
Console.WriteLine();
|
||||
foreach (var shape in Shapes.Skip(1))
|
||||
{
|
||||
var graphs = Enumerable.Range(0, instances).Select(i => Build(shape, salt: "bulk", instanceIndex: i)).ToList();
|
||||
ScriptCompileVerdictCache.Clear();
|
||||
var per = new Samples();
|
||||
var sw = Stopwatch.StartNew();
|
||||
foreach (var g in graphs)
|
||||
{
|
||||
per.Add(Samples.Time(() =>
|
||||
{
|
||||
var c = Flatten(flattener, g);
|
||||
var v = validator.Validate(c);
|
||||
if (!v.IsValid) throw new InvalidOperationException(v.SummarizeErrors(3));
|
||||
hasher.ComputeHash(c);
|
||||
}));
|
||||
}
|
||||
Console.WriteLine($"{shape.Name}: flatten+validate+hash of {instances} instances sharing one template, serial: " +
|
||||
$"{sw.Elapsed.TotalSeconds:F2} s total; per instance {per.Summary()}");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static FlattenedConfiguration Flatten(FlatteningService flattener, Graph g)
|
||||
{
|
||||
var r = flattener.Flatten(g.Instance, g.Chain, g.Compositions, g.ComposedChains, g.Connections);
|
||||
return r.IsSuccess ? r.Value : throw new InvalidOperationException(r.Error);
|
||||
}
|
||||
|
||||
private sealed record Graph(
|
||||
Instance Instance,
|
||||
IReadOnlyList<Template> Chain,
|
||||
IReadOnlyDictionary<int, IReadOnlyList<TemplateComposition>> Compositions,
|
||||
IReadOnlyDictionary<int, IReadOnlyList<Template>> ComposedChains,
|
||||
IReadOnlyDictionary<int, DataConnection> Connections);
|
||||
|
||||
/// <summary>
|
||||
/// A chain of <c>Depth</c> templates (each level adds attributes/alarms/scripts and
|
||||
/// overrides a fifth of its parent's attribute values), <c>Modules</c> feature modules
|
||||
/// composed on the base, each module composing one sub-module, and an instance that
|
||||
/// overrides 10% of the attributes and binds every data-sourced attribute to one connection.
|
||||
/// </summary>
|
||||
private static Graph Build(Shape s, string salt, int instanceIndex = 0)
|
||||
{
|
||||
var nextId = 1;
|
||||
var connection = new DataConnection("PLC-01", "OpcUa", 1) { Id = 1 };
|
||||
|
||||
Template NewTemplate(string name, int? parent) => new(name) { Id = nextId++, ParentTemplateId = parent };
|
||||
|
||||
void Populate(Template t, string prefix, int attrs, int alarms, int scripts)
|
||||
{
|
||||
for (var a = 0; a < attrs; a++)
|
||||
{
|
||||
t.Attributes.Add(new TemplateAttribute($"{prefix}Attr{a}")
|
||||
{
|
||||
DataType = a % 4 == 0 ? DataType.String : DataType.Double,
|
||||
Value = a % 4 == 0 ? "idle" : "0",
|
||||
DataSourceReference = a % 2 == 0 ? $"ns=2;s={prefix}.Attr{a}" : null,
|
||||
});
|
||||
}
|
||||
for (var al = 0; al < alarms; al++)
|
||||
{
|
||||
t.Alarms.Add(new TemplateAlarm($"{prefix}Alarm{al}")
|
||||
{
|
||||
TriggerType = AlarmTriggerType.RangeViolation,
|
||||
TriggerConfiguration = $"{{\"attributeName\":\"{prefix}Attr{al * 4 % Math.Max(attrs, 1) + 1}\",\"high\":100}}",
|
||||
PriorityLevel = 1 + al % 3,
|
||||
});
|
||||
}
|
||||
for (var sc = 0; sc < scripts; sc++)
|
||||
t.Scripts.Add(new TemplateScript($"{prefix}Script{sc}", ScriptBody($"{salt}:{prefix}:{sc}")) { Id = nextId++ });
|
||||
}
|
||||
|
||||
var chain = new List<Template>();
|
||||
Template? parentT = null;
|
||||
for (var level = 0; level < s.Depth; level++)
|
||||
{
|
||||
var t = NewTemplate($"Level{level}", parentT?.Id);
|
||||
Populate(t, $"L{level}", s.AttrsPerLevel, s.AlarmsPerLevel, s.ScriptsPerLevel);
|
||||
if (parentT is not null)
|
||||
{
|
||||
foreach (var inherited in parentT.Attributes.Where((_, i) => i % 5 == 0))
|
||||
t.Attributes.Add(new TemplateAttribute(inherited.Name) { DataType = inherited.DataType, Value = inherited.DataType == DataType.String ? "derived" : "1" });
|
||||
}
|
||||
chain.Insert(0, t); // most-derived first
|
||||
parentT = t;
|
||||
}
|
||||
|
||||
var baseTemplate = chain[^1];
|
||||
var compositions = new Dictionary<int, IReadOnlyList<TemplateComposition>>();
|
||||
var composedChains = new Dictionary<int, IReadOnlyList<Template>>();
|
||||
var baseModules = new List<TemplateComposition>();
|
||||
for (var m = 0; m < s.Modules; m++)
|
||||
{
|
||||
var module = NewTemplate($"Module{m}", null);
|
||||
Populate(module, "M", s.ModuleAttrs, s.ModuleAlarms, s.ModuleScripts);
|
||||
if (s.SubModuleAttrs > 0)
|
||||
{
|
||||
var sub = NewTemplate($"SubModule{m}", null);
|
||||
Populate(sub, "S", s.SubModuleAttrs, 0, 0);
|
||||
composedChains[sub.Id] = [sub];
|
||||
compositions[module.Id] = [new TemplateComposition("Sensor") { ComposedTemplateId = sub.Id, TemplateId = module.Id }];
|
||||
}
|
||||
composedChains[module.Id] = [module];
|
||||
baseModules.Add(new TemplateComposition($"Unit{m}") { ComposedTemplateId = module.Id, TemplateId = baseTemplate.Id });
|
||||
}
|
||||
if (baseModules.Count > 0) compositions[baseTemplate.Id] = baseModules;
|
||||
|
||||
var instance = new Instance($"Machine-{instanceIndex:D4}") { Id = 1 + instanceIndex, TemplateId = chain[0].Id, SiteId = 1 };
|
||||
var allAttrs = chain.SelectMany(t => t.Attributes).Select(a => a.Name).Distinct().ToList();
|
||||
foreach (var name in allAttrs.Where((_, i) => i % 10 == 0))
|
||||
instance.AttributeOverrides.Add(new InstanceAttributeOverride(name) { OverrideValue = "5" });
|
||||
foreach (var name in chain.SelectMany(t => t.Attributes).Where(a => a.DataSourceReference is not null).Select(a => a.Name).Distinct())
|
||||
instance.ConnectionBindings.Add(new InstanceConnectionBinding(name) { DataConnectionId = connection.Id });
|
||||
|
||||
return new Graph(instance, chain, compositions, composedChains, new Dictionary<int, DataConnection> { [connection.Id] = connection });
|
||||
}
|
||||
|
||||
/// <summary>~35 lines of ordinary script C#; the tag makes each script's code (the compile-cache key) unique.</summary>
|
||||
private static string ScriptBody(string tag) => $$"""
|
||||
// {{tag}}
|
||||
var readings = new List<double>();
|
||||
for (var i = 0; i < 24; i++)
|
||||
{
|
||||
readings.Add(Math.Sin(i / 3.0) * 10.0 + i);
|
||||
}
|
||||
var mean = readings.Average();
|
||||
var variance = readings.Select(r => (r - mean) * (r - mean)).Sum() / readings.Count;
|
||||
var stdDev = Math.Sqrt(variance);
|
||||
var outliers = readings.Where(r => Math.Abs(r - mean) > 2 * stdDev).ToList();
|
||||
var summary = new Dictionary<string, object>
|
||||
{
|
||||
["mean"] = mean,
|
||||
["stdDev"] = stdDev,
|
||||
["outliers"] = outliers.Count,
|
||||
["tag"] = "{{tag}}",
|
||||
};
|
||||
string state;
|
||||
if (outliers.Count == 0)
|
||||
{
|
||||
state = "stable";
|
||||
}
|
||||
else if (outliers.Count < 3)
|
||||
{
|
||||
state = "drifting";
|
||||
}
|
||||
else
|
||||
{
|
||||
state = "unstable";
|
||||
}
|
||||
var text = string.Join(",", summary.Select(kv => $"{kv.Key}={kv.Value}"));
|
||||
return state + ";" + text;
|
||||
""";
|
||||
}
|
||||
Reference in New Issue
Block a user