Joseph Doherty
2b80a6defa
New ResidualBench tool (eventlog, flatten, audit, audit-contention) and docker/measure-ui-responsiveness.sh. All four residuals pass; results in docs/plans/2026-11-10-wp4-residual-scope-results.md. Findings filed as #122, #123 and #124 (fixed in the following commit).
237 lines
12 KiB
C#
237 lines
12 KiB
C#
using System.Diagnostics;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
|
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
|
|
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.ResidualBench;
|
|
|
|
/// <summary>
|
|
/// [xc-5] Template flattening and validation complete within acceptable time for large
|
|
/// templates. Runs the production <see cref="FlatteningService"/>, <see cref="ValidationService"/>
|
|
/// (the deploy gate, including the Roslyn trust check + compile of every script) and
|
|
/// <see cref="RevisionHashService"/> in-process over synthetic template graphs of three sizes.
|
|
/// The configuration-database read that <c>FlatteningPipeline</c> does first is not in frame.
|
|
/// </summary>
|
|
internal static class FlattenBench
|
|
{
|
|
private sealed record Shape(
|
|
string Name, int Depth, int AttrsPerLevel, int AlarmsPerLevel, int ScriptsPerLevel,
|
|
int Modules, int ModuleAttrs, int ModuleAlarms, int ModuleScripts, int SubModuleAttrs);
|
|
|
|
private static readonly Shape[] Shapes =
|
|
[
|
|
new("typical", Depth: 2, AttrsPerLevel: 40, AlarmsPerLevel: 5, ScriptsPerLevel: 3,
|
|
Modules: 2, ModuleAttrs: 10, ModuleAlarms: 1, ModuleScripts: 1, SubModuleAttrs: 0),
|
|
new("large", Depth: 5, AttrsPerLevel: 100, AlarmsPerLevel: 10, ScriptsPerLevel: 6,
|
|
Modules: 10, ModuleAttrs: 50, ModuleAlarms: 5, ModuleScripts: 3, SubModuleAttrs: 10),
|
|
new("extra-large", Depth: 8, AttrsPerLevel: 250, AlarmsPerLevel: 25, ScriptsPerLevel: 15,
|
|
Modules: 20, ModuleAttrs: 100, ModuleAlarms: 10, ModuleScripts: 5, SubModuleAttrs: 20),
|
|
];
|
|
|
|
public static int Run(BenchArgs args)
|
|
{
|
|
var instances = args.GetInt("instances", 500);
|
|
var repeats = args.GetInt("repeats", 20);
|
|
args.RejectUnknown();
|
|
|
|
var flattener = new FlatteningService();
|
|
var validator = new ValidationService();
|
|
var hasher = new RevisionHashService();
|
|
|
|
// Warm the JIT and Roslyn once on a throwaway graph so the first shape doesn't pay it.
|
|
{
|
|
var warm = Build(Shapes[0], salt: "jit-warmup");
|
|
var w = flattener.Flatten(warm.Instance, warm.Chain, warm.Compositions, warm.ComposedChains, warm.Connections).Value;
|
|
validator.Validate(w);
|
|
hasher.ComputeHash(w);
|
|
}
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine("| shape | chain depth | flattened attrs / alarms / scripts | flatten P50 ms | validate, cold compile ms | validate, warm cache P50 ms | validate, compile off P50 ms | hash P50 ms | errors |");
|
|
Console.WriteLine("|---|---|---|---|---|---|---|---|---|");
|
|
foreach (var shape in Shapes)
|
|
{
|
|
var g = Build(shape, salt: shape.Name);
|
|
FlattenedConfiguration config = null!;
|
|
var flatten = new Samples();
|
|
for (var i = 0; i < repeats; i++)
|
|
flatten.Add(Samples.Time(() => config = Flatten(flattener, g)));
|
|
|
|
ScriptCompileVerdictCache.Clear();
|
|
ValidationResult result = null!;
|
|
var cold = Samples.Time(() => result = validator.Validate(config));
|
|
|
|
var warm = new Samples();
|
|
for (var i = 0; i < repeats; i++) warm.Add(Samples.Time(() => validator.Validate(config)));
|
|
var noCompile = new Samples();
|
|
for (var i = 0; i < repeats; i++) noCompile.Add(Samples.Time(() => validator.Validate(config, validateScriptCompilation: false)));
|
|
var hash = new Samples();
|
|
for (var i = 0; i < repeats; i++) hash.Add(Samples.Time(() => hasher.ComputeHash(config)));
|
|
|
|
Console.WriteLine($"| {shape.Name} | {shape.Depth} | {config.Attributes.Count:N0} / {config.Alarms.Count} / {config.Scripts.Count} | " +
|
|
$"{flatten.Percentile(50):F2} | {cold:F0} | {warm.Percentile(50):F2} | {noCompile.Percentile(50):F2} | {hash.Percentile(50):F2} | {result.Errors.Count} |");
|
|
if (result.Errors.Count > 0)
|
|
Console.WriteLine($" first errors: {result.SummarizeErrors(3)}");
|
|
}
|
|
|
|
// Bulk-deploy CPU: N instances of one template (scripts shared, so after the first
|
|
// instance every compile is a verdict-cache hit — the production shape).
|
|
Console.WriteLine();
|
|
foreach (var shape in Shapes.Skip(1))
|
|
{
|
|
var graphs = Enumerable.Range(0, instances).Select(i => Build(shape, salt: "bulk", instanceIndex: i)).ToList();
|
|
ScriptCompileVerdictCache.Clear();
|
|
var per = new Samples();
|
|
var sw = Stopwatch.StartNew();
|
|
foreach (var g in graphs)
|
|
{
|
|
per.Add(Samples.Time(() =>
|
|
{
|
|
var c = Flatten(flattener, g);
|
|
var v = validator.Validate(c);
|
|
if (!v.IsValid) throw new InvalidOperationException(v.SummarizeErrors(3));
|
|
hasher.ComputeHash(c);
|
|
}));
|
|
}
|
|
Console.WriteLine($"{shape.Name}: flatten+validate+hash of {instances} instances sharing one template, serial: " +
|
|
$"{sw.Elapsed.TotalSeconds:F2} s total; per instance {per.Summary()}");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
private static FlattenedConfiguration Flatten(FlatteningService flattener, Graph g)
|
|
{
|
|
var r = flattener.Flatten(g.Instance, g.Chain, g.Compositions, g.ComposedChains, g.Connections);
|
|
return r.IsSuccess ? r.Value : throw new InvalidOperationException(r.Error);
|
|
}
|
|
|
|
private sealed record Graph(
|
|
Instance Instance,
|
|
IReadOnlyList<Template> Chain,
|
|
IReadOnlyDictionary<int, IReadOnlyList<TemplateComposition>> Compositions,
|
|
IReadOnlyDictionary<int, IReadOnlyList<Template>> ComposedChains,
|
|
IReadOnlyDictionary<int, DataConnection> Connections);
|
|
|
|
/// <summary>
|
|
/// A chain of <c>Depth</c> templates (each level adds attributes/alarms/scripts and
|
|
/// overrides a fifth of its parent's attribute values), <c>Modules</c> feature modules
|
|
/// composed on the base, each module composing one sub-module, and an instance that
|
|
/// overrides 10% of the attributes and binds every data-sourced attribute to one connection.
|
|
/// </summary>
|
|
private static Graph Build(Shape s, string salt, int instanceIndex = 0)
|
|
{
|
|
var nextId = 1;
|
|
var connection = new DataConnection("PLC-01", "OpcUa", 1) { Id = 1 };
|
|
|
|
Template NewTemplate(string name, int? parent) => new(name) { Id = nextId++, ParentTemplateId = parent };
|
|
|
|
void Populate(Template t, string prefix, int attrs, int alarms, int scripts)
|
|
{
|
|
for (var a = 0; a < attrs; a++)
|
|
{
|
|
t.Attributes.Add(new TemplateAttribute($"{prefix}Attr{a}")
|
|
{
|
|
DataType = a % 4 == 0 ? DataType.String : DataType.Double,
|
|
Value = a % 4 == 0 ? "idle" : "0",
|
|
DataSourceReference = a % 2 == 0 ? $"ns=2;s={prefix}.Attr{a}" : null,
|
|
});
|
|
}
|
|
for (var al = 0; al < alarms; al++)
|
|
{
|
|
t.Alarms.Add(new TemplateAlarm($"{prefix}Alarm{al}")
|
|
{
|
|
TriggerType = AlarmTriggerType.RangeViolation,
|
|
TriggerConfiguration = $"{{\"attributeName\":\"{prefix}Attr{al * 4 % Math.Max(attrs, 1) + 1}\",\"high\":100}}",
|
|
PriorityLevel = 1 + al % 3,
|
|
});
|
|
}
|
|
for (var sc = 0; sc < scripts; sc++)
|
|
t.Scripts.Add(new TemplateScript($"{prefix}Script{sc}", ScriptBody($"{salt}:{prefix}:{sc}")) { Id = nextId++ });
|
|
}
|
|
|
|
var chain = new List<Template>();
|
|
Template? parentT = null;
|
|
for (var level = 0; level < s.Depth; level++)
|
|
{
|
|
var t = NewTemplate($"Level{level}", parentT?.Id);
|
|
Populate(t, $"L{level}", s.AttrsPerLevel, s.AlarmsPerLevel, s.ScriptsPerLevel);
|
|
if (parentT is not null)
|
|
{
|
|
foreach (var inherited in parentT.Attributes.Where((_, i) => i % 5 == 0))
|
|
t.Attributes.Add(new TemplateAttribute(inherited.Name) { DataType = inherited.DataType, Value = inherited.DataType == DataType.String ? "derived" : "1" });
|
|
}
|
|
chain.Insert(0, t); // most-derived first
|
|
parentT = t;
|
|
}
|
|
|
|
var baseTemplate = chain[^1];
|
|
var compositions = new Dictionary<int, IReadOnlyList<TemplateComposition>>();
|
|
var composedChains = new Dictionary<int, IReadOnlyList<Template>>();
|
|
var baseModules = new List<TemplateComposition>();
|
|
for (var m = 0; m < s.Modules; m++)
|
|
{
|
|
var module = NewTemplate($"Module{m}", null);
|
|
Populate(module, "M", s.ModuleAttrs, s.ModuleAlarms, s.ModuleScripts);
|
|
if (s.SubModuleAttrs > 0)
|
|
{
|
|
var sub = NewTemplate($"SubModule{m}", null);
|
|
Populate(sub, "S", s.SubModuleAttrs, 0, 0);
|
|
composedChains[sub.Id] = [sub];
|
|
compositions[module.Id] = [new TemplateComposition("Sensor") { ComposedTemplateId = sub.Id, TemplateId = module.Id }];
|
|
}
|
|
composedChains[module.Id] = [module];
|
|
baseModules.Add(new TemplateComposition($"Unit{m}") { ComposedTemplateId = module.Id, TemplateId = baseTemplate.Id });
|
|
}
|
|
if (baseModules.Count > 0) compositions[baseTemplate.Id] = baseModules;
|
|
|
|
var instance = new Instance($"Machine-{instanceIndex:D4}") { Id = 1 + instanceIndex, TemplateId = chain[0].Id, SiteId = 1 };
|
|
var allAttrs = chain.SelectMany(t => t.Attributes).Select(a => a.Name).Distinct().ToList();
|
|
foreach (var name in allAttrs.Where((_, i) => i % 10 == 0))
|
|
instance.AttributeOverrides.Add(new InstanceAttributeOverride(name) { OverrideValue = "5" });
|
|
foreach (var name in chain.SelectMany(t => t.Attributes).Where(a => a.DataSourceReference is not null).Select(a => a.Name).Distinct())
|
|
instance.ConnectionBindings.Add(new InstanceConnectionBinding(name) { DataConnectionId = connection.Id });
|
|
|
|
return new Graph(instance, chain, compositions, composedChains, new Dictionary<int, DataConnection> { [connection.Id] = connection });
|
|
}
|
|
|
|
/// <summary>~35 lines of ordinary script C#; the tag makes each script's code (the compile-cache key) unique.</summary>
|
|
private static string ScriptBody(string tag) => $$"""
|
|
// {{tag}}
|
|
var readings = new List<double>();
|
|
for (var i = 0; i < 24; i++)
|
|
{
|
|
readings.Add(Math.Sin(i / 3.0) * 10.0 + i);
|
|
}
|
|
var mean = readings.Average();
|
|
var variance = readings.Select(r => (r - mean) * (r - mean)).Sum() / readings.Count;
|
|
var stdDev = Math.Sqrt(variance);
|
|
var outliers = readings.Where(r => Math.Abs(r - mean) > 2 * stdDev).ToList();
|
|
var summary = new Dictionary<string, object>
|
|
{
|
|
["mean"] = mean,
|
|
["stdDev"] = stdDev,
|
|
["outliers"] = outliers.Count,
|
|
["tag"] = "{{tag}}",
|
|
};
|
|
string state;
|
|
if (outliers.Count == 0)
|
|
{
|
|
state = "stable";
|
|
}
|
|
else if (outliers.Count < 3)
|
|
{
|
|
state = "drifting";
|
|
}
|
|
else
|
|
{
|
|
state = "unstable";
|
|
}
|
|
var text = string.Join(",", summary.Select(kv => $"{kv.Key}={kv.Value}"));
|
|
return state + ";" + text;
|
|
""";
|
|
}
|