perf(deploy): flatten-session caching, bulk DeploySiteAsync, paged management queries

2026-08-14 21:14:22 -04:00
parent ee193cd2bb
commit 48b3c40a7f
59 changed files with 3051 additions and 231 deletions
@@ -0,0 +1,257 @@
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
+
+namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;
+
+/// <summary>
+/// Memoises the repository reads that <see cref="FlatteningPipeline"/> performs,
+/// for the lifetime of ONE flatten/validate session.
+///
+/// <para>
+/// Without a session every instance flatten re-walks its whole template
+/// inheritance chain (one query per link), re-loads the compositions of every
+/// template in that chain and of every composed chain it reaches, and re-issues
+/// the three session-global queries — shared scripts, the shared-schema library,
+/// and the target site's data connections. For a bulk deploy of N instances off
+/// the same template that is N× the same work; even a single instance re-loads a
+/// composed chain once per composing template.
+/// </para>
+///
+/// <para>
+/// <b>Cache key.</b> Templates are keyed on <c>(id, ITemplateGraphWatermark
+/// version)</c> and chain MEMBERSHIP additionally on the watermark's
+/// <see cref="ITemplateGraphWatermark.StructureVersion"/>, so a template edited
+/// mid-session is re-read rather than served from the memo. The session is also
+/// short-lived by construction — it is created per deploy/validate operation and
+/// discarded with it — so a cached graph can never outlive the operation that
+/// captured it. That is the design's "template state is captured at the time of
+/// flatten" guarantee, unchanged: the session narrows the capture window, it does
+/// not widen it.
+/// </para>
+///
+/// <para>
+/// <b>Threading.</b> A session is NOT thread-safe and must not be shared across
+/// threads. It caches entities materialised by one scoped <c>DbContext</c>, which
+/// is itself single-threaded, and it hands out tracked entities that the
+/// in-flight bundle importer relies on observing. <c>DeploySiteAsync</c> honours
+/// this by running the whole prepare (flatten) phase serially on one scope and
+/// parallelising only the site round-trips, which touch no repository at all.
+/// </para>
+/// </summary>
+public sealed class FlattenSession
+{
+    private readonly ITemplateGraphWatermark _watermark;
+
+    private readonly Dictionary<int, (long Version, Template Template)> _templates = new();
+    private readonly Dictionary<int, CachedChain> _chains = new();
+    private readonly Dictionary<int, (long Version, IReadOnlyList<TemplateComposition> Compositions)> _compositions = new();
+    private readonly Dictionary<int, IReadOnlyDictionary<int, DataConnection>> _dataConnections = new();
+
+    private IReadOnlyList<ResolvedScript>? _sharedScripts;
+    private IReadOnlyDictionary<string, string>? _schemaLibrary;
+
+    /// <summary>Initializes a session bound to the process-wide graph watermark.</summary>
+    /// <param name="watermark">Watermark supplying the per-template versions used as cache keys.</param>
+    public FlattenSession(ITemplateGraphWatermark watermark)
+    {
+        _watermark = watermark ?? throw new ArgumentNullException(nameof(watermark));
+    }
+
+    /// <summary>
+    /// Number of template-chain walks that actually hit the repository. Reset
+    /// never; used by tests to assert that N instances sharing a template load
+    /// that chain exactly once.
+    /// </summary>
+    public int ChainLoads { get; private set; }
+
+    /// <summary>Number of single-template repository reads that missed the memo.</summary>
+    public int TemplateLoads { get; private set; }
+
+    /// <summary>Number of per-template composition queries that missed the memo.</summary>
+    public int CompositionLoads { get; private set; }
+
+    /// <summary>Number of times the session-global queries (shared scripts, schema library) ran.</summary>
+    public int GlobalLoads { get; private set; }
+
+    /// <summary>Number of per-site data-connection queries that missed the memo.</summary>
+    public int DataConnectionLoads { get; private set; }
+
+    /// <summary>
+    /// Every template id this session resolved — the union of all inheritance and
+    /// composed chains it walked. The staleness fast path records the watermark
+    /// version of exactly this set, so a change to ANY template that fed a
+    /// flatten invalidates the memoised revision hash.
+    /// </summary>
+    public IReadOnlyCollection<int> VisitedTemplateIds => _templates.Keys;
+
+    /// <summary>
+    /// Returns the inheritance chain for <paramref name="templateId"/> (the
+    /// template itself, then each ancestor), loading it through
+    /// <paramref name="loadTemplate"/> on a miss.
+    /// </summary>
+    /// <param name="templateId">Root of the chain to build.</param>
+    /// <param name="loadTemplate">Repository read for one template with its children.</param>
+    /// <returns>The chain, ordered leaf-first.</returns>
+    public async Task<IReadOnlyList<Template>> GetChainAsync(
+        int templateId,
+        Func<int, Task<Template?>> loadTemplate)
+    {
+        var structure = _watermark.StructureVersion;
+        if (_chains.TryGetValue(templateId, out var cached) && IsChainCurrent(cached, structure))
+            return cached.Chain;
+
+        var chain = new List<Template>();
+        var memberVersions = new Dictionary<int, long>();
+        var currentId = (int?)templateId;
+
+        // Guard against a cyclic parent edge. Acyclicity is enforced on save, but
+        // this walk must not hang if a bad row ever reaches the database.
+        var seen = new HashSet<int>();
+        while (currentId.HasValue && seen.Add(currentId.Value))
+        {
+            memberVersions[currentId.Value] = _watermark.GetTemplateVersion(currentId.Value);
+            var template = await GetTemplateAsync(currentId.Value, loadTemplate);
+            if (template == null) break;
+            chain.Add(template);
+            currentId = template.ParentTemplateId;
+        }
+
+        ChainLoads++;
+        _chains[templateId] = new CachedChain(structure, memberVersions, chain);
+        return chain;
+    }
+
+    /// <summary>
+    /// A memoised chain plus the exact watermark readings it was built under.
+    /// </summary>
+    /// <param name="Structure">Structure version, covering chain MEMBERSHIP.</param>
+    /// <param name="MemberVersions">Per-template version of every link, covering chain CONTENTS.</param>
+    /// <param name="Chain">The chain itself, ordered leaf-first.</param>
+    private sealed record CachedChain(
+        long Structure,
+        IReadOnlyDictionary<int, long> MemberVersions,
+        IReadOnlyList<Template> Chain);
+
+    /// <summary>
+    /// A cached chain is current only when BOTH the graph shape and every member's
+    /// own version are unchanged.
+    ///
+    /// <para>
+    /// Both halves are load-bearing. Structure alone would miss an ordinary member
+    /// edit (an attribute added to a template already in the chain), which changes
+    /// what the flattener produces without changing which templates it walks.
+    /// Member versions alone would miss a re-parent, which changes membership
+    /// without touching any surviving member. Checking only one would let a session
+    /// serve a chain that no longer reflects the database.
+    /// </para>
+    /// </summary>
+    private bool IsChainCurrent(CachedChain cached, long structure)
+    {
+        if (cached.Structure != structure)
+            return false;
+
+        foreach (var (templateId, version) in cached.MemberVersions)
+        {
+            if (_watermark.GetTemplateVersion(templateId) != version)
+                return false;
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    /// Returns one template with its children, loading it through
+    /// <paramref name="loadTemplate"/> on a miss or when its watermark version moved.
+    /// </summary>
+    /// <param name="templateId">Template to read.</param>
+    /// <param name="loadTemplate">Repository read for one template with its children.</param>
+    /// <returns>The template, or <see langword="null"/> when it does not exist.</returns>
+    public async Task<Template?> GetTemplateAsync(int templateId, Func<int, Task<Template?>> loadTemplate)
+    {
+        var version = _watermark.GetTemplateVersion(templateId);
+        if (_templates.TryGetValue(templateId, out var cached) && cached.Version == version)
+            return cached.Template;
+
+        var template = await loadTemplate(templateId);
+        TemplateLoads++;
+        if (template != null)
+            _templates[templateId] = (version, template);
+        return template;
+    }
+
+    /// <summary>
+    /// Returns the composition rows declared directly on <paramref name="templateId"/>.
+    /// </summary>
+    /// <param name="templateId">Template whose compositions are read.</param>
+    /// <param name="loadCompositions">Repository read for one template's compositions.</param>
+    /// <returns>The composition rows.</returns>
+    public async Task<IReadOnlyList<TemplateComposition>> GetCompositionsAsync(
+        int templateId,
+        Func<int, Task<IReadOnlyList<TemplateComposition>>> loadCompositions)
+    {
+        var version = _watermark.GetTemplateVersion(templateId);
+        if (_compositions.TryGetValue(templateId, out var cached) && cached.Version == version)
+            return cached.Compositions;
+
+        var compositions = await loadCompositions(templateId);
+        CompositionLoads++;
+        _compositions[templateId] = (version, compositions);
+        return compositions;
+    }
+
+    /// <summary>
+    /// Returns the site's data connections keyed by id. Cached per site for the
+    /// session — a bulk site deploy resolves them once for the whole batch.
+    /// </summary>
+    /// <param name="siteId">Site whose connections are read.</param>
+    /// <param name="loadConnections">Repository read for a site's data connections.</param>
+    /// <returns>The site's connections keyed by database id.</returns>
+    public async Task<IReadOnlyDictionary<int, DataConnection>> GetDataConnectionsAsync(
+        int siteId,
+        Func<int, Task<IReadOnlyDictionary<int, DataConnection>>> loadConnections)
+    {
+        if (_dataConnections.TryGetValue(siteId, out var cached))
+            return cached;
+
+        var connections = await loadConnections(siteId);
+        DataConnectionLoads++;
+        _dataConnections[siteId] = connections;
+        return connections;
+    }
+
+    /// <summary>
+    /// Returns the shared-script set used for semantic validation. Hoisted to the
+    /// session so a bulk deploy issues this query once instead of once per instance.
+    /// </summary>
+    /// <param name="loadSharedScripts">Repository read for all shared scripts.</param>
+    /// <returns>The resolved shared scripts.</returns>
+    public async Task<IReadOnlyList<ResolvedScript>> GetSharedScriptsAsync(
+        Func<Task<IReadOnlyList<ResolvedScript>>> loadSharedScripts)
+    {
+        if (_sharedScripts != null)
+            return _sharedScripts;
+
+        _sharedScripts = await loadSharedScripts();
+        GlobalLoads++;
+        return _sharedScripts;
+    }
+
+    /// <summary>
+    /// Returns the shared JSON-Schema library as a name → schema-JSON map, hoisted
+    /// to the session for the same reason as the shared scripts.
+    /// </summary>
+    /// <param name="loadSchemaLibrary">Repository read building the schema library map.</param>
+    /// <returns>The schema library.</returns>
+    public async Task<IReadOnlyDictionary<string, string>> GetSchemaLibraryAsync(
+        Func<Task<IReadOnlyDictionary<string, string>>> loadSchemaLibrary)
+    {
+        if (_schemaLibrary != null)
+            return _schemaLibrary;
+
+        _schemaLibrary = await loadSchemaLibrary();
+        GlobalLoads++;
+        return _schemaLibrary;
+    }
+}