perf(site-runtime): warm the next startup batch during the stagger delay and compile a batch's distinct units in parallel (review 2026-10-21 01/SR-P3)

DeployCompileValidator.WarmCompileCache dedupes script bodies/trigger expressions across a batch and compiles them with Parallel.ForEach (half the cores). CreateInstanceActorBatch starts the next batch's warm as it arms the stagger timer; creation order and delay are unchanged. The warm's parsed configs are reused at creation.
2026-10-22 08:22:26 -04:00
parent dde22a3dd1
commit 999653a6b9
5 changed files with 256 additions and 29 deletions
@@ -292,8 +292,10 @@ public class DeploymentWarmThenGateTests : TestKit, IDisposable

        AwaitAssert(() =>
        {
-            // One compile, then repeated hits: the second pre-warm plus both PreStarts.
-            Assert.True(_compileCache.Hits >= 3,
+            // One compile, then repeated hits from both PreStarts. (The batch warm dedupes
+            // distinct bodies before compiling — review 2026-10-21 01/SR-P3 — so the second
+            // config no longer costs a warm-side lookup of its own.)
+            Assert.True(_compileCache.Hits >= 2,
                $"expected the pre-warmed body to be served from cache, saw {_compileCache.Hits} hits");
        }, TimeSpan.FromSeconds(20));

@@ -0,0 +1,104 @@
+using System.Text.Json;
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+using ZB.MOM.WW.ScadaBridge.TestSupport;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
+
+/// <summary>
+/// Review 2026-10-21 01/SR-P3: boot/failover compile warm was serial within a batch and
+/// serialised across batches — batch k+1's compiles did not start until batch k's actors existed
+/// AND the stagger delay had elapsed, so recovery time was the sum of every distinct compile plus
+/// every delay, on one thread. The next batch is now warmed while the current batch's delay runs,
+/// and a batch's distinct compile units are warmed in parallel.
+/// </summary>
+public class StartupBatchPrewarmTests : TestKit, IDisposable
+{
+    private readonly SiteStorageService _storage;
+    private readonly SiteScriptCompileCache _compileCache = new();
+    private readonly ScriptCompilationService _compilationService;
+    private readonly SharedScriptLibrary _sharedScriptLibrary;
+    private readonly TestLocalDb _localDb;
+
+    /// <summary>Creates a temporary LocalDb-backed site store and an observable compile cache.</summary>
+    public StartupBatchPrewarmTests()
+    {
+        _localDb = TestLocalDb.CreateTemp("startup-prewarm");
+        _storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
+        _storage.InitializeAsync().GetAwaiter().GetResult();
+        _compilationService = new ScriptCompilationService(
+            NullLogger<ScriptCompilationService>.Instance, _compileCache);
+        _sharedScriptLibrary = new SharedScriptLibrary(
+            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
+    }
+
+    /// <inheritdoc />
+    void IDisposable.Dispose()
+    {
+        Shutdown();
+        var path = _localDb.Path;
+        _localDb.Dispose();
+        TestLocalDb.DeleteFiles(path);
+    }
+
+    private static string ConfigWithScript(string name, int distinct) => JsonSerializer.Serialize(new FlattenedConfiguration
+    {
+        InstanceUniqueName = name,
+        Attributes = [new ResolvedAttribute { CanonicalName = "A", Value = "1", DataType = "Int32" }],
+        Scripts = [new ResolvedScript { CanonicalName = "S", Code = $"return {distinct} * 2;" }]
+    });
+
+    private async Task<bool> IsRunningAsync(IActorRef dm, string name)
+    {
+        var snapshot = await dm.Ask<DebugViewSnapshot>(
+            new DebugSnapshotRequest(name, Guid.NewGuid().ToString("N")), TimeSpan.FromSeconds(5));
+        return !snapshot.InstanceNotFound;
+    }
+
+    /// <summary>
+    /// Two one-instance batches with a long stagger delay: batch 2's script is compiled while the
+    /// delay after batch 1 is still running, i.e. BEFORE batch 2's Instance Actor exists. Before
+    /// the fix its compile only started once the delay had elapsed.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task NextBatch_IsWarmedDuringTheStaggerDelay()
+    {
+        await _storage.StoreDeployedConfigAsync("B0", ConfigWithScript("B0", 1001), "d0", "r0", true);
+        await _storage.StoreDeployedConfigAsync("B1", ConfigWithScript("B1", 1002), "d1", "r1", true);
+
+        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
+            _storage, _compilationService, _sharedScriptLibrary, null,
+            new SiteRuntimeOptions
+            {
+                StartupBatchSize = 1, StartupBatchDelayMs = 6000, InstanceSweepIntervalSeconds = 0
+            },
+            NullLogger<DeploymentManagerActor>.Instance,
+            dclManager: null, healthCollector: null, serviceProvider: null,
+            loggerFactory: null, configFetcher: null)));
+
+        // Batch 1 is up...
+        await AwaitAssertAsync(async () => Assert.True(await IsRunningAsync(dm, "B0") || await IsRunningAsync(dm, "B1")),
+            TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(50));
+
+        // ...and both scripts are compiled while exactly one instance is running.
+        await AwaitAssertAsync(() =>
+        {
+            Assert.True(_compileCache.Count >= 2, $"compile cache holds {_compileCache.Count} entries");
+            return Task.CompletedTask;
+        }, TimeSpan.FromSeconds(3), TimeSpan.FromMilliseconds(50));
+        Assert.False(await IsRunningAsync(dm, "B0") && await IsRunningAsync(dm, "B1"),
+            "the second batch was created early — the stagger delay must be unchanged");
+
+        // The stagger itself is unchanged: batch 2 still starts after the delay.
+        await AwaitAssertAsync(async () =>
+            Assert.True(await IsRunningAsync(dm, "B0") && await IsRunningAsync(dm, "B1")),
+            TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(100));
+    }
+}