dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(site-runtime): warm the next startup batch during the stagger delay and compile a batch's distinct units in parallel (review 2026-10-21 01/SR-P3)

Browse Source 
DeployCompileValidator.WarmCompileCache dedupes script bodies/trigger
expressions across a batch and compiles them with Parallel.ForEach (half the
cores). CreateInstanceActorBatch starts the next batch's warm as it arms the
stagger timer; creation order and delay are unchanged. The warm's parsed
configs are reused at creation.
This commit is contained in:
Joseph Doherty
2026-10-22 08:22:26 -04:00
parent dde22a3dd1
commit 999653a6b9
5 changed files with 256 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+84 -26
View File
@@ -396,7 +396,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Receive<DeployCompileWarmed>(HandleDeployCompileWarmed);
// A staggered startup batch's compiles have been pre-warmed off-thread;
// create the batch's Instance Actors now that their PreStart compiles are hits.
Receive<BatchCompileWarmed>(msg => CreateInstanceActorBatch(msg.Batch));
Receive<BatchCompileWarmed>(CreateInstanceActorBatch);
// Notify-and-fetch: central sends a small RefreshDeploymentCommand;
// the active singleton fetches the flattened config over HTTP, then reuses the
@@ -830,32 +830,72 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
var startIdx = state.NextIndex;
var endIdx = Math.Min(startIdx + batchSize, state.Configs.Count);
var validator = _deployCompileValidator;
var batchConfigs = new string[endIdx - startIdx];
for (var i = startIdx; i < endIdx; i++)
batchConfigs[i - startIdx] = state.Configs[i].ConfigJson;
// Review 2026-10-21 01/SR-P3: the previous batch normally started THIS batch's warm when
// it scheduled the stagger delay, so by now it is usually done. Only the first batch (or
// a batch whose prewarm belongs to a different boot load) warms from here.
var warm = _nextBatchWarm is { } prewarm &&
ReferenceEquals(prewarm.Configs, state.Configs) &&
prewarm.StartIndex == startIdx
? prewarm.Warm
: StartBatchWarm(state.Configs, startIdx, endIdx);
_nextBatchWarm = null;
Task.Run(() =>
warm.ContinueWith(t => new BatchCompileWarmed(msg, t.IsCompletedSuccessfully ? t.Result : null))
.PipeTo(Self);
}
/// <summary>
/// Upper bound on concurrent Roslyn compiles while warming a startup batch (01/SR-P3). Half
/// the cores: boot is a burst, but the node is also re-establishing DCL connections and
/// starting the batch already warmed, so the warm must not take every core.
/// </summary>
private static int StartupWarmParallelism => Math.Max(1, Environment.ProcessorCount / 2);
/// <summary>
/// The last startup batch's look-ahead warm: the boot load it belongs to (compared by
/// reference), the batch's first index, and the task that parses + warms it (01/SR-P3).
/// </summary>
private (List<DeployedInstance> Configs, int StartIndex, Task<FlattenedConfiguration?[]> Warm)? _nextBatchWarm;
/// <summary>
/// Parses one staggered startup batch's configs and warms their distinct compile units in
/// parallel, off the actor thread (review 2026-10-21 01/SR-P3). Returns the parsed configs
/// index-aligned with the batch (a null slot = did not parse) so creation reuses them instead
/// of deserializing again (01/SR-P5). Never faults: a failure just yields null slots, and the
/// Instance Actor's own compile/parse is the backstop exactly as before the warm existed.
/// </summary>
private Task<FlattenedConfiguration?[]> StartBatchWarm(List<DeployedInstance> configs, int startIdx, int endIdx)
{
var validator = _deployCompileValidator;
var batchJson = new string[endIdx - startIdx];
for (var i = startIdx; i < endIdx; i++)
batchJson[i - startIdx] = configs[i].ConfigJson;
var parallelism = StartupWarmParallelism;
return Task.Run(() =>
{
foreach (var configJson in batchConfigs)
var parsed = new FlattenedConfiguration?[batchJson.Length];
for (var i = 0; i < batchJson.Length; i++)
{
try
{
// Verdict discarded: startup does not gate on compile failures (a failing
// script is logged by the Instance Actor and leaves the rest running).
// This call exists only to populate the compile cache owned by the
// shared ScriptCompilationService (SiteScriptCompileCache).
validator.Validate(configJson);
}
catch
{
// Best-effort warm — a throw here just means that instance's PreStart
// compiles the hard way, exactly as it did before the warm-then-gate change.
}
if (DeployCompileValidator.TryParse(batchJson[i], out var config, out _))
parsed[i] = config;
}
return new BatchCompileWarmed(msg);
}).PipeTo(Self);
try
{
// Verdicts discarded: startup does not gate on compile failures (a failing
// script is logged by the Instance Actor and leaves the rest running). This
// exists only to populate the compile cache owned by the shared
// ScriptCompilationService (SiteScriptCompileCache).
validator.WarmCompileCache(parsed.OfType<FlattenedConfiguration>().ToList(), parallelism);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Startup batch compile warm threw; the Instance Actors will compile instead");
}
return parsed;
});
}
/// <summary>
@@ -863,8 +903,9 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// schedules the next batch, with a configurable delay between batches to prevent
/// reconnection storms on failover.
/// </summary>
private void CreateInstanceActorBatch(StartNextBatch msg)
private void CreateInstanceActorBatch(BatchCompileWarmed warmed)
{
var msg = warmed.Batch;
var state = msg.State;
var batchSize = _options.StartupBatchSize;
var startIdx = state.NextIndex;
@@ -878,8 +919,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
{
var config = state.Configs[i];
// CreateInstanceActor ensures this config's DCL connections (register row 85). A
// refused actor name is logged inside and skips just that instance (01/SR-C1).
CreateInstanceActor(config.InstanceUniqueName, config.ConfigJson, config.RevisionHash, out _);
// refused actor name is logged inside and skips just that instance (01/SR-C1). The
// warm's parsed config is reused rather than deserialized again (01/SR-P5).
var parsed = warmed.Parsed is { } all && i - startIdx < all.Length ? all[i - startIdx] : null;
CreateInstanceActor(
config.InstanceUniqueName, config.ConfigJson, config.RevisionHash, out _,
parsed: parsed);
}
UpdateInstanceCounts();
@@ -888,6 +933,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
if (endIdx < state.Configs.Count)
{
var nextState = new BatchState(state.Configs, endIdx);
// Review 2026-10-21 01/SR-P3: warm the NEXT batch now, while the stagger delay runs,
// instead of after it. Actor creation order and the delay are unchanged — the next
// batch is still created only once the timer fires AND its warm has landed — so the
// reconnection-storm protection is exactly as before; only the compiles moved earlier.
_nextBatchWarm = (state.Configs, endIdx, StartBatchWarm(
state.Configs, endIdx, Math.Min(endIdx + batchSize, state.Configs.Count)));
Timers.StartSingleTimer(
"startup-batch",
new StartNextBatch(nextState),
@@ -4058,7 +4111,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// Piped back to self once a staggered startup batch's compiles have been
/// pre-warmed, carrying the original batch message so actor creation resumes unchanged.
/// </summary>
internal sealed record BatchCompileWarmed(StartNextBatch Batch);
/// <param name="Batch">The batch message the warm ran for.</param>
/// <param name="Parsed">
/// The batch's configs parsed by the warm, index-aligned (01/SR-P5); null when the warm task
/// itself failed, in which case each config is parsed at creation instead.
/// </param>
internal sealed record BatchCompileWarmed(StartNextBatch Batch, FlattenedConfiguration?[]? Parsed = null);
/// <summary>
/// A lifecycle command queued because its instance's deploy was mid-compile-warm, with
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Deployment/DeployCompileValidator.cs
+63
View File
@@ -154,6 +154,69 @@ public sealed class DeployCompileValidator
return errors;
}
/// <summary>
/// Populates the shared compile cache for a whole startup batch (review 2026-10-21 01/SR-P3):
/// every DISTINCT script body and trigger expression across <paramref name="configs"/> is
/// compiled once, up to <paramref name="maxDegreeOfParallelism"/> at a time. Same compile units
/// as <see cref="Validate(FlattenedConfiguration, string?)"/> — scripts and alarm on-trigger
/// scripts against <c>ScriptGlobals</c>, Expression triggers against
/// <c>TriggerExpressionGlobals</c> — but deduplicated first, because the cache is keyed on
/// code: N instances of one template would otherwise race N identical cold compiles in
/// parallel. Best-effort: verdicts are discarded and a throwing unit is skipped, since every
/// Instance Actor's own compile remains the correctness backstop.
/// </summary>
/// <param name="configs">The batch's parsed configurations.</param>
/// <param name="maxDegreeOfParallelism">Upper bound on concurrent Roslyn compiles.</param>
public void WarmCompileCache(IReadOnlyList<FlattenedConfiguration> configs, int maxDegreeOfParallelism)
{
var units = new Dictionary<(bool IsTrigger, string Code), string>();
foreach (var config in configs)
{
foreach (var script in config.Scripts)
{
units.TryAdd((false, script.Code), script.CanonicalName);
AddTriggerUnit(units, script.TriggerType, script.TriggerConfiguration,
$"script-trigger-{script.CanonicalName}");
}
// Alarm on-trigger scripts compile the referenced SCRIPT's body against ScriptGlobals,
// i.e. the same cache key as that script — already a unit above.
foreach (var alarm in config.Alarms)
AddTriggerUnit(units, alarm.TriggerType, alarm.TriggerConfiguration,
$"alarm-trigger-expr-{alarm.CanonicalName}");
}
Parallel.ForEach(
units,
new ParallelOptions { MaxDegreeOfParallelism = Math.Max(1, maxDegreeOfParallelism) },
unit =>
{
try
{
if (unit.Key.IsTrigger)
_compilationService.CompileTriggerExpression(unit.Value, unit.Key.Code);
else
_compilationService.Compile(unit.Value, unit.Key.Code);
}
catch
{
// Best-effort warm — the Instance Actor's PreStart compiles it the hard way.
}
});
}
private static void AddTriggerUnit(
Dictionary<(bool IsTrigger, string Code), string> units,
string? triggerType, string? triggerConfigJson, string compileName)
{
if (!string.Equals(triggerType, "Expression", StringComparison.OrdinalIgnoreCase))
return;
var expression = TriggerExpressionGlobals.ExtractExpression(triggerConfigJson);
if (expression != null)
units.TryAdd((true, expression), compileName);
}
/// <summary>
/// Review 2026-10-21 01/SR-C1: every name that becomes an Akka actor name on the site — the
/// Instance Actor itself (<c>DeploymentManagerActor.CreateInstanceActor</c>) and its