perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,230 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 9 — warm-then-gate deploys and startup batch pre-warm.
|
||||
///
|
||||
/// <para>The site-side compile gate (S3) must stay synchronous on the Deployment Manager's
|
||||
/// thread, because redeploy-supersede and delete-during-redeploy both depend on strict mailbox
|
||||
/// FIFO. But the Roslyn compile it performs used to hold the singleton for the whole
|
||||
/// compilation, stalling every OTHER instance's commands behind one instance's scripts. WP3.1
|
||||
/// warms the compile off-thread first and then re-runs the gate as pure cache hits, with a
|
||||
/// per-instance in-flight guard preserving same-instance ordering.</para>
|
||||
///
|
||||
/// <para>These tests pin the ordering contract, not the timing: a command for the SAME instance
|
||||
/// arriving during a warm must be queued and applied after the deploy, a superseded deploy must
|
||||
/// answer its deployer instead of leaving it to Ask-timeout, and commands for DIFFERENT
|
||||
/// instances must not block each other.</para>
|
||||
///
|
||||
/// <para>Shares the <c>SiteScriptCompileCache</c> collection because the batch pre-warm test
|
||||
/// asserts on that process-wide cache's hit counter.</para>
|
||||
/// </summary>
|
||||
[Collection("SiteScriptCompileCache")]
|
||||
public class DeploymentWarmThenGateTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public DeploymentWarmThenGateTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("dm-warm-gate-test");
|
||||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedScriptLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null) =>
|
||||
ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage, _compilationService, _sharedScriptLibrary, null,
|
||||
new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance, null,
|
||||
healthCollector)));
|
||||
|
||||
/// <summary>
|
||||
/// Captures the deployed-instance count the Deployment Manager reports. The count is
|
||||
/// mutated only on the actor thread — <c>HandleDeploy</c> adds the instance name,
|
||||
/// <c>HandleDelete</c> removes it — so it is an exact, storage-race-free record of the
|
||||
/// order in which the two commands were APPLIED.
|
||||
/// </summary>
|
||||
private sealed class DeployedCountCollector : ISiteHealthCollector
|
||||
{
|
||||
public int LastDeployedCount { get; private set; }
|
||||
public void IncrementScriptError() { }
|
||||
public void IncrementAlarmError() { }
|
||||
public void IncrementDeadLetter() { }
|
||||
public void IncrementSiteAuditWriteFailures() { }
|
||||
public void IncrementAuditRedactionFailure() { }
|
||||
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
|
||||
public void RemoveConnection(string connectionName) { }
|
||||
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
|
||||
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
|
||||
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
|
||||
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
|
||||
public void SetInstanceCounts(int deployed, int enabled, int disabled) => LastDeployedCount = deployed;
|
||||
public void SetParkedMessageCount(int count) { }
|
||||
public void SetNodeHostname(string hostname) { }
|
||||
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
|
||||
public void SetActiveNode(bool isActive) { }
|
||||
public bool IsActiveNode => true;
|
||||
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
private static string ConfigJson(string instanceName, string? scriptCode = null) =>
|
||||
JsonSerializer.Serialize(new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
Attributes =
|
||||
[
|
||||
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
|
||||
],
|
||||
Scripts = scriptCode is null
|
||||
? []
|
||||
: [new ResolvedScript { CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call" }]
|
||||
});
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteArrivingDuringTheCompileWarm_IsQueuedAndAppliedAfterTheDeploy()
|
||||
{
|
||||
var health = new DeployedCountCollector();
|
||||
var dm = CreateDeploymentManager(health);
|
||||
await Task.Delay(500); // empty startup
|
||||
Assert.Equal(0, health.LastDeployedCount);
|
||||
|
||||
var deployProbe = CreateTestProbe();
|
||||
var deleteProbe = CreateTestProbe();
|
||||
|
||||
// Back-to-back on the mailbox: the delete lands while the deploy's compile warm is
|
||||
// still in flight, so it must be queued rather than racing ahead of the deploy.
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-1", "WarmPump", "h1", ConfigJson("WarmPump", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
deployProbe.Ref);
|
||||
dm.Tell(new DeleteInstanceCommand("del-1", "WarmPump", DateTimeOffset.UtcNow), deleteProbe.Ref);
|
||||
|
||||
var deploy = deployProbe.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(DeploymentStatus.Success, deploy.Status);
|
||||
|
||||
var delete = deleteProbe.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.True(delete.Success);
|
||||
|
||||
// Terminal in-memory state: the deploy applied FIRST (adding the instance) and the
|
||||
// delete applied SECOND (removing it), leaving the count at 0. Had the delete raced
|
||||
// ahead of the warm it would have removed nothing and the deploy would have left the
|
||||
// count at 1. This is the ordering signal rather than the SQLite row, because the
|
||||
// deploy's store and the delete's remove are independent background tasks whose
|
||||
// completion order the actor has never guaranteed (true before WP3.1 as well).
|
||||
AwaitAssert(() => Assert.Equal(0, health.LastDeployedCount), TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecondDeployDuringTheWarm_SupersedesTheFirst_AndAnswersItsDeployer()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
var first = CreateTestProbe();
|
||||
var second = CreateTestProbe();
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-a", "SupersedePump", "h1", ConfigJson("SupersedePump", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
first.Ref);
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-b", "SupersedePump", "h2", ConfigJson("SupersedePump", "return 2;"), "admin", DateTimeOffset.UtcNow),
|
||||
second.Ref);
|
||||
|
||||
// The displaced deployer is answered rather than left to time out its Ask.
|
||||
var superseded = first.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal("dep-a", superseded.DeploymentId);
|
||||
Assert.Equal(DeploymentStatus.Failed, superseded.Status);
|
||||
Assert.Contains("superseded", superseded.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var winner = second.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal("dep-b", winner.DeploymentId);
|
||||
Assert.Equal(DeploymentStatus.Success, winner.Status);
|
||||
|
||||
// Exactly one row, carrying the winning revision hash.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "SupersedePump");
|
||||
Assert.Equal("h2", row.RevisionHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploysForDifferentInstances_DoNotBlockEachOther()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
var probeX = CreateTestProbe();
|
||||
var probeY = CreateTestProbe();
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-x", "PumpX", "hx", ConfigJson("PumpX", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
probeX.Ref);
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-y", "PumpY", "hy", ConfigJson("PumpY", "return 2;"), "admin", DateTimeOffset.UtcNow),
|
||||
probeY.Ref);
|
||||
|
||||
// Both apply; the per-instance warm guard scopes to the instance, so a warm for X
|
||||
// never queues a command for Y.
|
||||
Assert.Equal(DeploymentStatus.Success,
|
||||
probeX.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
Assert.Equal(DeploymentStatus.Success,
|
||||
probeY.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpX");
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpY");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StaggeredStartup_PreWarmsEachBatchSoInstanceActorPreStartCompilesAreCacheHits()
|
||||
{
|
||||
// Two instances sharing one script body. The batch pre-warm compiles it once; the
|
||||
// second config's warm and BOTH Instance Actors' PreStart compiles are then hits.
|
||||
// Before WP3.1 every Instance Actor Roslyn-compiled its own scripts inside PreStart,
|
||||
// serialising a site's whole failover recovery behind compilation.
|
||||
const string sharedCode = "return 41 + 1;";
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"BatchOne", ConfigJson("BatchOne", sharedCode), "d1", "h1", true);
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true);
|
||||
|
||||
SiteScriptCompileCache.Clear();
|
||||
Assert.Equal(0, SiteScriptCompileCache.Hits);
|
||||
|
||||
CreateDeploymentManager();
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
// One compile, then repeated hits: the second pre-warm plus both PreStarts.
|
||||
Assert.True(SiteScriptCompileCache.Hits >= 3,
|
||||
$"expected the pre-warmed body to be served from cache, saw {SiteScriptCompileCache.Hits} hits");
|
||||
}, TimeSpan.FromSeconds(20));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user