using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.TestSupport; using System.Text.Json; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// WP3.1 test group 9 — warm-then-gate deploys and startup batch pre-warm. /// /// The site-side compile gate (S3) must stay synchronous on the Deployment Manager's /// thread, because redeploy-supersede and delete-during-redeploy both depend on strict mailbox /// FIFO. But the Roslyn compile it performs used to hold the singleton for the whole /// compilation, stalling every OTHER instance's commands behind one instance's scripts. WP3.1 /// warms the compile off-thread first and then re-runs the gate as pure cache hits, with a /// per-instance in-flight guard preserving same-instance ordering. /// /// These tests pin the ordering contract, not the timing: a command for the SAME instance /// arriving during a warm must be queued and applied after the deploy, a superseded deploy must /// answer its deployer instead of leaving it to Ask-timeout, and commands for DIFFERENT /// instances must not block each other. /// /// Shares the SiteScriptCompileCache collection because the batch pre-warm test /// asserts on that process-wide cache's hit counter. /// [Collection("SiteScriptCompileCache")] public class DeploymentWarmThenGateTests : TestKit, IDisposable { private readonly SiteStorageService _storage; private readonly ScriptCompilationService _compilationService; private readonly SharedScriptLibrary _sharedScriptLibrary; private readonly TestLocalDb _localDb; public DeploymentWarmThenGateTests() { _localDb = TestLocalDb.CreateTemp("dm-warm-gate-test"); _storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); _storage.InitializeAsync().GetAwaiter().GetResult(); _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedScriptLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); } void IDisposable.Dispose() { Shutdown(); var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null) => ActorOf(Props.Create(() => new DeploymentManagerActor( _storage, _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions(), NullLogger.Instance, null, healthCollector))); /// /// Captures the deployed-instance count the Deployment Manager reports. The count is /// mutated only on the actor thread — HandleDeploy adds the instance name, /// HandleDelete removes it — so it is an exact, storage-race-free record of the /// order in which the two commands were APPLIED. /// private sealed class DeployedCountCollector : ISiteHealthCollector { public int LastDeployedCount { get; private set; } public void IncrementScriptError() { } public void IncrementAlarmError() { } public void IncrementDeadLetter() { } public void IncrementSiteAuditWriteFailures() { } public void IncrementAuditRedactionFailure() { } public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { } public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { } public void RemoveConnection(string connectionName) { } public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { } public void UpdateConnectionEndpoint(string connectionName, string endpoint) { } public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { } public void SetStoreAndForwardDepths(IReadOnlyDictionary depths) { } public void SetInstanceCounts(int deployed, int enabled, int disabled) => LastDeployedCount = deployed; public void SetParkedMessageCount(int count) { } public void SetNodeHostname(string hostname) { } public void SetClusterNodes(IReadOnlyList nodes) { } public void SetActiveNode(bool isActive) { } public bool IsActiveNode => true; public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId) => throw new NotSupportedException(); } private static string ConfigJson(string instanceName, string? scriptCode = null) => JsonSerializer.Serialize(new FlattenedConfiguration { InstanceUniqueName = instanceName, Attributes = [ new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" } ], Scripts = scriptCode is null ? [] : [new ResolvedScript { CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call" }] }); [Fact] public async Task DeleteArrivingDuringTheCompileWarm_IsQueuedAndAppliedAfterTheDeploy() { var health = new DeployedCountCollector(); var dm = CreateDeploymentManager(health); await Task.Delay(500); // empty startup Assert.Equal(0, health.LastDeployedCount); var deployProbe = CreateTestProbe(); var deleteProbe = CreateTestProbe(); // Back-to-back on the mailbox: the delete lands while the deploy's compile warm is // still in flight, so it must be queued rather than racing ahead of the deploy. dm.Tell(new DeployInstanceCommand( "dep-1", "WarmPump", "h1", ConfigJson("WarmPump", "return 1;"), "admin", DateTimeOffset.UtcNow), deployProbe.Ref); dm.Tell(new DeleteInstanceCommand("del-1", "WarmPump", DateTimeOffset.UtcNow), deleteProbe.Ref); var deploy = deployProbe.ExpectMsg(TimeSpan.FromSeconds(15)); Assert.Equal(DeploymentStatus.Success, deploy.Status); var delete = deleteProbe.ExpectMsg(TimeSpan.FromSeconds(15)); Assert.True(delete.Success); // Terminal in-memory state: the deploy applied FIRST (adding the instance) and the // delete applied SECOND (removing it), leaving the count at 0. Had the delete raced // ahead of the warm it would have removed nothing and the deploy would have left the // count at 1. This is the ordering signal rather than the SQLite row, because the // deploy's store and the delete's remove are independent background tasks whose // completion order the actor has never guaranteed (true before WP3.1 as well). AwaitAssert(() => Assert.Equal(0, health.LastDeployedCount), TimeSpan.FromSeconds(10)); } [Fact] public async Task SecondDeployDuringTheWarm_SupersedesTheFirst_AndAnswersItsDeployer() { var dm = CreateDeploymentManager(); await Task.Delay(500); var first = CreateTestProbe(); var second = CreateTestProbe(); dm.Tell(new DeployInstanceCommand( "dep-a", "SupersedePump", "h1", ConfigJson("SupersedePump", "return 1;"), "admin", DateTimeOffset.UtcNow), first.Ref); dm.Tell(new DeployInstanceCommand( "dep-b", "SupersedePump", "h2", ConfigJson("SupersedePump", "return 2;"), "admin", DateTimeOffset.UtcNow), second.Ref); // The displaced deployer is answered rather than left to time out its Ask. var superseded = first.ExpectMsg(TimeSpan.FromSeconds(15)); Assert.Equal("dep-a", superseded.DeploymentId); Assert.Equal(DeploymentStatus.Failed, superseded.Status); Assert.Contains("superseded", superseded.ErrorMessage!, StringComparison.OrdinalIgnoreCase); var winner = second.ExpectMsg(TimeSpan.FromSeconds(15)); Assert.Equal("dep-b", winner.DeploymentId); Assert.Equal(DeploymentStatus.Success, winner.Status); // Exactly one row, carrying the winning revision hash. var configs = await _storage.GetAllDeployedConfigsAsync(); var row = Assert.Single(configs, c => c.InstanceUniqueName == "SupersedePump"); Assert.Equal("h2", row.RevisionHash); } [Fact] public async Task DeploysForDifferentInstances_DoNotBlockEachOther() { var dm = CreateDeploymentManager(); await Task.Delay(500); var probeX = CreateTestProbe(); var probeY = CreateTestProbe(); dm.Tell(new DeployInstanceCommand( "dep-x", "PumpX", "hx", ConfigJson("PumpX", "return 1;"), "admin", DateTimeOffset.UtcNow), probeX.Ref); dm.Tell(new DeployInstanceCommand( "dep-y", "PumpY", "hy", ConfigJson("PumpY", "return 2;"), "admin", DateTimeOffset.UtcNow), probeY.Ref); // Both apply; the per-instance warm guard scopes to the instance, so a warm for X // never queues a command for Y. Assert.Equal(DeploymentStatus.Success, probeX.ExpectMsg(TimeSpan.FromSeconds(15)).Status); Assert.Equal(DeploymentStatus.Success, probeY.ExpectMsg(TimeSpan.FromSeconds(15)).Status); var configs = await _storage.GetAllDeployedConfigsAsync(); Assert.Contains(configs, c => c.InstanceUniqueName == "PumpX"); Assert.Contains(configs, c => c.InstanceUniqueName == "PumpY"); } [Fact] public async Task StaggeredStartup_PreWarmsEachBatchSoInstanceActorPreStartCompilesAreCacheHits() { // Two instances sharing one script body. The batch pre-warm compiles it once; the // second config's warm and BOTH Instance Actors' PreStart compiles are then hits. // Before WP3.1 every Instance Actor Roslyn-compiled its own scripts inside PreStart, // serialising a site's whole failover recovery behind compilation. const string sharedCode = "return 41 + 1;"; await _storage.StoreDeployedConfigAsync( "BatchOne", ConfigJson("BatchOne", sharedCode), "d1", "h1", true); await _storage.StoreDeployedConfigAsync( "BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true); SiteScriptCompileCache.Clear(); Assert.Equal(0, SiteScriptCompileCache.Hits); CreateDeploymentManager(); AwaitAssert(() => { // One compile, then repeated hits: the second pre-warm plus both PreStarts. Assert.True(SiteScriptCompileCache.Hits >= 3, $"expected the pre-warmed body to be served from cache, saw {SiteScriptCompileCache.Hits} hits"); }, TimeSpan.FromSeconds(20)); } }