using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// WP3.1 test group 9 — warm-then-gate deploys and startup batch pre-warm.
///
/// The site-side compile gate (S3) must stay synchronous on the Deployment Manager's
/// thread, because redeploy-supersede and delete-during-redeploy both depend on strict mailbox
/// FIFO. But the Roslyn compile it performs used to hold the singleton for the whole
/// compilation, stalling every OTHER instance's commands behind one instance's scripts. WP3.1
/// warms the compile off-thread first and then re-runs the gate as pure cache hits, with a
/// per-instance in-flight guard preserving same-instance ordering.
///
/// These tests pin the ordering contract, not the timing: a command for the SAME instance
/// arriving during a warm must be queued and applied after the deploy, a superseded deploy must
/// answer its deployer instead of leaving it to Ask-timeout, and commands for DIFFERENT
/// instances must not block each other.
///
/// Shares the SiteScriptCompileCache collection because the batch pre-warm test
/// asserts on that process-wide cache's hit counter.
///
[Collection("SiteScriptCompileCache")]
public class DeploymentWarmThenGateTests : TestKit, IDisposable
{
private readonly SiteStorageService _storage;
private readonly ScriptCompilationService _compilationService;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly TestLocalDb _localDb;
public DeploymentWarmThenGateTests()
{
_localDb = TestLocalDb.CreateTemp("dm-warm-gate-test");
_storage = new SiteStorageService(_localDb.Db, NullLogger.Instance);
_storage.InitializeAsync().GetAwaiter().GetResult();
_compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedScriptLibrary = new SharedScriptLibrary(
_compilationService, NullLogger.Instance);
}
void IDisposable.Dispose()
{
Shutdown();
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null) =>
ActorOf(Props.Create(() => new DeploymentManagerActor(
_storage, _compilationService, _sharedScriptLibrary, null,
new SiteRuntimeOptions(), NullLogger.Instance, null,
healthCollector)));
///
/// Captures the deployed-instance count the Deployment Manager reports. The count is
/// mutated only on the actor thread — HandleDeploy adds the instance name,
/// HandleDelete removes it — so it is an exact, storage-race-free record of the
/// order in which the two commands were APPLIED.
///
private sealed class DeployedCountCollector : ISiteHealthCollector
{
public int LastDeployedCount { get; private set; }
public void IncrementScriptError() { }
public void IncrementAlarmError() { }
public void IncrementDeadLetter() { }
public void IncrementSiteAuditWriteFailures() { }
public void IncrementAuditRedactionFailure() { }
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
public void RemoveConnection(string connectionName) { }
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
public void SetStoreAndForwardDepths(IReadOnlyDictionary depths) { }
public void SetInstanceCounts(int deployed, int enabled, int disabled) => LastDeployedCount = deployed;
public void SetParkedMessageCount(int count) { }
public void SetNodeHostname(string hostname) { }
public void SetClusterNodes(IReadOnlyList nodes) { }
public void SetActiveNode(bool isActive) { }
public bool IsActiveNode => true;
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId)
=> throw new NotSupportedException();
}
private static string ConfigJson(string instanceName, string? scriptCode = null) =>
JsonSerializer.Serialize(new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
],
Scripts = scriptCode is null
? []
: [new ResolvedScript { CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call" }]
});
[Fact]
public async Task DeleteArrivingDuringTheCompileWarm_IsQueuedAndAppliedAfterTheDeploy()
{
var health = new DeployedCountCollector();
var dm = CreateDeploymentManager(health);
await Task.Delay(500); // empty startup
Assert.Equal(0, health.LastDeployedCount);
var deployProbe = CreateTestProbe();
var deleteProbe = CreateTestProbe();
// Back-to-back on the mailbox: the delete lands while the deploy's compile warm is
// still in flight, so it must be queued rather than racing ahead of the deploy.
dm.Tell(new DeployInstanceCommand(
"dep-1", "WarmPump", "h1", ConfigJson("WarmPump", "return 1;"), "admin", DateTimeOffset.UtcNow),
deployProbe.Ref);
dm.Tell(new DeleteInstanceCommand("del-1", "WarmPump", DateTimeOffset.UtcNow), deleteProbe.Ref);
var deploy = deployProbe.ExpectMsg(TimeSpan.FromSeconds(15));
Assert.Equal(DeploymentStatus.Success, deploy.Status);
var delete = deleteProbe.ExpectMsg(TimeSpan.FromSeconds(15));
Assert.True(delete.Success);
// Terminal in-memory state: the deploy applied FIRST (adding the instance) and the
// delete applied SECOND (removing it), leaving the count at 0. Had the delete raced
// ahead of the warm it would have removed nothing and the deploy would have left the
// count at 1. This is the ordering signal rather than the SQLite row, because the
// deploy's store and the delete's remove are independent background tasks whose
// completion order the actor has never guaranteed (true before WP3.1 as well).
AwaitAssert(() => Assert.Equal(0, health.LastDeployedCount), TimeSpan.FromSeconds(10));
}
[Fact]
public async Task SecondDeployDuringTheWarm_SupersedesTheFirst_AndAnswersItsDeployer()
{
var dm = CreateDeploymentManager();
await Task.Delay(500);
var first = CreateTestProbe();
var second = CreateTestProbe();
dm.Tell(new DeployInstanceCommand(
"dep-a", "SupersedePump", "h1", ConfigJson("SupersedePump", "return 1;"), "admin", DateTimeOffset.UtcNow),
first.Ref);
dm.Tell(new DeployInstanceCommand(
"dep-b", "SupersedePump", "h2", ConfigJson("SupersedePump", "return 2;"), "admin", DateTimeOffset.UtcNow),
second.Ref);
// The displaced deployer is answered rather than left to time out its Ask.
var superseded = first.ExpectMsg(TimeSpan.FromSeconds(15));
Assert.Equal("dep-a", superseded.DeploymentId);
Assert.Equal(DeploymentStatus.Failed, superseded.Status);
Assert.Contains("superseded", superseded.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
var winner = second.ExpectMsg(TimeSpan.FromSeconds(15));
Assert.Equal("dep-b", winner.DeploymentId);
Assert.Equal(DeploymentStatus.Success, winner.Status);
// Exactly one row, carrying the winning revision hash.
var configs = await _storage.GetAllDeployedConfigsAsync();
var row = Assert.Single(configs, c => c.InstanceUniqueName == "SupersedePump");
Assert.Equal("h2", row.RevisionHash);
}
[Fact]
public async Task DeploysForDifferentInstances_DoNotBlockEachOther()
{
var dm = CreateDeploymentManager();
await Task.Delay(500);
var probeX = CreateTestProbe();
var probeY = CreateTestProbe();
dm.Tell(new DeployInstanceCommand(
"dep-x", "PumpX", "hx", ConfigJson("PumpX", "return 1;"), "admin", DateTimeOffset.UtcNow),
probeX.Ref);
dm.Tell(new DeployInstanceCommand(
"dep-y", "PumpY", "hy", ConfigJson("PumpY", "return 2;"), "admin", DateTimeOffset.UtcNow),
probeY.Ref);
// Both apply; the per-instance warm guard scopes to the instance, so a warm for X
// never queues a command for Y.
Assert.Equal(DeploymentStatus.Success,
probeX.ExpectMsg(TimeSpan.FromSeconds(15)).Status);
Assert.Equal(DeploymentStatus.Success,
probeY.ExpectMsg(TimeSpan.FromSeconds(15)).Status);
var configs = await _storage.GetAllDeployedConfigsAsync();
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpX");
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpY");
}
[Fact]
public async Task StaggeredStartup_PreWarmsEachBatchSoInstanceActorPreStartCompilesAreCacheHits()
{
// Two instances sharing one script body. The batch pre-warm compiles it once; the
// second config's warm and BOTH Instance Actors' PreStart compiles are then hits.
// Before WP3.1 every Instance Actor Roslyn-compiled its own scripts inside PreStart,
// serialising a site's whole failover recovery behind compilation.
const string sharedCode = "return 41 + 1;";
await _storage.StoreDeployedConfigAsync(
"BatchOne", ConfigJson("BatchOne", sharedCode), "d1", "h1", true);
await _storage.StoreDeployedConfigAsync(
"BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true);
SiteScriptCompileCache.Clear();
Assert.Equal(0, SiteScriptCompileCache.Hits);
CreateDeploymentManager();
AwaitAssert(() =>
{
// One compile, then repeated hits: the second pre-warm plus both PreStarts.
Assert.True(SiteScriptCompileCache.Hits >= 3,
$"expected the pre-warmed body to be served from cache, saw {SiteScriptCompileCache.Hits} hits");
}, TimeSpan.FromSeconds(20));
}
}