using System.Collections.Concurrent; using System.Text.Json; using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; using ZB.MOM.WW.ScadaBridge.TestSupport; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// Regression tests for deferred register row 67 — startup reconcile healed the FILE but not /// the RUNTIME. /// /// /// Instance Actors are created from exactly two paths: the Deployment Manager's boot-time SQLite /// load, and an explicit deploy/enable command. 's /// node-STARTUP fetch runs AFTER that boot load has already read storage, so a config it healed /// into the file started nothing — the enabled, successfully-deployed instance stayed dead until /// the NEXT restart, whose load finally picked it up. Proven live on site-a 2026-08-15: /// "Reconcile: fetched + stored config" with no "InstanceActor started" after it, and a debug /// snapshot answering instanceNotFound. /// /// /// /// The fix is an raised by the reconcile pass and /// handled by the Deployment Manager singleton. These tests pin both halves plus the seam /// between them. /// /// public class ReconcileStartsInstanceTests : TestKit, IDisposable { private const string SiteIdentifier = "site-1"; private const string NodeId = "node-a"; private readonly SiteStorageService _storage; private readonly ScriptCompilationService _compilationService; private readonly SharedScriptLibrary _sharedScriptLibrary; private readonly TestLocalDb _localDb; public ReconcileStartsInstanceTests() { _localDb = TestLocalDb.CreateTemp("reconcile-start-test"); _storage = new SiteStorageService( _localDb.Db, NullLogger.Instance); _storage.InitializeAsync().GetAwaiter().GetResult(); _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedScriptLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); } void IDisposable.Dispose() { // Shut the actors down BEFORE the ILocalDb they write through is disposed. Shutdown(); var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } /// /// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read /// storage — its first ISiteHealthCollector.SetInstanceCounts report. /// /// /// Deferred register row 83. This used to be a bare Task.Delay(500). Every test /// here writes its deployed_configurations row AFTER the manager is constructed, and a /// row that lands before the boot load reads storage is picked up by that load — so the test /// would still pass, while proving the BOOT LOAD works rather than the startup-reconcile path /// it claims to cover. A sleep only makes that race unlikely; on a loaded machine the load can /// still be in flight when it expires, and the test silently changes what it proves instead of /// failing. The row-78 fix session hit exactly this trap and closed it in /// InstanceSweepSelfHealTests; this is the same gate, applied to row 67's suite. /// /// /// /// The sweep is disabled (InstanceSweepIntervalSeconds = 0) so these tests keep proving /// the row-67 path specifically — with the sweep /// armed, its periodic self-heal would independently start the same instances and the suite /// would no longer distinguish the two mechanisms. /// /// private IActorRef CreateDeploymentManager( IServiceProvider? serviceProvider = null, Func>>? configLoader = null, Func>? deployedConfigLoader = null) { var health = new DeploymentManagerStartupProbe(); var dm = ActorOf(Props.Create(() => new DeploymentManagerActor( _storage, _compilationService, _sharedScriptLibrary, null, new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 }, NullLogger.Instance, dclManager: null, healthCollector: health, serviceProvider: serviceProvider, loggerFactory: null, configFetcher: null, // In position: an expression tree rejects a named argument that skips a parameter. startupLoadRetryInterval: null, configLoader: configLoader, externalSystemCache: null, scriptScheduler: null, connectionDefinitionLoader: null, deployedConfigLoader: deployedConfigLoader))); AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25)); return dm; } private IActorRef CreateReconciliationActor( IActorRef siteCommunicationActor, IDeploymentConfigFetcher fetcher, IActorRef? deploymentManager = null) => ActorOf(Props.Create(() => new SiteReconciliationActor( _storage, fetcher, siteCommunicationActor, SiteIdentifier, NodeId, NullLogger.Instance, TimeSpan.FromMilliseconds(50), // initial delay — fast for tests TimeSpan.FromSeconds(5), // ask timeout deploymentManager))); private static string MakeConfigJson(string instanceName) { var config = new FlattenedConfiguration { InstanceUniqueName = instanceName, Attributes = [ new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" } ] }; return JsonSerializer.Serialize(config); } /// /// Asks the Deployment Manager for a debug snapshot — the exact probe that answered /// instanceNotFound on the live rig. InstanceNotFound == false means an /// Instance Actor is running and routable. /// private async Task InstanceIsRunningAsync(IActorRef deploymentManager, string instanceName) { var snapshot = await deploymentManager.Ask( new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()), TimeSpan.FromSeconds(5)); return !snapshot.InstanceNotFound; } // ── The reproduction ── /// /// THE row-67 reproduction, end to end over one shared storage file: the node boots with no /// config for an enabled instance (so the Deployment Manager's startup load starts nothing), /// then the startup reconcile pass fetches and stores that config. The instance must be /// RUNNING afterwards — not merely present in SQLite waiting for a second restart. /// /// /// Fails before the fix at the final assert: the config row is stored, but the debug /// snapshot still answers InstanceNotFound = true because nothing ever created the /// Instance Actor. /// /// [Fact] public async Task StartupReconcile_StoringConfigForEnabledInstance_StartsTheInstanceActor() { // Boot the Deployment Manager against EMPTY storage — this is the missing-config boot // (lost volume / first boot after a restore / rehomed data dir). var dm = CreateDeploymentManager(); // Baseline: the instance is genuinely not running. Assert.False(await InstanceIsRunningAsync(dm, "HealedPump")); // Now the startup reconcile pass runs and central reports the instance as a gap. const string instanceName = "HealedPump"; var configJson = MakeConfigJson(instanceName); var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson)); var commProbe = CreateTestProbe(); CreateReconciliationActor(commProbe, fetcher, dm); commProbe.ExpectMsg(TimeSpan.FromSeconds(5)); commProbe.Reply(new ReconcileSiteResponse( [new ReconcileGapItem(instanceName, "dep-healed", "rev-healed", true, "tok-healed")], [], "http://central:9000")); // The file heals... await AwaitAssertAsync(async () => { var configs = await _storage.GetAllDeployedConfigsAsync(); Assert.Contains(configs, c => c.InstanceUniqueName == instanceName); }, TimeSpan.FromSeconds(5)); // ...and so must the RUNTIME. This is the assertion that fails pre-fix. await AwaitAssertAsync(async () => Assert.True( await InstanceIsRunningAsync(dm, instanceName), "Reconcile stored the config but the InstanceActor was never started " + "— the instance would stay dead until the next restart (register row 67)"), TimeSpan.FromSeconds(10)); } /// /// The heal must not be silent: starting a reconcile-healed instance emits an operational /// deployment site event naming the instance, so an operator can see that a node /// booted without a config and self-repaired. /// [Fact] public async Task ReconcileStartedInstance_EmitsASiteEvent() { var siteLog = new FakeSiteEventLogger(); var dm = CreateDeploymentManager(new SingleServiceProvider(siteLog)); const string instanceName = "SurfacedPump"; await _storage.StoreDeployedConfigAsync( instanceName, MakeConfigJson(instanceName), "dep-s", "rev-s", true); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await AwaitAssertAsync(() => { Assert.Contains(siteLog.OfType("deployment"), r => r.InstanceId == instanceName && r.Source == "DeploymentManagerActor" && r.Message.Contains("self-heal", StringComparison.OrdinalIgnoreCase)); return Task.CompletedTask; }, TimeSpan.FromSeconds(10)); } // ── The reconcile half: which instances get a start request ── /// /// An ENABLED instance healed by the pass produces an ensure-started request to the /// Deployment Manager proxy. /// [Fact] public void ReconcilePass_RequestsStart_ForHealedEnabledInstance() { var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"B\"}")); var commProbe = CreateTestProbe(); var dmProbe = CreateTestProbe(); CreateReconciliationActor(commProbe, fetcher, dmProbe); commProbe.ExpectMsg(TimeSpan.FromSeconds(5)); commProbe.Reply(new ReconcileSiteResponse( [new ReconcileGapItem("B", "depB", "rev2", true, "tok-b")], [], "http://central:9000")); var ensure = dmProbe.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.Equal("B", ensure.InstanceUniqueName); Assert.Contains("reconcile", ensure.Reason, StringComparison.OrdinalIgnoreCase); } /// /// A DISABLED instance healed by the pass must NOT be started — a disabled instance /// legitimately has a config row and no running actor. /// [Fact] public void ReconcilePass_DoesNotRequestStart_ForHealedDisabledInstance() { var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"D\"}")); var commProbe = CreateTestProbe(); var dmProbe = CreateTestProbe(); CreateReconciliationActor(commProbe, fetcher, dmProbe); commProbe.ExpectMsg(TimeSpan.FromSeconds(5)); commProbe.Reply(new ReconcileSiteResponse( [new ReconcileGapItem("D", "depD", "revD", false, "tok-d")], [], "http://central:9000")); dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2)); } /// /// An item that failed to fetch was never stored, so it must not be started either. /// [Fact] public void ReconcilePass_DoesNotRequestStart_ForFailedFetch() { var fetcher = new FakeConfigFetcher(_ => Task.FromException(new DeploymentConfigFetchException("boom", isSuperseded: false))); var commProbe = CreateTestProbe(); var dmProbe = CreateTestProbe(); CreateReconciliationActor(commProbe, fetcher, dmProbe); commProbe.ExpectMsg(TimeSpan.FromSeconds(5)); commProbe.Reply(new ReconcileSiteResponse( [new ReconcileGapItem("Bad", "depBad", "revBad", true, "tok-bad")], [], "http://central:9000")); dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2)); } // ── The Deployment Manager half: the ensure-started contract ── /// /// The command is idempotent: both site nodes' reconcile actors legitimately raise it for /// the same instance, and a repeat must not disturb (or replace) the running actor. /// [Fact] public async Task EnsureInstanceStarted_IsIdempotent_WhenAlreadyRunning() { var dm = CreateDeploymentManager(); const string instanceName = "IdempotentPump"; dm.Tell(new DeployInstanceCommand( "dep-idem", instanceName, "sha256:abc", MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow)); ExpectMsg(TimeSpan.FromSeconds(5)); // Wait on the instance actually being routable, not on the clock. await AwaitAssertAsync(async () => Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10)); var before = await ResolveInstanceChildAsync(dm, instanceName); // Two redundant ensure-started requests, as the two nodes would send. This one IS a // settle wait: the assertion below is that nothing happened, which cannot be awaited on. dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await Task.Delay(1000); // Same actor ref — nothing was restarted or replaced, and it is still routable. var after = await ResolveInstanceChildAsync(dm, instanceName); Assert.Equal(before, after); Assert.True(await InstanceIsRunningAsync(dm, instanceName)); } /// /// The durable row is the authority on enabled/disabled state: a DISABLED instance stays /// stopped even if an ensure-started request arrives for it (e.g. a disable landed between /// the reconcile fetch and the start request). /// [Fact] public async Task EnsureInstanceStarted_DisabledInstance_StaysStopped() { var dm = CreateDeploymentManager(); const string instanceName = "DisabledPump"; await _storage.StoreDeployedConfigAsync( instanceName, MakeConfigJson(instanceName), "dep-d", "rev-d", false); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await Task.Delay(1500); Assert.False(await InstanceIsRunningAsync(dm, instanceName)); } /// /// An ensure-started request for an instance with no stored config on THIS node is a quiet /// no-op — the normal case on an unreplicated site whose peer node healed its own file. /// The Deployment Manager must not crash or invent an instance. /// [Fact] public async Task EnsureInstanceStarted_UnknownInstance_IsAQuietNoOp() { var dm = CreateDeploymentManager(); dm.Tell(new EnsureInstanceStartedCommand("NeverHeardOf", "startup reconcile")); await Task.Delay(1500); Assert.False(await InstanceIsRunningAsync(dm, "NeverHeardOf")); // The manager is still alive and serving other work. dm.Tell(new DeployInstanceCommand( "dep-after", "AfterPump", "sha256:abc", MakeConfigJson("AfterPump"), "admin", DateTimeOffset.UtcNow)); Assert.Equal( DeploymentStatus.Success, ExpectMsg(TimeSpan.FromSeconds(5)).Status); } // ── Row 101 applied to the heal: a result that outlived its snapshot ── /// /// The heal's storage read is asynchronous, so a lifecycle command can commit while it is /// in flight — and the READ ITSELF cannot notice, because it already saw the enabled row and /// neither disable nor delete leaves any of the three pipeline markers /// ShouldAttemptEnsureStart re-checks. Starting from that superseded snapshot is /// PERMANENT: the instance reconcile sweep walks store rows only, so it has no /// runtime→store stop leg that would ever take the ghost instance back down. /// /// /// This test holds the read open, commits a disable behind it, then releases it — the /// interleaving the fix's store-write-generation stamp exists to reject (arch-review /// 2026-08-19 P1-1). Fails before the fix: the stale enabled snapshot passes every /// guard and the Instance Actor is created for a disabled instance. /// /// [Fact] public async Task EnsureInstanceStarted_ResultLandingAfterADisable_IsDiscarded() { const string instanceName = "RacedDisablePump"; var loader = new GatedConfigLoader(_storage); var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync); await _storage.StoreDeployedConfigAsync( instanceName, MakeConfigJson(instanceName), "dep-rd", "rev-rd", true); // The heal reads the (still enabled) row and parks on the gate. loader.Arm(); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await loader.WaitUntilParkedAsync(); // The operator disables the instance; the write commits and the manager processes both // halves of the generation bump while the heal's snapshot is still in flight. dm.Tell(new DisableInstanceCommand("cmd-disable", instanceName, DateTimeOffset.UtcNow)); ExpectMsg(TimeSpan.FromSeconds(5)); await AwaitAssertAsync(async () => { var configs = await _storage.GetAllDeployedConfigsAsync(); Assert.False(configs.Single(c => c.InstanceUniqueName == instanceName).IsEnabled); }, TimeSpan.FromSeconds(5)); // Now let the superseded result land. loader.Release(); await Task.Delay(1500); Assert.False( await InstanceIsRunningAsync(dm, instanceName), "The heal started an instance the operator had just disabled — and nothing would " + "ever stop it again (arch-review 2026-08-19 P1-1)"); } /// /// The delete half of the same race. Worse than the disable case: the durable row is GONE, /// so the resurrected instance has no config row at all — the sweep cannot even see it to /// reason about, and the ghost survives until the node restarts. /// /// Fails before the fix at the final assert. /// [Fact] public async Task EnsureInstanceStarted_ResultLandingAfterADelete_IsDiscarded() { const string instanceName = "RacedDeletePump"; var loader = new GatedConfigLoader(_storage); var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync); await _storage.StoreDeployedConfigAsync( instanceName, MakeConfigJson(instanceName), "dep-rx", "rev-rx", true); loader.Arm(); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await loader.WaitUntilParkedAsync(); dm.Tell(new DeleteInstanceCommand("cmd-delete", instanceName, DateTimeOffset.UtcNow)); ExpectMsg(TimeSpan.FromSeconds(5)); await AwaitAssertAsync(async () => { var configs = await _storage.GetAllDeployedConfigsAsync(); Assert.DoesNotContain(configs, c => c.InstanceUniqueName == instanceName); }, TimeSpan.FromSeconds(5)); loader.Release(); await Task.Delay(1500); Assert.False( await InstanceIsRunningAsync(dm, instanceName), "The heal resurrected a deleted instance from a snapshot read before the delete " + "(arch-review 2026-08-19 P1-1)"); } /// /// The complement of the two tests above: with no mutation in the read's window the heal /// still starts the instance, so the generation check rejects superseded snapshots without /// disabling the row-67 repair itself. /// [Fact] public async Task EnsureInstanceStarted_ResultLandingWithNoInterveningWrite_StillStarts() { const string instanceName = "UnracedPump"; var loader = new GatedConfigLoader(_storage); var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync); await _storage.StoreDeployedConfigAsync( instanceName, MakeConfigJson(instanceName), "dep-ok", "rev-ok", true); loader.Arm(); dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile")); await loader.WaitUntilParkedAsync(); loader.Release(); await AwaitAssertAsync(async () => Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10)); } /// /// Resolves the Instance Actor child of the Deployment Manager by path — the direct /// "the InstanceActor exists" check, complementing the debug-snapshot probe. /// private async Task ResolveInstanceChildAsync(IActorRef deploymentManager, string instanceName) => await Sys.ActorSelection(deploymentManager.Path / instanceName) .ResolveOne(TimeSpan.FromSeconds(5)); /// /// A single-instance deployed-config loader that can be held open mid-read, so a test can /// commit a lifecycle write while a heal's snapshot is provably in flight rather than hoping /// the two race the right way round. /// /// /// Reads real storage every time and only parks AFTER the read, so the returned snapshot is /// exactly what production would have seen at that instant. Inert until /// is called, which keeps the Deployment Manager's boot load — a separate seam /// (_configLoader, register P1-4) untouched by this gate — unblocked. /// /// private sealed class GatedConfigLoader(SiteStorageService storage) { private TaskCompletionSource? _release; private readonly TaskCompletionSource _parked = new(TaskCreationOptions.RunContinuationsAsynchronously); public void Arm() => _release = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); public void Release() => _release!.TrySetResult(); public Task WaitUntilParkedAsync() => _parked.Task.WaitAsync(TimeSpan.FromSeconds(10)); public async Task LoadAsync(string instanceName) { var config = await storage.GetDeployedConfigAsync(instanceName); var gate = _release; if (gate != null) { _parked.TrySetResult(); await gate.Task; } return config; } } /// /// In-test fake : runs a per-deploymentId behavior and /// records every call thread-safely (the fetch runs on a pool thread). /// private sealed class FakeConfigFetcher : IDeploymentConfigFetcher { private readonly Func> _behavior; public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new(); public FakeConfigFetcher(Func> behavior) => _behavior = behavior; public async Task FetchAsync( string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct) { Calls.Enqueue((centralFetchBaseUrl, deploymentId, token)); await Task.Yield(); return await _behavior(deploymentId); } } }