65b3c23239
HandleEnsureInstanceStarted, HandleDeploymentStateQuery, and HandleEnable each called GetAllDeployedConfigsAsync — materializing every deployed config, including every ConfigJson blob — just to FirstOrDefault one row. A volume-restore reconcile healing hundreds of instances turned that into hundreds of full-table reads. Add SiteStorageService.GetDeployedConfigAsync(name) (single-row, full projection incl. ConfigJson, for callers that (re)start an Instance Actor) and GetDeployedConfigMetadataAsync(name) (single-row, ConfigJson-free projection, for callers that only need DeploymentId/RevisionHash/IsEnabled). Switch the three call sites; leave decisions/semantics identical. HandleEnsureInstanceStarted gets its own _deployedConfigLoader seam, deliberately separate from the existing _configLoader (still used by the boot-time load, which legitimately needs every row) so a test overriding one cannot accidentally gate the other. Generation fencing (StoreGenerationAtRead, Medium wave T1) is untouched: the generation is still captured on the actor thread before the read dispatches. Tests: new SiteStorageServiceTests lock in that both projections return the same values GetAllDeployedConfigsAsync().FirstOrDefault(...) did. The three row-101 ensure-started race tests (ReconcileStartsInstanceTests) are ported to the new seam and still pass, proving generation fencing survives the projection. Proof of failure: since this is a pure I/O-cost fix with no observable behavior change, the new/changed tests were confirmed to fail to even compile against the unfixed code (stash/unstash) — GetDeployedConfigAsync, GetDeployedConfigMetadataAsync, and the deployedConfigLoader ctor param don't exist there.
575 lines
25 KiB
C#
575 lines
25 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
|
|
|
/// <summary>
|
|
/// Regression tests for deferred register row 67 — <b>startup reconcile healed the FILE but not
|
|
/// the RUNTIME</b>.
|
|
///
|
|
/// <para>
|
|
/// Instance Actors are created from exactly two paths: the Deployment Manager's boot-time SQLite
|
|
/// load, and an explicit deploy/enable command. <see cref="SiteReconciliationActor"/>'s
|
|
/// node-STARTUP fetch runs AFTER that boot load has already read storage, so a config it healed
|
|
/// into the file started nothing — the enabled, successfully-deployed instance stayed dead until
|
|
/// the NEXT restart, whose load finally picked it up. Proven live on site-a 2026-08-15:
|
|
/// "Reconcile: fetched + stored config" with no "InstanceActor started" after it, and a debug
|
|
/// snapshot answering <c>instanceNotFound</c>.
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// The fix is an <see cref="EnsureInstanceStartedCommand"/> raised by the reconcile pass and
|
|
/// handled by the Deployment Manager singleton. These tests pin both halves plus the seam
|
|
/// between them.
|
|
/// </para>
|
|
/// </summary>
|
|
public class ReconcileStartsInstanceTests : TestKit, IDisposable
|
|
{
|
|
private const string SiteIdentifier = "site-1";
|
|
private const string NodeId = "node-a";
|
|
|
|
private readonly SiteStorageService _storage;
|
|
private readonly ScriptCompilationService _compilationService;
|
|
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
|
private readonly TestLocalDb _localDb;
|
|
|
|
public ReconcileStartsInstanceTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("reconcile-start-test");
|
|
_storage = new SiteStorageService(
|
|
_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
_storage.InitializeAsync().GetAwaiter().GetResult();
|
|
_compilationService = new ScriptCompilationService(
|
|
NullLogger<ScriptCompilationService>.Instance);
|
|
_sharedScriptLibrary = new SharedScriptLibrary(
|
|
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
|
}
|
|
|
|
void IDisposable.Dispose()
|
|
{
|
|
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
|
|
Shutdown();
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read
|
|
/// storage — its first <c>ISiteHealthCollector.SetInstanceCounts</c> report.
|
|
///
|
|
/// <para>
|
|
/// <b>Deferred register row 83.</b> This used to be a bare <c>Task.Delay(500)</c>. Every test
|
|
/// here writes its <c>deployed_configurations</c> row AFTER the manager is constructed, and a
|
|
/// row that lands before the boot load reads storage is picked up by that load — so the test
|
|
/// would still pass, while proving the BOOT LOAD works rather than the startup-reconcile path
|
|
/// it claims to cover. A sleep only makes that race unlikely; on a loaded machine the load can
|
|
/// still be in flight when it expires, and the test silently changes what it proves instead of
|
|
/// failing. The row-78 fix session hit exactly this trap and closed it in
|
|
/// <c>InstanceSweepSelfHealTests</c>; this is the same gate, applied to row 67's suite.
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// The sweep is disabled (<c>InstanceSweepIntervalSeconds = 0</c>) so these tests keep proving
|
|
/// the row-67 <see cref="EnsureInstanceStartedCommand"/> path specifically — with the sweep
|
|
/// armed, its periodic self-heal would independently start the same instances and the suite
|
|
/// would no longer distinguish the two mechanisms.
|
|
/// </para>
|
|
/// </summary>
|
|
private IActorRef CreateDeploymentManager(
|
|
IServiceProvider? serviceProvider = null,
|
|
Func<Task<List<DeployedInstance>>>? configLoader = null,
|
|
Func<string, Task<DeployedInstance?>>? deployedConfigLoader = null)
|
|
{
|
|
var health = new DeploymentManagerStartupProbe();
|
|
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
|
_storage,
|
|
_compilationService,
|
|
_sharedScriptLibrary,
|
|
null,
|
|
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
|
|
NullLogger<DeploymentManagerActor>.Instance,
|
|
dclManager: null,
|
|
healthCollector: health,
|
|
serviceProvider: serviceProvider,
|
|
loggerFactory: null,
|
|
configFetcher: null,
|
|
// In position: an expression tree rejects a named argument that skips a parameter.
|
|
startupLoadRetryInterval: null,
|
|
configLoader: configLoader,
|
|
externalSystemCache: null,
|
|
scriptScheduler: null,
|
|
connectionDefinitionLoader: null,
|
|
deployedConfigLoader: deployedConfigLoader)));
|
|
|
|
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
|
|
TimeSpan.FromMilliseconds(25));
|
|
return dm;
|
|
}
|
|
|
|
private IActorRef CreateReconciliationActor(
|
|
IActorRef siteCommunicationActor,
|
|
IDeploymentConfigFetcher fetcher,
|
|
IActorRef? deploymentManager = null) =>
|
|
ActorOf(Props.Create(() => new SiteReconciliationActor(
|
|
_storage,
|
|
fetcher,
|
|
siteCommunicationActor,
|
|
SiteIdentifier,
|
|
NodeId,
|
|
NullLogger<SiteReconciliationActor>.Instance,
|
|
TimeSpan.FromMilliseconds(50), // initial delay — fast for tests
|
|
TimeSpan.FromSeconds(5), // ask timeout
|
|
deploymentManager)));
|
|
|
|
private static string MakeConfigJson(string instanceName)
|
|
{
|
|
var config = new FlattenedConfiguration
|
|
{
|
|
InstanceUniqueName = instanceName,
|
|
Attributes =
|
|
[
|
|
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "42", DataType = "Int32" }
|
|
]
|
|
};
|
|
return JsonSerializer.Serialize(config);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Asks the Deployment Manager for a debug snapshot — the exact probe that answered
|
|
/// <c>instanceNotFound</c> on the live rig. <c>InstanceNotFound == false</c> means an
|
|
/// Instance Actor is running and routable.
|
|
/// </summary>
|
|
private async Task<bool> InstanceIsRunningAsync(IActorRef deploymentManager, string instanceName)
|
|
{
|
|
var snapshot = await deploymentManager.Ask<DebugViewSnapshot>(
|
|
new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()),
|
|
TimeSpan.FromSeconds(5));
|
|
return !snapshot.InstanceNotFound;
|
|
}
|
|
|
|
// ── The reproduction ──
|
|
|
|
/// <summary>
|
|
/// THE row-67 reproduction, end to end over one shared storage file: the node boots with no
|
|
/// config for an enabled instance (so the Deployment Manager's startup load starts nothing),
|
|
/// then the startup reconcile pass fetches and stores that config. The instance must be
|
|
/// RUNNING afterwards — not merely present in SQLite waiting for a second restart.
|
|
///
|
|
/// <para>
|
|
/// <b>Fails before the fix</b> at the final assert: the config row is stored, but the debug
|
|
/// snapshot still answers <c>InstanceNotFound = true</c> because nothing ever created the
|
|
/// Instance Actor.
|
|
/// </para>
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task StartupReconcile_StoringConfigForEnabledInstance_StartsTheInstanceActor()
|
|
{
|
|
// Boot the Deployment Manager against EMPTY storage — this is the missing-config boot
|
|
// (lost volume / first boot after a restore / rehomed data dir).
|
|
var dm = CreateDeploymentManager();
|
|
|
|
// Baseline: the instance is genuinely not running.
|
|
Assert.False(await InstanceIsRunningAsync(dm, "HealedPump"));
|
|
|
|
// Now the startup reconcile pass runs and central reports the instance as a gap.
|
|
const string instanceName = "HealedPump";
|
|
var configJson = MakeConfigJson(instanceName);
|
|
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson));
|
|
var commProbe = CreateTestProbe();
|
|
|
|
CreateReconciliationActor(commProbe, fetcher, dm);
|
|
|
|
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
|
commProbe.Reply(new ReconcileSiteResponse(
|
|
[new ReconcileGapItem(instanceName, "dep-healed", "rev-healed", true, "tok-healed")],
|
|
[],
|
|
"http://central:9000"));
|
|
|
|
// The file heals...
|
|
await AwaitAssertAsync(async () =>
|
|
{
|
|
var configs = await _storage.GetAllDeployedConfigsAsync();
|
|
Assert.Contains(configs, c => c.InstanceUniqueName == instanceName);
|
|
}, TimeSpan.FromSeconds(5));
|
|
|
|
// ...and so must the RUNTIME. This is the assertion that fails pre-fix.
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(
|
|
await InstanceIsRunningAsync(dm, instanceName),
|
|
"Reconcile stored the config but the InstanceActor was never started "
|
|
+ "— the instance would stay dead until the next restart (register row 67)"),
|
|
TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The heal must not be silent: starting a reconcile-healed instance emits an operational
|
|
/// <c>deployment</c> site event naming the instance, so an operator can see that a node
|
|
/// booted without a config and self-repaired.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ReconcileStartedInstance_EmitsASiteEvent()
|
|
{
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateDeploymentManager(new SingleServiceProvider(siteLog));
|
|
|
|
const string instanceName = "SurfacedPump";
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-s", "rev-s", true);
|
|
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
|
|
await AwaitAssertAsync(() =>
|
|
{
|
|
Assert.Contains(siteLog.OfType("deployment"), r =>
|
|
r.InstanceId == instanceName &&
|
|
r.Source == "DeploymentManagerActor" &&
|
|
r.Message.Contains("self-heal", StringComparison.OrdinalIgnoreCase));
|
|
return Task.CompletedTask;
|
|
}, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
// ── The reconcile half: which instances get a start request ──
|
|
|
|
/// <summary>
|
|
/// An ENABLED instance healed by the pass produces an ensure-started request to the
|
|
/// Deployment Manager proxy.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ReconcilePass_RequestsStart_ForHealedEnabledInstance()
|
|
{
|
|
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"B\"}"));
|
|
var commProbe = CreateTestProbe();
|
|
var dmProbe = CreateTestProbe();
|
|
|
|
CreateReconciliationActor(commProbe, fetcher, dmProbe);
|
|
|
|
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
|
commProbe.Reply(new ReconcileSiteResponse(
|
|
[new ReconcileGapItem("B", "depB", "rev2", true, "tok-b")],
|
|
[],
|
|
"http://central:9000"));
|
|
|
|
var ensure = dmProbe.ExpectMsg<EnsureInstanceStartedCommand>(TimeSpan.FromSeconds(10));
|
|
Assert.Equal("B", ensure.InstanceUniqueName);
|
|
Assert.Contains("reconcile", ensure.Reason, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A DISABLED instance healed by the pass must NOT be started — a disabled instance
|
|
/// legitimately has a config row and no running actor.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ReconcilePass_DoesNotRequestStart_ForHealedDisabledInstance()
|
|
{
|
|
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("{\"instanceUniqueName\":\"D\"}"));
|
|
var commProbe = CreateTestProbe();
|
|
var dmProbe = CreateTestProbe();
|
|
|
|
CreateReconciliationActor(commProbe, fetcher, dmProbe);
|
|
|
|
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
|
commProbe.Reply(new ReconcileSiteResponse(
|
|
[new ReconcileGapItem("D", "depD", "revD", false, "tok-d")],
|
|
[],
|
|
"http://central:9000"));
|
|
|
|
dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2));
|
|
}
|
|
|
|
/// <summary>
|
|
/// An item that failed to fetch was never stored, so it must not be started either.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ReconcilePass_DoesNotRequestStart_ForFailedFetch()
|
|
{
|
|
var fetcher = new FakeConfigFetcher(_ =>
|
|
Task.FromException<string>(new DeploymentConfigFetchException("boom", isSuperseded: false)));
|
|
var commProbe = CreateTestProbe();
|
|
var dmProbe = CreateTestProbe();
|
|
|
|
CreateReconciliationActor(commProbe, fetcher, dmProbe);
|
|
|
|
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
|
commProbe.Reply(new ReconcileSiteResponse(
|
|
[new ReconcileGapItem("Bad", "depBad", "revBad", true, "tok-bad")],
|
|
[],
|
|
"http://central:9000"));
|
|
|
|
dmProbe.ExpectNoMsg(TimeSpan.FromSeconds(2));
|
|
}
|
|
|
|
// ── The Deployment Manager half: the ensure-started contract ──
|
|
|
|
/// <summary>
|
|
/// The command is idempotent: both site nodes' reconcile actors legitimately raise it for
|
|
/// the same instance, and a repeat must not disturb (or replace) the running actor.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_IsIdempotent_WhenAlreadyRunning()
|
|
{
|
|
var dm = CreateDeploymentManager();
|
|
|
|
const string instanceName = "IdempotentPump";
|
|
dm.Tell(new DeployInstanceCommand(
|
|
"dep-idem", instanceName, "sha256:abc",
|
|
MakeConfigJson(instanceName), "admin", DateTimeOffset.UtcNow));
|
|
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
|
|
|
|
// Wait on the instance actually being routable, not on the clock.
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10));
|
|
|
|
var before = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// Two redundant ensure-started requests, as the two nodes would send. This one IS a
|
|
// settle wait: the assertion below is that nothing happened, which cannot be awaited on.
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
await Task.Delay(1000);
|
|
|
|
// Same actor ref — nothing was restarted or replaced, and it is still routable.
|
|
var after = await ResolveInstanceChildAsync(dm, instanceName);
|
|
Assert.Equal(before, after);
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The durable row is the authority on enabled/disabled state: a DISABLED instance stays
|
|
/// stopped even if an ensure-started request arrives for it (e.g. a disable landed between
|
|
/// the reconcile fetch and the start request).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_DisabledInstance_StaysStopped()
|
|
{
|
|
var dm = CreateDeploymentManager();
|
|
|
|
const string instanceName = "DisabledPump";
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-d", "rev-d", false);
|
|
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
await Task.Delay(1500);
|
|
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// An ensure-started request for an instance with no stored config on THIS node is a quiet
|
|
/// no-op — the normal case on an unreplicated site whose peer node healed its own file.
|
|
/// The Deployment Manager must not crash or invent an instance.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_UnknownInstance_IsAQuietNoOp()
|
|
{
|
|
var dm = CreateDeploymentManager();
|
|
|
|
dm.Tell(new EnsureInstanceStartedCommand("NeverHeardOf", "startup reconcile"));
|
|
await Task.Delay(1500);
|
|
|
|
Assert.False(await InstanceIsRunningAsync(dm, "NeverHeardOf"));
|
|
|
|
// The manager is still alive and serving other work.
|
|
dm.Tell(new DeployInstanceCommand(
|
|
"dep-after", "AfterPump", "sha256:abc",
|
|
MakeConfigJson("AfterPump"), "admin", DateTimeOffset.UtcNow));
|
|
Assert.Equal(
|
|
DeploymentStatus.Success,
|
|
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5)).Status);
|
|
}
|
|
|
|
// ── Row 101 applied to the heal: a result that outlived its snapshot ──
|
|
|
|
/// <summary>
|
|
/// The heal's storage read is asynchronous, so a lifecycle command can commit while it is
|
|
/// in flight — and the READ ITSELF cannot notice, because it already saw the enabled row and
|
|
/// neither disable nor delete leaves any of the three pipeline markers
|
|
/// <c>ShouldAttemptEnsureStart</c> re-checks. Starting from that superseded snapshot is
|
|
/// PERMANENT: the instance reconcile sweep walks store rows only, so it has no
|
|
/// runtime→store stop leg that would ever take the ghost instance back down.
|
|
///
|
|
/// <para>
|
|
/// This test holds the read open, commits a disable behind it, then releases it — the
|
|
/// interleaving the fix's store-write-generation stamp exists to reject (arch-review
|
|
/// 2026-08-19 P1-1). <b>Fails before the fix</b>: the stale enabled snapshot passes every
|
|
/// guard and the Instance Actor is created for a disabled instance.
|
|
/// </para>
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_ResultLandingAfterADisable_IsDiscarded()
|
|
{
|
|
const string instanceName = "RacedDisablePump";
|
|
var loader = new GatedConfigLoader(_storage);
|
|
var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-rd", "rev-rd", true);
|
|
|
|
// The heal reads the (still enabled) row and parks on the gate.
|
|
loader.Arm();
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
await loader.WaitUntilParkedAsync();
|
|
|
|
// The operator disables the instance; the write commits and the manager processes both
|
|
// halves of the generation bump while the heal's snapshot is still in flight.
|
|
dm.Tell(new DisableInstanceCommand("cmd-disable", instanceName, DateTimeOffset.UtcNow));
|
|
ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
|
|
await AwaitAssertAsync(async () =>
|
|
{
|
|
var configs = await _storage.GetAllDeployedConfigsAsync();
|
|
Assert.False(configs.Single(c => c.InstanceUniqueName == instanceName).IsEnabled);
|
|
}, TimeSpan.FromSeconds(5));
|
|
|
|
// Now let the superseded result land.
|
|
loader.Release();
|
|
await Task.Delay(1500);
|
|
|
|
Assert.False(
|
|
await InstanceIsRunningAsync(dm, instanceName),
|
|
"The heal started an instance the operator had just disabled — and nothing would "
|
|
+ "ever stop it again (arch-review 2026-08-19 P1-1)");
|
|
}
|
|
|
|
/// <summary>
|
|
/// The delete half of the same race. Worse than the disable case: the durable row is GONE,
|
|
/// so the resurrected instance has no config row at all — the sweep cannot even see it to
|
|
/// reason about, and the ghost survives until the node restarts.
|
|
///
|
|
/// <para><b>Fails before the fix</b> at the final assert.</para>
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_ResultLandingAfterADelete_IsDiscarded()
|
|
{
|
|
const string instanceName = "RacedDeletePump";
|
|
var loader = new GatedConfigLoader(_storage);
|
|
var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-rx", "rev-rx", true);
|
|
|
|
loader.Arm();
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
await loader.WaitUntilParkedAsync();
|
|
|
|
dm.Tell(new DeleteInstanceCommand("cmd-delete", instanceName, DateTimeOffset.UtcNow));
|
|
ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(5));
|
|
await AwaitAssertAsync(async () =>
|
|
{
|
|
var configs = await _storage.GetAllDeployedConfigsAsync();
|
|
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == instanceName);
|
|
}, TimeSpan.FromSeconds(5));
|
|
|
|
loader.Release();
|
|
await Task.Delay(1500);
|
|
|
|
Assert.False(
|
|
await InstanceIsRunningAsync(dm, instanceName),
|
|
"The heal resurrected a deleted instance from a snapshot read before the delete "
|
|
+ "(arch-review 2026-08-19 P1-1)");
|
|
}
|
|
|
|
/// <summary>
|
|
/// The complement of the two tests above: with no mutation in the read's window the heal
|
|
/// still starts the instance, so the generation check rejects superseded snapshots without
|
|
/// disabling the row-67 repair itself.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task EnsureInstanceStarted_ResultLandingWithNoInterveningWrite_StillStarts()
|
|
{
|
|
const string instanceName = "UnracedPump";
|
|
var loader = new GatedConfigLoader(_storage);
|
|
var dm = CreateDeploymentManager(deployedConfigLoader: loader.LoadAsync);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-ok", "rev-ok", true);
|
|
|
|
loader.Arm();
|
|
dm.Tell(new EnsureInstanceStartedCommand(instanceName, "startup reconcile"));
|
|
await loader.WaitUntilParkedAsync();
|
|
loader.Release();
|
|
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resolves the Instance Actor child of the Deployment Manager by path — the direct
|
|
/// "the InstanceActor exists" check, complementing the debug-snapshot probe.
|
|
/// </summary>
|
|
private async Task<IActorRef> ResolveInstanceChildAsync(IActorRef deploymentManager, string instanceName) =>
|
|
await Sys.ActorSelection(deploymentManager.Path / instanceName)
|
|
.ResolveOne(TimeSpan.FromSeconds(5));
|
|
|
|
/// <summary>
|
|
/// A single-instance deployed-config loader that can be held open mid-read, so a test can
|
|
/// commit a lifecycle write while a heal's snapshot is provably in flight rather than hoping
|
|
/// the two race the right way round.
|
|
///
|
|
/// <para>
|
|
/// Reads real storage every time and only parks AFTER the read, so the returned snapshot is
|
|
/// exactly what production would have seen at that instant. Inert until <see cref="Arm"/>
|
|
/// is called, which keeps the Deployment Manager's boot load — a separate seam
|
|
/// (<c>_configLoader</c>, register P1-4) untouched by this gate — unblocked.
|
|
/// </para>
|
|
/// </summary>
|
|
private sealed class GatedConfigLoader(SiteStorageService storage)
|
|
{
|
|
private TaskCompletionSource? _release;
|
|
private readonly TaskCompletionSource _parked =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public void Arm() => _release = new TaskCompletionSource(
|
|
TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public void Release() => _release!.TrySetResult();
|
|
|
|
public Task WaitUntilParkedAsync() => _parked.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
public async Task<DeployedInstance?> LoadAsync(string instanceName)
|
|
{
|
|
var config = await storage.GetDeployedConfigAsync(instanceName);
|
|
var gate = _release;
|
|
if (gate != null)
|
|
{
|
|
_parked.TrySetResult();
|
|
await gate.Task;
|
|
}
|
|
return config;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior and
|
|
/// records every call thread-safely (the fetch runs on a pool thread).
|
|
/// </summary>
|
|
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
|
|
{
|
|
private readonly Func<string, Task<string>> _behavior;
|
|
public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();
|
|
|
|
public FakeConfigFetcher(Func<string, Task<string>> behavior) => _behavior = behavior;
|
|
|
|
public async Task<string> FetchAsync(
|
|
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
|
{
|
|
Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
|
|
await Task.Yield();
|
|
return await _behavior(deploymentId);
|
|
}
|
|
}
|
|
}
|