fix(site-runtime): artifact-apply watchdog + queued-apply failure on stop, boot-skip site event, enable revert, batch Ask slack (review 2026-10-21 lane A L1-L4)

L1: an artifact apply holding the slot past 10 min (ctor-overridable) logs an
Error site event naming it without releasing the slot; PostStop answers
queued applies Failed. L2: a boot-batch instance whose actor cannot be created
emits an Error site event and leaves the deployed set. L3: a failed enable
reverts the committed is_enabled (generation-bumped both ends). L4: the routed
SetAttributes batch Ask is the 30 s per-write bound plus 10 s mailbox slack.
This commit is contained in:
Joseph Doherty
2026-10-22 09:06:28 -04:00
parent 099b2221d6
commit 6f6534ac7e
4 changed files with 158 additions and 12 deletions
@@ -48,7 +48,7 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
TestLocalDb.DeleteFiles(path);
}
private IActorRef CreateBootedDeploymentManager()
private IActorRef CreateBootedDeploymentManager(FakeSiteEventLogger? siteLog = null)
{
var health = new DeploymentManagerStartupProbe();
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
@@ -60,7 +60,7 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
NullLogger<DeploymentManagerActor>.Instance,
dclManager: null,
healthCollector: health,
serviceProvider: null,
serviceProvider: siteLog == null ? null : new SingleServiceProvider(siteLog),
loggerFactory: null,
configFetcher: null)));
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25));
@@ -196,4 +196,38 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
var result = ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
Assert.True(result.Success, result.ErrorMessage);
}
/// <summary>Lane A L3: a failed enable reverts the is_enabled flag it had already committed.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Enable_FailedCreate_RevertsIsEnabled()
{
await _storage.StoreDeployedConfigAsync("a b", ConfigJson("a b"), "d0", "r0", isEnabled: false);
var dm = CreateBootedDeploymentManager();
dm.Tell(new EnableInstanceCommand("cmd-1", "a b", DateTimeOffset.UtcNow));
Assert.False(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(10)).Success);
await AwaitAssertAsync(async () =>
Assert.False((await _storage.GetDeployedConfigMetadataAsync("a b"))!.IsEnabled),
TimeSpan.FromSeconds(10));
}
/// <summary>Lane A L2: a boot-batch skip is surfaced as an Error site event.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Boot_SkippedInvalidName_EmitsASiteEvent()
{
await _storage.StoreDeployedConfigAsync("a b", ConfigJson("a b"), "d0", "r0", isEnabled: true);
var siteLog = new FakeSiteEventLogger();
CreateBootedDeploymentManager(siteLog);
await AwaitAssertAsync(() =>
{
Assert.Contains(siteLog.OfType("deployment"), r =>
r.Severity == "Error" && r.InstanceId == "a b" &&
r.Message.Contains("could not be started at boot", StringComparison.Ordinal));
return Task.CompletedTask;
}, TimeSpan.FromSeconds(10));
}
}