fix(site-runtime): artifact-apply watchdog + queued-apply failure on stop, boot-skip site event, enable revert, batch Ask slack (review 2026-10-21 lane A L1-L4)

L1: an artifact apply holding the slot past 10 min (ctor-overridable) logs an Error site event naming it without releasing the slot; PostStop answers queued applies Failed. L2: a boot-batch instance whose actor cannot be created emits an Error site event and leaves the deployed set. L3: a failed enable reverts the committed is_enabled (generation-bumped both ends). L4: the routed SetAttributes batch Ask is the 30 s per-write bound plus 10 s mailbox slack.
2026-10-22 09:06:28 -04:00
parent 099b2221d6
commit 6f6534ac7e
4 changed files with 158 additions and 12 deletions
@@ -48,7 +48,7 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
        TestLocalDb.DeleteFiles(path);
    }

-    private IActorRef CreateBootedDeploymentManager()
+    private IActorRef CreateBootedDeploymentManager(FakeSiteEventLogger? siteLog = null)
    {
        var health = new DeploymentManagerStartupProbe();
        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
@@ -60,7 +60,7 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager: null,
            healthCollector: health,
-            serviceProvider: null,
+            serviceProvider: siteLog == null ? null : new SingleServiceProvider(siteLog),
            loggerFactory: null,
            configFetcher: null)));
        AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25));
@@ -196,4 +196,38 @@ public class DeploymentManagerActorNameTests : TestKit, IDisposable
        var result = ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
        Assert.True(result.Success, result.ErrorMessage);
    }
+
+    /// <summary>Lane A L3: a failed enable reverts the is_enabled flag it had already committed.</summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task Enable_FailedCreate_RevertsIsEnabled()
+    {
+        await _storage.StoreDeployedConfigAsync("a b", ConfigJson("a b"), "d0", "r0", isEnabled: false);
+        var dm = CreateBootedDeploymentManager();
+
+        dm.Tell(new EnableInstanceCommand("cmd-1", "a b", DateTimeOffset.UtcNow));
+        Assert.False(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(10)).Success);
+
+        await AwaitAssertAsync(async () =>
+            Assert.False((await _storage.GetDeployedConfigMetadataAsync("a b"))!.IsEnabled),
+            TimeSpan.FromSeconds(10));
+    }
+
+    /// <summary>Lane A L2: a boot-batch skip is surfaced as an Error site event.</summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task Boot_SkippedInvalidName_EmitsASiteEvent()
+    {
+        await _storage.StoreDeployedConfigAsync("a b", ConfigJson("a b"), "d0", "r0", isEnabled: true);
+        var siteLog = new FakeSiteEventLogger();
+        CreateBootedDeploymentManager(siteLog);
+
+        await AwaitAssertAsync(() =>
+        {
+            Assert.Contains(siteLog.OfType("deployment"), r =>
+                r.Severity == "Error" && r.InstanceId == "a b" &&
+                r.Message.Contains("could not be started at boot", StringComparison.Ordinal));
+            return Task.CompletedTask;
+        }, TimeSpan.FromSeconds(10));
+    }
 }
@@ -98,4 +98,34 @@ public class DeploymentManagerArtifactSerializationTests : TestKit, IDisposable
        Assert.True(_sharedScriptLibrary.Contains("Z"));
        Assert.DoesNotContain(_sharedScriptLibrary.GetRegisteredScriptNames(), n => n.StartsWith("Slow"));
    }
+
+    /// <summary>
+    /// Lane A L1: an apply that holds the slot past the watchdog is named in an Error site event,
+    /// and queued deploys are answered Failed when the manager stops.
+    /// </summary>
+    [Fact]
+    public void StuckApply_IsReported_AndQueuedAppliesAreFailedOnStop()
+    {
+        var siteLog = new FakeSiteEventLogger();
+        var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
+            _storage, _compilationService, _sharedScriptLibrary, null,
+            new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 0 },
+            NullLogger<DeploymentManagerActor>.Instance,
+            null, null, new SingleServiceProvider(siteLog), null, null, null, null, null, null, null, null,
+            null, null, TimeSpan.FromMilliseconds(200))));
+
+        var slowSet = Enumerable.Range(0, 60).Select(i => $"Stuck{i}").ToList();
+        var queuedSender = CreateTestProbe();
+        dm.Tell(SharedScripts("dep-slow", slowSet));
+        dm.Tell(SharedScripts("dep-queued", ["Q"]), queuedSender.Ref);
+
+        AwaitAssert(() => Assert.Contains(siteLog.OfType("deployment"), r =>
+                r.Severity == "Error" && r.Message.Contains("dep-slow", StringComparison.Ordinal)),
+            TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(50));
+
+        Sys.Stop(dm);
+        var failed = queuedSender.ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
+        Assert.Equal("dep-queued", failed.DeploymentId);
+        Assert.False(failed.Success);
+    }
 }