fix(reconcile): expiry-aware pending staging — expired rows no longer block self-heal

2026-06-26 17:23:26 -04:00
parent 6538216b0c
commit fd22f5ce0a
5 changed files with 156 additions and 21 deletions
@@ -141,7 +141,7 @@ public class ReconcileServiceTests

        // The existing pending row for inst-C carries the FIRST node's deploymentId + token.
        var nowUtc = DateTimeOffset.UtcNow;
-        _deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<CancellationToken>())
+        _deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<DateTimeOffset?>(), Arg.Any<CancellationToken>())
            .Returns(new PendingDeployment(
                "dep-C-existing", 3, "rev3-existing", "{\"cfg\":\"inst-C\"}",
                "tok-C-existing", nowUtc, nowUtc.AddMinutes(5)));
@@ -182,7 +182,7 @@ public class ReconcileServiceTests
            .Returns(false);

        // The pending row was purged between the stage attempt and the read.
-        _deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<CancellationToken>())
+        _deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<DateTimeOffset?>(), Arg.Any<CancellationToken>())
            .Returns((PendingDeployment?)null);

        var response = await CreateService().ReconcileAsync(Request(/* empty local inventory */));
@@ -189,6 +189,111 @@ public class PendingDeploymentRepositoryTests : IDisposable
        Assert.Null(row);
    }

+    [Fact]
+    public async Task StagePendingIfAbsent_ExpiredRowDifferentDeploymentId_DoesNotBlock_RemovesExpiredAndStagesLiveRow()
+    {
+        // Live-found bug: pending rows are only TTL-purged (the periodic purge is a deferred TODO),
+        // so an EXPIRED-but-unpurged row for the instance must NOT make staging report "a deploy is
+        // in flight" — it would hand the node an expired token (HTTP 404) and leave it unhealed.
+        // The expired row is removed and the fresh, live row is staged.
+        var instanceId = await SeedInstanceAsync("InstExp1");
+        var now = DateTimeOffset.UtcNow;
+
+        // Seed an EXPIRED pending row (different deploymentId), bypassing the repo.
+        _context.Set<PendingDeployment>().Add(new PendingDeployment(
+            "dep-expired", instanceId, "rev-old", "{\"old\":true}", "tok-old",
+            now.AddMinutes(-20), now.AddMinutes(-10)));
+        await _context.SaveChangesAsync();
+
+        var staged = await _repository.StagePendingIfAbsentAsync(
+            instanceId, "dep-fresh", "rev-fresh", "{\"fresh\":true}", "tok-fresh",
+            now, now.AddMinutes(10));
+
+        Assert.True(staged);
+        Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep-expired"));
+
+        var row = await _repository.GetPendingDeploymentByIdAsync("dep-fresh");
+        Assert.NotNull(row);
+        Assert.Equal(instanceId, row!.InstanceId);
+        Assert.Equal("tok-fresh", row.Token);
+        Assert.True(row.ExpiresAtUtc > now); // the new row is live
+    }
+
+    [Fact]
+    public async Task StagePendingIfAbsent_ExpiredRowSameDeploymentId_NoUniqueCollision_StagesNew()
+    {
+        // A fresh reconcile re-stages the snapshot's own DeploymentId. If an EXPIRED row already
+        // carries that same DeploymentId, staging must remove it FIRST to avoid colliding on the
+        // PendingDeployment.DeploymentId UNIQUE index — re-staging "D1" over an expired "D1" works.
+        var instanceId = await SeedInstanceAsync("InstExp2");
+        var now = DateTimeOffset.UtcNow;
+
+        _context.Set<PendingDeployment>().Add(new PendingDeployment(
+            "D1", instanceId, "rev-old", "{\"old\":true}", "tok-old",
+            now.AddMinutes(-20), now.AddMinutes(-10)));
+        await _context.SaveChangesAsync();
+
+        var staged = await _repository.StagePendingIfAbsentAsync(
+            instanceId, "D1", "rev-new", "{\"new\":true}", "tok-new",
+            now, now.AddMinutes(10));
+
+        Assert.True(staged);
+        var row = await _repository.GetPendingDeploymentByIdAsync("D1");
+        Assert.NotNull(row);
+        Assert.Equal("tok-new", row!.Token);
+        Assert.Equal("{\"new\":true}", row.ConfigurationJson);
+        Assert.True(row.ExpiresAtUtc > now);
+    }
+
+    [Fact]
+    public async Task StagePendingIfAbsent_LiveRow_Blocks_LeavesUntouched()
+    {
+        // A still-LIVE pending row (future expiry) signals a genuine in-flight deploy or a concurrent
+        // reconcile — staging must return false and leave it untouched (no supersession).
+        var instanceId = await SeedInstanceAsync("InstLive");
+        var now = DateTimeOffset.UtcNow;
+
+        _context.Set<PendingDeployment>().Add(new PendingDeployment(
+            "dep-live", instanceId, "rev-live", "{\"live\":true}", "tok-live",
+            now.AddMinutes(-1), now.AddMinutes(9)));
+        await _context.SaveChangesAsync();
+
+        var staged = await _repository.StagePendingIfAbsentAsync(
+            instanceId, "dep-new", "rev-new", "{\"new\":true}", "tok-new",
+            now, now.AddMinutes(10));
+
+        Assert.False(staged);
+        var existing = await _repository.GetPendingDeploymentByIdAsync("dep-live");
+        Assert.NotNull(existing);
+        Assert.Equal("tok-live", existing!.Token);
+        Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep-new"));
+    }
+
+    [Fact]
+    public async Task GetPendingDeploymentByInstanceId_WithNowUtc_ReturnsLiveRow_NotExpired()
+    {
+        // The by-instance getter must never hand back an EXPIRED row (whose token the config-fetch
+        // endpoint would 404). With nowUtc supplied, an expired row is filtered out; a live row is returned.
+        var expiredInst = await SeedInstanceAsync("InstGetExpired");
+        var liveInst = await SeedInstanceAsync("InstGetLive");
+        var now = DateTimeOffset.UtcNow;
+
+        _context.Set<PendingDeployment>().Add(new PendingDeployment(
+            "dep-getexp", expiredInst, "rev", "{}", "tok-exp",
+            now.AddMinutes(-20), now.AddMinutes(-10)));
+        _context.Set<PendingDeployment>().Add(new PendingDeployment(
+            "dep-getlive", liveInst, "rev", "{}", "tok-live",
+            now.AddMinutes(-1), now.AddMinutes(9)));
+        await _context.SaveChangesAsync();
+
+        // Expired row filtered out when nowUtc is supplied.
+        Assert.Null(await _repository.GetPendingDeploymentByInstanceIdAsync(expiredInst, now));
+        // Live row still returned.
+        var live = await _repository.GetPendingDeploymentByInstanceIdAsync(liveInst, now);
+        Assert.NotNull(live);
+        Assert.Equal("dep-getlive", live!.DeploymentId);
+    }
+
    [Fact]
    public async Task AddPendingDeployment_MultiplePriorRowsSameInstance_AllSuperseded()
    {