fix(site-secrets): prune secret_mirror_status rows of decommissioned or renamed nodes

Each node now re-writes its mirror-status row every SecretMirrorStatusRefreshInterval (10 min) as a heartbeat; SecretMirrorStatusReporter leaves off the health report and deletes any row not refreshed for SecretMirrorStatusStaleAfter (1 h, validated >= 2x the refresh). The delete is conditional on the checked_at it read, so a node that came back keeps its fresh row; a returning node re-writes its row on its first pass.
2026-09-29 10:56:53 -04:00
parent 2650f79472
commit 55e1741075
14 changed files with 244 additions and 13 deletions
@@ -695,6 +695,26 @@ public class DeployedSecretsMirrorTests : IDisposable
        Assert.Contains("kek-central", failure.Message);
    }

+    [Fact]
+    public async Task Status_UnchangedFailureSet_IsRefreshedOnceTheRefreshIntervalHasPassed()
+    {
+        // The row's checked_at is a heartbeat: SecretMirrorStatusReporter prunes rows that stop
+        // being refreshed, so a live node must re-write an unchanged row every refresh interval.
+        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
+        var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+        var first = Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc;
+
+        time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval - TimeSpan.FromSeconds(1);
+        await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Equal(first, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc);
+
+        time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval;
+        await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Equal(time.Now, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc);
+    }
+
    [Fact]
    public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
    {
@@ -40,7 +40,8 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
            "node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
        var collector = new SiteHealthCollector();
        var reporter = new SecretMirrorStatusReporter(
-            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
+            time: new FixedTime(at.AddMinutes(1)));

        await reporter.ProbeAsync();
        var report = collector.CollectReport("site-a");
@@ -92,4 +93,60 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
    }
+
+    // ── Stale rows (decommissioned / renamed nodes) ──
+
+    [Fact]
+    public async Task Probe_RowOlderThanStaleAfter_IsLeftOffTheReport_AndDeleted()
+    {
+        var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
+        await _storage.SetSecretMirrorStatusAsync("node-a", [], now.AddMinutes(-5));
+        await _storage.SetSecretMirrorStatusAsync(
+            "node-old", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], now.AddHours(-2));
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
+            staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
+
+        await reporter.ProbeAsync();
+
+        Assert.Equal(["node-a"], collector.CollectReport("site-a").SecretMirrorStatuses!.Select(s => s.NodeName));
+        Assert.Equal(["node-a"], (await _storage.GetSecretMirrorStatusesAsync()).Select(s => s.NodeName));
+    }
+
+    [Fact]
+    public async Task Probe_RowExactlyAtTheCutoff_IsKept()
+    {
+        var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
+        await _storage.SetSecretMirrorStatusAsync("node-b", [], now.AddHours(-1));
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
+            staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
+
+        await reporter.ProbeAsync();
+
+        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
+        Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
+    }
+
+    [Fact]
+    public async Task ConditionalDelete_DoesNotRemoveARowTheNodeRewroteAfterTheRead()
+    {
+        // The reporter read a stale checked_at, but the node came back and re-wrote its row
+        // before the delete ran: the fresh row must survive.
+        var stale = new DateTimeOffset(2026, 9, 29, 9, 0, 0, TimeSpan.Zero);
+        await _storage.SetSecretMirrorStatusAsync("node-b", [], stale);
+        await _storage.SetSecretMirrorStatusAsync("node-b", [], stale.AddHours(3));
+
+        Assert.False(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale));
+        Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
+        Assert.True(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale.AddHours(3)));
+        Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
+    }
+
+    private sealed class FixedTime(DateTimeOffset now) : TimeProvider
+    {
+        public override DateTimeOffset GetUtcNow() => now;
+    }
 }
@@ -243,4 +243,26 @@ public class SiteRuntimeOptionsValidatorTests
        Assert.True(result.Failed);
        Assert.Contains("ScriptFileAccess:MaxFileBytes", result.FailureMessage);
    }
+
+    [Fact]
+    public void ZeroSecretMirrorStatusRefreshInterval_IsRejected()
+    {
+        var result = Validate(new SiteRuntimeOptions { SecretMirrorStatusRefreshInterval = TimeSpan.Zero });
+
+        Assert.True(result.Failed);
+        Assert.Contains("SecretMirrorStatusRefreshInterval", result.FailureMessage);
+    }
+
+    [Fact]
+    public void SecretMirrorStatusStaleAfter_UnderTwoRefreshes_IsRejected()
+    {
+        var result = Validate(new SiteRuntimeOptions
+        {
+            SecretMirrorStatusRefreshInterval = TimeSpan.FromMinutes(10),
+            SecretMirrorStatusStaleAfter = TimeSpan.FromMinutes(15),
+        });
+
+        Assert.True(result.Failed);
+        Assert.Contains("SecretMirrorStatusStaleAfter", result.FailureMessage);
+    }
 }