fix(site-secrets): prune secret_mirror_status rows of decommissioned or renamed nodes

Each node now re-writes its mirror-status row every SecretMirrorStatusRefreshInterval
(10 min) as a heartbeat; SecretMirrorStatusReporter leaves off the health report and
deletes any row not refreshed for SecretMirrorStatusStaleAfter (1 h, validated >= 2x the
refresh). The delete is conditional on the checked_at it read, so a node that came back
keeps its fresh row; a returning node re-writes its row on its first pass.
This commit is contained in:
Joseph Doherty
2026-09-29 10:56:53 -04:00
parent 2650f79472
commit 55e1741075
14 changed files with 244 additions and 13 deletions
@@ -695,6 +695,26 @@ public class DeployedSecretsMirrorTests : IDisposable
Assert.Contains("kek-central", failure.Message);
}
[Fact]
public async Task Status_UnchangedFailureSet_IsRefreshedOnceTheRefreshIntervalHasPassed()
{
// The row's checked_at is a heartbeat: SecretMirrorStatusReporter prunes rows that stop
// being refreshed, so a live node must re-write an unchanged row every refresh interval.
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
await mirror.ReconcileAsync(CancellationToken.None);
var first = Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc;
time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval - TimeSpan.FromSeconds(1);
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(first, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc);
time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval;
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(time.Now, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc);
}
[Fact]
public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
{