fix(site-secrets): keep a live node's mirror-status row from losing a race with the peer's prune

Rig finding on the upgrade restart: both nodes' rows were older than the stale threshold
(the old build never refreshed them), so node-a's reporter pruned node-b's row while node-b
wrote a fresh one, and the replicated delete won. The reporter now prunes nothing during its
first refresh interval after start-up, and each mirror pass re-writes its own row if it no
longer holds what the mirror last wrote.
This commit is contained in:
Joseph Doherty
2026-09-29 11:06:26 -04:00
parent 55e1741075
commit 3a217b9484
7 changed files with 100 additions and 13 deletions
@@ -130,6 +130,27 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task Probe_DuringTheStartUpGrace_PrunesNothing_ThenPrunesAfterIt()
{
var time = new MutableTime(new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero));
await _storage.SetSecretMirrorStatusAsync("node-b", [], time.Now.AddHours(-2));
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
staleAfter: TimeSpan.FromHours(1), time: time, pruneGrace: TimeSpan.FromMinutes(10));
time.Now = time.Now.AddMinutes(9);
await reporter.ProbeAsync();
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
time.Now = time.Now.AddMinutes(1);
await reporter.ProbeAsync();
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task ConditionalDelete_DoesNotRemoveARowTheNodeRewroteAfterTheRead()
{
@@ -145,6 +166,12 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
private sealed class MutableTime(DateTimeOffset now) : TimeProvider
{
public DateTimeOffset Now { get; set; } = now;
public override DateTimeOffset GetUtcNow() => Now;
}
private sealed class FixedTime(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;