dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(site-secrets): keep a live node's mirror-status row from losing a race with the peer's prune

Browse Source 
Rig finding on the upgrade restart: both nodes' rows were older than the stale threshold
(the old build never refreshed them), so node-a's reporter pruned node-b's row while node-b
wrote a fresh one, and the replicated delete won. The reporter now prunes nothing during its
first refresh interval after start-up, and each mirror pass re-writes its own row if it no
longer holds what the mirror last wrote.
This commit is contained in:
Joseph Doherty
2026-09-29 11:06:26 -04:00
parent 55e1741075
commit 3a217b9484
7 changed files with 100 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs
+20
View File
@@ -715,6 +715,26 @@ public class DeployedSecretsMirrorTests : IDisposable
Assert.Equal(time.Now, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc);
}
[Fact]
public async Task Status_RowDeletedByAPeerPrune_IsRewrittenOnTheNextPass()
{
// Rig finding: when both nodes restart together, the peer's reporter can prune this
// node's old row concurrently with this node's fresh write, and the replicated delete
// wins. The mirror notices its row is gone and writes it again.
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
await mirror.ReconcileAsync(CancellationToken.None);
var written = Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc;
Assert.True(await storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", written));
time.Now = time.Now.AddSeconds(15);
await mirror.ReconcileAsync(CancellationToken.None);
var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
Assert.Equal("node-b", status.NodeName);
Assert.Equal(time.Now, status.CheckedAtUtc);
}
[Fact]
public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
{
tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/SecretMirrorStatusReporterTests.cs
+27
View File
@@ -130,6 +130,27 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task Probe_DuringTheStartUpGrace_PrunesNothing_ThenPrunesAfterIt()
{
var time = new MutableTime(new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero));
await _storage.SetSecretMirrorStatusAsync("node-b", [], time.Now.AddHours(-2));
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
staleAfter: TimeSpan.FromHours(1), time: time, pruneGrace: TimeSpan.FromMinutes(10));
time.Now = time.Now.AddMinutes(9);
await reporter.ProbeAsync();
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
time.Now = time.Now.AddMinutes(1);
await reporter.ProbeAsync();
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task ConditionalDelete_DoesNotRemoveARowTheNodeRewroteAfterTheRead()
{
@@ -145,6 +166,12 @@ public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
private sealed class MutableTime(DateTimeOffset now) : TimeProvider
{
public DateTimeOffset Now { get; set; } = now;
public override DateTimeOffset GetUtcNow() => Now;
}
private sealed class FixedTime(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;