Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/SecretMirrorStatusReporterTests.cs
T
Joseph Doherty 3a217b9484 fix(site-secrets): keep a live node's mirror-status row from losing a race with the peer's prune
Rig finding on the upgrade restart: both nodes' rows were older than the stale threshold
(the old build never refreshed them), so node-a's reporter pruned node-b's row while node-b
wrote a fresh one, and the replicated delete won. The reporter now prunes nothing during its
first refresh interval after start-up, and each mirror pass re-writes its own row if it no
longer holds what the mirror last wrote.
2026-09-29 11:06:26 -04:00

180 lines
7.5 KiB
C#

using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
using ZB.MOM.WW.ScadaBridge.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
/// <summary>
/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
/// </summary>
public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
{
private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
private SiteStorageService _storage = null!;
public async Task InitializeAsync()
{
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
await _storage.InitializeAsync();
}
public Task DisposeAsync() => Task.CompletedTask;
public void Dispose()
{
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
[Fact]
public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
{
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
await _storage.SetSecretMirrorStatusAsync(
"node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
time: new FixedTime(at.AddMinutes(1)));
await reporter.ProbeAsync();
var report = collector.CollectReport("site-a");
Assert.NotNull(report.SecretMirrorStatuses);
Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
}
[Fact]
public void BeforeAnyProbe_TheReportFieldIsNull()
{
var report = new SiteHealthCollector().CollectReport("site-a");
Assert.Null(report.SecretMirrorStatuses);
}
[Fact]
public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
{
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
await reporter.ProbeAsync();
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
}
[Fact]
public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
{
await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
await reporter.ProbeAsync();
await using (var connection = _storage.CreateConnection())
await using (var drop = connection.CreateCommand())
{
drop.CommandText = "DROP TABLE secret_mirror_status";
await drop.ExecuteNonQueryAsync();
}
await reporter.ProbeAsync();
// Point-in-time: not drained on collect, and not cleared by a failed read.
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
}
// ── Stale rows (decommissioned / renamed nodes) ──
[Fact]
public async Task Probe_RowOlderThanStaleAfter_IsLeftOffTheReport_AndDeleted()
{
var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
await _storage.SetSecretMirrorStatusAsync("node-a", [], now.AddMinutes(-5));
await _storage.SetSecretMirrorStatusAsync(
"node-old", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], now.AddHours(-2));
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
await reporter.ProbeAsync();
Assert.Equal(["node-a"], collector.CollectReport("site-a").SecretMirrorStatuses!.Select(s => s.NodeName));
Assert.Equal(["node-a"], (await _storage.GetSecretMirrorStatusesAsync()).Select(s => s.NodeName));
}
[Fact]
public async Task Probe_RowExactlyAtTheCutoff_IsKept()
{
var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
await _storage.SetSecretMirrorStatusAsync("node-b", [], now.AddHours(-1));
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
await reporter.ProbeAsync();
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task Probe_DuringTheStartUpGrace_PrunesNothing_ThenPrunesAfterIt()
{
var time = new MutableTime(new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero));
await _storage.SetSecretMirrorStatusAsync("node-b", [], time.Now.AddHours(-2));
var collector = new SiteHealthCollector();
var reporter = new SecretMirrorStatusReporter(
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
staleAfter: TimeSpan.FromHours(1), time: time, pruneGrace: TimeSpan.FromMinutes(10));
time.Now = time.Now.AddMinutes(9);
await reporter.ProbeAsync();
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
time.Now = time.Now.AddMinutes(1);
await reporter.ProbeAsync();
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
[Fact]
public async Task ConditionalDelete_DoesNotRemoveARowTheNodeRewroteAfterTheRead()
{
// The reporter read a stale checked_at, but the node came back and re-wrote its row
// before the delete ran: the fresh row must survive.
var stale = new DateTimeOffset(2026, 9, 29, 9, 0, 0, TimeSpan.Zero);
await _storage.SetSecretMirrorStatusAsync("node-b", [], stale);
await _storage.SetSecretMirrorStatusAsync("node-b", [], stale.AddHours(3));
Assert.False(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale));
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
Assert.True(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale.AddHours(3)));
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
}
private sealed class MutableTime(DateTimeOffset now) : TimeProvider
{
public DateTimeOffset Now { get; set; } = now;
public override DateTimeOffset GetUtcNow() => Now;
}
private sealed class FixedTime(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;
}
}