3a217b9484
Rig finding on the upgrade restart: both nodes' rows were older than the stale threshold (the old build never refreshed them), so node-a's reporter pruned node-b's row while node-b wrote a fresh one, and the replicated delete won. The reporter now prunes nothing during its first refresh interval after start-up, and each mirror pass re-writes its own row if it no longer holds what the mirror last wrote.
180 lines
7.5 KiB
C#
180 lines
7.5 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
|
|
|
|
/// <summary>
|
|
/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
|
|
/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
|
|
/// </summary>
|
|
public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
|
|
{
|
|
private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
|
|
private SiteStorageService _storage = null!;
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await _storage.InitializeAsync();
|
|
}
|
|
|
|
public Task DisposeAsync() => Task.CompletedTask;
|
|
|
|
public void Dispose()
|
|
{
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
|
|
{
|
|
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
|
|
await _storage.SetSecretMirrorStatusAsync(
|
|
"node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
|
|
time: new FixedTime(at.AddMinutes(1)));
|
|
|
|
await reporter.ProbeAsync();
|
|
var report = collector.CollectReport("site-a");
|
|
|
|
Assert.NotNull(report.SecretMirrorStatuses);
|
|
Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
|
|
Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
|
|
}
|
|
|
|
[Fact]
|
|
public void BeforeAnyProbe_TheReportFieldIsNull()
|
|
{
|
|
var report = new SiteHealthCollector().CollectReport("site-a");
|
|
|
|
Assert.Null(report.SecretMirrorStatuses);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
|
|
{
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
|
|
{
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
|
await reporter.ProbeAsync();
|
|
|
|
await using (var connection = _storage.CreateConnection())
|
|
await using (var drop = connection.CreateCommand())
|
|
{
|
|
drop.CommandText = "DROP TABLE secret_mirror_status";
|
|
await drop.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
// Point-in-time: not drained on collect, and not cleared by a failed read.
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
}
|
|
|
|
// ── Stale rows (decommissioned / renamed nodes) ──
|
|
|
|
[Fact]
|
|
public async Task Probe_RowOlderThanStaleAfter_IsLeftOffTheReport_AndDeleted()
|
|
{
|
|
var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], now.AddMinutes(-5));
|
|
await _storage.SetSecretMirrorStatusAsync(
|
|
"node-old", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], now.AddHours(-2));
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
|
|
staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
Assert.Equal(["node-a"], collector.CollectReport("site-a").SecretMirrorStatuses!.Select(s => s.NodeName));
|
|
Assert.Equal(["node-a"], (await _storage.GetSecretMirrorStatusesAsync()).Select(s => s.NodeName));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_RowExactlyAtTheCutoff_IsKept()
|
|
{
|
|
var now = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-b", [], now.AddHours(-1));
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
|
|
staleAfter: TimeSpan.FromHours(1), time: new FixedTime(now));
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_DuringTheStartUpGrace_PrunesNothing_ThenPrunesAfterIt()
|
|
{
|
|
var time = new MutableTime(new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero));
|
|
await _storage.SetSecretMirrorStatusAsync("node-b", [], time.Now.AddHours(-2));
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance,
|
|
staleAfter: TimeSpan.FromHours(1), time: time, pruneGrace: TimeSpan.FromMinutes(10));
|
|
|
|
time.Now = time.Now.AddMinutes(9);
|
|
await reporter.ProbeAsync();
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
|
|
|
time.Now = time.Now.AddMinutes(1);
|
|
await reporter.ProbeAsync();
|
|
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ConditionalDelete_DoesNotRemoveARowTheNodeRewroteAfterTheRead()
|
|
{
|
|
// The reporter read a stale checked_at, but the node came back and re-wrote its row
|
|
// before the delete ran: the fresh row must survive.
|
|
var stale = new DateTimeOffset(2026, 9, 29, 9, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-b", [], stale);
|
|
await _storage.SetSecretMirrorStatusAsync("node-b", [], stale.AddHours(3));
|
|
|
|
Assert.False(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale));
|
|
Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
|
Assert.True(await _storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", stale.AddHours(3)));
|
|
Assert.Empty(await _storage.GetSecretMirrorStatusesAsync());
|
|
}
|
|
|
|
private sealed class MutableTime(DateTimeOffset now) : TimeProvider
|
|
{
|
|
public DateTimeOffset Now { get; set; } = now;
|
|
public override DateTimeOffset GetUtcNow() => Now;
|
|
}
|
|
|
|
private sealed class FixedTime(DateTimeOffset now) : TimeProvider
|
|
{
|
|
public override DateTimeOffset GetUtcNow() => now;
|
|
}
|
|
}
|