Joseph Doherty
0289d51137
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
96 lines
3.6 KiB
C#
96 lines
3.6 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
|
|
|
|
/// <summary>
|
|
/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
|
|
/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
|
|
/// </summary>
|
|
public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
|
|
{
|
|
private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
|
|
private SiteStorageService _storage = null!;
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await _storage.InitializeAsync();
|
|
}
|
|
|
|
public Task DisposeAsync() => Task.CompletedTask;
|
|
|
|
public void Dispose()
|
|
{
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
|
|
{
|
|
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
|
|
await _storage.SetSecretMirrorStatusAsync(
|
|
"node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
|
|
|
await reporter.ProbeAsync();
|
|
var report = collector.CollectReport("site-a");
|
|
|
|
Assert.NotNull(report.SecretMirrorStatuses);
|
|
Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
|
|
Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
|
|
}
|
|
|
|
[Fact]
|
|
public void BeforeAnyProbe_TheReportFieldIsNull()
|
|
{
|
|
var report = new SiteHealthCollector().CollectReport("site-a");
|
|
|
|
Assert.Null(report.SecretMirrorStatuses);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
|
|
{
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
|
|
{
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
|
|
var collector = new SiteHealthCollector();
|
|
var reporter = new SecretMirrorStatusReporter(
|
|
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
|
await reporter.ProbeAsync();
|
|
|
|
await using (var connection = _storage.CreateConnection())
|
|
await using (var drop = connection.CreateCommand())
|
|
{
|
|
drop.CommandText = "DROP TABLE secret_mirror_status";
|
|
await drop.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await reporter.ProbeAsync();
|
|
|
|
// Point-in-time: not drained on collect, and not cleared by a failed read.
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
|
}
|
|
}
|