fix(site-secrets): report every site node's secret mirror failures on the health report (M1)

A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch that stops a deployed secret decrypting there) was only logged on that node: site health reports are sent by the active node only, so central never saw it. - New 12th LocalDb-replicated table secret_mirror_status (node_name PK, failures_json of {SecretName, Message}, checked_at; no FKs). Each node's mirror writes only its own row, and only when its failure set changes plus once per start-up (every write replicates via CDC). Best-effort: a failed write is a Warning and is retried next pass. Node key = INodeIdentityProvider NodeName, falling back to the machine name. - SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable SiteHealthReport.SecretMirrorStatuses (null = no data). - Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so null != empty); vendored CentralControl.cs regenerated; mapper both ways. - Central UI Health card: 'Secret mirror failing' badge + compact per-node node / secret / message list. - Tests: mirror status write-on-change/first-pass/failure isolation, storage round-trip, reporter/collector, DTO mapper round-trip incl. null/empty, bUnit, replicated-table set (twelve) + FK guard wording.
2026-09-28 21:33:10 -04:00
parent 23a139a46c
commit 0289d51137
24 changed files with 1718 additions and 100 deletions
@@ -0,0 +1,95 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
+using ZB.MOM.WW.ScadaBridge.TestSupport;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
+
+/// <summary>
+/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
+/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
+/// </summary>
+public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
+{
+    private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
+    private SiteStorageService _storage = null!;
+
+    public async Task InitializeAsync()
+    {
+        _storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
+        await _storage.InitializeAsync();
+    }
+
+    public Task DisposeAsync() => Task.CompletedTask;
+
+    public void Dispose()
+    {
+        var path = _localDb.Path;
+        _localDb.Dispose();
+        TestLocalDb.DeleteFiles(path);
+    }
+
+    [Fact]
+    public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
+    {
+        var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
+        await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
+        await _storage.SetSecretMirrorStatusAsync(
+            "node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+
+        await reporter.ProbeAsync();
+        var report = collector.CollectReport("site-a");
+
+        Assert.NotNull(report.SecretMirrorStatuses);
+        Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
+        Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
+    }
+
+    [Fact]
+    public void BeforeAnyProbe_TheReportFieldIsNull()
+    {
+        var report = new SiteHealthCollector().CollectReport("site-a");
+
+        Assert.Null(report.SecretMirrorStatuses);
+    }
+
+    [Fact]
+    public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
+    {
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+
+        await reporter.ProbeAsync();
+
+        Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
+    }
+
+    [Fact]
+    public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
+    {
+        await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+        await reporter.ProbeAsync();
+
+        await using (var connection = _storage.CreateConnection())
+        await using (var drop = connection.CreateCommand())
+        {
+            drop.CommandText = "DROP TABLE secret_mirror_status";
+            await drop.ExecuteNonQueryAsync();
+        }
+
+        await reporter.ProbeAsync();
+
+        // Point-in-time: not drained on collect, and not cleared by a failed read.
+        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
+        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
+    }
+}