fix(site-secrets): report every site node's secret mirror failures on the health report (M1)

A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch that stops a deployed secret decrypting there) was only logged on that node: site health reports are sent by the active node only, so central never saw it. - New 12th LocalDb-replicated table secret_mirror_status (node_name PK, failures_json of {SecretName, Message}, checked_at; no FKs). Each node's mirror writes only its own row, and only when its failure set changes plus once per start-up (every write replicates via CDC). Best-effort: a failed write is a Warning and is retried next pass. Node key = INodeIdentityProvider NodeName, falling back to the machine name. - SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable SiteHealthReport.SecretMirrorStatuses (null = no data). - Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so null != empty); vendored CentralControl.cs regenerated; mapper both ways. - Central UI Health card: 'Secret mirror failing' badge + compact per-node node / secret / message list. - Tests: mirror status write-on-change/first-pass/failure isolation, storage round-trip, reporter/collector, DTO mapper round-trip incl. null/empty, bUnit, replicated-table set (twelve) + FK guard wording.
2026-09-28 21:33:10 -04:00
parent 23a139a46c
commit 0289d51137
24 changed files with 1718 additions and 100 deletions
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
@@ -655,6 +656,132 @@ public class DeployedSecretsMirrorTests : IDisposable
        }
    }

+    // ── Per-node status row for central (review finding M1) ──
+
+    [Fact]
+    public async Task Status_FirstPassAfterStartup_WritesThisNodesRow_EvenWhenHealthy()
+    {
+        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
+        var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
+        Assert.Equal("node-b", status.NodeName);
+        Assert.Empty(status.Failures);
+        Assert.Equal(time.Now, status.CheckedAtUtc);
+    }
+
+    [Fact]
+    public async Task Status_UnchangedFailureSet_IsNotRewrittenOnLaterPasses()
+    {
+        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
+        var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
+        resolver.ThrowFor("a");
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+        var firstWrite = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
+
+        // Same failure on every later pass: the row is replicated, so it must not be rewritten.
+        time.Now = time.Now.AddMinutes(5);
+        await mirror.ReconcileAsync(CancellationToken.None);
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        var after = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
+        Assert.Equal(firstWrite.CheckedAtUtc, after.CheckedAtUtc);
+        var failure = Assert.Single(after.Failures);
+        Assert.Equal("a", failure.SecretName);
+        Assert.Contains("kek-central", failure.Message);
+    }
+
+    [Fact]
+    public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
+    {
+        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
+        var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-a"));
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Empty(Assert.Single(await storage.GetSecretMirrorStatusesAsync()).Failures);
+
+        // A decrypt failure appears -> row rewritten with the failure.
+        resolver.ThrowFor("a");
+        time.Now = time.Now.AddMinutes(1);
+        await mirror.ReconcileAsync(CancellationToken.None);
+        var failing = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
+        Assert.Equal("a", Assert.Single(failing.Failures).SecretName);
+        Assert.Equal(time.Now, failing.CheckedAtUtc);
+
+        // Healed -> row rewritten empty.
+        resolver.ClearFailures();
+        time.Now = time.Now.AddMinutes(1);
+        await mirror.ReconcileAsync(CancellationToken.None);
+        var healed = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
+        Assert.Empty(healed.Failures);
+        Assert.Equal(time.Now, healed.CheckedAtUtc);
+    }
+
+    [Fact]
+    public async Task Status_RowCarriesNamesAndMessagesOnly_NeverCiphertext()
+    {
+        var (mirror, storage, _, resolver, _) = await CreateAsync(nodeIdentity: new FixedNodeIdentity("node-a"));
+        resolver.ThrowFor("a");
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        await using var connection = storage.CreateConnection();
+        await using var command = connection.CreateCommand();
+        command.CommandText = "SELECT failures_json FROM secret_mirror_status WHERE node_name = 'node-a'";
+        var json = (string)(await command.ExecuteScalarAsync())!;
+        Assert.Contains("\"SecretName\":\"a\"", json);
+        Assert.DoesNotContain("Ciphertext", json);
+        Assert.DoesNotContain(Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5 }), json);
+    }
+
+    [Fact]
+    public async Task Status_WriteFailure_DoesNotBreakThePass_AndIsRetriedNextPass()
+    {
+        var logger = new CapturingLogger<DeployedSecretsMirror>();
+        var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, nodeIdentity: new FixedNodeIdentity("node-a"));
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+
+        await using (var connection = storage.CreateConnection())
+        await using (var drop = connection.CreateCommand())
+        {
+            drop.CommandText = "DROP TABLE secret_mirror_status";
+            await drop.ExecuteNonQueryAsync();
+        }
+
+        var r = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Equal(["a"], r.Applied);
+        Assert.True(store.Contains("a"));
+        Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("mirror status"));
+
+        // Table back (e.g. transient fault cleared): the unchanged failure set is still
+        // written, because the earlier write never landed.
+        await storage.InitializeAsync();
+        await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Equal("node-a", Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
+    }
+
+    [Fact]
+    public async Task Status_WithoutNodeIdentity_FallsBackToMachineName()
+    {
+        var (mirror, storage, _, _, _) = await CreateAsync();
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Equal(Environment.MachineName, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
+    }
+
+    private sealed class FixedNodeIdentity(string? name) : INodeIdentityProvider
+    {
+        public string? NodeName { get; } = name;
+    }
+
    // ── Harness ──

    private async Task<(
@@ -665,7 +792,8 @@ public class DeployedSecretsMirrorTests : IDisposable
        FakeSecretCacheInvalidator Invalidator)> CreateAsync(
        CapturingLogger<DeployedSecretsMirror>? logger = null,
        TimeSpan? interval = null,
-        TimeProvider? time = null)
+        TimeProvider? time = null,
+        INodeIdentityProvider? nodeIdentity = null)
    {
        var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        await storage.InitializeAsync();
@@ -685,7 +813,8 @@ public class DeployedSecretsMirrorTests : IDisposable
            invalidator,
            options,
            (ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
-            time ?? TimeProvider.System);
+            time ?? TimeProvider.System,
+            nodeIdentity);

        return (mirror, storage, store, resolver, invalidator);
    }
@@ -0,0 +1,95 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
+using ZB.MOM.WW.ScadaBridge.TestSupport;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
+
+/// <summary>
+/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
+/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
+/// </summary>
+public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
+{
+    private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
+    private SiteStorageService _storage = null!;
+
+    public async Task InitializeAsync()
+    {
+        _storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
+        await _storage.InitializeAsync();
+    }
+
+    public Task DisposeAsync() => Task.CompletedTask;
+
+    public void Dispose()
+    {
+        var path = _localDb.Path;
+        _localDb.Dispose();
+        TestLocalDb.DeleteFiles(path);
+    }
+
+    [Fact]
+    public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
+    {
+        var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
+        await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
+        await _storage.SetSecretMirrorStatusAsync(
+            "node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+
+        await reporter.ProbeAsync();
+        var report = collector.CollectReport("site-a");
+
+        Assert.NotNull(report.SecretMirrorStatuses);
+        Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
+        Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
+    }
+
+    [Fact]
+    public void BeforeAnyProbe_TheReportFieldIsNull()
+    {
+        var report = new SiteHealthCollector().CollectReport("site-a");
+
+        Assert.Null(report.SecretMirrorStatuses);
+    }
+
+    [Fact]
+    public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
+    {
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+
+        await reporter.ProbeAsync();
+
+        Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
+    }
+
+    [Fact]
+    public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
+    {
+        await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
+        var collector = new SiteHealthCollector();
+        var reporter = new SecretMirrorStatusReporter(
+            collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
+        await reporter.ProbeAsync();
+
+        await using (var connection = _storage.CreateConnection())
+        await using (var drop = connection.CreateCommand())
+        {
+            drop.CommandText = "DROP TABLE secret_mirror_status";
+            await drop.ExecuteNonQueryAsync();
+        }
+
+        await reporter.ProbeAsync();
+
+        // Point-in-time: not drained on collect, and not cleared by a failed read.
+        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
+        Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
+    }
+}