dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(site-secrets): report every site node's secret mirror failures on the health report (M1)

Browse Source 
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.

- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
  failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
  mirror writes only its own row, and only when its failure set changes plus
  once per start-up (every write replicates via CDC). Best-effort: a failed
  write is a Warning and is retried next pass. Node key = INodeIdentityProvider
  NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
  into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
  SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
  null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
  node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
  round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
  bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
Joseph Doherty
2026-09-28 21:33:10 -04:00
parent 23a139a46c
commit 0289d51137
24 changed files with 1718 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs
+131 -2
View File
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
@@ -655,6 +656,132 @@ public class DeployedSecretsMirrorTests : IDisposable
}
}
// ── Per-node status row for central (review finding M1) ──
[Fact]
public async Task Status_FirstPassAfterStartup_WritesThisNodesRow_EvenWhenHealthy()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
await mirror.ReconcileAsync(CancellationToken.None);
var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
Assert.Equal("node-b", status.NodeName);
Assert.Empty(status.Failures);
Assert.Equal(time.Now, status.CheckedAtUtc);
}
[Fact]
public async Task Status_UnchangedFailureSet_IsNotRewrittenOnLaterPasses()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
resolver.ThrowFor("a");
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
await mirror.ReconcileAsync(CancellationToken.None);
var firstWrite = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
// Same failure on every later pass: the row is replicated, so it must not be rewritten.
time.Now = time.Now.AddMinutes(5);
await mirror.ReconcileAsync(CancellationToken.None);
await mirror.ReconcileAsync(CancellationToken.None);
var after = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
Assert.Equal(firstWrite.CheckedAtUtc, after.CheckedAtUtc);
var failure = Assert.Single(after.Failures);
Assert.Equal("a", failure.SecretName);
Assert.Contains("kek-central", failure.Message);
}
[Fact]
public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-a"));
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Empty(Assert.Single(await storage.GetSecretMirrorStatusesAsync()).Failures);
// A decrypt failure appears -> row rewritten with the failure.
resolver.ThrowFor("a");
time.Now = time.Now.AddMinutes(1);
await mirror.ReconcileAsync(CancellationToken.None);
var failing = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
Assert.Equal("a", Assert.Single(failing.Failures).SecretName);
Assert.Equal(time.Now, failing.CheckedAtUtc);
// Healed -> row rewritten empty.
resolver.ClearFailures();
time.Now = time.Now.AddMinutes(1);
await mirror.ReconcileAsync(CancellationToken.None);
var healed = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
Assert.Empty(healed.Failures);
Assert.Equal(time.Now, healed.CheckedAtUtc);
}
[Fact]
public async Task Status_RowCarriesNamesAndMessagesOnly_NeverCiphertext()
{
var (mirror, storage, _, resolver, _) = await CreateAsync(nodeIdentity: new FixedNodeIdentity("node-a"));
resolver.ThrowFor("a");
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
await mirror.ReconcileAsync(CancellationToken.None);
await using var connection = storage.CreateConnection();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT failures_json FROM secret_mirror_status WHERE node_name = 'node-a'";
var json = (string)(await command.ExecuteScalarAsync())!;
Assert.Contains("\"SecretName\":\"a\"", json);
Assert.DoesNotContain("Ciphertext", json);
Assert.DoesNotContain(Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5 }), json);
}
[Fact]
public async Task Status_WriteFailure_DoesNotBreakThePass_AndIsRetriedNextPass()
{
var logger = new CapturingLogger<DeployedSecretsMirror>();
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, nodeIdentity: new FixedNodeIdentity("node-a"));
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
await using (var connection = storage.CreateConnection())
await using (var drop = connection.CreateCommand())
{
drop.CommandText = "DROP TABLE secret_mirror_status";
await drop.ExecuteNonQueryAsync();
}
var r = await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(["a"], r.Applied);
Assert.True(store.Contains("a"));
Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("mirror status"));
// Table back (e.g. transient fault cleared): the unchanged failure set is still
// written, because the earlier write never landed.
await storage.InitializeAsync();
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal("node-a", Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
}
[Fact]
public async Task Status_WithoutNodeIdentity_FallsBackToMachineName()
{
var (mirror, storage, _, _, _) = await CreateAsync();
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(Environment.MachineName, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
}
private sealed class FixedNodeIdentity(string? name) : INodeIdentityProvider
{
public string? NodeName { get; } = name;
}
// ── Harness ──
private async Task<(
@@ -665,7 +792,8 @@ public class DeployedSecretsMirrorTests : IDisposable
FakeSecretCacheInvalidator Invalidator)> CreateAsync(
CapturingLogger<DeployedSecretsMirror>? logger = null,
TimeSpan? interval = null,
TimeProvider? time = null)
TimeProvider? time = null,
INodeIdentityProvider? nodeIdentity = null)
{
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
await storage.InitializeAsync();
@@ -685,7 +813,8 @@ public class DeployedSecretsMirrorTests : IDisposable
invalidator,
options,
(ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
time ?? TimeProvider.System);
time ?? TimeProvider.System,
nodeIdentity);
return (mirror, storage, store, resolver, invalidator);
}