fix(site-secrets): report every site node's secret mirror failures on the health report (M1)

A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.

- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
  failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
  mirror writes only its own row, and only when its failure set changes plus
  once per start-up (every write replicates via CDC). Best-effort: a failed
  write is a Warning and is retried next pass. Node key = INodeIdentityProvider
  NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
  into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
  SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
  null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
  node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
  round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
  bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
Joseph Doherty
2026-09-28 21:33:10 -04:00
parent 23a139a46c
commit 0289d51137
24 changed files with 1718 additions and 100 deletions
@@ -369,6 +369,7 @@ public class CentralControlDtoMapperTests
Assert.Null(roundTripped.LocalDbReplicationConnected);
Assert.Null(roundTripped.LocalDbOplogBacklog);
Assert.Null(roundTripped.StartupReconcilerActive);
Assert.Null(roundTripped.SecretMirrorStatuses);
}
[Fact]
@@ -384,10 +385,14 @@ public class CentralControlDtoMapperTests
DataConnectionEndpoints = new Dictionary<string, string>(),
DataConnectionTagQuality = new Dictionary<string, TagQualityCounts>(),
ClusterNodes = [],
SecretMirrorStatuses = [],
};
var roundTripped = CentralControlDtoMapper.FromDto(Wire(CentralControlDtoMapper.ToDto(original)));
Assert.NotNull(roundTripped.SecretMirrorStatuses);
Assert.Empty(roundTripped.SecretMirrorStatuses);
Assert.NotNull(roundTripped.DataConnectionEndpoints);
Assert.Empty(roundTripped.DataConnectionEndpoints);
Assert.NotNull(roundTripped.DataConnectionTagQuality);
@@ -559,6 +564,18 @@ public class CentralControlDtoMapperTests
DetachedScriptThreads = 2,
ScriptRunShedCount = 13,
SiteStreamAlarmDropCount = 41L,
// Review finding M1: every node's secret mirror status, standby included.
SecretMirrorStatuses =
[
new SecretMirrorNodeStatus("node-a", SampleInstant, []),
new SecretMirrorNodeStatus(
"node-b",
SampleInstant.AddMinutes(-3),
[
new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node"),
new SecretMirrorFailure("MX-KEY", "could not be confirmed applied"),
]),
],
};
/// <summary>
@@ -611,6 +628,17 @@ public class CentralControlDtoMapperTests
Assert.Equal(expected.DetachedScriptThreads, actual.DetachedScriptThreads);
Assert.Equal(expected.ScriptRunShedCount, actual.ScriptRunShedCount);
Assert.Equal(expected.SiteStreamAlarmDropCount, actual.SiteStreamAlarmDropCount);
Assert.Equal(expected.SecretMirrorStatuses is null, actual.SecretMirrorStatuses is null);
if (expected.SecretMirrorStatuses is not null)
{
Assert.Equal(expected.SecretMirrorStatuses.Count, actual.SecretMirrorStatuses!.Count);
for (var i = 0; i < expected.SecretMirrorStatuses.Count; i++)
{
Assert.Equal(expected.SecretMirrorStatuses[i].NodeName, actual.SecretMirrorStatuses[i].NodeName);
Assert.Equal(expected.SecretMirrorStatuses[i].CheckedAtUtc, actual.SecretMirrorStatuses[i].CheckedAtUtc);
Assert.Equal(expected.SecretMirrorStatuses[i].Failures, actual.SecretMirrorStatuses[i].Failures);
}
}
}
/// <summary>