fix(site-secrets): report every site node's secret mirror failures on the health report (M1)

A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.

- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
  failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
  mirror writes only its own row, and only when its failure set changes plus
  once per start-up (every write replicates via CDC). Best-effort: a failed
  write is a Warning and is retried next pass. Node key = INodeIdentityProvider
  NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
  into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
  SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
  null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
  node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
  round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
  bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
Joseph Doherty
2026-09-28 21:33:10 -04:00
parent 23a139a46c
commit 0289d51137
24 changed files with 1718 additions and 100 deletions
@@ -212,7 +212,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
// ---- helpers ----------------------------------------------------------------------
/// <summary>
/// The eleven replicated tables, plus the two deliberately-unregistered central-only
/// The twelve replicated tables, plus the two deliberately-unregistered central-only
/// notification tables, plus the node-local (also deliberately-unregistered)
/// <c>mirrored_secrets</c> ledger.
/// </summary>
@@ -221,7 +221,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state",
"deployed_secrets",
"deployed_secrets", "secret_mirror_status",
"notification_lists", "smtp_configurations", "mirrored_secrets",
];
@@ -230,7 +230,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state",
"deployed_secrets",
"deployed_secrets", "secret_mirror_status",
];
private static void AssertCaptureTriggersCoverTheReplicatedTables(ILocalDb db)
@@ -15,8 +15,9 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <c>RegisterReplicated</c> time, which on a site node is a BOOT failure of the replicating
/// pair. This test moves that discovery from the first replicated boot to the build: it walks
/// the real site schema (every table <c>SiteLocalDbSetup.OnReady</c> creates) and asserts none
/// of the eleven <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
/// added for site-secrets artifact deploy) has such an inbound FK.
/// of the twelve <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
/// added for site-secrets artifact deploy, and <c>secret_mirror_status</c>, added by review
/// finding M1) has such an inbound FK.
/// </summary>
/// <remarks>
/// The schema is built with an UNREPLICATED config on purpose: a replicated one would make
@@ -254,7 +254,7 @@ public class SiteLocalDbWiringTests : IDisposable
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "notification_lists",
"data_connection_definitions", "smtp_configurations", "native_alarm_state",
"deployed_secrets", "mirrored_secrets",
"deployed_secrets", "mirrored_secrets", "secret_mirror_status",
})
{
Assert.Contains(table, tables);
@@ -262,9 +262,10 @@ public class SiteLocalDbWiringTests : IDisposable
}
[Fact]
public void Site_LocalDb_RegistersExactlyTheElevenReplicatedTables()
public void Site_LocalDb_RegistersExactlyTheTwelveReplicatedTables()
{
// The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets.
// The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets
// and review finding M1's per-node secret_mirror_status.
// This is an EXACTLY check rather than a Contains in both directions, and both
// directions are load-bearing.
//
@@ -285,8 +286,8 @@ public class SiteLocalDbWiringTests : IDisposable
// data_connection_definitions precedes database_connections.
"OperationTracking", "data_connection_definitions", "database_connections",
"deployed_configurations", "deployed_secrets", "external_systems",
"native_alarm_state", "sf_messages", "shared_scripts", "site_events",
"static_attribute_overrides",
"native_alarm_state", "secret_mirror_status", "sf_messages", "shared_scripts",
"site_events", "static_attribute_overrides",
],
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray());
}
@@ -334,6 +335,7 @@ public class SiteLocalDbWiringTests : IDisposable
["instance_unique_name", "source_canonical_name", "source_reference"],
db.ReplicatedTables["native_alarm_state"].PkColumns);
Assert.Equal(["name"], db.ReplicatedTables["deployed_secrets"].PkColumns);
Assert.Equal(["node_name"], db.ReplicatedTables["secret_mirror_status"].PkColumns);
}
private static HashSet<string> TableNames(ILocalDb db)