fix(site-secrets): report every site node's secret mirror failures on the health report (M1)
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
@@ -3,6 +3,7 @@ using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
@@ -924,6 +925,87 @@ public class SiteStorageService
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
// ── Secret mirror status (review finding M1 — LocalDb-replicated, one row per node) ──
|
||||
|
||||
/// <summary>
|
||||
/// Records <paramref name="nodeName"/>'s current deployed-secrets mirror failures in the
|
||||
/// replicated <c>secret_mirror_status</c> table (UPSERT on the node name). An empty
|
||||
/// <paramref name="failures"/> list records a healthy mirror. Every call is a replicated
|
||||
/// write, so callers write only when the failure set changes.
|
||||
/// </summary>
|
||||
/// <param name="nodeName">This node's name — the row key.</param>
|
||||
/// <param name="failures">The node's current failures (names and messages only, never values).</param>
|
||||
/// <param name="checkedAtUtc">When the recorded failure set was observed.</param>
|
||||
/// <returns>A task that completes when the row has been stored.</returns>
|
||||
public async Task SetSecretMirrorStatusAsync(
|
||||
string nodeName, IReadOnlyList<SecretMirrorFailure> failures, DateTimeOffset checkedAtUtc)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(nodeName);
|
||||
ArgumentNullException.ThrowIfNull(failures);
|
||||
|
||||
await using var connection = OpenConnection();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = @"
|
||||
INSERT INTO secret_mirror_status (node_name, failures_json, checked_at)
|
||||
VALUES (@nodeName, @failures, @checkedAt)
|
||||
ON CONFLICT(node_name) DO UPDATE SET
|
||||
failures_json = excluded.failures_json,
|
||||
checked_at = excluded.checked_at";
|
||||
|
||||
command.Parameters.AddWithValue("@nodeName", nodeName);
|
||||
command.Parameters.AddWithValue("@failures", JsonSerializer.Serialize(failures));
|
||||
command.Parameters.AddWithValue("@checkedAt", checkedAtUtc.ToUniversalTime().ToString("O"));
|
||||
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns every node's row from <c>secret_mirror_status</c>, ordered by node name. A row
|
||||
/// whose <c>failures_json</c> cannot be parsed is returned with a single synthetic failure
|
||||
/// saying so, rather than failing the whole read or silently reading as healthy.
|
||||
/// </summary>
|
||||
/// <returns>A task that resolves to every node's recorded mirror status.</returns>
|
||||
public async Task<IReadOnlyList<SecretMirrorNodeStatus>> GetSecretMirrorStatusesAsync()
|
||||
{
|
||||
await using var connection = OpenConnection();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText =
|
||||
"SELECT node_name, failures_json, checked_at FROM secret_mirror_status ORDER BY node_name";
|
||||
|
||||
var results = new List<SecretMirrorNodeStatus>();
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
var nodeName = reader.GetString(0);
|
||||
IReadOnlyList<SecretMirrorFailure> failures;
|
||||
try
|
||||
{
|
||||
failures = JsonSerializer.Deserialize<List<SecretMirrorFailure>>(reader.GetString(1))
|
||||
?? [];
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or NotSupportedException)
|
||||
{
|
||||
failures =
|
||||
[
|
||||
new SecretMirrorFailure(
|
||||
"(unreadable)",
|
||||
$"Mirror status row for node '{nodeName}' could not be parsed ({ex.GetType().Name})"),
|
||||
];
|
||||
}
|
||||
|
||||
var checkedAt = DateTimeOffset.TryParse(
|
||||
reader.GetString(2), null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsed)
|
||||
? parsed
|
||||
: DateTimeOffset.MinValue;
|
||||
|
||||
results.Add(new SecretMirrorNodeStatus(nodeName, checkedAt, failures));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Shared Script CRUD ──
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user