fix(site-secrets): report every site node's secret mirror failures on the health report (M1)
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
@@ -198,6 +198,19 @@ public interface ISiteHealthCollector
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest per-node deployed-secrets mirror statuses (review finding M1) used
|
||||
/// by the <c>SecretMirrorStatuses</c> field of the next <see cref="CollectReport"/> call.
|
||||
/// Refreshed periodically by the site-side <c>SecretMirrorStatusReporter</c> from the
|
||||
/// LocalDb-replicated <c>secret_mirror_status</c> table, so the ACTIVE node's report
|
||||
/// carries the STANDBY node's mirror failures too. Point-in-time: NOT reset on collect.
|
||||
/// </summary>
|
||||
/// <param name="statuses">Every node's row, or <see langword="null"/> for "no data".</param>
|
||||
void SetSecretMirrorStatuses(IReadOnlyList<SecretMirrorNodeStatus>? statuses)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest script-execution-scheduler gauges (queue depth, busy
|
||||
/// thread count, and age in seconds of the oldest in-flight script) used by
|
||||
|
||||
@@ -32,6 +32,10 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
// -1 = reconciler SKIPPED (no IDeploymentConfigFetcher)
|
||||
// Null must stay distinct from -1: "we don't know" is not "self-heal is missing".
|
||||
private volatile int _startupReconcilerActive;
|
||||
// Review finding M1: every site node's deployed-secrets mirror status, refreshed by the
|
||||
// SiteRuntime SecretMirrorStatusReporter. Null = the reporter has not run (or its read
|
||||
// failed before it ever succeeded) — "no data", distinct from an empty list.
|
||||
private volatile IReadOnlyList<SecretMirrorNodeStatus>? _secretMirrorStatuses;
|
||||
private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
|
||||
private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
|
||||
private readonly ConcurrentDictionary<string, string> _connectionEndpoints = new();
|
||||
@@ -130,6 +134,12 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
_startupReconcilerActive = active ? 1 : -1;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetSecretMirrorStatuses(IReadOnlyList<SecretMirrorNodeStatus>? statuses)
|
||||
{
|
||||
_secretMirrorStatuses = statuses?.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
|
||||
{
|
||||
@@ -333,7 +343,9 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount),
|
||||
// Tri-state: 0 stays null ("not reported"), which must never collapse into
|
||||
// false ("self-heal is missing"). Register row 73.
|
||||
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0
|
||||
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0,
|
||||
// M1: point-in-time, not drained; null stays null ("no data").
|
||||
SecretMirrorStatuses = _secretMirrorStatuses
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user