fix(site-secrets): report every site node's secret mirror failures on the health report (M1)
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
@@ -369,6 +369,7 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Null(roundTripped.LocalDbReplicationConnected);
|
||||
Assert.Null(roundTripped.LocalDbOplogBacklog);
|
||||
Assert.Null(roundTripped.StartupReconcilerActive);
|
||||
Assert.Null(roundTripped.SecretMirrorStatuses);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -384,10 +385,14 @@ public class CentralControlDtoMapperTests
|
||||
DataConnectionEndpoints = new Dictionary<string, string>(),
|
||||
DataConnectionTagQuality = new Dictionary<string, TagQualityCounts>(),
|
||||
ClusterNodes = [],
|
||||
SecretMirrorStatuses = [],
|
||||
};
|
||||
|
||||
var roundTripped = CentralControlDtoMapper.FromDto(Wire(CentralControlDtoMapper.ToDto(original)));
|
||||
|
||||
Assert.NotNull(roundTripped.SecretMirrorStatuses);
|
||||
Assert.Empty(roundTripped.SecretMirrorStatuses);
|
||||
|
||||
Assert.NotNull(roundTripped.DataConnectionEndpoints);
|
||||
Assert.Empty(roundTripped.DataConnectionEndpoints);
|
||||
Assert.NotNull(roundTripped.DataConnectionTagQuality);
|
||||
@@ -559,6 +564,18 @@ public class CentralControlDtoMapperTests
|
||||
DetachedScriptThreads = 2,
|
||||
ScriptRunShedCount = 13,
|
||||
SiteStreamAlarmDropCount = 41L,
|
||||
// Review finding M1: every node's secret mirror status, standby included.
|
||||
SecretMirrorStatuses =
|
||||
[
|
||||
new SecretMirrorNodeStatus("node-a", SampleInstant, []),
|
||||
new SecretMirrorNodeStatus(
|
||||
"node-b",
|
||||
SampleInstant.AddMinutes(-3),
|
||||
[
|
||||
new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node"),
|
||||
new SecretMirrorFailure("MX-KEY", "could not be confirmed applied"),
|
||||
]),
|
||||
],
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
@@ -611,6 +628,17 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Equal(expected.DetachedScriptThreads, actual.DetachedScriptThreads);
|
||||
Assert.Equal(expected.ScriptRunShedCount, actual.ScriptRunShedCount);
|
||||
Assert.Equal(expected.SiteStreamAlarmDropCount, actual.SiteStreamAlarmDropCount);
|
||||
Assert.Equal(expected.SecretMirrorStatuses is null, actual.SecretMirrorStatuses is null);
|
||||
if (expected.SecretMirrorStatuses is not null)
|
||||
{
|
||||
Assert.Equal(expected.SecretMirrorStatuses.Count, actual.SecretMirrorStatuses!.Count);
|
||||
for (var i = 0; i < expected.SecretMirrorStatuses.Count; i++)
|
||||
{
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].NodeName, actual.SecretMirrorStatuses[i].NodeName);
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].CheckedAtUtc, actual.SecretMirrorStatuses[i].CheckedAtUtc);
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].Failures, actual.SecretMirrorStatuses[i].Failures);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user