fix(site-secrets): report every site node's secret mirror failures on the health report (M1)
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
This commit is contained in:
@@ -275,6 +275,14 @@
|
||||
No self-heal
|
||||
</span>
|
||||
}
|
||||
@if (HasSecretMirrorFailures(state))
|
||||
{
|
||||
<span class="badge bg-warning text-dark"
|
||||
data-test="secret-mirror-badge"
|
||||
title="A site node could not apply and/or decrypt one or more deployed secrets. A standby node's failures surface here before a failover would expose them — see the details in the card body.">
|
||||
Secret mirror failing
|
||||
</span>
|
||||
}
|
||||
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
|
||||
{
|
||||
<small class="text-muted">offline since @changedAt.ToString("u")</small>
|
||||
@@ -302,6 +310,26 @@
|
||||
@if (state.LatestReport != null)
|
||||
{
|
||||
var report = state.LatestReport;
|
||||
@if (HasSecretMirrorFailures(state))
|
||||
{
|
||||
@* Review finding M1: every site node's deployed-secrets mirror failures,
|
||||
the standby's included (carried on the active node's report). *@
|
||||
<div class="alert alert-warning py-2 px-3 mb-3 small" data-test="secret-mirror-failures" role="alert">
|
||||
<strong>Deployed secrets failing on a site node</strong>
|
||||
<ul class="mb-0 ps-3">
|
||||
@foreach (var node in report.SecretMirrorStatuses!.Where(n => n.Failures.Count > 0))
|
||||
{
|
||||
@foreach (var failure in node.Failures)
|
||||
{
|
||||
<li>
|
||||
<span class="font-monospace">@node.NodeName</span> ·
|
||||
<span class="font-monospace">@failure.SecretName</span>: @failure.Message
|
||||
</li>
|
||||
}
|
||||
}
|
||||
</ul>
|
||||
</div>
|
||||
}
|
||||
<div class="row g-3">
|
||||
@* Each logical group below sits in its own card, with the group's
|
||||
head promoted to a real card-header, instead of h6 rules stacked
|
||||
@@ -590,6 +618,15 @@
|
||||
state.IsOnline
|
||||
&& state.LatestReport?.StartupReconcilerActive == false;
|
||||
|
||||
// Review finding M1: warn when any site node (the standby included) reports deployed
|
||||
// secrets it cannot apply/decrypt. Null ("no data": central self-report, pre-M1 site) and
|
||||
// nodes with an empty failure list are healthy. Offline sites are skipped like the other
|
||||
// warning badges — the Offline badge is the signal that matters there.
|
||||
private static bool HasSecretMirrorFailures(SiteHealthState state) =>
|
||||
state.IsOnline
|
||||
&& state.LatestReport?.SecretMirrorStatuses is { } statuses
|
||||
&& statuses.Any(n => n.Failures.Count > 0);
|
||||
|
||||
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
|
||||
// Fewer than two means no standby, which disables the manual-failover control. This is
|
||||
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
/// <summary>
|
||||
/// One site node's latest deployed-secrets mirror outcome, as carried on
|
||||
/// <see cref="SiteHealthReport.SecretMirrorStatuses"/> (review finding M1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every site node runs its own <c>DeployedSecretsMirror</c> and records its current
|
||||
/// failure set in the LocalDb-replicated <c>secret_mirror_status</c> table, keyed by node
|
||||
/// name; the ACTIVE node reads every row and ships them on its health report, so a
|
||||
/// STANDBY node's failures (most importantly a master-key mismatch that stops a deployed
|
||||
/// secret decrypting there) reach central before a failover exposes them. Carries names
|
||||
/// and human-readable messages only — never a secret value or ciphertext.
|
||||
/// </remarks>
|
||||
/// <param name="NodeName">The reporting node's semantic name (<c>node-a</c>/<c>node-b</c>), or its host name when unconfigured.</param>
|
||||
/// <param name="CheckedAtUtc">When that node last recorded a CHANGE in its failure set (or its first pass after start-up) — not the time of its latest pass.</param>
|
||||
/// <param name="Failures">That node's current mirror failures; empty when the node's mirror is healthy.</param>
|
||||
public sealed record SecretMirrorNodeStatus(
|
||||
string NodeName,
|
||||
DateTimeOffset CheckedAtUtc,
|
||||
IReadOnlyList<SecretMirrorFailure> Failures);
|
||||
|
||||
/// <summary>A single deployed secret that failed to apply and/or decrypt on one site node.</summary>
|
||||
/// <param name="SecretName">The secret's name.</param>
|
||||
/// <param name="Message">A human-readable failure message — never includes the secret's value.</param>
|
||||
public sealed record SecretMirrorFailure(string SecretName, string Message);
|
||||
@@ -173,6 +173,20 @@ public record SiteHealthReport(
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public bool? StartupReconcilerActive { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Every site node's latest deployed-secrets mirror outcome (review finding M1) — one
|
||||
/// entry per node that has recorded a row in the replicated <c>secret_mirror_status</c>
|
||||
/// table, including the STANDBY node, whose own health report is never sent.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Null = no data</b>: a central self-report, a pre-M1 site build, a site whose
|
||||
/// reporter has not run yet, or a failed read. An EMPTY list means the feature is wired
|
||||
/// but no node has recorded a status yet. A node entry with an empty
|
||||
/// <see cref="SecretMirrorNodeStatus.Failures"/> list is a healthy mirror. Names and
|
||||
/// messages only — never secret values or ciphertext.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<SecretMirrorNodeStatus>? SecretMirrorStatuses { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -559,6 +559,32 @@ public static class CentralControlDtoMapper
|
||||
dto.SiteAuditBacklog = snapshot;
|
||||
}
|
||||
|
||||
// Review finding M1: wrapper presence keeps null ("no data") distinct from empty.
|
||||
if (msg.SecretMirrorStatuses is { } mirrorStatuses)
|
||||
{
|
||||
var wrapper = new SecretMirrorStatusListDto();
|
||||
foreach (var node in mirrorStatuses)
|
||||
{
|
||||
var nodeDto = new SecretMirrorNodeStatusDto
|
||||
{
|
||||
NodeName = node.NodeName,
|
||||
CheckedAtUtc = Timestamp.FromDateTimeOffset(node.CheckedAtUtc),
|
||||
};
|
||||
foreach (var failure in node.Failures)
|
||||
{
|
||||
nodeDto.Failures.Add(new SecretMirrorFailureDto
|
||||
{
|
||||
SecretName = failure.SecretName,
|
||||
Message = failure.Message,
|
||||
});
|
||||
}
|
||||
|
||||
wrapper.Nodes.Add(nodeDto);
|
||||
}
|
||||
|
||||
dto.SecretMirrorStatuses = wrapper;
|
||||
}
|
||||
|
||||
return dto;
|
||||
}
|
||||
|
||||
@@ -620,6 +646,19 @@ public static class CentralControlDtoMapper
|
||||
OnDiskBytes: snapshot.OnDiskBytes);
|
||||
}
|
||||
|
||||
List<SecretMirrorNodeStatus>? mirrorStatuses = null;
|
||||
if (dto.SecretMirrorStatuses is { } mirrorWrapper)
|
||||
{
|
||||
mirrorStatuses = new List<SecretMirrorNodeStatus>(mirrorWrapper.Nodes.Count);
|
||||
foreach (var node in mirrorWrapper.Nodes)
|
||||
{
|
||||
mirrorStatuses.Add(new SecretMirrorNodeStatus(
|
||||
node.NodeName,
|
||||
node.CheckedAtUtc?.ToDateTimeOffset() ?? DateTimeOffset.MinValue,
|
||||
node.Failures.Select(f => new SecretMirrorFailure(f.SecretName, f.Message)).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
return new SiteHealthReport(
|
||||
SiteId: dto.SiteId,
|
||||
SequenceNumber: dto.SequenceNumber,
|
||||
@@ -657,6 +696,7 @@ public static class CentralControlDtoMapper
|
||||
DetachedScriptThreads = dto.DetachedScriptThreads,
|
||||
ScriptRunShedCount = dto.ScriptRunShedCount,
|
||||
SiteStreamAlarmDropCount = dto.SiteStreamAlarmDropCount,
|
||||
SecretMirrorStatuses = mirrorStatuses,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -193,6 +193,25 @@ message NodeStatusListDto {
|
||||
repeated NodeStatusDto nodes = 1;
|
||||
}
|
||||
|
||||
// Review finding M1: one site node's deployed-secrets mirror failures. Names and
|
||||
// human-readable messages only — never a secret value or ciphertext.
|
||||
message SecretMirrorFailureDto {
|
||||
string secret_name = 1;
|
||||
string message = 2;
|
||||
}
|
||||
|
||||
message SecretMirrorNodeStatusDto {
|
||||
string node_name = 1;
|
||||
google.protobuf.Timestamp checked_at_utc = 2;
|
||||
repeated SecretMirrorFailureDto failures = 3;
|
||||
}
|
||||
|
||||
// Presence wrapper (same reason as NodeStatusListDto): absent = null ("no data"),
|
||||
// present-but-empty = the feature is wired and no node has recorded a status yet.
|
||||
message SecretMirrorStatusListDto {
|
||||
repeated SecretMirrorNodeStatusDto nodes = 1;
|
||||
}
|
||||
|
||||
// Site -> Central: periodic site health report. Mirrors Commons SiteHealthReport.
|
||||
// Additive-only evolution: field numbers are never reused.
|
||||
message SiteHealthReportDto {
|
||||
@@ -245,6 +264,10 @@ message SiteHealthReportDto {
|
||||
// dedicated alarm publish queue because it was at capacity. Point-in-time
|
||||
// (not reset on collect), like site_event_log_write_failures.
|
||||
int64 site_stream_alarm_drop_count = 32;
|
||||
// Review finding M1: every site node's deployed-secrets mirror status (the standby's
|
||||
// included — it is read from a LocalDb-replicated table on the active node).
|
||||
// Absent means null ("no data": central self-report, pre-M1 site, or not read yet).
|
||||
SecretMirrorStatusListDto secret_mirror_statuses = 33;
|
||||
}
|
||||
|
||||
// Central -> Site: health report ack, so a lost report is observable.
|
||||
|
||||
@@ -198,6 +198,19 @@ public interface ISiteHealthCollector
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest per-node deployed-secrets mirror statuses (review finding M1) used
|
||||
/// by the <c>SecretMirrorStatuses</c> field of the next <see cref="CollectReport"/> call.
|
||||
/// Refreshed periodically by the site-side <c>SecretMirrorStatusReporter</c> from the
|
||||
/// LocalDb-replicated <c>secret_mirror_status</c> table, so the ACTIVE node's report
|
||||
/// carries the STANDBY node's mirror failures too. Point-in-time: NOT reset on collect.
|
||||
/// </summary>
|
||||
/// <param name="statuses">Every node's row, or <see langword="null"/> for "no data".</param>
|
||||
void SetSecretMirrorStatuses(IReadOnlyList<SecretMirrorNodeStatus>? statuses)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest script-execution-scheduler gauges (queue depth, busy
|
||||
/// thread count, and age in seconds of the oldest in-flight script) used by
|
||||
|
||||
@@ -32,6 +32,10 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
// -1 = reconciler SKIPPED (no IDeploymentConfigFetcher)
|
||||
// Null must stay distinct from -1: "we don't know" is not "self-heal is missing".
|
||||
private volatile int _startupReconcilerActive;
|
||||
// Review finding M1: every site node's deployed-secrets mirror status, refreshed by the
|
||||
// SiteRuntime SecretMirrorStatusReporter. Null = the reporter has not run (or its read
|
||||
// failed before it ever succeeded) — "no data", distinct from an empty list.
|
||||
private volatile IReadOnlyList<SecretMirrorNodeStatus>? _secretMirrorStatuses;
|
||||
private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
|
||||
private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
|
||||
private readonly ConcurrentDictionary<string, string> _connectionEndpoints = new();
|
||||
@@ -130,6 +134,12 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
_startupReconcilerActive = active ? 1 : -1;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetSecretMirrorStatuses(IReadOnlyList<SecretMirrorNodeStatus>? statuses)
|
||||
{
|
||||
_secretMirrorStatuses = statuses?.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
|
||||
{
|
||||
@@ -333,7 +343,9 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount),
|
||||
// Tri-state: 0 stays null ("not reported"), which must never collapse into
|
||||
// false ("self-heal is missing"). Register row 73.
|
||||
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0
|
||||
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0,
|
||||
// M1: point-in-time, not drained; null stays null ("no data").
|
||||
SecretMirrorStatuses = _secretMirrorStatuses
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
public static class SiteLocalDbSetup
|
||||
{
|
||||
/// <summary>
|
||||
/// The eleven tables that participate in replication — registered when this node has a peer,
|
||||
/// The twelve tables that participate in replication — registered when this node has a peer,
|
||||
/// and deregistered (stale triggers dropped) when it does not.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
@@ -69,6 +69,13 @@ public static class SiteLocalDbSetup
|
||||
/// <c>deployed_secrets</c> row, and replicating the ledger would make one node's progress
|
||||
/// masquerade as the other's.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <c>secret_mirror_status</c> is the twelfth entry (review finding M1). It is per-node
|
||||
/// DATA — one row per node, keyed by <c>node_name</c>, each written only by its own node —
|
||||
/// but it is replicated precisely so the active node can read the standby's row and put
|
||||
/// it on the site health report (only the active node sends one). Explicit primary key,
|
||||
/// no FK, no BLOB columns.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static readonly string[] ReplicatedTables =
|
||||
[
|
||||
@@ -83,6 +90,7 @@ public static class SiteLocalDbSetup
|
||||
"data_connection_definitions",
|
||||
"native_alarm_state",
|
||||
"deployed_secrets",
|
||||
"secret_mirror_status",
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
@@ -140,8 +148,9 @@ public static class SiteLocalDbSetup
|
||||
// ELEVENTH entry to ReplicatedTables on an already-replicating site streams a
|
||||
// snapshot of only that one table, where through 0.2.x the flag was a single
|
||||
// per-database bit and the same edit would have re-streamed all eleven in full, in
|
||||
// both directions. Nothing about the first boot changes: enabling replication on a
|
||||
// site that has been running without it seeds all eleven at once, the flagged set is
|
||||
// both directions (the same holds for secret_mirror_status, the twelfth, added by
|
||||
// review finding M1). Nothing about the first boot changes: enabling replication on a
|
||||
// site that has been running without it seeds all twelve at once, the flagged set is
|
||||
// therefore every registered table, and the library sends that as an ordinary full
|
||||
// snapshot. A hypothetical "register the whole list in one call" API would silently
|
||||
// give the coarse behaviour back.
|
||||
|
||||
@@ -3,8 +3,8 @@ using Microsoft.Data.Sqlite;
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// DDL for the site's eleven configuration tables (nine original plus
|
||||
/// <c>deployed_secrets</c> and <c>mirrored_secrets</c>), extracted from
|
||||
/// DDL for the site's twelve configuration tables (nine original plus
|
||||
/// <c>deployed_secrets</c>, <c>mirrored_secrets</c> and <c>secret_mirror_status</c>), extracted from
|
||||
/// <see cref="SiteStorageService"/> so it can be applied by whoever owns the database
|
||||
/// file.
|
||||
/// <para>
|
||||
@@ -23,7 +23,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
public static class SiteStorageSchema
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates the eleven site configuration tables when absent, and additively upgrades
|
||||
/// Creates the twelve site configuration tables when absent, and additively upgrades
|
||||
/// tables created by an older build. Idempotent — safe to run on every startup.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open connection to the database that should hold the tables.</param>
|
||||
@@ -136,6 +136,20 @@ public static class SiteStorageSchema
|
||||
revision INTEGER NOT NULL,
|
||||
mirrored_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Per-node deployed-secrets mirror status (review finding M1): one row per site
|
||||
-- node, keyed by node name, holding that node's CURRENT mirror failures as a JSON
|
||||
-- array of {SecretName, Message} (names and messages only — never values or
|
||||
-- ciphertext). Unlike mirrored_secrets this IS replicated
|
||||
-- (SiteLocalDbSetup.ReplicatedTables): each node writes only its own row, and the
|
||||
-- replication is exactly what lets the ACTIVE node's health report carry the
|
||||
-- STANDBY node's failures to central. Written only when a node's failure set
|
||||
-- changes (plus once per start-up), never per mirror tick. No foreign keys.
|
||||
CREATE TABLE IF NOT EXISTS secret_mirror_status (
|
||||
node_name TEXT PRIMARY KEY,
|
||||
failures_json TEXT NOT NULL,
|
||||
checked_at TEXT NOT NULL
|
||||
);
|
||||
";
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
@@ -924,6 +925,87 @@ public class SiteStorageService
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
// ── Secret mirror status (review finding M1 — LocalDb-replicated, one row per node) ──
|
||||
|
||||
/// <summary>
|
||||
/// Records <paramref name="nodeName"/>'s current deployed-secrets mirror failures in the
|
||||
/// replicated <c>secret_mirror_status</c> table (UPSERT on the node name). An empty
|
||||
/// <paramref name="failures"/> list records a healthy mirror. Every call is a replicated
|
||||
/// write, so callers write only when the failure set changes.
|
||||
/// </summary>
|
||||
/// <param name="nodeName">This node's name — the row key.</param>
|
||||
/// <param name="failures">The node's current failures (names and messages only, never values).</param>
|
||||
/// <param name="checkedAtUtc">When the recorded failure set was observed.</param>
|
||||
/// <returns>A task that completes when the row has been stored.</returns>
|
||||
public async Task SetSecretMirrorStatusAsync(
|
||||
string nodeName, IReadOnlyList<SecretMirrorFailure> failures, DateTimeOffset checkedAtUtc)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(nodeName);
|
||||
ArgumentNullException.ThrowIfNull(failures);
|
||||
|
||||
await using var connection = OpenConnection();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = @"
|
||||
INSERT INTO secret_mirror_status (node_name, failures_json, checked_at)
|
||||
VALUES (@nodeName, @failures, @checkedAt)
|
||||
ON CONFLICT(node_name) DO UPDATE SET
|
||||
failures_json = excluded.failures_json,
|
||||
checked_at = excluded.checked_at";
|
||||
|
||||
command.Parameters.AddWithValue("@nodeName", nodeName);
|
||||
command.Parameters.AddWithValue("@failures", JsonSerializer.Serialize(failures));
|
||||
command.Parameters.AddWithValue("@checkedAt", checkedAtUtc.ToUniversalTime().ToString("O"));
|
||||
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns every node's row from <c>secret_mirror_status</c>, ordered by node name. A row
|
||||
/// whose <c>failures_json</c> cannot be parsed is returned with a single synthetic failure
|
||||
/// saying so, rather than failing the whole read or silently reading as healthy.
|
||||
/// </summary>
|
||||
/// <returns>A task that resolves to every node's recorded mirror status.</returns>
|
||||
public async Task<IReadOnlyList<SecretMirrorNodeStatus>> GetSecretMirrorStatusesAsync()
|
||||
{
|
||||
await using var connection = OpenConnection();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText =
|
||||
"SELECT node_name, failures_json, checked_at FROM secret_mirror_status ORDER BY node_name";
|
||||
|
||||
var results = new List<SecretMirrorNodeStatus>();
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
var nodeName = reader.GetString(0);
|
||||
IReadOnlyList<SecretMirrorFailure> failures;
|
||||
try
|
||||
{
|
||||
failures = JsonSerializer.Deserialize<List<SecretMirrorFailure>>(reader.GetString(1))
|
||||
?? [];
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or NotSupportedException)
|
||||
{
|
||||
failures =
|
||||
[
|
||||
new SecretMirrorFailure(
|
||||
"(unreadable)",
|
||||
$"Mirror status row for node '{nodeName}' could not be parsed ({ex.GetType().Name})"),
|
||||
];
|
||||
}
|
||||
|
||||
var checkedAt = DateTimeOffset.TryParse(
|
||||
reader.GetString(2), null, System.Globalization.DateTimeStyles.RoundtripKind, out var parsed)
|
||||
? parsed
|
||||
: DateTimeOffset.MinValue;
|
||||
|
||||
results.Add(new SecretMirrorNodeStatus(nodeName, checkedAt, failures));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Shared Script CRUD ──
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
@@ -153,6 +155,18 @@ public sealed record MirrorResult(
|
||||
/// so a LATER failure — even one with an identical fingerprint — logs at Error again. This state
|
||||
/// is in-memory only (per node process), which matches the ledger's own node-local scope.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status row for central (review finding M1).</b> A failure on the STANDBY node used to be
|
||||
/// visible only in that node's own log — site health reports are sent by the active node only.
|
||||
/// After each pass this mirror records its node's current failure set (secret name + message,
|
||||
/// never a value) in the LocalDb-replicated <c>secret_mirror_status</c> table, keyed by node
|
||||
/// name; <c>SecretMirrorStatusReporter</c> on the active node reads every node's row onto the
|
||||
/// health report. Because every write replicates via CDC, the row is written only when the
|
||||
/// failure set (compared as sorted name+message pairs) differs from what this process last
|
||||
/// wrote — plus once on the first pass after start-up, so a restart refreshes a stale row. The
|
||||
/// write is best-effort: a failure logs a Warning (repeats at Debug) and is retried on the next
|
||||
/// pass, and never affects the pass's own <see cref="MirrorResult"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DeployedSecretsMirror : BackgroundService
|
||||
{
|
||||
@@ -165,6 +179,17 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
private readonly IOptions<SiteRuntimeOptions> _options;
|
||||
private readonly ILogger<DeployedSecretsMirror> _logger;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly string _nodeName;
|
||||
|
||||
/// <summary>
|
||||
/// The failure set (sorted name+message pairs) this process last wrote successfully to
|
||||
/// <c>secret_mirror_status</c>, or <see langword="null"/> before the first successful write
|
||||
/// — which is what forces the once-per-start-up write (review finding M1).
|
||||
/// </summary>
|
||||
private IReadOnlyList<SecretMirrorFailure>? _lastWrittenStatus;
|
||||
|
||||
/// <summary>Whether a failed status write has already been logged at Warning (de-spam).</summary>
|
||||
private bool _statusWriteFailureLogged;
|
||||
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private bool _disposed;
|
||||
@@ -199,6 +224,11 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
/// <param name="options">Supplies <see cref="SiteRuntimeOptions.DeployedSecretsMirrorInterval"/>.</param>
|
||||
/// <param name="logger">Logger instance. Never logs secret values — names, revisions, and KEK ids only.</param>
|
||||
/// <param name="time">Clock driving the reconcile loop's <see cref="PeriodicTimer"/> and the tombstone-recovery stamp.</param>
|
||||
/// <param name="nodeIdentity">
|
||||
/// Supplies this node's name (<c>node-a</c>/<c>node-b</c>), the key of its
|
||||
/// <c>secret_mirror_status</c> row. Optional: when absent or unconfigured, the machine name
|
||||
/// is used instead.
|
||||
/// </param>
|
||||
public DeployedSecretsMirror(
|
||||
SiteStorageService storage,
|
||||
ISecretStore store,
|
||||
@@ -206,7 +236,8 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
ISecretCacheInvalidator? invalidator,
|
||||
IOptions<SiteRuntimeOptions> options,
|
||||
ILogger<DeployedSecretsMirror> logger,
|
||||
TimeProvider time)
|
||||
TimeProvider time,
|
||||
INodeIdentityProvider? nodeIdentity = null)
|
||||
{
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
@@ -215,6 +246,7 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_time = time ?? throw new ArgumentNullException(nameof(time));
|
||||
_nodeName = nodeIdentity?.NodeName ?? Environment.MachineName;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -314,6 +346,8 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
rows.Count(r => !r.IsRemoved) + unreadable.Count(u => !u.IsRemoved), failed.Count);
|
||||
}
|
||||
|
||||
await RecordStatusIfChangedAsync(failedNames, failed).ConfigureAwait(false);
|
||||
|
||||
return result;
|
||||
}
|
||||
finally
|
||||
@@ -322,6 +356,42 @@ public sealed class DeployedSecretsMirror : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes this node's <c>secret_mirror_status</c> row when the pass's failure set differs
|
||||
/// from the last one this process wrote (or nothing has been written yet since start-up).
|
||||
/// Best-effort: never throws, so the pass result is unaffected by a status-write fault.
|
||||
/// See the class remarks (review finding M1).
|
||||
/// </summary>
|
||||
private async Task RecordStatusIfChangedAsync(IReadOnlyList<string> failedNames, IReadOnlyList<string> failed)
|
||||
{
|
||||
var current = failedNames
|
||||
.Select((name, i) => new SecretMirrorFailure(name, failed[i]))
|
||||
.OrderBy(f => f.SecretName, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.Message, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
if (_lastWrittenStatus is not null && _lastWrittenStatus.SequenceEqual(current))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _storage.SetSecretMirrorStatusAsync(_nodeName, current, _time.GetUtcNow()).ConfigureAwait(false);
|
||||
_lastWrittenStatus = current;
|
||||
_statusWriteFailureLogged = false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Log(
|
||||
_statusWriteFailureLogged ? LogLevel.Debug : LogLevel.Warning,
|
||||
ex,
|
||||
"Failed to record the deployed secrets mirror status for node {NodeName}; retrying next pass",
|
||||
_nodeName);
|
||||
_statusWriteFailureLogged = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles a tombstoned <c>deployed_secrets</c> row: deletes it from this node's store, but
|
||||
/// only if this node had previously ledgered it as mirrored — an out-of-band, same-named
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Review finding M1: site-side hosted service that periodically reads EVERY node's row from
|
||||
/// the LocalDb-replicated <c>secret_mirror_status</c> table and pushes the list into
|
||||
/// <see cref="ISiteHealthCollector.SetSecretMirrorStatuses"/>, so the next health report
|
||||
/// carries each site node's deployed-secrets mirror failures — the standby's included.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Runs on every site node (the read is a cheap local SELECT), but only the active node's
|
||||
/// report is ever sent (<c>HealthReportSender</c>), which is exactly why the status lives in a
|
||||
/// REPLICATED table: the standby writes its row, CDC carries it to the active node, and this
|
||||
/// reporter on the active node forwards it. A site pair WITHOUT LocalDb replication reports
|
||||
/// only the active node's row — the same prerequisite the deployed secrets themselves have.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Lives in SiteRuntime (which already references HealthMonitoring and owns
|
||||
/// <see cref="SiteStorageService"/>), following <c>SiteStreamAlarmDropReporter</c>: immediate
|
||||
/// first probe, fixed cadence, exceptions logged and swallowed. A failed read leaves the
|
||||
/// previous snapshot on the collector rather than clearing it.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SecretMirrorStatusReporter : BackgroundService
|
||||
{
|
||||
/// <summary>Default poll cadence (30 s), matching the site health report interval.</summary>
|
||||
internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly ISiteHealthCollector _collector;
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly ILogger<SecretMirrorStatusReporter> _logger;
|
||||
private readonly TimeSpan _pollInterval;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="SecretMirrorStatusReporter"/>.</summary>
|
||||
/// <param name="collector">The site health collector receiving the statuses.</param>
|
||||
/// <param name="storage">Site storage exposing <c>secret_mirror_status</c>.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="pollInterval">Poll interval override; defaults to <see cref="DefaultPollInterval"/> (30 s).</param>
|
||||
public SecretMirrorStatusReporter(
|
||||
ISiteHealthCollector collector,
|
||||
SiteStorageService storage,
|
||||
ILogger<SecretMirrorStatusReporter> logger,
|
||||
TimeSpan? pollInterval = null)
|
||||
{
|
||||
_collector = collector ?? throw new ArgumentNullException(nameof(collector));
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_pollInterval = pollInterval ?? DefaultPollInterval;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await ProbeAsync().ConfigureAwait(false);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await ProbeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads every node's mirror status and pushes it onto the collector; never throws.
|
||||
/// Public so a test can drive one deterministic probe.
|
||||
/// </summary>
|
||||
/// <returns>A task that completes when the probe has finished.</returns>
|
||||
public async Task ProbeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var statuses = await _storage.GetSecretMirrorStatusesAsync().ConfigureAwait(false);
|
||||
_collector.SetSecretMirrorStatuses(statuses);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Catch-all is deliberate: health reporting must never take the node down, and
|
||||
// the loop must survive so the next tick retries.
|
||||
_logger.LogWarning(ex, "SecretMirrorStatusReporter probe failed; next tick will retry.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -140,6 +140,13 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<DeployedSecretsMirror>();
|
||||
services.AddHostedService(sp => sp.GetRequiredService<DeployedSecretsMirror>());
|
||||
|
||||
// Review finding M1: forward every node's replicated secret_mirror_status row onto the
|
||||
// site health report, so the ACTIVE node's report carries the STANDBY's mirror failures.
|
||||
services.AddHostedService(sp => new SecretMirrorStatusReporter(
|
||||
sp.GetRequiredService<HealthMonitoring.ISiteHealthCollector>(),
|
||||
sp.GetRequiredService<SiteStorageService>(),
|
||||
sp.GetRequiredService<ILogger<SecretMirrorStatusReporter>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
@@ -559,6 +559,70 @@ public class HealthPageTests : BunitContext
|
||||
StartupReconcilerActive = active,
|
||||
};
|
||||
|
||||
// ----- Review finding M1: per-node deployed-secrets mirror failures ------------------
|
||||
//
|
||||
// The standby's failures ride the active node's report; the card must name the node,
|
||||
// the secret and the message. Null ("no data") and all-healthy nodes render nothing.
|
||||
|
||||
[Fact]
|
||||
public void Renders_SecretMirrorFailures_LabelledByNode()
|
||||
{
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = ReportWithReconciler("site-a", active: true) with
|
||||
{
|
||||
SecretMirrorStatuses =
|
||||
[
|
||||
new SecretMirrorNodeStatus("node-a", DateTimeOffset.UtcNow, []),
|
||||
new SecretMirrorNodeStatus(
|
||||
"node-b",
|
||||
DateTimeOffset.UtcNow,
|
||||
[new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")]),
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() =>
|
||||
{
|
||||
Assert.Contains("data-test=\"secret-mirror-badge\"", cut.Markup);
|
||||
var panel = cut.Find("[data-test=secret-mirror-failures]");
|
||||
Assert.Contains("node-b", panel.TextContent);
|
||||
Assert.Contains("OPC-PWD", panel.TextContent);
|
||||
Assert.Contains("did not decrypt on this node", panel.TextContent);
|
||||
Assert.DoesNotContain("node-a", panel.TextContent);
|
||||
});
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)] // null = "no data" (central self-report, pre-M1 site)
|
||||
[InlineData(true)] // every node healthy
|
||||
public void DoesNotRender_SecretMirrorWarning_WhenNoNodeIsFailing(bool reported)
|
||||
{
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = ReportWithReconciler("site-a", active: true) with
|
||||
{
|
||||
SecretMirrorStatuses = reported
|
||||
? [new SecretMirrorNodeStatus("node-a", DateTimeOffset.UtcNow, [])]
|
||||
: null,
|
||||
},
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
||||
Assert.DoesNotContain("data-test=\"secret-mirror-badge\"", cut.Markup);
|
||||
Assert.DoesNotContain("data-test=\"secret-mirror-failures\"", cut.Markup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Renders_OfflineSince_ForOfflineSiteWithStatusChange()
|
||||
{
|
||||
|
||||
@@ -369,6 +369,7 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Null(roundTripped.LocalDbReplicationConnected);
|
||||
Assert.Null(roundTripped.LocalDbOplogBacklog);
|
||||
Assert.Null(roundTripped.StartupReconcilerActive);
|
||||
Assert.Null(roundTripped.SecretMirrorStatuses);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -384,10 +385,14 @@ public class CentralControlDtoMapperTests
|
||||
DataConnectionEndpoints = new Dictionary<string, string>(),
|
||||
DataConnectionTagQuality = new Dictionary<string, TagQualityCounts>(),
|
||||
ClusterNodes = [],
|
||||
SecretMirrorStatuses = [],
|
||||
};
|
||||
|
||||
var roundTripped = CentralControlDtoMapper.FromDto(Wire(CentralControlDtoMapper.ToDto(original)));
|
||||
|
||||
Assert.NotNull(roundTripped.SecretMirrorStatuses);
|
||||
Assert.Empty(roundTripped.SecretMirrorStatuses);
|
||||
|
||||
Assert.NotNull(roundTripped.DataConnectionEndpoints);
|
||||
Assert.Empty(roundTripped.DataConnectionEndpoints);
|
||||
Assert.NotNull(roundTripped.DataConnectionTagQuality);
|
||||
@@ -559,6 +564,18 @@ public class CentralControlDtoMapperTests
|
||||
DetachedScriptThreads = 2,
|
||||
ScriptRunShedCount = 13,
|
||||
SiteStreamAlarmDropCount = 41L,
|
||||
// Review finding M1: every node's secret mirror status, standby included.
|
||||
SecretMirrorStatuses =
|
||||
[
|
||||
new SecretMirrorNodeStatus("node-a", SampleInstant, []),
|
||||
new SecretMirrorNodeStatus(
|
||||
"node-b",
|
||||
SampleInstant.AddMinutes(-3),
|
||||
[
|
||||
new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node"),
|
||||
new SecretMirrorFailure("MX-KEY", "could not be confirmed applied"),
|
||||
]),
|
||||
],
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
@@ -611,6 +628,17 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Equal(expected.DetachedScriptThreads, actual.DetachedScriptThreads);
|
||||
Assert.Equal(expected.ScriptRunShedCount, actual.ScriptRunShedCount);
|
||||
Assert.Equal(expected.SiteStreamAlarmDropCount, actual.SiteStreamAlarmDropCount);
|
||||
Assert.Equal(expected.SecretMirrorStatuses is null, actual.SecretMirrorStatuses is null);
|
||||
if (expected.SecretMirrorStatuses is not null)
|
||||
{
|
||||
Assert.Equal(expected.SecretMirrorStatuses.Count, actual.SecretMirrorStatuses!.Count);
|
||||
for (var i = 0; i < expected.SecretMirrorStatuses.Count; i++)
|
||||
{
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].NodeName, actual.SecretMirrorStatuses[i].NodeName);
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].CheckedAtUtc, actual.SecretMirrorStatuses[i].CheckedAtUtc);
|
||||
Assert.Equal(expected.SecretMirrorStatuses[i].Failures, actual.SecretMirrorStatuses[i].Failures);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Review finding M1 — <c>SiteHealthReport.SecretMirrorStatuses</c> carries every site node's
|
||||
/// deployed-secrets mirror failures (the standby's included) on the active node's report.
|
||||
/// </summary>
|
||||
public class SecretMirrorStatusHealthFieldTests
|
||||
{
|
||||
[Fact]
|
||||
public void CollectReport_NeverTold_LeavesTheFieldNull()
|
||||
{
|
||||
Assert.Null(new SiteHealthCollector().CollectReport("TestSite").SecretMirrorStatuses);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_CarriesTheLatestSnapshot_AndDoesNotDrainIt()
|
||||
{
|
||||
var collector = new SiteHealthCollector();
|
||||
collector.SetSecretMirrorStatuses(
|
||||
[
|
||||
new SecretMirrorNodeStatus("node-a", DateTimeOffset.UtcNow, []),
|
||||
new SecretMirrorNodeStatus(
|
||||
"node-b", DateTimeOffset.UtcNow, [new SecretMirrorFailure("S1", "did not decrypt")]),
|
||||
]);
|
||||
|
||||
var first = collector.CollectReport("TestSite").SecretMirrorStatuses!;
|
||||
var second = collector.CollectReport("TestSite").SecretMirrorStatuses!;
|
||||
|
||||
Assert.Equal(["node-a", "node-b"], first.Select(s => s.NodeName));
|
||||
Assert.Equal(2, second.Count); // point-in-time, not interval-drained
|
||||
Assert.Equal("S1", Assert.Single(first[1].Failures).SecretName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SetNull_ResetsToNoData()
|
||||
{
|
||||
var collector = new SiteHealthCollector();
|
||||
collector.SetSecretMirrorStatuses([]);
|
||||
collector.SetSecretMirrorStatuses(null);
|
||||
|
||||
Assert.Null(collector.CollectReport("TestSite").SecretMirrorStatuses);
|
||||
}
|
||||
}
|
||||
@@ -212,7 +212,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
|
||||
// ---- helpers ----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// The eleven replicated tables, plus the two deliberately-unregistered central-only
|
||||
/// The twelve replicated tables, plus the two deliberately-unregistered central-only
|
||||
/// notification tables, plus the node-local (also deliberately-unregistered)
|
||||
/// <c>mirrored_secrets</c> ledger.
|
||||
/// </summary>
|
||||
@@ -221,7 +221,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
|
||||
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
|
||||
"static_attribute_overrides", "shared_scripts", "external_systems",
|
||||
"database_connections", "data_connection_definitions", "native_alarm_state",
|
||||
"deployed_secrets",
|
||||
"deployed_secrets", "secret_mirror_status",
|
||||
"notification_lists", "smtp_configurations", "mirrored_secrets",
|
||||
];
|
||||
|
||||
@@ -230,7 +230,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
|
||||
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
|
||||
"static_attribute_overrides", "shared_scripts", "external_systems",
|
||||
"database_connections", "data_connection_definitions", "native_alarm_state",
|
||||
"deployed_secrets",
|
||||
"deployed_secrets", "secret_mirror_status",
|
||||
];
|
||||
|
||||
private static void AssertCaptureTriggersCoverTheReplicatedTables(ILocalDb db)
|
||||
|
||||
@@ -15,8 +15,9 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
/// <c>RegisterReplicated</c> time, which on a site node is a BOOT failure of the replicating
|
||||
/// pair. This test moves that discovery from the first replicated boot to the build: it walks
|
||||
/// the real site schema (every table <c>SiteLocalDbSetup.OnReady</c> creates) and asserts none
|
||||
/// of the eleven <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
|
||||
/// added for site-secrets artifact deploy) has such an inbound FK.
|
||||
/// of the twelve <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
|
||||
/// added for site-secrets artifact deploy, and <c>secret_mirror_status</c>, added by review
|
||||
/// finding M1) has such an inbound FK.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The schema is built with an UNREPLICATED config on purpose: a replicated one would make
|
||||
|
||||
@@ -254,7 +254,7 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
|
||||
"external_systems", "database_connections", "notification_lists",
|
||||
"data_connection_definitions", "smtp_configurations", "native_alarm_state",
|
||||
"deployed_secrets", "mirrored_secrets",
|
||||
"deployed_secrets", "mirrored_secrets", "secret_mirror_status",
|
||||
})
|
||||
{
|
||||
Assert.Contains(table, tables);
|
||||
@@ -262,9 +262,10 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_LocalDb_RegistersExactlyTheElevenReplicatedTables()
|
||||
public void Site_LocalDb_RegistersExactlyTheTwelveReplicatedTables()
|
||||
{
|
||||
// The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets.
|
||||
// The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets
|
||||
// and review finding M1's per-node secret_mirror_status.
|
||||
// This is an EXACTLY check rather than a Contains in both directions, and both
|
||||
// directions are load-bearing.
|
||||
//
|
||||
@@ -285,8 +286,8 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
// data_connection_definitions precedes database_connections.
|
||||
"OperationTracking", "data_connection_definitions", "database_connections",
|
||||
"deployed_configurations", "deployed_secrets", "external_systems",
|
||||
"native_alarm_state", "sf_messages", "shared_scripts", "site_events",
|
||||
"static_attribute_overrides",
|
||||
"native_alarm_state", "secret_mirror_status", "sf_messages", "shared_scripts",
|
||||
"site_events", "static_attribute_overrides",
|
||||
],
|
||||
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray());
|
||||
}
|
||||
@@ -334,6 +335,7 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
["instance_unique_name", "source_canonical_name", "source_reference"],
|
||||
db.ReplicatedTables["native_alarm_state"].PkColumns);
|
||||
Assert.Equal(["name"], db.ReplicatedTables["deployed_secrets"].PkColumns);
|
||||
Assert.Equal(["node_name"], db.ReplicatedTables["secret_mirror_status"].PkColumns);
|
||||
}
|
||||
|
||||
private static HashSet<string> TableNames(ILocalDb db)
|
||||
|
||||
+49
@@ -1,6 +1,7 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
@@ -303,6 +304,54 @@ public class DeployedSecretsStorageTests : IAsyncLifetime, IDisposable
|
||||
Assert.Equal("A", mirrored.Name);
|
||||
}
|
||||
|
||||
// ── secret_mirror_status (review finding M1) ──
|
||||
|
||||
[Fact]
|
||||
public async Task SecretMirrorStatus_RoundTrips_OneRowPerNode_OrderedByNodeName()
|
||||
{
|
||||
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
||||
await _storage.SetSecretMirrorStatusAsync(
|
||||
"node-b", [new SecretMirrorFailure("A", "did not decrypt")], at);
|
||||
await _storage.SetSecretMirrorStatusAsync("node-a", [], at.AddMinutes(1));
|
||||
|
||||
var rows = await _storage.GetSecretMirrorStatusesAsync();
|
||||
|
||||
Assert.Equal(["node-a", "node-b"], rows.Select(r => r.NodeName));
|
||||
Assert.Empty(rows[0].Failures);
|
||||
Assert.Equal(at.AddMinutes(1), rows[0].CheckedAtUtc);
|
||||
var failure = Assert.Single(rows[1].Failures);
|
||||
Assert.Equal(new SecretMirrorFailure("A", "did not decrypt"), failure);
|
||||
Assert.Equal(at, rows[1].CheckedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecretMirrorStatus_Set_UpsertsTheNodesRow()
|
||||
{
|
||||
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
||||
await _storage.SetSecretMirrorStatusAsync("node-a", [new SecretMirrorFailure("A", "x")], at);
|
||||
await _storage.SetSecretMirrorStatusAsync("node-a", [], at.AddHours(1));
|
||||
|
||||
var row = Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Empty(row.Failures);
|
||||
Assert.Equal(at.AddHours(1), row.CheckedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecretMirrorStatus_UnparseableRow_ReadsAsAFailure_NotAsHealthy()
|
||||
{
|
||||
await using (var connection = _storage.CreateConnection())
|
||||
await using (var insert = connection.CreateCommand())
|
||||
{
|
||||
insert.CommandText =
|
||||
"INSERT INTO secret_mirror_status (node_name, failures_json, checked_at) VALUES ('node-b', '{not json', '2026-09-28T00:00:00.0000000+00:00')";
|
||||
await insert.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
var row = Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Equal("node-b", row.NodeName);
|
||||
Assert.Single(row.Failures);
|
||||
}
|
||||
|
||||
private static SecretArtifact Art(string name, long revision) => new(
|
||||
Name: name,
|
||||
ContentType: "application/octet-stream",
|
||||
|
||||
@@ -23,14 +23,14 @@ public class SiteStorageSchemaTests
|
||||
SiteStorageSchema.Apply(connection);
|
||||
SiteStorageSchema.Apply(connection); // second run must not throw
|
||||
|
||||
// Every table is named explicitly on purpose — a loop asserting "11 tables"
|
||||
// Every table is named explicitly on purpose — a loop asserting "12 tables"
|
||||
// would still pass if one were renamed.
|
||||
foreach (var table in new[]
|
||||
{
|
||||
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
|
||||
"external_systems", "database_connections", "notification_lists",
|
||||
"data_connection_definitions", "smtp_configurations", "native_alarm_state",
|
||||
"deployed_secrets", "mirrored_secrets",
|
||||
"deployed_secrets", "mirrored_secrets", "secret_mirror_status",
|
||||
})
|
||||
{
|
||||
Assert.True(TableExists(connection, table), $"missing table: {table}");
|
||||
|
||||
+131
-2
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
||||
@@ -655,6 +656,132 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-node status row for central (review finding M1) ──
|
||||
|
||||
[Fact]
|
||||
public async Task Status_FirstPassAfterStartup_WritesThisNodesRow_EvenWhenHealthy()
|
||||
{
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Equal("node-b", status.NodeName);
|
||||
Assert.Empty(status.Failures);
|
||||
Assert.Equal(time.Now, status.CheckedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Status_UnchangedFailureSet_IsNotRewrittenOnLaterPasses()
|
||||
{
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
|
||||
resolver.ThrowFor("a");
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
var firstWrite = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
||||
|
||||
// Same failure on every later pass: the row is replicated, so it must not be rewritten.
|
||||
time.Now = time.Now.AddMinutes(5);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
var after = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Equal(firstWrite.CheckedAtUtc, after.CheckedAtUtc);
|
||||
var failure = Assert.Single(after.Failures);
|
||||
Assert.Equal("a", failure.SecretName);
|
||||
Assert.Contains("kek-central", failure.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
|
||||
{
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-a"));
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
Assert.Empty(Assert.Single(await storage.GetSecretMirrorStatusesAsync()).Failures);
|
||||
|
||||
// A decrypt failure appears -> row rewritten with the failure.
|
||||
resolver.ThrowFor("a");
|
||||
time.Now = time.Now.AddMinutes(1);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
var failing = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Equal("a", Assert.Single(failing.Failures).SecretName);
|
||||
Assert.Equal(time.Now, failing.CheckedAtUtc);
|
||||
|
||||
// Healed -> row rewritten empty.
|
||||
resolver.ClearFailures();
|
||||
time.Now = time.Now.AddMinutes(1);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
var healed = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
||||
Assert.Empty(healed.Failures);
|
||||
Assert.Equal(time.Now, healed.CheckedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Status_RowCarriesNamesAndMessagesOnly_NeverCiphertext()
|
||||
{
|
||||
var (mirror, storage, _, resolver, _) = await CreateAsync(nodeIdentity: new FixedNodeIdentity("node-a"));
|
||||
resolver.ThrowFor("a");
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
await using var connection = storage.CreateConnection();
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "SELECT failures_json FROM secret_mirror_status WHERE node_name = 'node-a'";
|
||||
var json = (string)(await command.ExecuteScalarAsync())!;
|
||||
Assert.Contains("\"SecretName\":\"a\"", json);
|
||||
Assert.DoesNotContain("Ciphertext", json);
|
||||
Assert.DoesNotContain(Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5 }), json);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Status_WriteFailure_DoesNotBreakThePass_AndIsRetriedNextPass()
|
||||
{
|
||||
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
||||
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, nodeIdentity: new FixedNodeIdentity("node-a"));
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
|
||||
await using (var connection = storage.CreateConnection())
|
||||
await using (var drop = connection.CreateCommand())
|
||||
{
|
||||
drop.CommandText = "DROP TABLE secret_mirror_status";
|
||||
await drop.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(["a"], r.Applied);
|
||||
Assert.True(store.Contains("a"));
|
||||
Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("mirror status"));
|
||||
|
||||
// Table back (e.g. transient fault cleared): the unchanged failure set is still
|
||||
// written, because the earlier write never landed.
|
||||
await storage.InitializeAsync();
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
Assert.Equal("node-a", Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Status_WithoutNodeIdentity_FallsBackToMachineName()
|
||||
{
|
||||
var (mirror, storage, _, _, _) = await CreateAsync();
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(Environment.MachineName, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
|
||||
}
|
||||
|
||||
private sealed class FixedNodeIdentity(string? name) : INodeIdentityProvider
|
||||
{
|
||||
public string? NodeName { get; } = name;
|
||||
}
|
||||
|
||||
// ── Harness ──
|
||||
|
||||
private async Task<(
|
||||
@@ -665,7 +792,8 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
FakeSecretCacheInvalidator Invalidator)> CreateAsync(
|
||||
CapturingLogger<DeployedSecretsMirror>? logger = null,
|
||||
TimeSpan? interval = null,
|
||||
TimeProvider? time = null)
|
||||
TimeProvider? time = null,
|
||||
INodeIdentityProvider? nodeIdentity = null)
|
||||
{
|
||||
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
@@ -685,7 +813,8 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
invalidator,
|
||||
options,
|
||||
(ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
|
||||
time ?? TimeProvider.System);
|
||||
time ?? TimeProvider.System,
|
||||
nodeIdentity);
|
||||
|
||||
return (mirror, storage, store, resolver, invalidator);
|
||||
}
|
||||
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Review finding M1: <see cref="SecretMirrorStatusReporter"/> forwards every node's
|
||||
/// replicated <c>secret_mirror_status</c> row onto the site health report via the collector.
|
||||
/// </summary>
|
||||
public class SecretMirrorStatusReporterTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly TestLocalDb _localDb = TestLocalDb.CreateTemp("secret-mirror-status");
|
||||
private SiteStorageService _storage = null!;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
await _storage.InitializeAsync();
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_PushesEveryNodesRow_OntoTheNextReport()
|
||||
{
|
||||
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
||||
await _storage.SetSecretMirrorStatusAsync("node-a", [], at);
|
||||
await _storage.SetSecretMirrorStatusAsync(
|
||||
"node-b", [new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")], at);
|
||||
var collector = new SiteHealthCollector();
|
||||
var reporter = new SecretMirrorStatusReporter(
|
||||
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
||||
|
||||
await reporter.ProbeAsync();
|
||||
var report = collector.CollectReport("site-a");
|
||||
|
||||
Assert.NotNull(report.SecretMirrorStatuses);
|
||||
Assert.Equal(["node-a", "node-b"], report.SecretMirrorStatuses!.Select(s => s.NodeName));
|
||||
Assert.Equal("OPC-PWD", Assert.Single(report.SecretMirrorStatuses[1].Failures).SecretName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BeforeAnyProbe_TheReportFieldIsNull()
|
||||
{
|
||||
var report = new SiteHealthCollector().CollectReport("site-a");
|
||||
|
||||
Assert.Null(report.SecretMirrorStatuses);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_NoRowsYet_PushesAnEmptyList_NotNull()
|
||||
{
|
||||
var collector = new SiteHealthCollector();
|
||||
var reporter = new SecretMirrorStatusReporter(
|
||||
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
||||
|
||||
await reporter.ProbeAsync();
|
||||
|
||||
Assert.Empty(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ReadFailure_KeepsThePreviousSnapshot_AndDoesNotThrow()
|
||||
{
|
||||
await _storage.SetSecretMirrorStatusAsync("node-a", [], DateTimeOffset.UtcNow);
|
||||
var collector = new SiteHealthCollector();
|
||||
var reporter = new SecretMirrorStatusReporter(
|
||||
collector, _storage, NullLogger<SecretMirrorStatusReporter>.Instance);
|
||||
await reporter.ProbeAsync();
|
||||
|
||||
await using (var connection = _storage.CreateConnection())
|
||||
await using (var drop = connection.CreateCommand())
|
||||
{
|
||||
drop.CommandText = "DROP TABLE secret_mirror_status";
|
||||
await drop.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
await reporter.ProbeAsync();
|
||||
|
||||
// Point-in-time: not drained on collect, and not cleared by a failed read.
|
||||
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
||||
Assert.Single(collector.CollectReport("site-a").SecretMirrorStatuses!);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user