Joseph Doherty
0289d51137
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
370 lines
15 KiB
C#
370 lines
15 KiB
C#
using System.Text.Json;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Persistence;
|
|
|
|
/// <summary>
|
|
/// Task 8, site-secrets artifact deploy: site-local <c>deployed_secrets</c>
|
|
/// (LocalDb-replicated ciphertext) and <c>mirrored_secrets</c> (node-local ledger) stores.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Backed by a real temp-file LocalDb: <see cref="SiteStorageService"/> takes an
|
|
/// <c>ILocalDb</c> rather than a connection string, and LocalDb has no in-memory mode.
|
|
/// </remarks>
|
|
public class DeployedSecretsStorageTests : IAsyncLifetime, IDisposable
|
|
{
|
|
private readonly TestLocalDb _localDb;
|
|
private SiteStorageService _storage = null!;
|
|
|
|
public DeployedSecretsStorageTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("deployed-secrets");
|
|
}
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await _storage.InitializeAsync();
|
|
}
|
|
|
|
public Task DisposeAsync() => Task.CompletedTask;
|
|
|
|
public void Dispose()
|
|
{
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
// ── deployed_secrets: full-set upsert / soft-delete reconciliation ──
|
|
|
|
[Fact]
|
|
public async Task Upsert_IsFullSet_MarksAbsentRowsRemoved()
|
|
{
|
|
var s = _storage;
|
|
await s.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
|
|
var (applied, removed) = await s.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
|
|
|
|
Assert.Equal(["A"], applied);
|
|
Assert.Equal(["B"], removed);
|
|
|
|
var rows = await s.GetDeployedSecretsAsync();
|
|
Assert.Equal(2, rows.Single(r => r.Name == "A").Secret.Revision);
|
|
Assert.True(rows.Single(r => r.Name == "B").IsRemoved);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Upsert_FirstCall_AppliesEveryRow_AndRemovesNothing()
|
|
{
|
|
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
|
|
[Art("A", 1), Art("B", 1)], "d1");
|
|
|
|
Assert.Equal(["A", "B"], applied);
|
|
Assert.Empty(removed);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Upsert_ReAppliedRow_IsNotReportedAsRemovedTwice()
|
|
{
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
|
|
var (_, removedFirst) = await _storage.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
|
|
Assert.Equal(["B"], removedFirst);
|
|
|
|
// B stays removed and is not re-reported as "newly" removed on a later call that
|
|
// still omits it.
|
|
var (_, removedSecond) = await _storage.UpsertDeployedSecretsAsync([Art("A", 3)], "d3");
|
|
Assert.Empty(removedSecond);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Upsert_RowThatWasRemoved_CanBeReAppliedAndFlipsBackToNotRemoved()
|
|
{
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d2"); // B removed
|
|
|
|
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
|
|
[Art("A", 1), Art("B", 2)], "d3");
|
|
|
|
Assert.Equal(["A", "B"], applied);
|
|
Assert.Empty(removed);
|
|
|
|
var rows = await _storage.GetDeployedSecretsAsync();
|
|
var b = rows.Single(r => r.Name == "B");
|
|
Assert.False(b.IsRemoved);
|
|
Assert.Equal(2, b.Secret.Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Upsert_EmptySet_MarksEveryExistingRowRemoved()
|
|
{
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
|
|
|
|
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync([], "d2");
|
|
|
|
Assert.Empty(applied);
|
|
Assert.Equal(["A", "B"], removed.OrderBy(n => n, StringComparer.Ordinal));
|
|
|
|
var rows = await _storage.GetDeployedSecretsAsync();
|
|
Assert.All(rows, r => Assert.True(r.IsRemoved));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetDeployedSecretsAsync_RoundTripsEveryFieldOfTheSecretArtifact()
|
|
{
|
|
var art = Art("Full", 7);
|
|
await _storage.UpsertDeployedSecretsAsync([art], "dep-full");
|
|
|
|
var row = Assert.Single(await _storage.GetDeployedSecretsAsync());
|
|
|
|
Assert.Equal("Full", row.Name);
|
|
Assert.Equal("dep-full", row.DeploymentId);
|
|
Assert.False(row.IsRemoved);
|
|
Assert.Equal(art.Name, row.Secret.Name);
|
|
Assert.Equal(art.ContentType, row.Secret.ContentType);
|
|
Assert.Equal(art.Ciphertext, row.Secret.Ciphertext);
|
|
Assert.Equal(art.Nonce, row.Secret.Nonce);
|
|
Assert.Equal(art.Tag, row.Secret.Tag);
|
|
Assert.Equal(art.WrappedDek, row.Secret.WrappedDek);
|
|
Assert.Equal(art.WrapNonce, row.Secret.WrapNonce);
|
|
Assert.Equal(art.WrapTag, row.Secret.WrapTag);
|
|
Assert.Equal(art.KekId, row.Secret.KekId);
|
|
Assert.Equal(art.Revision, row.Secret.Revision);
|
|
Assert.Equal(art.CreatedUtc, row.Secret.CreatedUtc);
|
|
Assert.Equal(art.UpdatedUtc, row.Secret.UpdatedUtc);
|
|
Assert.Equal(art.Description, row.Secret.Description);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetDeployedSecretsAsync_IncludesRemovedRows()
|
|
{
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
|
|
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed
|
|
|
|
var rows = await _storage.GetDeployedSecretsAsync();
|
|
|
|
var a = Assert.Single(rows);
|
|
Assert.Equal("A", a.Name);
|
|
Assert.True(a.IsRemoved);
|
|
}
|
|
|
|
// ── security: row_json holds only base64 ciphertext, never plaintext ──
|
|
|
|
[Fact]
|
|
public async Task RowJson_DeserializesToSecretArtifact_WithNoExtraProperties()
|
|
{
|
|
// Guards against ever widening SecretArtifact (or row_json) with a plaintext
|
|
// field: this walks the RAW stored JSON, not the deserialized object, so a
|
|
// stray extra property would be caught here even though JsonSerializer.Deserialize
|
|
// silently ignores unknown members by default.
|
|
var art = Art("Sec1", 3);
|
|
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec");
|
|
|
|
var rowJson = await ReadRawRowJsonAsync("Sec1");
|
|
|
|
// SecretArtifact is a record, but records only get member-wise Equals for reference
|
|
// types via THEIR OWN Equals — and Array does not override Equals, so byte[] members
|
|
// compare by reference. Assert.Equal(art, roundTripped) would therefore fail even on
|
|
// a byte-for-byte-identical round trip; compare field-by-field instead.
|
|
var roundTripped = JsonSerializer.Deserialize<SecretArtifact>(rowJson);
|
|
Assert.NotNull(roundTripped);
|
|
Assert.Equal(art.Name, roundTripped!.Name);
|
|
Assert.Equal(art.ContentType, roundTripped.ContentType);
|
|
Assert.Equal(art.Ciphertext, roundTripped.Ciphertext);
|
|
Assert.Equal(art.Nonce, roundTripped.Nonce);
|
|
Assert.Equal(art.Tag, roundTripped.Tag);
|
|
Assert.Equal(art.WrappedDek, roundTripped.WrappedDek);
|
|
Assert.Equal(art.WrapNonce, roundTripped.WrapNonce);
|
|
Assert.Equal(art.WrapTag, roundTripped.WrapTag);
|
|
Assert.Equal(art.KekId, roundTripped.KekId);
|
|
Assert.Equal(art.Revision, roundTripped.Revision);
|
|
Assert.Equal(art.CreatedUtc, roundTripped.CreatedUtc);
|
|
Assert.Equal(art.UpdatedUtc, roundTripped.UpdatedUtc);
|
|
Assert.Equal(art.Description, roundTripped.Description);
|
|
|
|
using var document = JsonDocument.Parse(rowJson);
|
|
var propertyNames = document.RootElement.EnumerateObject()
|
|
.Select(p => p.Name)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var expected = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
nameof(SecretArtifact.Name), nameof(SecretArtifact.ContentType),
|
|
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
|
|
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
|
|
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
|
|
nameof(SecretArtifact.KekId), nameof(SecretArtifact.Revision),
|
|
nameof(SecretArtifact.CreatedUtc), nameof(SecretArtifact.UpdatedUtc),
|
|
nameof(SecretArtifact.Description),
|
|
};
|
|
|
|
Assert.Equal(expected, propertyNames);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RowJson_EncodesEveryByteArrayField_AsBase64Text()
|
|
{
|
|
// byte[] serializes as base64 via System.Text.Json by default. Asserted at the raw
|
|
// JSON level (JsonValueKind.String, not an array of numbers) so a future custom
|
|
// converter that changed the wire shape would be caught here.
|
|
var art = Art("Sec2", 1);
|
|
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec2");
|
|
|
|
var rowJson = await ReadRawRowJsonAsync("Sec2");
|
|
using var document = JsonDocument.Parse(rowJson);
|
|
var root = document.RootElement;
|
|
|
|
foreach (var byteArrayField in new[]
|
|
{
|
|
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
|
|
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
|
|
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
|
|
})
|
|
{
|
|
var property = root.GetProperty(byteArrayField);
|
|
Assert.Equal(JsonValueKind.String, property.ValueKind);
|
|
|
|
// Base64.TryFromBase64String throws on malformed input — proves it really is
|
|
// base64 text, not just any string.
|
|
Assert.True(Convert.TryFromBase64String(property.GetString()!, new byte[64], out _));
|
|
}
|
|
}
|
|
|
|
private async Task<string> ReadRawRowJsonAsync(string name)
|
|
{
|
|
await using var connection = _storage.CreateConnection();
|
|
await using var command = connection.CreateCommand();
|
|
command.CommandText = "SELECT row_json FROM deployed_secrets WHERE name = @name";
|
|
command.Parameters.AddWithValue("@name", name);
|
|
return (string)(await command.ExecuteScalarAsync())!;
|
|
}
|
|
|
|
// ── mirrored_secrets: node-local ledger ──
|
|
|
|
[Fact]
|
|
public async Task MirroredSecrets_SetThenGet_RoundTrips()
|
|
{
|
|
await _storage.SetMirroredSecretAsync("Sec1", 3);
|
|
|
|
var rows = await _storage.GetMirroredSecretsAsync();
|
|
|
|
var row = Assert.Single(rows);
|
|
Assert.Equal("Sec1", row.Name);
|
|
Assert.Equal(3, row.Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MirroredSecrets_Set_IsUpsert()
|
|
{
|
|
await _storage.SetMirroredSecretAsync("Sec1", 1);
|
|
await _storage.SetMirroredSecretAsync("Sec1", 2);
|
|
|
|
var rows = await _storage.GetMirroredSecretsAsync();
|
|
|
|
var row = Assert.Single(rows);
|
|
Assert.Equal(2, row.Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MirroredSecrets_Remove_DeletesTheLedgerRow()
|
|
{
|
|
await _storage.SetMirroredSecretAsync("Sec1", 1);
|
|
|
|
await _storage.RemoveMirroredSecretAsync("Sec1");
|
|
|
|
Assert.Empty(await _storage.GetMirroredSecretsAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MirroredSecrets_Remove_OfAnUnknownName_IsANoOp()
|
|
{
|
|
await _storage.RemoveMirroredSecretAsync("NeverMirrored");
|
|
|
|
Assert.Empty(await _storage.GetMirroredSecretsAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MirroredSecrets_IsIndependentOf_DeployedSecrets()
|
|
{
|
|
// mirroring is a separate, node-local action from the site-wide deployed_secrets
|
|
// reconcile — removing a deployed secret must not touch the mirrored ledger, and
|
|
// vice versa. Mirroring/un-mirroring in response to a removal is the caller's job.
|
|
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
|
|
await _storage.SetMirroredSecretAsync("A", 1);
|
|
|
|
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed from deployed_secrets
|
|
|
|
var deployed = Assert.Single(await _storage.GetDeployedSecretsAsync());
|
|
Assert.True(deployed.IsRemoved);
|
|
|
|
var mirrored = Assert.Single(await _storage.GetMirroredSecretsAsync());
|
|
Assert.Equal("A", mirrored.Name);
|
|
}
|
|
|
|
// ── secret_mirror_status (review finding M1) ──
|
|
|
|
[Fact]
|
|
public async Task SecretMirrorStatus_RoundTrips_OneRowPerNode_OrderedByNodeName()
|
|
{
|
|
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync(
|
|
"node-b", [new SecretMirrorFailure("A", "did not decrypt")], at);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], at.AddMinutes(1));
|
|
|
|
var rows = await _storage.GetSecretMirrorStatusesAsync();
|
|
|
|
Assert.Equal(["node-a", "node-b"], rows.Select(r => r.NodeName));
|
|
Assert.Empty(rows[0].Failures);
|
|
Assert.Equal(at.AddMinutes(1), rows[0].CheckedAtUtc);
|
|
var failure = Assert.Single(rows[1].Failures);
|
|
Assert.Equal(new SecretMirrorFailure("A", "did not decrypt"), failure);
|
|
Assert.Equal(at, rows[1].CheckedAtUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SecretMirrorStatus_Set_UpsertsTheNodesRow()
|
|
{
|
|
var at = new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [new SecretMirrorFailure("A", "x")], at);
|
|
await _storage.SetSecretMirrorStatusAsync("node-a", [], at.AddHours(1));
|
|
|
|
var row = Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Empty(row.Failures);
|
|
Assert.Equal(at.AddHours(1), row.CheckedAtUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SecretMirrorStatus_UnparseableRow_ReadsAsAFailure_NotAsHealthy()
|
|
{
|
|
await using (var connection = _storage.CreateConnection())
|
|
await using (var insert = connection.CreateCommand())
|
|
{
|
|
insert.CommandText =
|
|
"INSERT INTO secret_mirror_status (node_name, failures_json, checked_at) VALUES ('node-b', '{not json', '2026-09-28T00:00:00.0000000+00:00')";
|
|
await insert.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
var row = Assert.Single(await _storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Equal("node-b", row.NodeName);
|
|
Assert.Single(row.Failures);
|
|
}
|
|
|
|
private static SecretArtifact Art(string name, long revision) => new(
|
|
Name: name,
|
|
ContentType: "application/octet-stream",
|
|
Ciphertext: [1, 2, 3, 4, 5],
|
|
Nonce: [10, 11, 12],
|
|
Tag: [20, 21, 22, 23],
|
|
WrappedDek: [30, 31, 32, 33, 34, 35],
|
|
WrapNonce: [40, 41, 42],
|
|
WrapTag: [50, 51, 52, 53],
|
|
KekId: "kek-1",
|
|
Revision: revision,
|
|
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
|
|
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
|
|
Description: "test secret");
|
|
}
|