feat(site): deployed_secrets (LocalDb-replicated ciphertext) + node-local mirrored_secrets ledger

This commit is contained in:
Joseph Doherty
2026-09-28 18:36:49 -04:00
parent 437b02a43c
commit a35ee663c8
8 changed files with 662 additions and 22 deletions
@@ -104,6 +104,21 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
Assert.DoesNotContain(triggers, t => t.StartsWith("__localdb_smtp_configurations_", StringComparison.Ordinal));
}
[Fact]
public void ReplicatedNode_InstallsNoCaptureTriggersOnTheNodeLocalMirroredSecretsLedger()
{
// mirrored_secrets records which names THIS node has mirrored into its own secret
// store — per-node fact, not site-wide state. Capturing it would replicate one node's
// mirroring progress onto the other, so it must stay uncaptured even on a fully
// replicated node.
var db = BuildDatabase(Config(apiKey: "cdc-test-key"));
var triggers = CaptureTriggers(db);
Assert.DoesNotContain(triggers, t => t.StartsWith("__localdb_mirrored_secrets_", StringComparison.Ordinal));
Assert.DoesNotContain("mirrored_secrets", db.ReplicatedTables.Keys);
}
[Fact]
public void UnreplicatedNode_DropsTriggersLeftByAPreviouslyReplicatedBuild()
{
@@ -196,13 +211,18 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
// ---- helpers ----------------------------------------------------------------------
/// <summary>The ten replicated tables plus the two deliberately-unregistered ones.</summary>
/// <summary>
/// The eleven replicated tables, plus the two deliberately-unregistered central-only
/// notification tables, plus the node-local (also deliberately-unregistered)
/// <c>mirrored_secrets</c> ledger.
/// </summary>
private static readonly string[] AllSiteTables =
[
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state",
"notification_lists", "smtp_configurations",
"deployed_secrets",
"notification_lists", "smtp_configurations", "mirrored_secrets",
];
private static readonly string[] ReplicatedTables =
@@ -210,6 +230,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
"OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state",
"deployed_secrets",
];
private static void AssertCaptureTriggersCoverTheReplicatedTables(ILocalDb db)
@@ -15,7 +15,8 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <c>RegisterReplicated</c> time, which on a site node is a BOOT failure of the replicating
/// pair. This test moves that discovery from the first replicated boot to the build: it walks
/// the real site schema (every table <c>SiteLocalDbSetup.OnReady</c> creates) and asserts none
/// of the ten <see cref="SiteLocalDbSetup.ReplicatedTables"/> has such an inbound FK.
/// of the eleven <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
/// added for site-secrets artifact deploy) has such an inbound FK.
/// </summary>
/// <remarks>
/// The schema is built with an UNREPLICATED config on purpose: a replicated one would make
@@ -245,7 +245,7 @@ public class SiteLocalDbWiringTests : IDisposable
var tables = TableNames(db);
// One from each Phase 1 schema, then every Phase 2 table.
// One from each Phase 1 schema, then every Phase 2 table plus the site-secrets tables.
Assert.Contains("OperationTracking", tables);
Assert.Contains("site_events", tables);
Assert.Contains("sf_messages", tables);
@@ -254,6 +254,7 @@ public class SiteLocalDbWiringTests : IDisposable
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "notification_lists",
"data_connection_definitions", "smtp_configurations", "native_alarm_state",
"deployed_secrets", "mirrored_secrets",
})
{
Assert.Contains(table, tables);
@@ -261,19 +262,21 @@ public class SiteLocalDbWiringTests : IDisposable
}
[Fact]
public void Site_LocalDb_RegistersExactlyTheTenReplicatedTables()
public void Site_LocalDb_RegistersExactlyTheElevenReplicatedTables()
{
// The Phase 2 cutover. This is an EXACTLY check rather than a Contains in both
// directions, and both directions are load-bearing.
// The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets.
// This is an EXACTLY check rather than a Contains in both directions, and both
// directions are load-bearing.
//
// Too few: the table silently stops replicating. Storage still works, every other
// test still passes, and the pair just quietly diverges — the failure mode Phase 1
// existed to end.
//
// Too many: notification_lists and smtp_configurations must NOT be here. They are
// permanently empty by design, and registering them would open a standing
// replication channel whose only historical payload was plaintext SMTP passwords.
// A Contains-based test would never catch that.
// Too many: notification_lists, smtp_configurations, and mirrored_secrets must NOT
// be here. notification_lists/smtp_configurations are permanently empty by design
// (their only historical payload was a plaintext SMTP password); mirrored_secrets is
// node-local fact (which names THIS node has mirrored), not site-wide state. A
// Contains-based test would never catch either.
var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.Equal(
@@ -281,8 +284,9 @@ public class SiteLocalDbWiringTests : IDisposable
// Ordinal order: '_' (0x5F) sorts before 'b' (0x62), so
// data_connection_definitions precedes database_connections.
"OperationTracking", "data_connection_definitions", "database_connections",
"deployed_configurations", "external_systems", "native_alarm_state",
"sf_messages", "shared_scripts", "site_events", "static_attribute_overrides",
"deployed_configurations", "deployed_secrets", "external_systems",
"native_alarm_state", "sf_messages", "shared_scripts", "site_events",
"static_attribute_overrides",
],
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray());
}
@@ -300,6 +304,17 @@ public class SiteLocalDbWiringTests : IDisposable
Assert.DoesNotContain("smtp_configurations", db.ReplicatedTables.Keys);
}
[Fact]
public void Site_LocalDb_DoesNotReplicateTheNodeLocalMirroredSecretsLedger()
{
// mirrored_secrets exists (SiteStorageSchema creates it) but must never be captured:
// it records which names THIS node has mirrored into its own secret store, which is
// per-node fact, not site-wide state — see SiteLocalDbSetup.ReplicatedTables.
var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.DoesNotContain("mirrored_secrets", db.ReplicatedTables.Keys);
}
[Fact]
public void Site_LocalDb_ReplicatedPhase2TablesHaveTheirExpectedPrimaryKeys()
{
@@ -318,6 +333,7 @@ public class SiteLocalDbWiringTests : IDisposable
Assert.Equal(
["instance_unique_name", "source_canonical_name", "source_reference"],
db.ReplicatedTables["native_alarm_state"].PkColumns);
Assert.Equal(["name"], db.ReplicatedTables["deployed_secrets"].PkColumns);
}
private static HashSet<string> TableNames(ILocalDb db)
@@ -0,0 +1,320 @@
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Persistence;
/// <summary>
/// Task 8, site-secrets artifact deploy: site-local <c>deployed_secrets</c>
/// (LocalDb-replicated ciphertext) and <c>mirrored_secrets</c> (node-local ledger) stores.
/// </summary>
/// <remarks>
/// Backed by a real temp-file LocalDb: <see cref="SiteStorageService"/> takes an
/// <c>ILocalDb</c> rather than a connection string, and LocalDb has no in-memory mode.
/// </remarks>
public class DeployedSecretsStorageTests : IAsyncLifetime, IDisposable
{
private readonly TestLocalDb _localDb;
private SiteStorageService _storage = null!;
public DeployedSecretsStorageTests()
{
_localDb = TestLocalDb.CreateTemp("deployed-secrets");
}
public async Task InitializeAsync()
{
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
await _storage.InitializeAsync();
}
public Task DisposeAsync() => Task.CompletedTask;
public void Dispose()
{
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
// ── deployed_secrets: full-set upsert / soft-delete reconciliation ──
[Fact]
public async Task Upsert_IsFullSet_MarksAbsentRowsRemoved()
{
var s = _storage;
await s.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (applied, removed) = await s.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
Assert.Equal(["A"], applied);
Assert.Equal(["B"], removed);
var rows = await s.GetDeployedSecretsAsync();
Assert.Equal(2, rows.Single(r => r.Name == "A").Secret.Revision);
Assert.True(rows.Single(r => r.Name == "B").IsRemoved);
}
[Fact]
public async Task Upsert_FirstCall_AppliesEveryRow_AndRemovesNothing()
{
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
[Art("A", 1), Art("B", 1)], "d1");
Assert.Equal(["A", "B"], applied);
Assert.Empty(removed);
}
[Fact]
public async Task Upsert_ReAppliedRow_IsNotReportedAsRemovedTwice()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (_, removedFirst) = await _storage.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
Assert.Equal(["B"], removedFirst);
// B stays removed and is not re-reported as "newly" removed on a later call that
// still omits it.
var (_, removedSecond) = await _storage.UpsertDeployedSecretsAsync([Art("A", 3)], "d3");
Assert.Empty(removedSecond);
}
[Fact]
public async Task Upsert_RowThatWasRemoved_CanBeReAppliedAndFlipsBackToNotRemoved()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d2"); // B removed
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
[Art("A", 1), Art("B", 2)], "d3");
Assert.Equal(["A", "B"], applied);
Assert.Empty(removed);
var rows = await _storage.GetDeployedSecretsAsync();
var b = rows.Single(r => r.Name == "B");
Assert.False(b.IsRemoved);
Assert.Equal(2, b.Secret.Revision);
}
[Fact]
public async Task Upsert_EmptySet_MarksEveryExistingRowRemoved()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync([], "d2");
Assert.Empty(applied);
Assert.Equal(["A", "B"], removed.OrderBy(n => n, StringComparer.Ordinal));
var rows = await _storage.GetDeployedSecretsAsync();
Assert.All(rows, r => Assert.True(r.IsRemoved));
}
[Fact]
public async Task GetDeployedSecretsAsync_RoundTripsEveryFieldOfTheSecretArtifact()
{
var art = Art("Full", 7);
await _storage.UpsertDeployedSecretsAsync([art], "dep-full");
var row = Assert.Single(await _storage.GetDeployedSecretsAsync());
Assert.Equal("Full", row.Name);
Assert.Equal("dep-full", row.DeploymentId);
Assert.False(row.IsRemoved);
Assert.Equal(art.Name, row.Secret.Name);
Assert.Equal(art.ContentType, row.Secret.ContentType);
Assert.Equal(art.Ciphertext, row.Secret.Ciphertext);
Assert.Equal(art.Nonce, row.Secret.Nonce);
Assert.Equal(art.Tag, row.Secret.Tag);
Assert.Equal(art.WrappedDek, row.Secret.WrappedDek);
Assert.Equal(art.WrapNonce, row.Secret.WrapNonce);
Assert.Equal(art.WrapTag, row.Secret.WrapTag);
Assert.Equal(art.KekId, row.Secret.KekId);
Assert.Equal(art.Revision, row.Secret.Revision);
Assert.Equal(art.CreatedUtc, row.Secret.CreatedUtc);
Assert.Equal(art.UpdatedUtc, row.Secret.UpdatedUtc);
Assert.Equal(art.Description, row.Secret.Description);
}
[Fact]
public async Task GetDeployedSecretsAsync_IncludesRemovedRows()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed
var rows = await _storage.GetDeployedSecretsAsync();
var a = Assert.Single(rows);
Assert.Equal("A", a.Name);
Assert.True(a.IsRemoved);
}
// ── security: row_json holds only base64 ciphertext, never plaintext ──
[Fact]
public async Task RowJson_DeserializesToSecretArtifact_WithNoExtraProperties()
{
// Guards against ever widening SecretArtifact (or row_json) with a plaintext
// field: this walks the RAW stored JSON, not the deserialized object, so a
// stray extra property would be caught here even though JsonSerializer.Deserialize
// silently ignores unknown members by default.
var art = Art("Sec1", 3);
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec");
var rowJson = await ReadRawRowJsonAsync("Sec1");
// SecretArtifact is a record, but records only get member-wise Equals for reference
// types via THEIR OWN Equals — and Array does not override Equals, so byte[] members
// compare by reference. Assert.Equal(art, roundTripped) would therefore fail even on
// a byte-for-byte-identical round trip; compare field-by-field instead.
var roundTripped = JsonSerializer.Deserialize<SecretArtifact>(rowJson);
Assert.NotNull(roundTripped);
Assert.Equal(art.Name, roundTripped!.Name);
Assert.Equal(art.ContentType, roundTripped.ContentType);
Assert.Equal(art.Ciphertext, roundTripped.Ciphertext);
Assert.Equal(art.Nonce, roundTripped.Nonce);
Assert.Equal(art.Tag, roundTripped.Tag);
Assert.Equal(art.WrappedDek, roundTripped.WrappedDek);
Assert.Equal(art.WrapNonce, roundTripped.WrapNonce);
Assert.Equal(art.WrapTag, roundTripped.WrapTag);
Assert.Equal(art.KekId, roundTripped.KekId);
Assert.Equal(art.Revision, roundTripped.Revision);
Assert.Equal(art.CreatedUtc, roundTripped.CreatedUtc);
Assert.Equal(art.UpdatedUtc, roundTripped.UpdatedUtc);
Assert.Equal(art.Description, roundTripped.Description);
using var document = JsonDocument.Parse(rowJson);
var propertyNames = document.RootElement.EnumerateObject()
.Select(p => p.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var expected = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
nameof(SecretArtifact.Name), nameof(SecretArtifact.ContentType),
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
nameof(SecretArtifact.KekId), nameof(SecretArtifact.Revision),
nameof(SecretArtifact.CreatedUtc), nameof(SecretArtifact.UpdatedUtc),
nameof(SecretArtifact.Description),
};
Assert.Equal(expected, propertyNames);
}
[Fact]
public async Task RowJson_EncodesEveryByteArrayField_AsBase64Text()
{
// byte[] serializes as base64 via System.Text.Json by default. Asserted at the raw
// JSON level (JsonValueKind.String, not an array of numbers) so a future custom
// converter that changed the wire shape would be caught here.
var art = Art("Sec2", 1);
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec2");
var rowJson = await ReadRawRowJsonAsync("Sec2");
using var document = JsonDocument.Parse(rowJson);
var root = document.RootElement;
foreach (var byteArrayField in new[]
{
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
})
{
var property = root.GetProperty(byteArrayField);
Assert.Equal(JsonValueKind.String, property.ValueKind);
// Base64.TryFromBase64String throws on malformed input — proves it really is
// base64 text, not just any string.
Assert.True(Convert.TryFromBase64String(property.GetString()!, new byte[64], out _));
}
}
private async Task<string> ReadRawRowJsonAsync(string name)
{
await using var connection = _storage.CreateConnection();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT row_json FROM deployed_secrets WHERE name = @name";
command.Parameters.AddWithValue("@name", name);
return (string)(await command.ExecuteScalarAsync())!;
}
// ── mirrored_secrets: node-local ledger ──
[Fact]
public async Task MirroredSecrets_SetThenGet_RoundTrips()
{
await _storage.SetMirroredSecretAsync("Sec1", 3);
var rows = await _storage.GetMirroredSecretsAsync();
var row = Assert.Single(rows);
Assert.Equal("Sec1", row.Name);
Assert.Equal(3, row.Revision);
}
[Fact]
public async Task MirroredSecrets_Set_IsUpsert()
{
await _storage.SetMirroredSecretAsync("Sec1", 1);
await _storage.SetMirroredSecretAsync("Sec1", 2);
var rows = await _storage.GetMirroredSecretsAsync();
var row = Assert.Single(rows);
Assert.Equal(2, row.Revision);
}
[Fact]
public async Task MirroredSecrets_Remove_DeletesTheLedgerRow()
{
await _storage.SetMirroredSecretAsync("Sec1", 1);
await _storage.RemoveMirroredSecretAsync("Sec1");
Assert.Empty(await _storage.GetMirroredSecretsAsync());
}
[Fact]
public async Task MirroredSecrets_Remove_OfAnUnknownName_IsANoOp()
{
await _storage.RemoveMirroredSecretAsync("NeverMirrored");
Assert.Empty(await _storage.GetMirroredSecretsAsync());
}
[Fact]
public async Task MirroredSecrets_IsIndependentOf_DeployedSecrets()
{
// mirroring is a separate, node-local action from the site-wide deployed_secrets
// reconcile — removing a deployed secret must not touch the mirrored ledger, and
// vice versa. Mirroring/un-mirroring in response to a removal is the caller's job.
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
await _storage.SetMirroredSecretAsync("A", 1);
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed from deployed_secrets
var deployed = Assert.Single(await _storage.GetDeployedSecretsAsync());
Assert.True(deployed.IsRemoved);
var mirrored = Assert.Single(await _storage.GetMirroredSecretsAsync());
Assert.Equal("A", mirrored.Name);
}
private static SecretArtifact Art(string name, long revision) => new(
Name: name,
ContentType: "application/octet-stream",
Ciphertext: [1, 2, 3, 4, 5],
Nonce: [10, 11, 12],
Tag: [20, 21, 22, 23],
WrappedDek: [30, 31, 32, 33, 34, 35],
WrapNonce: [40, 41, 42],
WrapTag: [50, 51, 52, 53],
KekId: "kek-1",
Revision: revision,
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
Description: "test secret");
}
@@ -23,13 +23,14 @@ public class SiteStorageSchemaTests
SiteStorageSchema.Apply(connection);
SiteStorageSchema.Apply(connection); // second run must not throw
// Every table is named explicitly on purpose — a loop asserting "9 tables"
// Every table is named explicitly on purpose — a loop asserting "11 tables"
// would still pass if one were renamed.
foreach (var table in new[]
{
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "notification_lists",
"data_connection_definitions", "smtp_configurations", "native_alarm_state",
"deployed_secrets", "mirrored_secrets",
})
{
Assert.True(TableExists(connection, table), $"missing table: {table}");