feat(site): deployed_secrets (LocalDb-replicated ciphertext) + node-local mirrored_secrets ledger

This commit is contained in:
Joseph Doherty
2026-09-28 18:36:49 -04:00
parent 437b02a43c
commit a35ee663c8
8 changed files with 662 additions and 22 deletions
@@ -48,14 +48,27 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
public static class SiteLocalDbSetup public static class SiteLocalDbSetup
{ {
/// <summary> /// <summary>
/// The ten tables that participate in replication — registered when this node has a peer, /// The eleven tables that participate in replication — registered when this node has a peer,
/// and deregistered (stale triggers dropped) when it does not. /// and deregistered (stale triggers dropped) when it does not.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// <para>
/// <c>notification_lists</c> and <c>smtp_configurations</c> are created by the schema but /// <c>notification_lists</c> and <c>smtp_configurations</c> are created by the schema but
/// deliberately absent here; see the rationale in <see cref="OnReady"/>. The list is shared by /// deliberately absent here; see the rationale in <see cref="OnReady"/>. The list is shared by
/// both branches on purpose: a table added to one and not the other would either register /// both branches on purpose: a table added to one and not the other would either register
/// without ever being cleaned up, or be cleaned up on a node that never registered it. /// without ever being cleaned up, or be cleaned up on a node that never registered it.
/// </para>
/// <para>
/// <c>deployed_secrets</c> (ciphertext-only, see <c>SecretArtifact</c>) is the eleventh
/// entry, added for site-secrets artifact deploy. It has an explicit primary key
/// (<c>name</c>) and no FK, so it registers like any other member of this list.
/// <c>mirrored_secrets</c> is deliberately NOT here, for the same reason
/// <c>notification_lists</c>/<c>smtp_configurations</c> are not: which names THIS node has
/// mirrored into its own local secret store is per-node fact, not site-wide state — the
/// two site nodes can be at different points in mirroring the same replicated
/// <c>deployed_secrets</c> row, and replicating the ledger would make one node's progress
/// masquerade as the other's.
/// </para>
/// </remarks> /// </remarks>
internal static readonly string[] ReplicatedTables = internal static readonly string[] ReplicatedTables =
[ [
@@ -69,6 +82,7 @@ public static class SiteLocalDbSetup
"database_connections", "database_connections",
"data_connection_definitions", "data_connection_definitions",
"native_alarm_state", "native_alarm_state",
"deployed_secrets",
]; ];
/// <summary> /// <summary>
@@ -122,11 +136,12 @@ public static class SiteLocalDbSetup
// //
// The PER-TABLE loop is load-bearing as of LocalDb 0.3.0, not just tidy. The snapshot // The PER-TABLE loop is load-bearing as of LocalDb 0.3.0, not just tidy. The snapshot
// flag baselining raises is now per-table, and it is raised by the call that actually // flag baselining raises is now per-table, and it is raised by the call that actually
// seeded something — so adding an ELEVENTH table to ReplicatedTables on an already // seeded something — so site-secrets artifact deploy adding deployed_secrets as an
// replicating site now streams a snapshot of that one table, where through 0.2.x the // ELEVENTH entry to ReplicatedTables on an already-replicating site streams a
// flag was a single per-database bit and the same edit re-streamed all eleven in full, // snapshot of only that one table, where through 0.2.x the flag was a single
// in both directions. Nothing about the first boot changes: enabling replication on a // per-database bit and the same edit would have re-streamed all eleven in full, in
// site that has been running without it seeds all ten at once, the flagged set is // both directions. Nothing about the first boot changes: enabling replication on a
// site that has been running without it seeds all eleven at once, the flagged set is
// therefore every registered table, and the library sends that as an ordinary full // therefore every registered table, and the library sends that as an ordinary full
// snapshot. A hypothetical "register the whole list in one call" API would silently // snapshot. A hypothetical "register the whole list in one call" API would silently
// give the coarse behaviour back. // give the coarse behaviour back.
@@ -3,7 +3,8 @@ using Microsoft.Data.Sqlite;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
/// <summary> /// <summary>
/// DDL for the site's nine configuration tables, extracted from /// DDL for the site's eleven configuration tables (nine original plus
/// <c>deployed_secrets</c> and <c>mirrored_secrets</c>), extracted from
/// <see cref="SiteStorageService"/> so it can be applied by whoever owns the database /// <see cref="SiteStorageService"/> so it can be applied by whoever owns the database
/// file. /// file.
/// <para> /// <para>
@@ -22,7 +23,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
public static class SiteStorageSchema public static class SiteStorageSchema
{ {
/// <summary> /// <summary>
/// Creates the nine site configuration tables when absent, and additively upgrades /// Creates the eleven site configuration tables when absent, and additively upgrades
/// tables created by an older build. Idempotent — safe to run on every startup. /// tables created by an older build. Idempotent — safe to run on every startup.
/// </summary> /// </summary>
/// <param name="connection">An open connection to the database that should hold the tables.</param> /// <param name="connection">An open connection to the database that should hold the tables.</param>
@@ -111,6 +112,30 @@ public static class SiteStorageSchema
metadata_json TEXT, metadata_json TEXT,
PRIMARY KEY (instance_unique_name, source_canonical_name, source_reference) PRIMARY KEY (instance_unique_name, source_canonical_name, source_reference)
); );
-- Deployed secrets (ciphertext only — see SecretArtifact). Replicated between the
-- two site nodes by LocalDb CDC (SiteLocalDbSetup.ReplicatedTables). A full-set
-- upsert soft-deletes rows absent from the incoming set via is_removed, never a
-- mass DELETE on a replicated table (that would ship as a destructive resync).
CREATE TABLE IF NOT EXISTS deployed_secrets (
name TEXT PRIMARY KEY,
row_json TEXT NOT NULL,
revision INTEGER NOT NULL,
updated_utc TEXT NOT NULL,
is_removed INTEGER NOT NULL DEFAULT 0,
deployment_id TEXT NOT NULL,
applied_at TEXT NOT NULL
);
-- Node-local ledger of which secret names THIS node has mirrored into its own
-- secret store. Deliberately NOT in SiteLocalDbSetup.ReplicatedTables: mirroring
-- is a per-node action against a per-node secret store, so which names are
-- mirrored (and at what revision) is node-specific, not site-wide state.
CREATE TABLE IF NOT EXISTS mirrored_secrets (
name TEXT PRIMARY KEY,
revision INTEGER NOT NULL,
mirrored_at TEXT NOT NULL
);
"; ";
command.ExecuteNonQuery(); command.ExecuteNonQuery();
} }
@@ -1,6 +1,8 @@
using System.Text.Json;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
@@ -658,6 +660,230 @@ public class SiteStorageService
_logger.LogDebug("Cleared native alarm state for {Instance}", instanceName); _logger.LogDebug("Cleared native alarm state for {Instance}", instanceName);
} }
// ── Deployed Secrets store (LocalDb-replicated ciphertext) ──
/// <summary>
/// Full-set upsert of the site's deployed secrets: every <see cref="SecretArtifact"/> in
/// <paramref name="secrets"/> is written (<c>is_removed = 0</c>), and every row not named
/// in the set is soft-deleted (<c>is_removed = 1</c>) rather than actually removed.
/// </summary>
/// <remarks>
/// <para>
/// Central ships the complete set of secrets referenced by a site's artifacts on every
/// artifact deploy, mirroring the reconcile contract of
/// <see cref="DeleteRowsExceptAsync"/> for the other artifact tables. Unlike those tables,
/// <c>deployed_secrets</c> is never hard-deleted: it is one of
/// <c>SiteLocalDbSetup.ReplicatedTables</c>, and a mass <c>DELETE</c> on a replicated table
/// would be captured and shipped to the peer as a destructive resync — the same hazard
/// <c>StoreAndForwardStorage.ReplaceAllAsync</c> was deleted for in LocalDb Phase 2. A row
/// that a later deploy re-adds simply flips back to <c>is_removed = 0</c>.
/// </para>
/// <para>
/// <c>row_json</c> is <c>JsonSerializer.Serialize(secret)</c> — the <see cref="SecretArtifact"/>
/// record verbatim, so its <c>byte[]</c> ciphertext fields serialize as base64 and nothing
/// but ciphertext ever lands in this column.
/// </para>
/// </remarks>
/// <param name="secrets">The complete set of secrets that should exist on this site.</param>
/// <param name="deploymentId">The deployment identifier that applied this set.</param>
/// <returns>
/// A task that resolves to the names that were applied (inserted or updated) and the
/// names that were newly marked removed by this call (rows that were previously
/// <c>is_removed = 0</c>; a name already removed by an earlier call is not repeated here).
/// </returns>
public async Task<(IReadOnlyList<string> Applied, IReadOnlyList<string> Removed)> UpsertDeployedSecretsAsync(
IReadOnlyList<SecretArtifact> secrets, string deploymentId)
{
ArgumentNullException.ThrowIfNull(secrets);
ArgumentException.ThrowIfNullOrWhiteSpace(deploymentId);
await using var connection = OpenConnection();
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync();
var appliedAt = DateTimeOffset.UtcNow.ToString("O");
var applied = new List<string>(secrets.Count);
await using (var upsert = connection.CreateCommand())
{
upsert.Transaction = transaction;
upsert.CommandText = @"
INSERT INTO deployed_secrets (name, row_json, revision, updated_utc, is_removed, deployment_id, applied_at)
VALUES (@name, @json, @revision, @updatedUtc, 0, @deploymentId, @appliedAt)
ON CONFLICT(name) DO UPDATE SET
row_json = excluded.row_json,
revision = excluded.revision,
updated_utc = excluded.updated_utc,
is_removed = 0,
deployment_id = excluded.deployment_id,
applied_at = excluded.applied_at";
var pName = upsert.Parameters.Add("@name", SqliteType.Text);
var pJson = upsert.Parameters.Add("@json", SqliteType.Text);
var pRevision = upsert.Parameters.Add("@revision", SqliteType.Integer);
var pUpdatedUtc = upsert.Parameters.Add("@updatedUtc", SqliteType.Text);
var pDeploymentId = upsert.Parameters.Add("@deploymentId", SqliteType.Text);
var pAppliedAt = upsert.Parameters.Add("@appliedAt", SqliteType.Text);
pDeploymentId.Value = deploymentId;
pAppliedAt.Value = appliedAt;
foreach (var secret in secrets)
{
pName.Value = secret.Name;
pJson.Value = JsonSerializer.Serialize(secret);
pRevision.Value = secret.Revision;
pUpdatedUtc.Value = secret.UpdatedUtc.ToString("O");
await upsert.ExecuteNonQueryAsync();
applied.Add(secret.Name);
}
}
// Names present in the incoming set are never candidates for removal, including
// one that was previously soft-deleted and has just been re-applied above.
var keepParams = secrets.Select((s, i) => (Key: $"@k{i}", Value: s.Name)).ToList();
var predicate = keepParams.Count == 0
? " WHERE is_removed = 0"
: $" WHERE is_removed = 0 AND name NOT IN ({string.Join(", ", keepParams.Select(p => p.Key))})";
var removed = new List<string>();
await using (var select = connection.CreateCommand())
{
select.Transaction = transaction;
select.CommandText = $"SELECT name FROM deployed_secrets{predicate}";
foreach (var (key, value) in keepParams)
select.Parameters.AddWithValue(key, value);
await using var reader = await select.ExecuteReaderAsync();
while (await reader.ReadAsync())
removed.Add(reader.GetString(0));
}
if (removed.Count > 0)
{
await using var markRemoved = connection.CreateCommand();
markRemoved.Transaction = transaction;
markRemoved.CommandText = $"UPDATE deployed_secrets SET is_removed = 1{predicate}";
foreach (var (key, value) in keepParams)
markRemoved.Parameters.AddWithValue(key, value);
await markRemoved.ExecuteNonQueryAsync();
}
await transaction.CommitAsync();
_logger.LogInformation(
"Deployed secrets reconcile for deployment {DeploymentId}: applied {AppliedCount}, newly removed {RemovedCount}",
deploymentId, applied.Count, removed.Count);
return (applied, removed);
}
/// <summary>
/// Returns every row in <c>deployed_secrets</c>, including soft-deleted (<c>is_removed</c>)
/// ones, with each row's <see cref="SecretArtifact"/> round-tripped from <c>row_json</c>.
/// </summary>
/// <returns>A task that resolves to every stored deployed-secret row.</returns>
public async Task<IReadOnlyList<DeployedSecretRow>> GetDeployedSecretsAsync()
{
await using var connection = OpenConnection();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT name, row_json, is_removed, deployment_id FROM deployed_secrets";
var results = new List<DeployedSecretRow>();
await using var reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
var name = reader.GetString(0);
var rowJson = reader.GetString(1);
var secret = JsonSerializer.Deserialize<SecretArtifact>(rowJson)
?? throw new InvalidOperationException($"deployed_secrets row '{name}' has invalid row_json");
results.Add(new DeployedSecretRow(
name,
secret,
reader.GetInt64(2) != 0,
reader.GetString(3)));
}
return results;
}
// ── Mirrored Secrets ledger (node-local only — deliberately NOT LocalDb-replicated) ──
/// <summary>
/// Returns every name this node has recorded as mirrored into its own secret store.
/// </summary>
/// <remarks>
/// <c>mirrored_secrets</c> is not in <c>SiteLocalDbSetup.ReplicatedTables</c> on purpose:
/// mirroring writes a name into THIS node's own local secret store, so which names are
/// mirrored — and at what revision — is per-node fact, not site-wide state. Replicating it
/// would tell this node's ledger about the OTHER node's mirroring progress, which is not
/// what it records.
/// </remarks>
/// <returns>A task that resolves to every mirrored-secret ledger row.</returns>
public async Task<IReadOnlyList<MirroredSecretRow>> GetMirroredSecretsAsync()
{
await using var connection = OpenConnection();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT name, revision, mirrored_at FROM mirrored_secrets";
var results = new List<MirroredSecretRow>();
await using var reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
results.Add(new MirroredSecretRow(
reader.GetString(0),
reader.GetInt64(1),
DateTimeOffset.Parse(reader.GetString(2), null, System.Globalization.DateTimeStyles.RoundtripKind)));
}
return results;
}
/// <summary>
/// Records that this node has mirrored <paramref name="name"/> at <paramref name="revision"/>
/// into its own secret store. Uses UPSERT semantics.
/// </summary>
/// <param name="name">The mirrored secret's name.</param>
/// <param name="revision">The revision that was mirrored.</param>
/// <returns>A task that completes when the ledger row has been stored or updated.</returns>
public async Task SetMirroredSecretAsync(string name, long revision)
{
await using var connection = OpenConnection();
await using var command = connection.CreateCommand();
command.CommandText = @"
INSERT INTO mirrored_secrets (name, revision, mirrored_at)
VALUES (@name, @revision, @mirroredAt)
ON CONFLICT(name) DO UPDATE SET
revision = excluded.revision,
mirrored_at = excluded.mirrored_at";
command.Parameters.AddWithValue("@name", name);
command.Parameters.AddWithValue("@revision", revision);
command.Parameters.AddWithValue("@mirroredAt", DateTimeOffset.UtcNow.ToString("O"));
await command.ExecuteNonQueryAsync();
}
/// <summary>
/// Removes the mirrored-secret ledger row for <paramref name="name"/> (e.g. once the
/// secret is no longer deployed to this site and this node has un-mirrored it).
/// </summary>
/// <param name="name">The mirrored secret's name.</param>
/// <returns>A task that completes when the ledger row has been deleted.</returns>
public async Task RemoveMirroredSecretAsync(string name)
{
await using var connection = OpenConnection();
await using var command = connection.CreateCommand();
command.CommandText = "DELETE FROM mirrored_secrets WHERE name = @name";
command.Parameters.AddWithValue("@name", name);
await command.ExecuteNonQueryAsync();
}
// ── Shared Script CRUD ── // ── Shared Script CRUD ──
/// <summary> /// <summary>
@@ -1095,3 +1321,18 @@ public class StoredDataConnectionDefinition
/// </summary> /// </summary>
public record NativeAlarmRow( public record NativeAlarmRow(
string SourceReference, string ConditionJson, DateTimeOffset LastTransitionAt, string? MetadataJson = null); string SourceReference, string ConditionJson, DateTimeOffset LastTransitionAt, string? MetadataJson = null);
/// <summary>
/// A single row from the site-local, LocalDb-replicated <c>deployed_secrets</c> table.
/// <paramref name="Secret"/> is the <see cref="SecretArtifact"/> round-tripped from
/// <c>row_json</c> — ciphertext only, never plaintext. <paramref name="IsRemoved"/> is true
/// for a soft-deleted row (see <see cref="SiteStorageService.UpsertDeployedSecretsAsync"/>);
/// the row is retained, not hard-deleted, because this table replicates.
/// </summary>
public record DeployedSecretRow(string Name, SecretArtifact Secret, bool IsRemoved, string DeploymentId);
/// <summary>
/// A single row from the site-local, node-local (NOT LocalDb-replicated) <c>mirrored_secrets</c>
/// ledger — the names this node has mirrored into its own secret store, and at what revision.
/// </summary>
public record MirroredSecretRow(string Name, long Revision, DateTimeOffset MirroredAt);
@@ -104,6 +104,21 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
Assert.DoesNotContain(triggers, t => t.StartsWith("__localdb_smtp_configurations_", StringComparison.Ordinal)); Assert.DoesNotContain(triggers, t => t.StartsWith("__localdb_smtp_configurations_", StringComparison.Ordinal));
} }
[Fact]
public void ReplicatedNode_InstallsNoCaptureTriggersOnTheNodeLocalMirroredSecretsLedger()
{
// mirrored_secrets records which names THIS node has mirrored into its own secret
// store — per-node fact, not site-wide state. Capturing it would replicate one node's
// mirroring progress onto the other, so it must stay uncaptured even on a fully
// replicated node.
var db = BuildDatabase(Config(apiKey: "cdc-test-key"));
var triggers = CaptureTriggers(db);
Assert.DoesNotContain(triggers, t => t.StartsWith("__localdb_mirrored_secrets_", StringComparison.Ordinal));
Assert.DoesNotContain("mirrored_secrets", db.ReplicatedTables.Keys);
}
[Fact] [Fact]
public void UnreplicatedNode_DropsTriggersLeftByAPreviouslyReplicatedBuild() public void UnreplicatedNode_DropsTriggersLeftByAPreviouslyReplicatedBuild()
{ {
@@ -196,13 +211,18 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
// ---- helpers ---------------------------------------------------------------------- // ---- helpers ----------------------------------------------------------------------
/// <summary>The ten replicated tables plus the two deliberately-unregistered ones.</summary> /// <summary>
/// The eleven replicated tables, plus the two deliberately-unregistered central-only
/// notification tables, plus the node-local (also deliberately-unregistered)
/// <c>mirrored_secrets</c> ledger.
/// </summary>
private static readonly string[] AllSiteTables = private static readonly string[] AllSiteTables =
[ [
"OperationTracking", "site_events", "sf_messages", "deployed_configurations", "OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems", "static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state", "database_connections", "data_connection_definitions", "native_alarm_state",
"notification_lists", "smtp_configurations", "deployed_secrets",
"notification_lists", "smtp_configurations", "mirrored_secrets",
]; ];
private static readonly string[] ReplicatedTables = private static readonly string[] ReplicatedTables =
@@ -210,6 +230,7 @@ public class SiteLocalDbCdcRegistrationTests : IDisposable
"OperationTracking", "site_events", "sf_messages", "deployed_configurations", "OperationTracking", "site_events", "sf_messages", "deployed_configurations",
"static_attribute_overrides", "shared_scripts", "external_systems", "static_attribute_overrides", "shared_scripts", "external_systems",
"database_connections", "data_connection_definitions", "native_alarm_state", "database_connections", "data_connection_definitions", "native_alarm_state",
"deployed_secrets",
]; ];
private static void AssertCaptureTriggersCoverTheReplicatedTables(ILocalDb db) private static void AssertCaptureTriggersCoverTheReplicatedTables(ILocalDb db)
@@ -15,7 +15,8 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <c>RegisterReplicated</c> time, which on a site node is a BOOT failure of the replicating /// <c>RegisterReplicated</c> time, which on a site node is a BOOT failure of the replicating
/// pair. This test moves that discovery from the first replicated boot to the build: it walks /// pair. This test moves that discovery from the first replicated boot to the build: it walks
/// the real site schema (every table <c>SiteLocalDbSetup.OnReady</c> creates) and asserts none /// the real site schema (every table <c>SiteLocalDbSetup.OnReady</c> creates) and asserts none
/// of the ten <see cref="SiteLocalDbSetup.ReplicatedTables"/> has such an inbound FK. /// of the eleven <see cref="SiteLocalDbSetup.ReplicatedTables"/> (incl. <c>deployed_secrets</c>,
/// added for site-secrets artifact deploy) has such an inbound FK.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// The schema is built with an UNREPLICATED config on purpose: a replicated one would make /// The schema is built with an UNREPLICATED config on purpose: a replicated one would make
@@ -245,7 +245,7 @@ public class SiteLocalDbWiringTests : IDisposable
var tables = TableNames(db); var tables = TableNames(db);
// One from each Phase 1 schema, then every Phase 2 table. // One from each Phase 1 schema, then every Phase 2 table plus the site-secrets tables.
Assert.Contains("OperationTracking", tables); Assert.Contains("OperationTracking", tables);
Assert.Contains("site_events", tables); Assert.Contains("site_events", tables);
Assert.Contains("sf_messages", tables); Assert.Contains("sf_messages", tables);
@@ -254,6 +254,7 @@ public class SiteLocalDbWiringTests : IDisposable
"deployed_configurations", "static_attribute_overrides", "shared_scripts", "deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "notification_lists", "external_systems", "database_connections", "notification_lists",
"data_connection_definitions", "smtp_configurations", "native_alarm_state", "data_connection_definitions", "smtp_configurations", "native_alarm_state",
"deployed_secrets", "mirrored_secrets",
}) })
{ {
Assert.Contains(table, tables); Assert.Contains(table, tables);
@@ -261,19 +262,21 @@ public class SiteLocalDbWiringTests : IDisposable
} }
[Fact] [Fact]
public void Site_LocalDb_RegistersExactlyTheTenReplicatedTables() public void Site_LocalDb_RegistersExactlyTheElevenReplicatedTables()
{ {
// The Phase 2 cutover. This is an EXACTLY check rather than a Contains in both // The Phase 2 cutover, extended by site-secrets artifact deploy's deployed_secrets.
// directions, and both directions are load-bearing. // This is an EXACTLY check rather than a Contains in both directions, and both
// directions are load-bearing.
// //
// Too few: the table silently stops replicating. Storage still works, every other // Too few: the table silently stops replicating. Storage still works, every other
// test still passes, and the pair just quietly diverges — the failure mode Phase 1 // test still passes, and the pair just quietly diverges — the failure mode Phase 1
// existed to end. // existed to end.
// //
// Too many: notification_lists and smtp_configurations must NOT be here. They are // Too many: notification_lists, smtp_configurations, and mirrored_secrets must NOT
// permanently empty by design, and registering them would open a standing // be here. notification_lists/smtp_configurations are permanently empty by design
// replication channel whose only historical payload was plaintext SMTP passwords. // (their only historical payload was a plaintext SMTP password); mirrored_secrets is
// A Contains-based test would never catch that. // node-local fact (which names THIS node has mirrored), not site-wide state. A
// Contains-based test would never catch either.
var db = _host.Services.GetRequiredService<ILocalDb>(); var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.Equal( Assert.Equal(
@@ -281,8 +284,9 @@ public class SiteLocalDbWiringTests : IDisposable
// Ordinal order: '_' (0x5F) sorts before 'b' (0x62), so // Ordinal order: '_' (0x5F) sorts before 'b' (0x62), so
// data_connection_definitions precedes database_connections. // data_connection_definitions precedes database_connections.
"OperationTracking", "data_connection_definitions", "database_connections", "OperationTracking", "data_connection_definitions", "database_connections",
"deployed_configurations", "external_systems", "native_alarm_state", "deployed_configurations", "deployed_secrets", "external_systems",
"sf_messages", "shared_scripts", "site_events", "static_attribute_overrides", "native_alarm_state", "sf_messages", "shared_scripts", "site_events",
"static_attribute_overrides",
], ],
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray()); db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray());
} }
@@ -300,6 +304,17 @@ public class SiteLocalDbWiringTests : IDisposable
Assert.DoesNotContain("smtp_configurations", db.ReplicatedTables.Keys); Assert.DoesNotContain("smtp_configurations", db.ReplicatedTables.Keys);
} }
[Fact]
public void Site_LocalDb_DoesNotReplicateTheNodeLocalMirroredSecretsLedger()
{
// mirrored_secrets exists (SiteStorageSchema creates it) but must never be captured:
// it records which names THIS node has mirrored into its own secret store, which is
// per-node fact, not site-wide state — see SiteLocalDbSetup.ReplicatedTables.
var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.DoesNotContain("mirrored_secrets", db.ReplicatedTables.Keys);
}
[Fact] [Fact]
public void Site_LocalDb_ReplicatedPhase2TablesHaveTheirExpectedPrimaryKeys() public void Site_LocalDb_ReplicatedPhase2TablesHaveTheirExpectedPrimaryKeys()
{ {
@@ -318,6 +333,7 @@ public class SiteLocalDbWiringTests : IDisposable
Assert.Equal( Assert.Equal(
["instance_unique_name", "source_canonical_name", "source_reference"], ["instance_unique_name", "source_canonical_name", "source_reference"],
db.ReplicatedTables["native_alarm_state"].PkColumns); db.ReplicatedTables["native_alarm_state"].PkColumns);
Assert.Equal(["name"], db.ReplicatedTables["deployed_secrets"].PkColumns);
} }
private static HashSet<string> TableNames(ILocalDb db) private static HashSet<string> TableNames(ILocalDb db)
@@ -0,0 +1,320 @@
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Persistence;
/// <summary>
/// Task 8, site-secrets artifact deploy: site-local <c>deployed_secrets</c>
/// (LocalDb-replicated ciphertext) and <c>mirrored_secrets</c> (node-local ledger) stores.
/// </summary>
/// <remarks>
/// Backed by a real temp-file LocalDb: <see cref="SiteStorageService"/> takes an
/// <c>ILocalDb</c> rather than a connection string, and LocalDb has no in-memory mode.
/// </remarks>
public class DeployedSecretsStorageTests : IAsyncLifetime, IDisposable
{
private readonly TestLocalDb _localDb;
private SiteStorageService _storage = null!;
public DeployedSecretsStorageTests()
{
_localDb = TestLocalDb.CreateTemp("deployed-secrets");
}
public async Task InitializeAsync()
{
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
await _storage.InitializeAsync();
}
public Task DisposeAsync() => Task.CompletedTask;
public void Dispose()
{
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
// ── deployed_secrets: full-set upsert / soft-delete reconciliation ──
[Fact]
public async Task Upsert_IsFullSet_MarksAbsentRowsRemoved()
{
var s = _storage;
await s.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (applied, removed) = await s.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
Assert.Equal(["A"], applied);
Assert.Equal(["B"], removed);
var rows = await s.GetDeployedSecretsAsync();
Assert.Equal(2, rows.Single(r => r.Name == "A").Secret.Revision);
Assert.True(rows.Single(r => r.Name == "B").IsRemoved);
}
[Fact]
public async Task Upsert_FirstCall_AppliesEveryRow_AndRemovesNothing()
{
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
[Art("A", 1), Art("B", 1)], "d1");
Assert.Equal(["A", "B"], applied);
Assert.Empty(removed);
}
[Fact]
public async Task Upsert_ReAppliedRow_IsNotReportedAsRemovedTwice()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (_, removedFirst) = await _storage.UpsertDeployedSecretsAsync([Art("A", 2)], "d2");
Assert.Equal(["B"], removedFirst);
// B stays removed and is not re-reported as "newly" removed on a later call that
// still omits it.
var (_, removedSecond) = await _storage.UpsertDeployedSecretsAsync([Art("A", 3)], "d3");
Assert.Empty(removedSecond);
}
[Fact]
public async Task Upsert_RowThatWasRemoved_CanBeReAppliedAndFlipsBackToNotRemoved()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d2"); // B removed
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync(
[Art("A", 1), Art("B", 2)], "d3");
Assert.Equal(["A", "B"], applied);
Assert.Empty(removed);
var rows = await _storage.GetDeployedSecretsAsync();
var b = rows.Single(r => r.Name == "B");
Assert.False(b.IsRemoved);
Assert.Equal(2, b.Secret.Revision);
}
[Fact]
public async Task Upsert_EmptySet_MarksEveryExistingRowRemoved()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1), Art("B", 1)], "d1");
var (applied, removed) = await _storage.UpsertDeployedSecretsAsync([], "d2");
Assert.Empty(applied);
Assert.Equal(["A", "B"], removed.OrderBy(n => n, StringComparer.Ordinal));
var rows = await _storage.GetDeployedSecretsAsync();
Assert.All(rows, r => Assert.True(r.IsRemoved));
}
[Fact]
public async Task GetDeployedSecretsAsync_RoundTripsEveryFieldOfTheSecretArtifact()
{
var art = Art("Full", 7);
await _storage.UpsertDeployedSecretsAsync([art], "dep-full");
var row = Assert.Single(await _storage.GetDeployedSecretsAsync());
Assert.Equal("Full", row.Name);
Assert.Equal("dep-full", row.DeploymentId);
Assert.False(row.IsRemoved);
Assert.Equal(art.Name, row.Secret.Name);
Assert.Equal(art.ContentType, row.Secret.ContentType);
Assert.Equal(art.Ciphertext, row.Secret.Ciphertext);
Assert.Equal(art.Nonce, row.Secret.Nonce);
Assert.Equal(art.Tag, row.Secret.Tag);
Assert.Equal(art.WrappedDek, row.Secret.WrappedDek);
Assert.Equal(art.WrapNonce, row.Secret.WrapNonce);
Assert.Equal(art.WrapTag, row.Secret.WrapTag);
Assert.Equal(art.KekId, row.Secret.KekId);
Assert.Equal(art.Revision, row.Secret.Revision);
Assert.Equal(art.CreatedUtc, row.Secret.CreatedUtc);
Assert.Equal(art.UpdatedUtc, row.Secret.UpdatedUtc);
Assert.Equal(art.Description, row.Secret.Description);
}
[Fact]
public async Task GetDeployedSecretsAsync_IncludesRemovedRows()
{
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed
var rows = await _storage.GetDeployedSecretsAsync();
var a = Assert.Single(rows);
Assert.Equal("A", a.Name);
Assert.True(a.IsRemoved);
}
// ── security: row_json holds only base64 ciphertext, never plaintext ──
[Fact]
public async Task RowJson_DeserializesToSecretArtifact_WithNoExtraProperties()
{
// Guards against ever widening SecretArtifact (or row_json) with a plaintext
// field: this walks the RAW stored JSON, not the deserialized object, so a
// stray extra property would be caught here even though JsonSerializer.Deserialize
// silently ignores unknown members by default.
var art = Art("Sec1", 3);
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec");
var rowJson = await ReadRawRowJsonAsync("Sec1");
// SecretArtifact is a record, but records only get member-wise Equals for reference
// types via THEIR OWN Equals — and Array does not override Equals, so byte[] members
// compare by reference. Assert.Equal(art, roundTripped) would therefore fail even on
// a byte-for-byte-identical round trip; compare field-by-field instead.
var roundTripped = JsonSerializer.Deserialize<SecretArtifact>(rowJson);
Assert.NotNull(roundTripped);
Assert.Equal(art.Name, roundTripped!.Name);
Assert.Equal(art.ContentType, roundTripped.ContentType);
Assert.Equal(art.Ciphertext, roundTripped.Ciphertext);
Assert.Equal(art.Nonce, roundTripped.Nonce);
Assert.Equal(art.Tag, roundTripped.Tag);
Assert.Equal(art.WrappedDek, roundTripped.WrappedDek);
Assert.Equal(art.WrapNonce, roundTripped.WrapNonce);
Assert.Equal(art.WrapTag, roundTripped.WrapTag);
Assert.Equal(art.KekId, roundTripped.KekId);
Assert.Equal(art.Revision, roundTripped.Revision);
Assert.Equal(art.CreatedUtc, roundTripped.CreatedUtc);
Assert.Equal(art.UpdatedUtc, roundTripped.UpdatedUtc);
Assert.Equal(art.Description, roundTripped.Description);
using var document = JsonDocument.Parse(rowJson);
var propertyNames = document.RootElement.EnumerateObject()
.Select(p => p.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var expected = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
nameof(SecretArtifact.Name), nameof(SecretArtifact.ContentType),
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
nameof(SecretArtifact.KekId), nameof(SecretArtifact.Revision),
nameof(SecretArtifact.CreatedUtc), nameof(SecretArtifact.UpdatedUtc),
nameof(SecretArtifact.Description),
};
Assert.Equal(expected, propertyNames);
}
[Fact]
public async Task RowJson_EncodesEveryByteArrayField_AsBase64Text()
{
// byte[] serializes as base64 via System.Text.Json by default. Asserted at the raw
// JSON level (JsonValueKind.String, not an array of numbers) so a future custom
// converter that changed the wire shape would be caught here.
var art = Art("Sec2", 1);
await _storage.UpsertDeployedSecretsAsync([art], "dep-sec2");
var rowJson = await ReadRawRowJsonAsync("Sec2");
using var document = JsonDocument.Parse(rowJson);
var root = document.RootElement;
foreach (var byteArrayField in new[]
{
nameof(SecretArtifact.Ciphertext), nameof(SecretArtifact.Nonce),
nameof(SecretArtifact.Tag), nameof(SecretArtifact.WrappedDek),
nameof(SecretArtifact.WrapNonce), nameof(SecretArtifact.WrapTag),
})
{
var property = root.GetProperty(byteArrayField);
Assert.Equal(JsonValueKind.String, property.ValueKind);
// Base64.TryFromBase64String throws on malformed input — proves it really is
// base64 text, not just any string.
Assert.True(Convert.TryFromBase64String(property.GetString()!, new byte[64], out _));
}
}
private async Task<string> ReadRawRowJsonAsync(string name)
{
await using var connection = _storage.CreateConnection();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT row_json FROM deployed_secrets WHERE name = @name";
command.Parameters.AddWithValue("@name", name);
return (string)(await command.ExecuteScalarAsync())!;
}
// ── mirrored_secrets: node-local ledger ──
[Fact]
public async Task MirroredSecrets_SetThenGet_RoundTrips()
{
await _storage.SetMirroredSecretAsync("Sec1", 3);
var rows = await _storage.GetMirroredSecretsAsync();
var row = Assert.Single(rows);
Assert.Equal("Sec1", row.Name);
Assert.Equal(3, row.Revision);
}
[Fact]
public async Task MirroredSecrets_Set_IsUpsert()
{
await _storage.SetMirroredSecretAsync("Sec1", 1);
await _storage.SetMirroredSecretAsync("Sec1", 2);
var rows = await _storage.GetMirroredSecretsAsync();
var row = Assert.Single(rows);
Assert.Equal(2, row.Revision);
}
[Fact]
public async Task MirroredSecrets_Remove_DeletesTheLedgerRow()
{
await _storage.SetMirroredSecretAsync("Sec1", 1);
await _storage.RemoveMirroredSecretAsync("Sec1");
Assert.Empty(await _storage.GetMirroredSecretsAsync());
}
[Fact]
public async Task MirroredSecrets_Remove_OfAnUnknownName_IsANoOp()
{
await _storage.RemoveMirroredSecretAsync("NeverMirrored");
Assert.Empty(await _storage.GetMirroredSecretsAsync());
}
[Fact]
public async Task MirroredSecrets_IsIndependentOf_DeployedSecrets()
{
// mirroring is a separate, node-local action from the site-wide deployed_secrets
// reconcile — removing a deployed secret must not touch the mirrored ledger, and
// vice versa. Mirroring/un-mirroring in response to a removal is the caller's job.
await _storage.UpsertDeployedSecretsAsync([Art("A", 1)], "d1");
await _storage.SetMirroredSecretAsync("A", 1);
await _storage.UpsertDeployedSecretsAsync([], "d2"); // A removed from deployed_secrets
var deployed = Assert.Single(await _storage.GetDeployedSecretsAsync());
Assert.True(deployed.IsRemoved);
var mirrored = Assert.Single(await _storage.GetMirroredSecretsAsync());
Assert.Equal("A", mirrored.Name);
}
private static SecretArtifact Art(string name, long revision) => new(
Name: name,
ContentType: "application/octet-stream",
Ciphertext: [1, 2, 3, 4, 5],
Nonce: [10, 11, 12],
Tag: [20, 21, 22, 23],
WrappedDek: [30, 31, 32, 33, 34, 35],
WrapNonce: [40, 41, 42],
WrapTag: [50, 51, 52, 53],
KekId: "kek-1",
Revision: revision,
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
Description: "test secret");
}
@@ -23,13 +23,14 @@ public class SiteStorageSchemaTests
SiteStorageSchema.Apply(connection); SiteStorageSchema.Apply(connection);
SiteStorageSchema.Apply(connection); // second run must not throw SiteStorageSchema.Apply(connection); // second run must not throw
// Every table is named explicitly on purpose — a loop asserting "9 tables" // Every table is named explicitly on purpose — a loop asserting "11 tables"
// would still pass if one were renamed. // would still pass if one were renamed.
foreach (var table in new[] foreach (var table in new[]
{ {
"deployed_configurations", "static_attribute_overrides", "shared_scripts", "deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "notification_lists", "external_systems", "database_connections", "notification_lists",
"data_connection_definitions", "smtp_configurations", "native_alarm_state", "data_connection_definitions", "smtp_configurations", "native_alarm_state",
"deployed_secrets", "mirrored_secrets",
}) })
{ {
Assert.True(TableExists(connection, table), $"missing table: {table}"); Assert.True(TableExists(connection, table), $"missing table: {table}");