feat(secrets): sqlite secret store (overwrite-in-place, tombstone, manifest, LWW)

This commit is contained in:
Joseph Doherty
2026-07-15 16:50:59 -04:00
parent e99a8731e9
commit 10455ec53b
3 changed files with 530 additions and 1 deletions
@@ -8,7 +8,7 @@ public enum MasterKeySource
/// <summary>Read a base64-encoded key from an environment variable.</summary>
Environment,
/// <summary>Read the key from a file (raw 32 bytes, or base64/hex text).</summary>
/// <summary>Read the key from a file (raw 32 bytes, or base64 text).</summary>
File,
/// <summary>Unprotect a DPAPI-sealed blob file (Windows / <see cref="System.Security.Cryptography.DataProtectionScope.CurrentUser"/>).</summary>
@@ -0,0 +1,302 @@
using System.Data;
using System.Globalization;
using Microsoft.Data.Sqlite;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.Secrets.Sqlite;
/// <summary>
/// SQLite-backed <see cref="ISecretStore"/>. Persists the envelope-encrypted
/// <see cref="StoredSecret"/> rows and their safe <see cref="SecretMetadata"/> projection,
/// using parameterized commands throughout. Local writes go through
/// <see cref="UpsertAsync"/> (which bumps the revision); replicated rows go through
/// <see cref="ApplyReplicatedAsync"/> (last-writer-wins, applied verbatim).
/// </summary>
/// <remarks>Timestamps are stored as round-trippable ISO-8601 (<c>"O"</c>) TEXT;
/// <c>content_type</c> as the enum name; <c>is_deleted</c> as a 0/1 integer.</remarks>
public sealed class SqliteSecretStore(SecretsSqliteConnectionFactory connectionFactory) : ISecretStore
{
// All columns of the secret table, in schema order, for full-row reads.
private const string AllColumns =
"name, description, content_type, ciphertext, nonce, tag, wrapped_dek, wrap_nonce, wrap_tag, " +
"kek_id, revision, is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by";
// The safe metadata projection — deliberately excludes every ciphertext / crypto BLOB column.
private const string MetadataColumns =
"name, description, content_type, kek_id, revision, is_deleted, created_utc, updated_utc, created_by, updated_by";
/// <inheritdoc />
public async Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct)
{
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = $"SELECT {AllColumns} FROM secret WHERE name = $name;";
command.Parameters.AddWithValue("$name", name.Value);
await using SqliteDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (!await reader.ReadAsync(ct).ConfigureAwait(false))
{
return null;
}
return ReadStoredSecret(reader);
}
/// <inheritdoc />
public async Task UpsertAsync(StoredSecret row, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(row);
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
string now = DateTimeOffset.UtcNow.ToString("O");
// INSERT establishes revision 0 / created==updated==now; ON CONFLICT overwrites the
// crypto material in place, bumps the revision, refreshes updated_utc/updated_by, and
// clears any tombstone — but deliberately preserves created_utc / created_by.
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = """
INSERT INTO secret (
name, description, content_type, ciphertext, nonce, tag,
wrapped_dek, wrap_nonce, wrap_tag, kek_id, revision,
is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by)
VALUES (
$name, $description, $content_type, $ciphertext, $nonce, $tag,
$wrapped_dek, $wrap_nonce, $wrap_tag, $kek_id, 0,
0, NULL, $now, $now, $created_by, $updated_by)
ON CONFLICT(name) DO UPDATE SET
description = excluded.description,
content_type = excluded.content_type,
ciphertext = excluded.ciphertext,
nonce = excluded.nonce,
tag = excluded.tag,
wrapped_dek = excluded.wrapped_dek,
wrap_nonce = excluded.wrap_nonce,
wrap_tag = excluded.wrap_tag,
kek_id = excluded.kek_id,
revision = secret.revision + 1,
updated_utc = $now,
updated_by = excluded.updated_by,
is_deleted = 0,
deleted_utc = NULL;
""";
BindCryptoColumns(command, row);
command.Parameters.AddWithValue("$now", now);
command.Parameters.AddWithValue("$created_by", (object?)row.CreatedBy ?? DBNull.Value);
command.Parameters.AddWithValue("$updated_by", (object?)row.UpdatedBy ?? DBNull.Value);
await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
{
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
string now = DateTimeOffset.UtcNow.ToString("O");
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = """
UPDATE secret SET
is_deleted = 1,
deleted_utc = $now,
revision = revision + 1,
updated_utc = $now,
updated_by = $actor
WHERE name = $name AND is_deleted = 0;
""";
command.Parameters.AddWithValue("$now", now);
command.Parameters.AddWithValue("$actor", (object?)actor ?? DBNull.Value);
command.Parameters.AddWithValue("$name", name.Value);
int rowsAffected = await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
return rowsAffected > 0;
}
/// <inheritdoc />
public async Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct)
{
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText =
$"SELECT {MetadataColumns} FROM secret WHERE ($include_deleted OR is_deleted = 0) ORDER BY name;";
command.Parameters.AddWithValue("$include_deleted", includeDeleted ? 1 : 0);
var results = new List<SecretMetadata>();
await using SqliteDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
results.Add(new SecretMetadata
{
Name = new SecretName(reader.GetString(0)),
Description = reader.IsDBNull(1) ? null : reader.GetString(1),
ContentType = Enum.Parse<SecretContentType>(reader.GetString(2)),
KekId = reader.GetString(3),
Revision = reader.GetInt64(4),
IsDeleted = reader.GetInt64(5) != 0,
CreatedUtc = ParseUtc(reader.GetString(6)),
UpdatedUtc = ParseUtc(reader.GetString(7)),
CreatedBy = reader.IsDBNull(8) ? null : reader.GetString(8),
UpdatedBy = reader.IsDBNull(9) ? null : reader.GetString(9),
});
}
return results;
}
/// <inheritdoc />
public async Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct)
{
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = "SELECT name, revision, updated_utc, is_deleted FROM secret ORDER BY name;";
var results = new List<SecretManifestEntry>();
await using SqliteDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
results.Add(new SecretManifestEntry
{
Name = new SecretName(reader.GetString(0)),
Revision = reader.GetInt64(1),
UpdatedUtc = ParseUtc(reader.GetString(2)),
IsDeleted = reader.GetInt64(3) != 0,
});
}
return results;
}
/// <inheritdoc />
public async Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(row);
await using SqliteConnection connection =
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
await using SqliteTransaction transaction = (SqliteTransaction)
await connection.BeginTransactionAsync(IsolationLevel.Serializable, ct).ConfigureAwait(false);
// Read the local (updated_utc, revision) so we can apply last-writer-wins.
await using (SqliteCommand read = connection.CreateCommand())
{
read.Transaction = transaction;
read.CommandText = "SELECT updated_utc, revision FROM secret WHERE name = $name;";
read.Parameters.AddWithValue("$name", row.Name.Value);
await using SqliteDataReader reader = await read.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (await reader.ReadAsync(ct).ConfigureAwait(false))
{
DateTimeOffset localUpdated = ParseUtc(reader.GetString(0));
long localRevision = reader.GetInt64(1);
// Incoming wins only if strictly newer by (updated_utc, then revision).
bool incomingIsNewer =
row.UpdatedUtc > localUpdated ||
(row.UpdatedUtc == localUpdated && row.Revision > localRevision);
if (!incomingIsNewer)
{
await transaction.CommitAsync(ct).ConfigureAwait(false);
return;
}
}
}
// Apply the incoming row VERBATIM — its own revision, timestamps, tombstone flag, and
// crypto material — with no revision bump (this is the replication path, not a local write).
await using (SqliteCommand upsert = connection.CreateCommand())
{
upsert.Transaction = transaction;
upsert.CommandText = """
INSERT INTO secret (
name, description, content_type, ciphertext, nonce, tag,
wrapped_dek, wrap_nonce, wrap_tag, kek_id, revision,
is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by)
VALUES (
$name, $description, $content_type, $ciphertext, $nonce, $tag,
$wrapped_dek, $wrap_nonce, $wrap_tag, $kek_id, $revision,
$is_deleted, $deleted_utc, $created_utc, $updated_utc, $created_by, $updated_by)
ON CONFLICT(name) DO UPDATE SET
description = excluded.description,
content_type = excluded.content_type,
ciphertext = excluded.ciphertext,
nonce = excluded.nonce,
tag = excluded.tag,
wrapped_dek = excluded.wrapped_dek,
wrap_nonce = excluded.wrap_nonce,
wrap_tag = excluded.wrap_tag,
kek_id = excluded.kek_id,
revision = excluded.revision,
is_deleted = excluded.is_deleted,
deleted_utc = excluded.deleted_utc,
created_utc = excluded.created_utc,
updated_utc = excluded.updated_utc,
created_by = excluded.created_by,
updated_by = excluded.updated_by;
""";
BindCryptoColumns(upsert, row);
upsert.Parameters.AddWithValue("$revision", row.Revision);
upsert.Parameters.AddWithValue("$is_deleted", row.IsDeleted ? 1 : 0);
upsert.Parameters.AddWithValue("$deleted_utc", (object?)row.DeletedUtc?.ToString("O") ?? DBNull.Value);
upsert.Parameters.AddWithValue("$created_utc", row.CreatedUtc.ToString("O"));
upsert.Parameters.AddWithValue("$updated_utc", row.UpdatedUtc.ToString("O"));
upsert.Parameters.AddWithValue("$created_by", (object?)row.CreatedBy ?? DBNull.Value);
upsert.Parameters.AddWithValue("$updated_by", (object?)row.UpdatedBy ?? DBNull.Value);
await upsert.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
await transaction.CommitAsync(ct).ConfigureAwait(false);
}
// Binds the identity, description, content-type, KEK id, and all six crypto BLOB columns
// shared by every insert path.
private static void BindCryptoColumns(SqliteCommand command, StoredSecret row)
{
command.Parameters.AddWithValue("$name", row.Name.Value);
command.Parameters.AddWithValue("$description", (object?)row.Description ?? DBNull.Value);
command.Parameters.AddWithValue("$content_type", row.ContentType.ToString());
command.Parameters.AddWithValue("$ciphertext", row.Ciphertext);
command.Parameters.AddWithValue("$nonce", row.Nonce);
command.Parameters.AddWithValue("$tag", row.Tag);
command.Parameters.AddWithValue("$wrapped_dek", row.WrappedDek);
command.Parameters.AddWithValue("$wrap_nonce", row.WrapNonce);
command.Parameters.AddWithValue("$wrap_tag", row.WrapTag);
command.Parameters.AddWithValue("$kek_id", row.KekId);
}
private static StoredSecret ReadStoredSecret(SqliteDataReader reader) => new()
{
Name = new SecretName(reader.GetString(0)),
Description = reader.IsDBNull(1) ? null : reader.GetString(1),
ContentType = Enum.Parse<SecretContentType>(reader.GetString(2)),
Ciphertext = reader.GetFieldValue<byte[]>(3),
Nonce = reader.GetFieldValue<byte[]>(4),
Tag = reader.GetFieldValue<byte[]>(5),
WrappedDek = reader.GetFieldValue<byte[]>(6),
WrapNonce = reader.GetFieldValue<byte[]>(7),
WrapTag = reader.GetFieldValue<byte[]>(8),
KekId = reader.GetString(9),
Revision = reader.GetInt64(10),
IsDeleted = reader.GetInt64(11) != 0,
DeletedUtc = reader.IsDBNull(12) ? null : ParseUtc(reader.GetString(12)),
CreatedUtc = ParseUtc(reader.GetString(13)),
UpdatedUtc = ParseUtc(reader.GetString(14)),
CreatedBy = reader.IsDBNull(15) ? null : reader.GetString(15),
UpdatedBy = reader.IsDBNull(16) ? null : reader.GetString(16),
};
private static DateTimeOffset ParseUtc(string value) =>
DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
}
@@ -0,0 +1,227 @@
using Microsoft.Data.Sqlite;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Sqlite;
namespace ZB.MOM.WW.Secrets.Tests.Sqlite;
public sealed class SqliteSecretStoreTests : IAsyncLifetime, IDisposable
{
private readonly string _dbPath =
Path.Combine(Path.GetTempPath(), $"zb-secrets-store-{Guid.NewGuid():N}.db");
private readonly SecretsSqliteConnectionFactory _factory;
private readonly SqliteSecretStore _store;
public SqliteSecretStoreTests()
{
_factory = new SecretsSqliteConnectionFactory(_dbPath);
_store = new SqliteSecretStore(_factory);
}
public async Task InitializeAsync() =>
await new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None);
public Task DisposeAsync() => Task.CompletedTask;
private static StoredSecret MakeSecret(
string name,
long revision = 0,
byte[]? ciphertext = null,
DateTimeOffset? updatedUtc = null,
DateTimeOffset? createdUtc = null,
bool isDeleted = false,
DateTimeOffset? deletedUtc = null,
string? createdBy = "alice",
string? updatedBy = "alice") => new()
{
Name = new SecretName(name),
Description = "desc",
ContentType = SecretContentType.ConnectionString,
Ciphertext = ciphertext ?? [1, 2, 3],
Nonce = [4, 5, 6],
Tag = [7, 8, 9],
WrappedDek = [10, 11, 12],
WrapNonce = [13, 14, 15],
WrapTag = [16, 17, 18],
KekId = "kek-1",
Revision = revision,
IsDeleted = isDeleted,
DeletedUtc = deletedUtc,
CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
CreatedBy = createdBy,
UpdatedBy = updatedBy,
};
[Fact]
public async Task Upsert_Then_Get_RoundTrips()
{
StoredSecret row = MakeSecret("app/db-conn");
await _store.UpsertAsync(row, CancellationToken.None);
StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
Assert.NotNull(got);
Assert.Equal("app/db-conn", got!.Name.Value);
Assert.Equal("desc", got.Description);
Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
Assert.Equal("kek-1", got.KekId);
Assert.Equal(0, got.Revision);
Assert.False(got.IsDeleted);
Assert.Null(got.DeletedUtc);
Assert.Equal("alice", got.CreatedBy);
Assert.Equal("alice", got.UpdatedBy);
}
[Fact]
public async Task Get_ReturnsNull_WhenAbsent()
{
StoredSecret? got = await _store.GetAsync(new SecretName("nope"), CancellationToken.None);
Assert.Null(got);
}
[Fact]
public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
{
StoredSecret first = MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice");
await _store.UpsertAsync(first, CancellationToken.None);
StoredSecret afterFirst = (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
Assert.Equal(0, afterFirst.Revision);
DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
// Second write: different crypto bytes and a different actor.
StoredSecret second = MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob");
await _store.UpsertAsync(second, CancellationToken.None);
StoredSecret afterSecond = (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
Assert.Equal(1, afterSecond.Revision);
Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
// created_utc / created_by are preserved from the original insert.
Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
Assert.Equal("alice", afterSecond.CreatedBy);
// updated_by reflects the second write.
Assert.Equal("bob", afterSecond.UpdatedBy);
Assert.False(afterSecond.IsDeleted);
}
[Fact]
public async Task List_ExcludesTombstoned_ByDefault()
{
await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
IReadOnlyList<SecretMetadata> visible = await _store.ListAsync(includeDeleted: false, CancellationToken.None);
Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
Assert.Contains(visible, m => m.Name.Value == "keep");
IReadOnlyList<SecretMetadata> all = await _store.ListAsync(includeDeleted: true, CancellationToken.None);
Assert.Contains(all, m => m.Name.Value == "gone");
Assert.Contains(all, m => m.Name.Value == "keep");
// Compile-time proof the projection carries no byte[] members: SecretMetadata is the element type.
SecretMetadata sample = all[0];
Assert.NotNull(sample);
}
[Fact]
public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
{
await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
bool first = await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None);
Assert.True(first);
StoredSecret tombstoned = (await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
Assert.True(tombstoned.IsDeleted);
Assert.NotNull(tombstoned.DeletedUtc);
Assert.Equal(1, tombstoned.Revision);
Assert.Equal("carol", tombstoned.UpdatedBy);
// Deleting an already-tombstoned row returns false.
bool second = await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None);
Assert.False(second);
// Deleting an unknown name returns false.
bool unknown = await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None);
Assert.False(unknown);
}
[Fact]
public async Task GetManifest_ReturnsAllRows()
{
await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
Assert.Equal(2, manifest.Count);
SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
Assert.False(a.IsDeleted);
Assert.Equal(0, a.Revision);
Assert.True(b.IsDeleted);
Assert.Equal(1, b.Revision);
}
[Fact]
public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
{
DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
// Seed a local row at revision 5 / T2.
StoredSecret seed = MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1);
await _store.ApplyReplicatedAsync(seed, CancellationToken.None);
StoredSecret afterSeed = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
Assert.Equal(5, afterSeed.Revision);
// Newer row (revision 6 / T3) is applied verbatim (revision NOT bumped past 6).
StoredSecret newer = MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1);
await _store.ApplyReplicatedAsync(newer, CancellationToken.None);
StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
Assert.Equal(6, afterNewer.Revision);
Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
Assert.Equal(t3, afterNewer.UpdatedUtc);
// Stale row (revision 4 / T1) is ignored.
StoredSecret stale = MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1);
await _store.ApplyReplicatedAsync(stale, CancellationToken.None);
StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
Assert.Equal(6, afterStale.Revision);
Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
}
public void Dispose()
{
// Drop pooled connections so the WAL/-shm/-wal sidecars release before we delete.
SqliteConnection.ClearAllPools();
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (IOException)
{
// Best-effort temp cleanup; a leaked temp file is not a test failure.
}
}
}
}