Joseph Doherty
dd0a846b64
Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
366 lines
15 KiB
C#
366 lines
15 KiB
C#
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
|
|
|
|
/// <summary>
|
|
/// The SQLite store's test suite, ported case-for-case onto the SQL-Server store.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Ported rather than newly written on purpose. The two stores must be behaviourally identical —
|
|
/// a cluster can hold both at once — so the strongest available check is that the assertions
|
|
/// written against SQLite pass unchanged here. Anywhere the T-SQL diverges in observable behaviour,
|
|
/// one of these fails.
|
|
/// </remarks>
|
|
[Collection("live-sqlserver")]
|
|
public sealed class SqlServerSecretStoreLiveTests : IAsyncLifetime
|
|
{
|
|
private readonly string _schema = LiveSqlServer.NewSchemaName();
|
|
private SqlServerSecretStore _store = null!;
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
if (!LiveSqlServer.IsEnabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
|
await new SqlServerSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
|
_store = new SqlServerSecretStore(factory);
|
|
}
|
|
|
|
public Task DisposeAsync() => LiveSqlServer.DropSchemaAsync(_schema);
|
|
|
|
private static StoredSecret MakeSecret(
|
|
string name,
|
|
long revision = 0,
|
|
byte[]? ciphertext = null,
|
|
DateTimeOffset? updatedUtc = null,
|
|
DateTimeOffset? createdUtc = null,
|
|
bool isDeleted = false,
|
|
DateTimeOffset? deletedUtc = null,
|
|
string? createdBy = "alice",
|
|
string? updatedBy = "alice") => new()
|
|
{
|
|
Name = new SecretName(name),
|
|
Description = "desc",
|
|
ContentType = SecretContentType.ConnectionString,
|
|
Ciphertext = ciphertext ?? [1, 2, 3],
|
|
Nonce = [4, 5, 6],
|
|
Tag = [7, 8, 9],
|
|
WrappedDek = [10, 11, 12],
|
|
WrapNonce = [13, 14, 15],
|
|
WrapTag = [16, 17, 18],
|
|
KekId = "kek-1",
|
|
Revision = revision,
|
|
IsDeleted = isDeleted,
|
|
DeletedUtc = deletedUtc,
|
|
CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
|
|
UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
|
|
CreatedBy = createdBy,
|
|
UpdatedBy = updatedBy,
|
|
};
|
|
|
|
[SkippableFact]
|
|
public async Task Upsert_Then_Get_RoundTrips()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/db-conn"), CancellationToken.None);
|
|
|
|
StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
|
|
|
|
Assert.NotNull(got);
|
|
Assert.Equal("app/db-conn", got!.Name.Value);
|
|
Assert.Equal("desc", got.Description);
|
|
Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
|
|
Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
|
|
Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
|
|
Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
|
|
Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
|
|
Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
|
|
Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
|
|
Assert.Equal("kek-1", got.KekId);
|
|
Assert.Equal(0, got.Revision);
|
|
Assert.False(got.IsDeleted);
|
|
Assert.Null(got.DeletedUtc);
|
|
Assert.Equal("alice", got.CreatedBy);
|
|
Assert.Equal("alice", got.UpdatedBy);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Get_ReturnsNull_WhenAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
Assert.Null(await _store.GetAsync(new SecretName("nope"), CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice"),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterFirst =
|
|
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
|
Assert.Equal(0, afterFirst.Revision);
|
|
DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob"),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterSecond =
|
|
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
|
Assert.Equal(1, afterSecond.Revision);
|
|
Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
|
|
Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
|
|
Assert.Equal("alice", afterSecond.CreatedBy);
|
|
Assert.Equal("bob", afterSecond.UpdatedBy);
|
|
Assert.False(afterSecond.IsDeleted);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task List_ExcludesTombstoned_ByDefault()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
|
|
|
|
IReadOnlyList<SecretMetadata> visible =
|
|
await _store.ListAsync(includeDeleted: false, CancellationToken.None);
|
|
Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
|
|
Assert.Contains(visible, m => m.Name.Value == "keep");
|
|
|
|
IReadOnlyList<SecretMetadata> all =
|
|
await _store.ListAsync(includeDeleted: true, CancellationToken.None);
|
|
Assert.Contains(all, m => m.Name.Value == "gone");
|
|
Assert.Contains(all, m => m.Name.Value == "keep");
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
|
|
|
|
Assert.True(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
|
|
|
StoredSecret tombstoned =
|
|
(await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
|
|
Assert.True(tombstoned.IsDeleted);
|
|
Assert.NotNull(tombstoned.DeletedUtc);
|
|
Assert.Equal(1, tombstoned.Revision);
|
|
Assert.Equal("carol", tombstoned.UpdatedBy);
|
|
|
|
Assert.False(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
|
Assert.False(await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetManifest_ReturnsAllRows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
|
|
|
|
IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(2, manifest.Count);
|
|
SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
|
|
SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
|
|
Assert.False(a.IsDeleted);
|
|
Assert.Equal(0, a.Revision);
|
|
Assert.True(b.IsDeleted);
|
|
Assert.Equal(1, b.Revision);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
|
DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
|
|
DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
Assert.Equal(5, (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!.Revision);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
|
Assert.Equal(6, afterNewer.Revision);
|
|
Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
|
|
Assert.Equal(t3, afterNewer.UpdatedUtc);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
|
Assert.Equal(6, afterStale.Revision);
|
|
Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)),
|
|
CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
|
|
|
|
StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
|
Assert.Equal(1, before.Revision);
|
|
|
|
bool applied = await _store.ApplyRewrapAsync(
|
|
before with
|
|
{
|
|
WrappedDek = [90, 91, 92],
|
|
WrapNonce = [93, 94, 95],
|
|
WrapTag = [96, 97, 98],
|
|
KekId = "kek-2",
|
|
},
|
|
before.WrappedDek,
|
|
CancellationToken.None);
|
|
Assert.True(applied);
|
|
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
|
Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
|
|
Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
|
|
Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
|
|
Assert.Equal("kek-2", after.KekId);
|
|
Assert.Equal(before.Revision, after.Revision);
|
|
Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
|
|
Assert.Equal(before.CreatedUtc, after.CreatedUtc);
|
|
Assert.Equal(before.UpdatedBy, after.UpdatedBy);
|
|
Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
|
|
Assert.Equal(before.Nonce, after.Nonce);
|
|
Assert.Equal(before.Tag, after.Tag);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_RewrapsTombstonedRows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
|
|
|
|
StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
|
Assert.True(tomb.IsDeleted);
|
|
|
|
Assert.True(await _store.ApplyRewrapAsync(
|
|
tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
|
tomb.WrappedDek,
|
|
CancellationToken.None));
|
|
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
|
Assert.Equal("kek-2", after.KekId);
|
|
Assert.True(after.IsDeleted);
|
|
Assert.Equal(tomb.Revision, after.Revision);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
|
|
|
|
Assert.False(await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/cas", ciphertext: [1, 2, 3]), CancellationToken.None);
|
|
StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
|
|
CancellationToken.None);
|
|
|
|
bool applied = await _store.ApplyRewrapAsync(
|
|
before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
|
before.WrappedDek,
|
|
CancellationToken.None);
|
|
|
|
Assert.False(applied);
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
|
Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
|
|
Assert.Equal("kek-1", after.KekId);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetMany_returns_only_the_requested_rows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("a/one"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("b/two"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("c/three"), CancellationToken.None);
|
|
|
|
IReadOnlyList<StoredSecret> rows = await _store.GetManyAsync(
|
|
[new SecretName("a/one"), new SecretName("c/three"), new SecretName("absent/name")],
|
|
CancellationToken.None);
|
|
|
|
Assert.Equal(2, rows.Count);
|
|
Assert.Contains(rows, r => r.Name.Value == "a/one");
|
|
Assert.Contains(rows, r => r.Name.Value == "c/three");
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetMany_short_circuits_on_an_empty_request()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
// Guards the IN () clause: an empty list would otherwise produce invalid T-SQL.
|
|
Assert.Empty(await _store.GetManyAsync([], CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Migrator_is_idempotent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
|
var migrator = new SqlServerSecretsStoreMigrator(factory);
|
|
|
|
// Every node runs this at every startup — re-running must be a no-op, not an error.
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
await _store.UpsertAsync(MakeSecret("survives/migration"), CancellationToken.None);
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
Assert.NotNull(await _store.GetAsync(new SecretName("survives/migration"), CancellationToken.None));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Serializes the live suite: the tests share one database server and each provisions its own
|
|
/// schema, so running them in parallel would multiply connection load for no benefit.
|
|
/// </summary>
|
|
[CollectionDefinition("live-sqlserver", DisableParallelization = true)]
|
|
public sealed class LiveSqlServerCollection;
|