feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)

Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 04:08:23 -04:00
parent e46060fada
commit dd0a846b64
55 changed files with 4848 additions and 51 deletions
@@ -0,0 +1,68 @@
+using Microsoft.Data.SqlClient;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
+
+/// <summary>
+/// Shared connection details for the env-gated live SQL-Server suite.
+/// </summary>
+/// <remarks>
+/// Gated on <c>SECRETS_SQLSERVER_CONNSTR</c> and skips cleanly when it is unset, matching the family
+/// live-test idiom — the offline suite must stay runnable on any machine. Set it to a database the
+/// test is allowed to create and drop a schema in, for example:
+/// <c>Server=10.100.0.35,31433;Database=ZbSecretsTest;User Id=sa;Password=...;TrustServerCertificate=True</c>.
+/// </remarks>
+public static class LiveSqlServer
+{
+    /// <summary>Environment variable holding the live connection string.</summary>
+    public const string ConnectionStringVariable = "SECRETS_SQLSERVER_CONNSTR";
+
+    /// <summary>The configured connection string, or <see langword="null"/> when the suite is not enabled.</summary>
+    public static string? ConnectionString =>
+        Environment.GetEnvironmentVariable(ConnectionStringVariable);
+
+    /// <summary>Whether the live suite should run.</summary>
+    public static bool IsEnabled => !string.IsNullOrWhiteSpace(ConnectionString);
+
+    /// <summary>Skip reason shown when the suite is not enabled.</summary>
+    public const string SkipReason =
+        "Live SQL-Server suite disabled; set SECRETS_SQLSERVER_CONNSTR to enable.";
+
+    /// <summary>
+    /// Builds options against a throwaway schema so parallel runs and repeat runs cannot collide,
+    /// and so a failed run leaves no trace in a shared database.
+    /// </summary>
+    /// <param name="schemaName">The unique schema name for this test.</param>
+    /// <returns>Options targeting that schema.</returns>
+    public static SqlServerSecretsOptions OptionsFor(string schemaName) => new()
+    {
+        ConnectionString = ConnectionString!,
+        SchemaName = schemaName,
+    };
+
+    /// <summary>Generates a unique, allow-list-legal schema name for one test class.</summary>
+    /// <returns>A fresh schema name.</returns>
+    public static string NewSchemaName() => $"zbtest_{Guid.NewGuid():N}"[..32];
+
+    /// <summary>Drops the throwaway schema and its tables.</summary>
+    /// <param name="schemaName">The schema to drop.</param>
+    /// <returns>A task that completes when the schema is gone.</returns>
+    public static async Task DropSchemaAsync(string schemaName)
+    {
+        if (!IsEnabled)
+        {
+            return;
+        }
+
+        await using var connection = new SqlConnection(ConnectionString);
+        await connection.OpenAsync();
+
+        await using SqlCommand command = connection.CreateCommand();
+        command.CommandText = $"""
+            IF OBJECT_ID(N'[{schemaName}].[secret]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[secret];
+            IF OBJECT_ID(N'[{schemaName}].[schema_version]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[schema_version];
+            IF EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'{schemaName}') EXEC(N'DROP SCHEMA [{schemaName}]');
+            """;
+
+        await command.ExecuteNonQueryAsync();
+    }
+}
@@ -0,0 +1,365 @@
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
+
+/// <summary>
+/// The SQLite store's test suite, ported case-for-case onto the SQL-Server store.
+/// </summary>
+/// <remarks>
+/// Ported rather than newly written on purpose. The two stores must be behaviourally identical —
+/// a cluster can hold both at once — so the strongest available check is that the assertions
+/// written against SQLite pass unchanged here. Anywhere the T-SQL diverges in observable behaviour,
+/// one of these fails.
+/// </remarks>
+[Collection("live-sqlserver")]
+public sealed class SqlServerSecretStoreLiveTests : IAsyncLifetime
+{
+    private readonly string _schema = LiveSqlServer.NewSchemaName();
+    private SqlServerSecretStore _store = null!;
+
+    public async Task InitializeAsync()
+    {
+        if (!LiveSqlServer.IsEnabled)
+        {
+            return;
+        }
+
+        var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
+        await new SqlServerSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
+        _store = new SqlServerSecretStore(factory);
+    }
+
+    public Task DisposeAsync() => LiveSqlServer.DropSchemaAsync(_schema);
+
+    private static StoredSecret MakeSecret(
+        string name,
+        long revision = 0,
+        byte[]? ciphertext = null,
+        DateTimeOffset? updatedUtc = null,
+        DateTimeOffset? createdUtc = null,
+        bool isDeleted = false,
+        DateTimeOffset? deletedUtc = null,
+        string? createdBy = "alice",
+        string? updatedBy = "alice") => new()
+        {
+            Name = new SecretName(name),
+            Description = "desc",
+            ContentType = SecretContentType.ConnectionString,
+            Ciphertext = ciphertext ?? [1, 2, 3],
+            Nonce = [4, 5, 6],
+            Tag = [7, 8, 9],
+            WrappedDek = [10, 11, 12],
+            WrapNonce = [13, 14, 15],
+            WrapTag = [16, 17, 18],
+            KekId = "kek-1",
+            Revision = revision,
+            IsDeleted = isDeleted,
+            DeletedUtc = deletedUtc,
+            CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
+            UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
+            CreatedBy = createdBy,
+            UpdatedBy = updatedBy,
+        };
+
+    [SkippableFact]
+    public async Task Upsert_Then_Get_RoundTrips()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/db-conn"), CancellationToken.None);
+
+        StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
+
+        Assert.NotNull(got);
+        Assert.Equal("app/db-conn", got!.Name.Value);
+        Assert.Equal("desc", got.Description);
+        Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
+        Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
+        Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
+        Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
+        Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
+        Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
+        Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
+        Assert.Equal("kek-1", got.KekId);
+        Assert.Equal(0, got.Revision);
+        Assert.False(got.IsDeleted);
+        Assert.Null(got.DeletedUtc);
+        Assert.Equal("alice", got.CreatedBy);
+        Assert.Equal("alice", got.UpdatedBy);
+    }
+
+    [SkippableFact]
+    public async Task Get_ReturnsNull_WhenAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        Assert.Null(await _store.GetAsync(new SecretName("nope"), CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice"),
+            CancellationToken.None);
+
+        StoredSecret afterFirst =
+            (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
+        Assert.Equal(0, afterFirst.Revision);
+        DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob"),
+            CancellationToken.None);
+
+        StoredSecret afterSecond =
+            (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
+        Assert.Equal(1, afterSecond.Revision);
+        Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
+        Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
+        Assert.Equal("alice", afterSecond.CreatedBy);
+        Assert.Equal("bob", afterSecond.UpdatedBy);
+        Assert.False(afterSecond.IsDeleted);
+    }
+
+    [SkippableFact]
+    public async Task List_ExcludesTombstoned_ByDefault()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
+
+        IReadOnlyList<SecretMetadata> visible =
+            await _store.ListAsync(includeDeleted: false, CancellationToken.None);
+        Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
+        Assert.Contains(visible, m => m.Name.Value == "keep");
+
+        IReadOnlyList<SecretMetadata> all =
+            await _store.ListAsync(includeDeleted: true, CancellationToken.None);
+        Assert.Contains(all, m => m.Name.Value == "gone");
+        Assert.Contains(all, m => m.Name.Value == "keep");
+    }
+
+    [SkippableFact]
+    public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
+
+        Assert.True(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
+
+        StoredSecret tombstoned =
+            (await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
+        Assert.True(tombstoned.IsDeleted);
+        Assert.NotNull(tombstoned.DeletedUtc);
+        Assert.Equal(1, tombstoned.Revision);
+        Assert.Equal("carol", tombstoned.UpdatedBy);
+
+        Assert.False(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
+        Assert.False(await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task GetManifest_ReturnsAllRows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
+
+        IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
+
+        Assert.Equal(2, manifest.Count);
+        SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
+        SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
+        Assert.False(a.IsDeleted);
+        Assert.Equal(0, a.Revision);
+        Assert.True(b.IsDeleted);
+        Assert.Equal(1, b.Revision);
+    }
+
+    [SkippableFact]
+    public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
+        DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
+        DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1),
+            CancellationToken.None);
+
+        Assert.Equal(5, (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!.Revision);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1),
+            CancellationToken.None);
+
+        StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
+        Assert.Equal(6, afterNewer.Revision);
+        Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
+        Assert.Equal(t3, afterNewer.UpdatedUtc);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1),
+            CancellationToken.None);
+
+        StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
+        Assert.Equal(6, afterStale.Revision);
+        Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)),
+            CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
+
+        StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
+        Assert.Equal(1, before.Revision);
+
+        bool applied = await _store.ApplyRewrapAsync(
+            before with
+            {
+                WrappedDek = [90, 91, 92],
+                WrapNonce = [93, 94, 95],
+                WrapTag = [96, 97, 98],
+                KekId = "kek-2",
+            },
+            before.WrappedDek,
+            CancellationToken.None);
+        Assert.True(applied);
+
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
+        Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
+        Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
+        Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
+        Assert.Equal("kek-2", after.KekId);
+        Assert.Equal(before.Revision, after.Revision);
+        Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
+        Assert.Equal(before.CreatedUtc, after.CreatedUtc);
+        Assert.Equal(before.UpdatedBy, after.UpdatedBy);
+        Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
+        Assert.Equal(before.Nonce, after.Nonce);
+        Assert.Equal(before.Tag, after.Tag);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_RewrapsTombstonedRows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
+
+        StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
+        Assert.True(tomb.IsDeleted);
+
+        Assert.True(await _store.ApplyRewrapAsync(
+            tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
+            tomb.WrappedDek,
+            CancellationToken.None));
+
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
+        Assert.Equal("kek-2", after.KekId);
+        Assert.True(after.IsDeleted);
+        Assert.Equal(tomb.Revision, after.Revision);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
+
+        Assert.False(await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/cas", ciphertext: [1, 2, 3]), CancellationToken.None);
+        StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
+
+        await _store.UpsertAsync(
+            MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
+            CancellationToken.None);
+
+        bool applied = await _store.ApplyRewrapAsync(
+            before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
+            before.WrappedDek,
+            CancellationToken.None);
+
+        Assert.False(applied);
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
+        Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
+        Assert.Equal("kek-1", after.KekId);
+    }
+
+    [SkippableFact]
+    public async Task GetMany_returns_only_the_requested_rows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("a/one"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("b/two"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("c/three"), CancellationToken.None);
+
+        IReadOnlyList<StoredSecret> rows = await _store.GetManyAsync(
+            [new SecretName("a/one"), new SecretName("c/three"), new SecretName("absent/name")],
+            CancellationToken.None);
+
+        Assert.Equal(2, rows.Count);
+        Assert.Contains(rows, r => r.Name.Value == "a/one");
+        Assert.Contains(rows, r => r.Name.Value == "c/three");
+    }
+
+    [SkippableFact]
+    public async Task GetMany_short_circuits_on_an_empty_request()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        // Guards the IN () clause: an empty list would otherwise produce invalid T-SQL.
+        Assert.Empty(await _store.GetManyAsync([], CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task Migrator_is_idempotent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
+        var migrator = new SqlServerSecretsStoreMigrator(factory);
+
+        // Every node runs this at every startup — re-running must be a no-op, not an error.
+        await migrator.MigrateAsync(CancellationToken.None);
+        await migrator.MigrateAsync(CancellationToken.None);
+
+        await _store.UpsertAsync(MakeSecret("survives/migration"), CancellationToken.None);
+        await migrator.MigrateAsync(CancellationToken.None);
+
+        Assert.NotNull(await _store.GetAsync(new SecretName("survives/migration"), CancellationToken.None));
+    }
+}
+
+/// <summary>
+/// Serializes the live suite: the tests share one database server and each provisions its own
+/// schema, so running them in parallel would multiply connection load for no benefit.
+/// </summary>
+[CollectionDefinition("live-sqlserver", DisableParallelization = true)]
+public sealed class LiveSqlServerCollection;