feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)

Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 04:08:23 -04:00
parent e46060fada
commit dd0a846b64
55 changed files with 4848 additions and 51 deletions
@@ -0,0 +1,111 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.Replication;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer.DependencyInjection;
+using ZB.MOM.WW.Secrets.Sqlite;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.DependencyInjection;
+
+/// <summary>
+/// Container-resolution tests for both SQL-Server topologies.
+/// </summary>
+/// <remarks>
+/// These exist because a code review caught that neither mode could resolve at all: the decorators
+/// ask for the concrete <see cref="SqliteSecretStore"/>, which the core package only registered
+/// behind <see cref="ISecretStore"/>. Every unit test passed and both modes were dead on arrival,
+/// because nothing built a provider. Resolving the graph is the assertion that matters.
+/// </remarks>
+public sealed class AddZbSecretsSqlServerTests
+{
+    private static IConfiguration Config(string sqlitePath) =>
+        new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
+        {
+            ["Secrets:SqlitePath"] = sqlitePath,
+            ["Secrets:RunMigrationsOnStartup"] = "false",
+            ["Secrets:MasterKey:Source"] = "Environment",
+            ["Secrets:MasterKey:EnvVarName"] = "ZB_SECRETS_TEST_KEY",
+            ["Secrets:SqlServer:ConnectionString"] = "Server=unused;Database=x;",
+        }).Build();
+
+    private static string TempDb() => Path.Combine(Path.GetTempPath(), $"zb-di-{Guid.NewGuid():N}.db");
+
+    [Fact]
+    public void SharedStore_mode_resolves_the_SqlServer_store_as_ISecretStore()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddZbSecretsSqlServerStore(Config(TempDb()));
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+
+        // The SQL-Server store displaces SQLite — no decorator, because there is nothing to replicate.
+        Assert.IsType<SqlServerSecretStore>(provider.GetRequiredService<ISecretStore>());
+        Assert.IsType<SqlServerSecretsStoreMigrator>(provider.GetRequiredService<ISecretsStoreMigrator>());
+    }
+
+    [Fact]
+    public void HubReplication_mode_resolves_a_decorated_local_store()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddZbSecretsSqlServerReplication(Config(TempDb()));
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+
+        Assert.IsType<ReplicatingSecretStore>(provider.GetRequiredService<ISecretStore>());
+        Assert.IsType<SqlServerSecretReplicator>(provider.GetRequiredService<ISecretReplicator>());
+        // The local store is still provisioned by the core SQLite migrator, not the hub's.
+        Assert.IsType<SqliteSecretsStoreMigrator>(provider.GetRequiredService<ISecretsStoreMigrator>());
+    }
+
+    [Fact]
+    public void HubReplication_mode_registers_both_the_hub_migration_and_the_sync_service()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddZbSecretsSqlServerReplication(Config(TempDb()));
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+        IHostedService[] hosted = [.. provider.GetServices<IHostedService>()];
+
+        // The hub schema is nobody's "own" store, so every node must provision it at startup too.
+        Assert.Contains(hosted, h => h is SqlServerSecretSyncService);
+        Assert.Single(hosted, h => h.GetType().Name == "SqlServerHubMigrationHostedService");
+    }
+
+    [Fact]
+    public void The_decorator_wraps_the_undecorated_local_store_rather_than_itself()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddZbSecretsSqlServerReplication(Config(TempDb()));
+
+        using ServiceProvider provider = services.BuildServiceProvider();
+
+        // Guards against the self-referential registration that would stack-overflow on first use.
+        var decorated = (ReplicatingSecretStore)provider.GetRequiredService<ISecretStore>();
+        Assert.NotNull(decorated);
+        Assert.NotSame(decorated, provider.GetRequiredService<SqliteSecretStore>());
+    }
+
+    [Fact]
+    public void A_missing_connection_string_fails_at_registration_not_at_first_resolve()
+    {
+        IConfiguration config = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["Secrets:SqlitePath"] = TempDb(),
+            })
+            .Build();
+
+        var services = new ServiceCollection();
+        services.AddLogging();
+
+        InvalidOperationException ex = Assert.Throws<InvalidOperationException>(
+            () => services.AddZbSecretsSqlServerStore(config));
+
+        Assert.Contains("ConnectionString", ex.Message, StringComparison.Ordinal);
+    }
+}
@@ -0,0 +1,51 @@
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer;
+using ZB.MOM.WW.Secrets.Sqlite;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
+
+/// <summary>
+/// An <see cref="ISecretReplicationHub"/> backed by a real SQLite store, standing in for the shared
+/// SQL-Server hub in offline convergence tests.
+/// </summary>
+/// <remarks>
+/// Deliberately backed by a real store rather than a dictionary: the behaviour under test is
+/// last-writer-wins convergence, which lives in <c>ApplyReplicatedAsync</c>. A hand-rolled fake would
+/// be re-implementing the very logic the tests exist to check, and would happily agree with a broken
+/// reconciler. The live SQL-Server tests then confirm the T-SQL store behaves the same way.
+/// </remarks>
+/// <param name="store">The SQLite store standing in for the hub database.</param>
+public sealed class SqliteBackedHub(SqliteSecretStore store) : ISecretReplicationHub
+{
+    /// <summary>Number of times the sweep asked this hub for rows — asserts the fetch was skipped.</summary>
+    public int GetManyCallCount { get; private set; }
+
+    /// <inheritdoc />
+    public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
+        store.GetManifestAsync(ct);
+
+    /// <inheritdoc />
+    public async Task<IReadOnlyList<StoredSecret>> GetManyAsync(
+        IReadOnlyList<SecretName> names, CancellationToken ct)
+    {
+        GetManyCallCount++;
+
+        var rows = new List<StoredSecret>(names.Count);
+
+        foreach (SecretName name in names)
+        {
+            StoredSecret? row = await store.GetAsync(name, ct);
+
+            if (row is not null)
+            {
+                rows.Add(row);
+            }
+        }
+
+        return rows;
+    }
+
+    /// <inheritdoc />
+    public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) =>
+        store.ApplyReplicatedAsync(row, ct);
+}
@@ -0,0 +1,68 @@
+using Microsoft.Data.SqlClient;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
+
+/// <summary>
+/// Shared connection details for the env-gated live SQL-Server suite.
+/// </summary>
+/// <remarks>
+/// Gated on <c>SECRETS_SQLSERVER_CONNSTR</c> and skips cleanly when it is unset, matching the family
+/// live-test idiom — the offline suite must stay runnable on any machine. Set it to a database the
+/// test is allowed to create and drop a schema in, for example:
+/// <c>Server=10.100.0.35,31433;Database=ZbSecretsTest;User Id=sa;Password=...;TrustServerCertificate=True</c>.
+/// </remarks>
+public static class LiveSqlServer
+{
+    /// <summary>Environment variable holding the live connection string.</summary>
+    public const string ConnectionStringVariable = "SECRETS_SQLSERVER_CONNSTR";
+
+    /// <summary>The configured connection string, or <see langword="null"/> when the suite is not enabled.</summary>
+    public static string? ConnectionString =>
+        Environment.GetEnvironmentVariable(ConnectionStringVariable);
+
+    /// <summary>Whether the live suite should run.</summary>
+    public static bool IsEnabled => !string.IsNullOrWhiteSpace(ConnectionString);
+
+    /// <summary>Skip reason shown when the suite is not enabled.</summary>
+    public const string SkipReason =
+        "Live SQL-Server suite disabled; set SECRETS_SQLSERVER_CONNSTR to enable.";
+
+    /// <summary>
+    /// Builds options against a throwaway schema so parallel runs and repeat runs cannot collide,
+    /// and so a failed run leaves no trace in a shared database.
+    /// </summary>
+    /// <param name="schemaName">The unique schema name for this test.</param>
+    /// <returns>Options targeting that schema.</returns>
+    public static SqlServerSecretsOptions OptionsFor(string schemaName) => new()
+    {
+        ConnectionString = ConnectionString!,
+        SchemaName = schemaName,
+    };
+
+    /// <summary>Generates a unique, allow-list-legal schema name for one test class.</summary>
+    /// <returns>A fresh schema name.</returns>
+    public static string NewSchemaName() => $"zbtest_{Guid.NewGuid():N}"[..32];
+
+    /// <summary>Drops the throwaway schema and its tables.</summary>
+    /// <param name="schemaName">The schema to drop.</param>
+    /// <returns>A task that completes when the schema is gone.</returns>
+    public static async Task DropSchemaAsync(string schemaName)
+    {
+        if (!IsEnabled)
+        {
+            return;
+        }
+
+        await using var connection = new SqlConnection(ConnectionString);
+        await connection.OpenAsync();
+
+        await using SqlCommand command = connection.CreateCommand();
+        command.CommandText = $"""
+            IF OBJECT_ID(N'[{schemaName}].[secret]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[secret];
+            IF OBJECT_ID(N'[{schemaName}].[schema_version]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[schema_version];
+            IF EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'{schemaName}') EXEC(N'DROP SCHEMA [{schemaName}]');
+            """;
+
+        await command.ExecuteNonQueryAsync();
+    }
+}
@@ -0,0 +1,365 @@
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
+
+/// <summary>
+/// The SQLite store's test suite, ported case-for-case onto the SQL-Server store.
+/// </summary>
+/// <remarks>
+/// Ported rather than newly written on purpose. The two stores must be behaviourally identical —
+/// a cluster can hold both at once — so the strongest available check is that the assertions
+/// written against SQLite pass unchanged here. Anywhere the T-SQL diverges in observable behaviour,
+/// one of these fails.
+/// </remarks>
+[Collection("live-sqlserver")]
+public sealed class SqlServerSecretStoreLiveTests : IAsyncLifetime
+{
+    private readonly string _schema = LiveSqlServer.NewSchemaName();
+    private SqlServerSecretStore _store = null!;
+
+    public async Task InitializeAsync()
+    {
+        if (!LiveSqlServer.IsEnabled)
+        {
+            return;
+        }
+
+        var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
+        await new SqlServerSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
+        _store = new SqlServerSecretStore(factory);
+    }
+
+    public Task DisposeAsync() => LiveSqlServer.DropSchemaAsync(_schema);
+
+    private static StoredSecret MakeSecret(
+        string name,
+        long revision = 0,
+        byte[]? ciphertext = null,
+        DateTimeOffset? updatedUtc = null,
+        DateTimeOffset? createdUtc = null,
+        bool isDeleted = false,
+        DateTimeOffset? deletedUtc = null,
+        string? createdBy = "alice",
+        string? updatedBy = "alice") => new()
+        {
+            Name = new SecretName(name),
+            Description = "desc",
+            ContentType = SecretContentType.ConnectionString,
+            Ciphertext = ciphertext ?? [1, 2, 3],
+            Nonce = [4, 5, 6],
+            Tag = [7, 8, 9],
+            WrappedDek = [10, 11, 12],
+            WrapNonce = [13, 14, 15],
+            WrapTag = [16, 17, 18],
+            KekId = "kek-1",
+            Revision = revision,
+            IsDeleted = isDeleted,
+            DeletedUtc = deletedUtc,
+            CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
+            UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
+            CreatedBy = createdBy,
+            UpdatedBy = updatedBy,
+        };
+
+    [SkippableFact]
+    public async Task Upsert_Then_Get_RoundTrips()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/db-conn"), CancellationToken.None);
+
+        StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
+
+        Assert.NotNull(got);
+        Assert.Equal("app/db-conn", got!.Name.Value);
+        Assert.Equal("desc", got.Description);
+        Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
+        Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
+        Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
+        Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
+        Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
+        Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
+        Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
+        Assert.Equal("kek-1", got.KekId);
+        Assert.Equal(0, got.Revision);
+        Assert.False(got.IsDeleted);
+        Assert.Null(got.DeletedUtc);
+        Assert.Equal("alice", got.CreatedBy);
+        Assert.Equal("alice", got.UpdatedBy);
+    }
+
+    [SkippableFact]
+    public async Task Get_ReturnsNull_WhenAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        Assert.Null(await _store.GetAsync(new SecretName("nope"), CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice"),
+            CancellationToken.None);
+
+        StoredSecret afterFirst =
+            (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
+        Assert.Equal(0, afterFirst.Revision);
+        DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob"),
+            CancellationToken.None);
+
+        StoredSecret afterSecond =
+            (await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
+        Assert.Equal(1, afterSecond.Revision);
+        Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
+        Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
+        Assert.Equal("alice", afterSecond.CreatedBy);
+        Assert.Equal("bob", afterSecond.UpdatedBy);
+        Assert.False(afterSecond.IsDeleted);
+    }
+
+    [SkippableFact]
+    public async Task List_ExcludesTombstoned_ByDefault()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
+
+        IReadOnlyList<SecretMetadata> visible =
+            await _store.ListAsync(includeDeleted: false, CancellationToken.None);
+        Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
+        Assert.Contains(visible, m => m.Name.Value == "keep");
+
+        IReadOnlyList<SecretMetadata> all =
+            await _store.ListAsync(includeDeleted: true, CancellationToken.None);
+        Assert.Contains(all, m => m.Name.Value == "gone");
+        Assert.Contains(all, m => m.Name.Value == "keep");
+    }
+
+    [SkippableFact]
+    public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
+
+        Assert.True(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
+
+        StoredSecret tombstoned =
+            (await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
+        Assert.True(tombstoned.IsDeleted);
+        Assert.NotNull(tombstoned.DeletedUtc);
+        Assert.Equal(1, tombstoned.Revision);
+        Assert.Equal("carol", tombstoned.UpdatedBy);
+
+        Assert.False(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
+        Assert.False(await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task GetManifest_ReturnsAllRows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
+
+        IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
+
+        Assert.Equal(2, manifest.Count);
+        SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
+        SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
+        Assert.False(a.IsDeleted);
+        Assert.Equal(0, a.Revision);
+        Assert.True(b.IsDeleted);
+        Assert.Equal(1, b.Revision);
+    }
+
+    [SkippableFact]
+    public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
+        DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
+        DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1),
+            CancellationToken.None);
+
+        Assert.Equal(5, (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!.Revision);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1),
+            CancellationToken.None);
+
+        StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
+        Assert.Equal(6, afterNewer.Revision);
+        Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
+        Assert.Equal(t3, afterNewer.UpdatedUtc);
+
+        await _store.ApplyReplicatedAsync(
+            MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1),
+            CancellationToken.None);
+
+        StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
+        Assert.Equal(6, afterStale.Revision);
+        Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(
+            MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)),
+            CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
+
+        StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
+        Assert.Equal(1, before.Revision);
+
+        bool applied = await _store.ApplyRewrapAsync(
+            before with
+            {
+                WrappedDek = [90, 91, 92],
+                WrapNonce = [93, 94, 95],
+                WrapTag = [96, 97, 98],
+                KekId = "kek-2",
+            },
+            before.WrappedDek,
+            CancellationToken.None);
+        Assert.True(applied);
+
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
+        Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
+        Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
+        Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
+        Assert.Equal("kek-2", after.KekId);
+        Assert.Equal(before.Revision, after.Revision);
+        Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
+        Assert.Equal(before.CreatedUtc, after.CreatedUtc);
+        Assert.Equal(before.UpdatedBy, after.UpdatedBy);
+        Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
+        Assert.Equal(before.Nonce, after.Nonce);
+        Assert.Equal(before.Tag, after.Tag);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_RewrapsTombstonedRows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
+        await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
+
+        StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
+        Assert.True(tomb.IsDeleted);
+
+        Assert.True(await _store.ApplyRewrapAsync(
+            tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
+            tomb.WrappedDek,
+            CancellationToken.None));
+
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
+        Assert.Equal("kek-2", after.KekId);
+        Assert.True(after.IsDeleted);
+        Assert.Equal(tomb.Revision, after.Revision);
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
+
+        Assert.False(await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("app/cas", ciphertext: [1, 2, 3]), CancellationToken.None);
+        StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
+
+        await _store.UpsertAsync(
+            MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
+            CancellationToken.None);
+
+        bool applied = await _store.ApplyRewrapAsync(
+            before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
+            before.WrappedDek,
+            CancellationToken.None);
+
+        Assert.False(applied);
+        StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
+        Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
+        Assert.Equal("kek-1", after.KekId);
+    }
+
+    [SkippableFact]
+    public async Task GetMany_returns_only_the_requested_rows()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        await _store.UpsertAsync(MakeSecret("a/one"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("b/two"), CancellationToken.None);
+        await _store.UpsertAsync(MakeSecret("c/three"), CancellationToken.None);
+
+        IReadOnlyList<StoredSecret> rows = await _store.GetManyAsync(
+            [new SecretName("a/one"), new SecretName("c/three"), new SecretName("absent/name")],
+            CancellationToken.None);
+
+        Assert.Equal(2, rows.Count);
+        Assert.Contains(rows, r => r.Name.Value == "a/one");
+        Assert.Contains(rows, r => r.Name.Value == "c/three");
+    }
+
+    [SkippableFact]
+    public async Task GetMany_short_circuits_on_an_empty_request()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        // Guards the IN () clause: an empty list would otherwise produce invalid T-SQL.
+        Assert.Empty(await _store.GetManyAsync([], CancellationToken.None));
+    }
+
+    [SkippableFact]
+    public async Task Migrator_is_idempotent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
+        var migrator = new SqlServerSecretsStoreMigrator(factory);
+
+        // Every node runs this at every startup — re-running must be a no-op, not an error.
+        await migrator.MigrateAsync(CancellationToken.None);
+        await migrator.MigrateAsync(CancellationToken.None);
+
+        await _store.UpsertAsync(MakeSecret("survives/migration"), CancellationToken.None);
+        await migrator.MigrateAsync(CancellationToken.None);
+
+        Assert.NotNull(await _store.GetAsync(new SecretName("survives/migration"), CancellationToken.None));
+    }
+}
+
+/// <summary>
+/// Serializes the live suite: the tests share one database server and each provisions its own
+/// schema, so running them in parallel would multiply connection load for no benefit.
+/// </summary>
+[CollectionDefinition("live-sqlserver", DisableParallelization = true)]
+public sealed class LiveSqlServerCollection;
@@ -0,0 +1,205 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
+using ZB.MOM.WW.Secrets.Sqlite;
+
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
+
+/// <summary>
+/// Convergence behaviour of the bidirectional hub sweep, exercised against real stores on both
+/// sides so the assertions are about actual persisted state rather than mock interactions.
+/// </summary>
+public sealed class SqlServerSecretSyncServiceTests : IDisposable
+{
+    private readonly List<string> _dbPaths = [];
+    private readonly SqliteSecretStore _local;
+    private readonly SqliteSecretStore _hubStore;
+    private readonly SqliteBackedHub _hub;
+
+    public SqlServerSecretSyncServiceTests()
+    {
+        _local = CreateStore();
+        _hubStore = CreateStore();
+        _hub = new SqliteBackedHub(_hubStore);
+    }
+
+    private SqliteSecretStore CreateStore()
+    {
+        string path = Path.Combine(Path.GetTempPath(), $"zb-sync-{Guid.NewGuid():N}.db");
+        _dbPaths.Add(path);
+        var factory = new SecretsSqliteConnectionFactory(path);
+        new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
+        return new SqliteSecretStore(factory);
+    }
+
+    private SqlServerSecretSyncService CreateService() => new(
+        _local,
+        _hub,
+        cacheInvalidator: null,
+        Options.Create(new SqlServerSecretsOptions { ConnectionString = "Server=unused;" }),
+        NullLogger<SqlServerSecretSyncService>.Instance);
+
+    private static StoredSecret Row(string name, byte marker) => new()
+    {
+        Name = new SecretName(name),
+        ContentType = SecretContentType.Text,
+        Ciphertext = [marker],
+        Nonce = [1],
+        Tag = [2],
+        WrappedDek = [3],
+        WrapNonce = [4],
+        WrapTag = [5],
+        KekId = "sha256:test",
+        Revision = 0,
+        CreatedUtc = DateTimeOffset.UtcNow,
+        UpdatedUtc = DateTimeOffset.UtcNow,
+    };
+
+    [Fact]
+    public async Task Sweep_pulls_a_secret_the_node_has_never_seen()
+    {
+        await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None);
+
+        (int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
+
+        Assert.Equal(1, pulled);
+        Assert.Equal(0, pushed);
+
+        StoredSecret? local = await _local.GetAsync(new SecretName("db/password"), CancellationToken.None);
+        Assert.NotNull(local);
+        Assert.Equal(0xAA, local!.Ciphertext[0]);
+    }
+
+    [Fact]
+    public async Task Sweep_pushes_a_local_write_the_hub_never_received()
+    {
+        // The case that makes best-effort publishing safe: the publish failed (hub was down), so the
+        // hub has never heard of this name and can never ask for it. Only a push sweep repairs this.
+        await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
+
+        (int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
+
+        Assert.Equal(0, pulled);
+        Assert.Equal(1, pushed);
+
+        StoredSecret? hub = await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None);
+        Assert.NotNull(hub);
+        Assert.Equal(0xBB, hub!.Ciphertext[0]);
+    }
+
+    [Fact]
+    public async Task Sweep_preserves_the_originating_revision_when_pushing()
+    {
+        // Pushed verbatim, not upserted: if the hub stamped its own revision the row would look newer
+        // than the writer's copy and bounce straight back on the next sweep.
+        await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
+        await _local.UpsertAsync(Row("api/key", 0xCC), CancellationToken.None);
+
+        StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
+        Assert.Equal(1, localRow.Revision);
+
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
+        Assert.Equal(localRow.Revision, hubRow.Revision);
+        Assert.Equal(localRow.UpdatedUtc, hubRow.UpdatedUtc);
+    }
+
+    [Fact]
+    public async Task Sweep_converges_both_directions_in_one_pass()
+    {
+        await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None);
+        await _hubStore.UpsertAsync(Row("only/hub", 0x22), CancellationToken.None);
+
+        (int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
+
+        Assert.Equal(1, pulled);
+        Assert.Equal(1, pushed);
+
+        Assert.NotNull(await _local.GetAsync(new SecretName("only/hub"), CancellationToken.None));
+        Assert.NotNull(await _hubStore.GetAsync(new SecretName("only/local"), CancellationToken.None));
+    }
+
+    [Fact]
+    public async Task Sweep_is_idempotent_so_a_converged_pair_stays_quiet()
+    {
+        await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
+
+        SqlServerSecretSyncService service = CreateService();
+        await service.SweepAsync(CancellationToken.None);
+
+        (int pulled, int pushed) = await service.SweepAsync(CancellationToken.None);
+
+        // A steady-state cluster must not write on every tick — that would churn the hub forever.
+        Assert.Equal(0, pulled);
+        Assert.Equal(0, pushed);
+    }
+
+    [Fact]
+    public async Task Sweep_propagates_a_delete_as_a_tombstone()
+    {
+        await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        await _local.DeleteAsync(new SecretName("api/key"), "tester", CancellationToken.None);
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
+        Assert.True(hubRow.IsDeleted);
+    }
+
+    [Fact]
+    public async Task Sweep_does_not_resurrect_a_secret_deleted_on_the_hub()
+    {
+        await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        await _hubStore.DeleteAsync(new SecretName("api/key"), "admin", CancellationToken.None);
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
+        Assert.True(localRow.IsDeleted);
+    }
+
+    [Fact]
+    public async Task Sweep_skips_the_hub_fetch_when_nothing_is_newer()
+    {
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        Assert.Equal(0, _hub.GetManyCallCount);
+    }
+
+    [Fact]
+    public async Task Sweep_lets_the_newer_side_win_a_conflicting_write()
+    {
+        await _local.UpsertAsync(Row("api/key", 0x11), CancellationToken.None);
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        // Both sides then write independently; the hub's write is later.
+        await _local.UpsertAsync(Row("api/key", 0x22), CancellationToken.None);
+        await Task.Delay(10);
+        await _hubStore.UpsertAsync(Row("api/key", 0x33), CancellationToken.None);
+
+        await CreateService().SweepAsync(CancellationToken.None);
+
+        StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
+        Assert.Equal(0x33, localRow.Ciphertext[0]);
+    }
+
+    public void Dispose()
+    {
+        Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
+
+        foreach (string db in _dbPaths)
+        {
+            foreach (string path in new[] { db, db + "-wal", db + "-shm" })
+            {
+                if (File.Exists(path))
+                {
+                    File.Delete(path);
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,96 @@
+namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
+
+/// <summary>
+/// The schema-name allow-list is the injection boundary for the one value that cannot be
+/// parameterized (a T-SQL object name), so it gets adversarial coverage rather than a happy path.
+/// </summary>
+public sealed class SqlServerSecretsOptionsTests
+{
+    [Theory]
+    [InlineData("zbsecrets")]
+    [InlineData("ZbSecrets")]
+    [InlineData("_private")]
+    [InlineData("schema_1")]
+    public void SchemaName_accepts_plain_identifiers(string value)
+    {
+        var options = new SqlServerSecretsOptions { SchemaName = value };
+
+        Assert.Equal(value, options.SchemaName);
+    }
+
+    [Theory]
+    // The bracket-escape trick that would otherwise break out of the [quoting] at the use site.
+    [InlineData("evil]]; DROP TABLE secret; --")]
+    [InlineData("dbo.secret")]
+    [InlineData("with space")]
+    [InlineData("quote'name")]
+    [InlineData("semi;colon")]
+    [InlineData("1leading_digit")]
+    [InlineData("")]
+    [InlineData("   ")]
+    public void SchemaName_rejects_anything_that_is_not_a_plain_identifier(string value)
+    {
+        Assert.Throws<ArgumentException>(() => new SqlServerSecretsOptions { SchemaName = value });
+    }
+
+    [Fact]
+    public void SchemaName_rejects_an_over_long_identifier()
+    {
+        Assert.Throws<ArgumentException>(() =>
+            new SqlServerSecretsOptions { SchemaName = new string('a', 129) });
+    }
+
+    [Fact]
+    public void Validate_rejects_a_missing_connection_string()
+    {
+        InvalidOperationException ex =
+            Assert.Throws<InvalidOperationException>(() => new SqlServerSecretsOptions().Validate());
+
+        Assert.Contains("ConnectionString", ex.Message, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public void Validate_rejects_a_non_positive_sync_interval()
+    {
+        var options = new SqlServerSecretsOptions
+        {
+            ConnectionString = "Server=x;",
+            SyncInterval = TimeSpan.Zero,
+        };
+
+        Assert.Throws<InvalidOperationException>(options.Validate);
+    }
+
+    [Fact]
+    public void Validate_rejects_a_non_positive_command_timeout()
+    {
+        var options = new SqlServerSecretsOptions
+        {
+            ConnectionString = "Server=x;",
+            CommandTimeout = TimeSpan.FromSeconds(-1),
+        };
+
+        Assert.Throws<InvalidOperationException>(options.Validate);
+    }
+
+    [Fact]
+    public void Validate_accepts_a_fully_configured_instance()
+    {
+        var options = new SqlServerSecretsOptions { ConnectionString = "Server=x;Database=y;" };
+
+        options.Validate();
+    }
+
+    [Fact]
+    public void CreateSchemaDdl_quotes_the_schema_name_everywhere_it_appears()
+    {
+        string ddl = SqlServerSecretsSchema.CreateSchemaDdl("zbsecrets");
+
+        Assert.Contains("[zbsecrets].[secret]", ddl, StringComparison.Ordinal);
+        Assert.Contains("[zbsecrets].[schema_version]", ddl, StringComparison.Ordinal);
+        // Every DDL statement is guarded, so re-running the migration on a provisioned database is
+        // a no-op rather than an error.
+        Assert.Contains("IF NOT EXISTS", ddl, StringComparison.Ordinal);
+        Assert.Contains("IS NULL", ddl, StringComparison.Ordinal);
+    }
+}
@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="Xunit.SkippableFact" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.Logging" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\ZB.MOM.WW.Secrets.Replicator.SqlServer\ZB.MOM.WW.Secrets.Replicator.SqlServer.csproj" />
+  </ItemGroup>
+
+</Project>