Files
Joseph Doherty dd0a846b64 feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)
Secrets were per-node SQLite, so a secret written on one node was invisible to
the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka
replicator" fork to build only the former; both are built here so the choice is
a deployment decision (availability vs partition tolerance) rather than a
library limitation.

Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a
local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over
distributed pub/sub). Core gains ISecretsStoreMigrator, one shared
SecretLastWriterWins predicate so no two stores can disagree on a tie, the
transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real
gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was
inert and local writes would not have propagated at all.

Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real
SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural
divergence between the stores fails) and a 9-test in-process 2-node Akka
cluster over real remoting. A post-build review caught six defects, all fixed
and now covered: both replication modes could not resolve from the container
(no test had built one), an unbounded fetch that broke past SQL Server's
2100-parameter cap, a poison row that aborted the rest of its batch forever,
Enum.Parse on peer input that could restart the actor in a loop, null crypto
blobs crossing the trust boundary, and a silently dropped pull-read failure.

Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 04:08:23 -04:00

206 lines
7.6 KiB
C#

using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
using ZB.MOM.WW.Secrets.Sqlite;
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
/// <summary>
/// Convergence behaviour of the bidirectional hub sweep, exercised against real stores on both
/// sides so the assertions are about actual persisted state rather than mock interactions.
/// </summary>
public sealed class SqlServerSecretSyncServiceTests : IDisposable
{
private readonly List<string> _dbPaths = [];
private readonly SqliteSecretStore _local;
private readonly SqliteSecretStore _hubStore;
private readonly SqliteBackedHub _hub;
public SqlServerSecretSyncServiceTests()
{
_local = CreateStore();
_hubStore = CreateStore();
_hub = new SqliteBackedHub(_hubStore);
}
private SqliteSecretStore CreateStore()
{
string path = Path.Combine(Path.GetTempPath(), $"zb-sync-{Guid.NewGuid():N}.db");
_dbPaths.Add(path);
var factory = new SecretsSqliteConnectionFactory(path);
new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
return new SqliteSecretStore(factory);
}
private SqlServerSecretSyncService CreateService() => new(
_local,
_hub,
cacheInvalidator: null,
Options.Create(new SqlServerSecretsOptions { ConnectionString = "Server=unused;" }),
NullLogger<SqlServerSecretSyncService>.Instance);
private static StoredSecret Row(string name, byte marker) => new()
{
Name = new SecretName(name),
ContentType = SecretContentType.Text,
Ciphertext = [marker],
Nonce = [1],
Tag = [2],
WrappedDek = [3],
WrapNonce = [4],
WrapTag = [5],
KekId = "sha256:test",
Revision = 0,
CreatedUtc = DateTimeOffset.UtcNow,
UpdatedUtc = DateTimeOffset.UtcNow,
};
[Fact]
public async Task Sweep_pulls_a_secret_the_node_has_never_seen()
{
await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(1, pulled);
Assert.Equal(0, pushed);
StoredSecret? local = await _local.GetAsync(new SecretName("db/password"), CancellationToken.None);
Assert.NotNull(local);
Assert.Equal(0xAA, local!.Ciphertext[0]);
}
[Fact]
public async Task Sweep_pushes_a_local_write_the_hub_never_received()
{
// The case that makes best-effort publishing safe: the publish failed (hub was down), so the
// hub has never heard of this name and can never ask for it. Only a push sweep repairs this.
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(0, pulled);
Assert.Equal(1, pushed);
StoredSecret? hub = await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None);
Assert.NotNull(hub);
Assert.Equal(0xBB, hub!.Ciphertext[0]);
}
[Fact]
public async Task Sweep_preserves_the_originating_revision_when_pushing()
{
// Pushed verbatim, not upserted: if the hub stamped its own revision the row would look newer
// than the writer's copy and bounce straight back on the next sweep.
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await _local.UpsertAsync(Row("api/key", 0xCC), CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(1, localRow.Revision);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(localRow.Revision, hubRow.Revision);
Assert.Equal(localRow.UpdatedUtc, hubRow.UpdatedUtc);
}
[Fact]
public async Task Sweep_converges_both_directions_in_one_pass()
{
await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None);
await _hubStore.UpsertAsync(Row("only/hub", 0x22), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(1, pulled);
Assert.Equal(1, pushed);
Assert.NotNull(await _local.GetAsync(new SecretName("only/hub"), CancellationToken.None));
Assert.NotNull(await _hubStore.GetAsync(new SecretName("only/local"), CancellationToken.None));
}
[Fact]
public async Task Sweep_is_idempotent_so_a_converged_pair_stays_quiet()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
SqlServerSecretSyncService service = CreateService();
await service.SweepAsync(CancellationToken.None);
(int pulled, int pushed) = await service.SweepAsync(CancellationToken.None);
// A steady-state cluster must not write on every tick — that would churn the hub forever.
Assert.Equal(0, pulled);
Assert.Equal(0, pushed);
}
[Fact]
public async Task Sweep_propagates_a_delete_as_a_tombstone()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
await _local.DeleteAsync(new SecretName("api/key"), "tester", CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.True(hubRow.IsDeleted);
}
[Fact]
public async Task Sweep_does_not_resurrect_a_secret_deleted_on_the_hub()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
await _hubStore.DeleteAsync(new SecretName("api/key"), "admin", CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.True(localRow.IsDeleted);
}
[Fact]
public async Task Sweep_skips_the_hub_fetch_when_nothing_is_newer()
{
await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(0, _hub.GetManyCallCount);
}
[Fact]
public async Task Sweep_lets_the_newer_side_win_a_conflicting_write()
{
await _local.UpsertAsync(Row("api/key", 0x11), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
// Both sides then write independently; the hub's write is later.
await _local.UpsertAsync(Row("api/key", 0x22), CancellationToken.None);
await Task.Delay(10);
await _hubStore.UpsertAsync(Row("api/key", 0x33), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(0x33, localRow.Ciphertext[0]);
}
public void Dispose()
{
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (string db in _dbPaths)
{
foreach (string path in new[] { db, db + "-wal", db + "-shm" })
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}
}