feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)
Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
+81
@@ -0,0 +1,81 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The DTO is the trust boundary for anything a peer sends. Every rejection here must surface as an
|
||||
/// <see cref="ArgumentException"/>, because that is the exception type the receiving actor filters
|
||||
/// on — anything else escapes its handler, restarts the actor, and (since the row is redelivered)
|
||||
/// loops.
|
||||
/// </summary>
|
||||
public sealed class HostileWireInputTests
|
||||
{
|
||||
private static SecretRowDto Valid() => new()
|
||||
{
|
||||
Name = "app/ok",
|
||||
ContentType = "Text",
|
||||
Ciphertext = [1],
|
||||
Nonce = [2],
|
||||
Tag = [3],
|
||||
WrappedDek = [4],
|
||||
WrapNonce = [5],
|
||||
WrapTag = [6],
|
||||
KekId = "sha256:x",
|
||||
Revision = 0,
|
||||
CreatedUtc = "2026-01-01T00:00:00.0000000+00:00",
|
||||
UpdatedUtc = "2026-01-01T00:00:00.0000000+00:00",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void A_valid_row_materializes()
|
||||
{
|
||||
StoredSecret row = Valid().ToStoredSecret();
|
||||
|
||||
Assert.Equal("app/ok", row.Name.Value);
|
||||
Assert.Equal(SecretContentType.Text, row.ContentType);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("../../etc/passwd")]
|
||||
[InlineData("/rooted")]
|
||||
[InlineData("has space")]
|
||||
[InlineData("")]
|
||||
public void A_malformed_name_is_rejected(string name)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => (Valid() with { Name = name }).ToStoredSecret());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// An in-range number names no member — Enum.Parse would happily produce (SecretContentType)4096
|
||||
// and the store would persist it for later readers that assume a defined value.
|
||||
[InlineData("4096")]
|
||||
// Overflows the underlying type: Enum.Parse throws OverflowException, which is NOT an
|
||||
// ArgumentException, so it would escape the actor's filter and restart it.
|
||||
[InlineData("99999999999999999999")]
|
||||
[InlineData("NotAContentType")]
|
||||
[InlineData("")]
|
||||
public void An_unrecognized_content_type_is_rejected_as_an_ArgumentException(string contentType)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(
|
||||
() => (Valid() with { ContentType = contentType }).ToStoredSecret());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_null_crypto_blob_is_rejected_at_the_boundary()
|
||||
{
|
||||
// `required byte[]` means "present in the payload", not "non-null" — an explicit JSON null
|
||||
// satisfies it. Caught here, or it blows up deep in the store's parameter binding instead.
|
||||
Assert.Throws<ArgumentException>(
|
||||
() => (Valid() with { Ciphertext = null! }).ToStoredSecret());
|
||||
Assert.Throws<ArgumentException>(
|
||||
() => (Valid() with { WrappedDek = null! }).ToStoredSecret());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_malformed_timestamp_is_rejected()
|
||||
{
|
||||
Assert.ThrowsAny<FormatException>(
|
||||
() => (Valid() with { UpdatedUtc = "not-a-date" }).ToStoredSecret());
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Serialization;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Wire-contract tests. Secret ciphertext crosses the network as these bytes, so the round-trip is
|
||||
/// asserted field-by-field rather than by equality alone — a silently dropped crypto field would
|
||||
/// produce a row that stores fine and fails to decrypt much later, on another node.
|
||||
/// </summary>
|
||||
public sealed class SecretReplicationSerializerTests : TestKit
|
||||
{
|
||||
public SecretReplicationSerializerTests()
|
||||
: base(AkkaSecretsReplication.SerializationConfig)
|
||||
{
|
||||
}
|
||||
|
||||
private static StoredSecret Row(string name) => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
Description = "a description",
|
||||
ContentType = SecretContentType.ConnectionString,
|
||||
Ciphertext = [1, 2, 3, 250],
|
||||
Nonce = [4, 5, 6],
|
||||
Tag = [7, 8, 9],
|
||||
WrappedDek = [10, 11, 12],
|
||||
WrapNonce = [13, 14, 15],
|
||||
WrapTag = [16, 17, 18],
|
||||
KekId = "sha256:abc",
|
||||
Revision = 42,
|
||||
IsDeleted = true,
|
||||
DeletedUtc = new DateTimeOffset(2026, 3, 2, 1, 0, 0, TimeSpan.Zero),
|
||||
CreatedUtc = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
||||
UpdatedUtc = new DateTimeOffset(2026, 2, 1, 0, 0, 0, TimeSpan.Zero),
|
||||
CreatedBy = "alice",
|
||||
UpdatedBy = "bob",
|
||||
};
|
||||
|
||||
private object RoundTrip(object message)
|
||||
{
|
||||
Serializer serializer = Sys.Serialization.FindSerializerFor(message);
|
||||
Assert.IsType<SecretReplicationSerializer>(serializer);
|
||||
|
||||
byte[] bytes = serializer.ToBinary(message);
|
||||
string manifest = ((SerializerWithStringManifest)serializer).Manifest(message);
|
||||
|
||||
return ((SerializerWithStringManifest)serializer).FromBinary(bytes, manifest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecretRowsMessage_round_trips_every_field()
|
||||
{
|
||||
StoredSecret original = Row("app/db-conn");
|
||||
var message = new SecretRowsMessage([SecretRowDto.FromStoredSecret(original)]);
|
||||
|
||||
var restored = (SecretRowsMessage)RoundTrip(message);
|
||||
StoredSecret result = restored.Rows.Single().ToStoredSecret();
|
||||
|
||||
Assert.Equal(original.Name.Value, result.Name.Value);
|
||||
Assert.Equal(original.Description, result.Description);
|
||||
Assert.Equal(original.ContentType, result.ContentType);
|
||||
Assert.Equal(original.Ciphertext, result.Ciphertext);
|
||||
Assert.Equal(original.Nonce, result.Nonce);
|
||||
Assert.Equal(original.Tag, result.Tag);
|
||||
Assert.Equal(original.WrappedDek, result.WrappedDek);
|
||||
Assert.Equal(original.WrapNonce, result.WrapNonce);
|
||||
Assert.Equal(original.WrapTag, result.WrapTag);
|
||||
Assert.Equal(original.KekId, result.KekId);
|
||||
Assert.Equal(original.Revision, result.Revision);
|
||||
Assert.Equal(original.IsDeleted, result.IsDeleted);
|
||||
Assert.Equal(original.DeletedUtc, result.DeletedUtc);
|
||||
Assert.Equal(original.CreatedUtc, result.CreatedUtc);
|
||||
Assert.Equal(original.UpdatedUtc, result.UpdatedUtc);
|
||||
Assert.Equal(original.CreatedBy, result.CreatedBy);
|
||||
Assert.Equal(original.UpdatedBy, result.UpdatedBy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Timestamps_survive_the_wire_exactly()
|
||||
{
|
||||
// Last-writer-wins compares UpdatedUtc for EQUALITY to break revision ties. A serializer that
|
||||
// rounded to the millisecond would turn a tie into a spurious "newer" and two nodes would
|
||||
// then overwrite each other forever, so tick-exactness is a correctness requirement.
|
||||
StoredSecret original = Row("app/precise") with
|
||||
{
|
||||
UpdatedUtc = new DateTimeOffset(2026, 2, 1, 12, 34, 56, TimeSpan.Zero).AddTicks(1234567),
|
||||
};
|
||||
|
||||
var restored = (SecretRowsMessage)RoundTrip(
|
||||
new SecretRowsMessage([SecretRowDto.FromStoredSecret(original)]));
|
||||
|
||||
Assert.Equal(original.UpdatedUtc, restored.Rows.Single().ToStoredSecret().UpdatedUtc);
|
||||
Assert.Equal(original.UpdatedUtc.Ticks, restored.Rows.Single().ToStoredSecret().UpdatedUtc.Ticks);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManifestAnnounce_round_trips()
|
||||
{
|
||||
var message = new SecretManifestAnnounce(
|
||||
[
|
||||
SecretManifestEntryDto.FromEntry(new SecretManifestEntry
|
||||
{
|
||||
Name = new SecretName("a/one"),
|
||||
Revision = 3,
|
||||
UpdatedUtc = new DateTimeOffset(2026, 5, 5, 5, 5, 5, TimeSpan.Zero),
|
||||
IsDeleted = true,
|
||||
}),
|
||||
]);
|
||||
|
||||
var restored = (SecretManifestAnnounce)RoundTrip(message);
|
||||
SecretManifestEntry entry = restored.Entries.Single().ToEntry();
|
||||
|
||||
Assert.Equal("a/one", entry.Name.Value);
|
||||
Assert.Equal(3, entry.Revision);
|
||||
Assert.True(entry.IsDeleted);
|
||||
Assert.Equal(new DateTimeOffset(2026, 5, 5, 5, 5, 5, TimeSpan.Zero), entry.UpdatedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PullRequest_round_trips()
|
||||
{
|
||||
var restored = (SecretPullRequest)RoundTrip(new SecretPullRequest(["a/one", "b/two"]));
|
||||
|
||||
Assert.Equal(["a/one", "b/two"], restored.Names);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unknown_manifest_fails_loudly_rather_than_guessing()
|
||||
{
|
||||
// A mixed-version cluster must fail on an unrecognized payload, not mis-deserialize a secret.
|
||||
var serializer = (SerializerWithStringManifest)Sys.Serialization
|
||||
.FindSerializerFor(new SecretPullRequest([]));
|
||||
|
||||
Assert.Throws<System.Runtime.Serialization.SerializationException>(
|
||||
() => serializer.FromBinary([1, 2, 3], "zbs-from-the-future-v9"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_wire_contract_carries_no_field_that_could_hold_key_material()
|
||||
{
|
||||
// Structural guard on the trust boundary: the DTO's surface is fixed and reviewed. If someone
|
||||
// adds a property to SecretRowDto, this test fails and forces a deliberate look at whether
|
||||
// the new field is safe to put on the wire.
|
||||
string[] properties = [.. typeof(SecretRowDto)
|
||||
.GetProperties()
|
||||
.Select(p => p.Name)
|
||||
.Order(StringComparer.Ordinal)];
|
||||
|
||||
Assert.Equal(
|
||||
[
|
||||
"Ciphertext", "ContentType", "CreatedBy", "CreatedUtc", "DeletedUtc", "Description",
|
||||
"IsDeleted", "KekId", "Name", "Nonce", "Revision", "Tag", "UpdatedBy", "UpdatedUtc",
|
||||
"WrapNonce", "WrapTag", "WrappedDek",
|
||||
],
|
||||
properties);
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Two real cluster nodes, two real SQLite stores, real remoting over loopback — the gate the G-7
|
||||
/// design called for, run in-process so it belongs to the ordinary offline suite instead of needing
|
||||
/// a deployed rig.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing here is faked: each node runs its own <see cref="ActorSystem"/> with Akka.Remote on its
|
||||
/// own port, they form a cluster, and rows travel through the actual serializer and the actual
|
||||
/// distributed pub/sub mediator. A mock-based test of this behaviour would prove almost nothing —
|
||||
/// the failure modes worth catching (self-echo, sender loss across the mediator, tombstones that do
|
||||
/// not propagate, a partition that never re-converges) only appear when messages really cross nodes.
|
||||
/// </remarks>
|
||||
public sealed class TwoNodeClusterReplicationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly List<string> _dbPaths = [];
|
||||
private ActorSystem _systemA = null!;
|
||||
private ActorSystem _systemB = null!;
|
||||
private SqliteSecretStore _storeA = null!;
|
||||
private SqliteSecretStore _storeB = null!;
|
||||
private ISecretStore _writableA = null!;
|
||||
private IActorRef _replicatorB = null!;
|
||||
|
||||
// Short so anti-entropy assertions do not dominate the suite's runtime.
|
||||
private static readonly TimeSpan AnnounceInterval = TimeSpan.FromMilliseconds(300);
|
||||
private static readonly TimeSpan Patience = TimeSpan.FromSeconds(20);
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_storeA = CreateStore();
|
||||
_storeB = CreateStore();
|
||||
|
||||
// Port 0 lets the OS assign free ports, so parallel test runs cannot collide.
|
||||
_systemA = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(port: 0));
|
||||
int portA = ClusterPort(_systemA);
|
||||
|
||||
// Node A seeds itself; node B joins it.
|
||||
_systemA.Dispose();
|
||||
_systemA = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(portA, seedPort: portA));
|
||||
_systemB = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(port: 0, seedPort: portA));
|
||||
|
||||
await AwaitClusterUpAsync(_systemA, expectedMembers: 2);
|
||||
await AwaitClusterUpAsync(_systemB, expectedMembers: 2);
|
||||
|
||||
IActorRef replicatorA = _systemA.ActorOf(
|
||||
SecretReplicationActor.Props(_storeA, null, AnnounceInterval), "zb-secret-replication");
|
||||
_replicatorB = _systemB.ActorOf(
|
||||
SecretReplicationActor.Props(_storeB, null, AnnounceInterval), "zb-secret-replication");
|
||||
|
||||
_writableA = new ReplicatingSecretStore(
|
||||
_storeA, new AkkaSecretReplicator(replicatorA), NullLogger<ReplicatingSecretStore>.Instance);
|
||||
|
||||
// The mediator gossips subscriptions between nodes; publishing before both sides are
|
||||
// registered would silently drop the message.
|
||||
await AwaitBothSubscribedAsync();
|
||||
}
|
||||
|
||||
private SqliteSecretStore CreateStore()
|
||||
{
|
||||
string path = Path.Combine(Path.GetTempPath(), $"zb-akka-{Guid.NewGuid():N}.db");
|
||||
_dbPaths.Add(path);
|
||||
var factory = new SecretsSqliteConnectionFactory(path);
|
||||
new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
|
||||
return new SqliteSecretStore(factory);
|
||||
}
|
||||
|
||||
private static Config ClusterConfig(int port, int? seedPort = null)
|
||||
{
|
||||
string seeds = seedPort is null
|
||||
? "[]"
|
||||
: $"[\"akka.tcp://zb-secrets-cluster@127.0.0.1:{seedPort}\"]";
|
||||
|
||||
return ConfigurationFactory.ParseString($$"""
|
||||
akka {
|
||||
loglevel = WARNING
|
||||
actor.provider = cluster
|
||||
remote.dot-netty.tcp {
|
||||
hostname = "127.0.0.1"
|
||||
public-hostname = "127.0.0.1"
|
||||
port = {{port}}
|
||||
}
|
||||
cluster {
|
||||
seed-nodes = {{seeds}}
|
||||
downing-provider-class = "Akka.Cluster.SBR.SplitBrainResolverProvider"
|
||||
}
|
||||
}
|
||||
""").WithFallback(AkkaSecretsReplication.SerializationConfig);
|
||||
}
|
||||
|
||||
private static int ClusterPort(ActorSystem system) =>
|
||||
Cluster.Get(system).SelfAddress.Port
|
||||
?? throw new InvalidOperationException("Cluster address has no port.");
|
||||
|
||||
private static async Task AwaitClusterUpAsync(ActorSystem system, int expectedMembers)
|
||||
{
|
||||
Cluster cluster = Cluster.Get(system);
|
||||
DateTime deadline = DateTime.UtcNow + Patience;
|
||||
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Cluster did not reach {expectedMembers} Up members within {Patience}.");
|
||||
}
|
||||
|
||||
// Writes a throwaway secret on A and waits for it to land on B. That round trip is the only
|
||||
// reliable signal that pub/sub subscriptions have gossiped across both nodes.
|
||||
private async Task AwaitBothSubscribedAsync()
|
||||
{
|
||||
var probe = new SecretName("warmup/probe");
|
||||
DateTime deadline = DateTime.UtcNow + Patience;
|
||||
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
await _writableA.UpsertAsync(Row("warmup/probe", 0x01), CancellationToken.None);
|
||||
|
||||
if (await WaitForAsync(_storeB, probe, _ => true, TimeSpan.FromMilliseconds(500)))
|
||||
{
|
||||
await _storeA.DeleteAsync(probe, "warmup", CancellationToken.None);
|
||||
await _storeB.DeleteAsync(probe, "warmup", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException("Nodes did not establish pub/sub subscriptions in time.");
|
||||
}
|
||||
|
||||
private static StoredSecret Row(string name, byte marker) => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
ContentType = SecretContentType.Text,
|
||||
Ciphertext = [marker],
|
||||
Nonce = [1],
|
||||
Tag = [2],
|
||||
WrappedDek = [3],
|
||||
WrapNonce = [4],
|
||||
WrapTag = [5],
|
||||
KekId = "sha256:shared-kek",
|
||||
Revision = 0,
|
||||
CreatedUtc = DateTimeOffset.UtcNow,
|
||||
UpdatedUtc = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
private static async Task<bool> WaitForAsync(
|
||||
ISecretStore store, SecretName name, Func<StoredSecret, bool> predicate, TimeSpan? timeout = null)
|
||||
{
|
||||
DateTime deadline = DateTime.UtcNow + (timeout ?? Patience);
|
||||
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
StoredSecret? row = await store.GetAsync(name, CancellationToken.None);
|
||||
|
||||
if (row is not null && predicate(row))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_secret_written_on_node_A_becomes_resolvable_on_node_B()
|
||||
{
|
||||
var name = new SecretName("db/password");
|
||||
await _writableA.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0xAA),
|
||||
"Node B never received the secret written on node A.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_ciphertext_arrives_byte_identical()
|
||||
{
|
||||
var name = new SecretName("app/exact");
|
||||
await _writableA.UpsertAsync(Row("app/exact", 0x7F), CancellationToken.None);
|
||||
|
||||
Assert.True(await WaitForAsync(_storeB, name, _ => true));
|
||||
|
||||
StoredSecret onA = (await _storeA.GetAsync(name, CancellationToken.None))!;
|
||||
StoredSecret onB = (await _storeB.GetAsync(name, CancellationToken.None))!;
|
||||
|
||||
Assert.Equal(onA.Ciphertext, onB.Ciphertext);
|
||||
Assert.Equal(onA.WrappedDek, onB.WrappedDek);
|
||||
Assert.Equal(onA.KekId, onB.KekId);
|
||||
// Verbatim: the peer keeps the originating node's revision and timestamp, or the row would
|
||||
// look newer on B than on A and bounce back.
|
||||
Assert.Equal(onA.Revision, onB.Revision);
|
||||
Assert.Equal(onA.UpdatedUtc, onB.UpdatedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_delete_on_node_A_tombstones_the_row_on_node_B()
|
||||
{
|
||||
var name = new SecretName("app/doomed");
|
||||
await _writableA.UpsertAsync(Row("app/doomed", 0xBB), CancellationToken.None);
|
||||
Assert.True(await WaitForAsync(_storeB, name, _ => true));
|
||||
|
||||
await _writableA.DeleteAsync(name, "operator", CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeB, name, r => r.IsDeleted),
|
||||
"The delete never propagated to node B.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_update_overwrites_the_earlier_value_on_the_peer()
|
||||
{
|
||||
var name = new SecretName("app/rotating");
|
||||
await _writableA.UpsertAsync(Row("app/rotating", 0x01), CancellationToken.None);
|
||||
Assert.True(await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x01));
|
||||
|
||||
await _writableA.UpsertAsync(Row("app/rotating", 0x02), CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x02 && r.Revision == 1),
|
||||
"Node B did not converge on the updated value.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Anti_entropy_repairs_a_write_that_never_got_broadcast()
|
||||
{
|
||||
// Simulates a dropped publish — the write goes straight to node A's store, bypassing the
|
||||
// replicator entirely, so the ONLY route to node B is the periodic manifest exchange. This is
|
||||
// the case that makes best-effort publishing safe.
|
||||
var name = new SecretName("app/missed-broadcast");
|
||||
await _storeA.UpsertAsync(Row("app/missed-broadcast", 0xCC), CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0xCC),
|
||||
"Anti-entropy failed to repair a write that was never broadcast.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Anti_entropy_pulls_in_the_other_direction_too()
|
||||
{
|
||||
// The mirror case: node B holds a row node A has never heard of. A cannot request a name it
|
||||
// does not know exists, so this only converges if the exchange pushes as well as pulls.
|
||||
var name = new SecretName("app/born-on-b");
|
||||
await _storeB.UpsertAsync(Row("app/born-on-b", 0xDD), CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeA, name, r => r.Ciphertext[0] == 0xDD),
|
||||
"Node A never learned about a secret that originated on node B.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_node_that_joins_late_catches_up_on_everything()
|
||||
{
|
||||
// Partition-heal / cold-start: the whole reason to choose this transport over a shared store.
|
||||
await _writableA.UpsertAsync(Row("bulk/one", 0x11), CancellationToken.None);
|
||||
await _writableA.UpsertAsync(Row("bulk/two", 0x22), CancellationToken.None);
|
||||
await _writableA.UpsertAsync(Row("bulk/three", 0x33), CancellationToken.None);
|
||||
|
||||
SqliteSecretStore lateStore = CreateStore();
|
||||
_systemB.ActorOf(
|
||||
SecretReplicationActor.Props(lateStore, null, AnnounceInterval), "late-joiner");
|
||||
|
||||
Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/one"), r => r.Ciphertext[0] == 0x11));
|
||||
Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/two"), r => r.Ciphertext[0] == 0x22));
|
||||
Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/three"), r => r.Ciphertext[0] == 0x33));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_broadcast_round_trip_does_not_drift_the_originating_nodes_revision()
|
||||
{
|
||||
// Note on what this does and does NOT prove: the revision would also stay put if the self-echo
|
||||
// filter were removed, because re-applying an identical row ties on (updated_utc, revision)
|
||||
// and last-writer-wins rejects a tie. So this covers the drift symptom via defence in depth,
|
||||
// not the filter itself. The filter is what stops the echo being forwarded onward, which
|
||||
// needs three nodes to observe — worth adding if a third node ever joins this rig.
|
||||
var name = new SecretName("app/no-echo");
|
||||
await _writableA.UpsertAsync(Row("app/no-echo", 0xEE), CancellationToken.None);
|
||||
Assert.True(await WaitForAsync(_storeB, name, _ => true));
|
||||
|
||||
await Task.Delay(AnnounceInterval * 4);
|
||||
|
||||
StoredSecret onA = (await _storeA.GetAsync(name, CancellationToken.None))!;
|
||||
Assert.Equal(0, onA.Revision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_malformed_row_from_a_peer_is_discarded_without_stalling_replication()
|
||||
{
|
||||
// Peer input is untrusted: a path-traversing name must be dropped, and — critically — the
|
||||
// actor must keep serving everything else afterwards.
|
||||
_replicatorB.Tell(new Protocol.SecretRowsMessage(
|
||||
[
|
||||
new Protocol.SecretRowDto
|
||||
{
|
||||
Name = "../../etc/passwd",
|
||||
ContentType = "Text",
|
||||
Ciphertext = [1], Nonce = [1], Tag = [1],
|
||||
WrappedDek = [1], WrapNonce = [1], WrapTag = [1],
|
||||
KekId = "sha256:x",
|
||||
Revision = 0,
|
||||
CreatedUtc = DateTimeOffset.UtcNow.ToString("O"),
|
||||
UpdatedUtc = DateTimeOffset.UtcNow.ToString("O"),
|
||||
},
|
||||
]));
|
||||
|
||||
var name = new SecretName("app/after-malformed");
|
||||
await _writableA.UpsertAsync(Row("app/after-malformed", 0x5A), CancellationToken.None);
|
||||
|
||||
Assert.True(
|
||||
await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x5A),
|
||||
"Replication stalled after a malformed row was received.");
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await CoordinatedShutdown.Get(_systemA).Run(CoordinatedShutdown.ClrExitReason.Instance);
|
||||
await CoordinatedShutdown.Get(_systemB).Run(CoordinatedShutdown.ClrExitReason.Instance);
|
||||
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
|
||||
foreach (string db in _dbPaths)
|
||||
{
|
||||
foreach (string path in new[] { db, db + "-wal", db + "-shm" })
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort temp cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
<PackageReference Include="Akka.TestKit.Xunit2" />
|
||||
<PackageReference Include="Akka.Remote" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\ZB.MOM.WW.Secrets.Replicator.AkkaDotNet\ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.Replicator.SqlServer.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// Container-resolution tests for both SQL-Server topologies.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These exist because a code review caught that neither mode could resolve at all: the decorators
|
||||
/// ask for the concrete <see cref="SqliteSecretStore"/>, which the core package only registered
|
||||
/// behind <see cref="ISecretStore"/>. Every unit test passed and both modes were dead on arrival,
|
||||
/// because nothing built a provider. Resolving the graph is the assertion that matters.
|
||||
/// </remarks>
|
||||
public sealed class AddZbSecretsSqlServerTests
|
||||
{
|
||||
private static IConfiguration Config(string sqlitePath) =>
|
||||
new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Secrets:SqlitePath"] = sqlitePath,
|
||||
["Secrets:RunMigrationsOnStartup"] = "false",
|
||||
["Secrets:MasterKey:Source"] = "Environment",
|
||||
["Secrets:MasterKey:EnvVarName"] = "ZB_SECRETS_TEST_KEY",
|
||||
["Secrets:SqlServer:ConnectionString"] = "Server=unused;Database=x;",
|
||||
}).Build();
|
||||
|
||||
private static string TempDb() => Path.Combine(Path.GetTempPath(), $"zb-di-{Guid.NewGuid():N}.db");
|
||||
|
||||
[Fact]
|
||||
public void SharedStore_mode_resolves_the_SqlServer_store_as_ISecretStore()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZbSecretsSqlServerStore(Config(TempDb()));
|
||||
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
|
||||
// The SQL-Server store displaces SQLite — no decorator, because there is nothing to replicate.
|
||||
Assert.IsType<SqlServerSecretStore>(provider.GetRequiredService<ISecretStore>());
|
||||
Assert.IsType<SqlServerSecretsStoreMigrator>(provider.GetRequiredService<ISecretsStoreMigrator>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HubReplication_mode_resolves_a_decorated_local_store()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZbSecretsSqlServerReplication(Config(TempDb()));
|
||||
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
|
||||
Assert.IsType<ReplicatingSecretStore>(provider.GetRequiredService<ISecretStore>());
|
||||
Assert.IsType<SqlServerSecretReplicator>(provider.GetRequiredService<ISecretReplicator>());
|
||||
// The local store is still provisioned by the core SQLite migrator, not the hub's.
|
||||
Assert.IsType<SqliteSecretsStoreMigrator>(provider.GetRequiredService<ISecretsStoreMigrator>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HubReplication_mode_registers_both_the_hub_migration_and_the_sync_service()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZbSecretsSqlServerReplication(Config(TempDb()));
|
||||
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
IHostedService[] hosted = [.. provider.GetServices<IHostedService>()];
|
||||
|
||||
// The hub schema is nobody's "own" store, so every node must provision it at startup too.
|
||||
Assert.Contains(hosted, h => h is SqlServerSecretSyncService);
|
||||
Assert.Single(hosted, h => h.GetType().Name == "SqlServerHubMigrationHostedService");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_decorator_wraps_the_undecorated_local_store_rather_than_itself()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddZbSecretsSqlServerReplication(Config(TempDb()));
|
||||
|
||||
using ServiceProvider provider = services.BuildServiceProvider();
|
||||
|
||||
// Guards against the self-referential registration that would stack-overflow on first use.
|
||||
var decorated = (ReplicatingSecretStore)provider.GetRequiredService<ISecretStore>();
|
||||
Assert.NotNull(decorated);
|
||||
Assert.NotSame(decorated, provider.GetRequiredService<SqliteSecretStore>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_missing_connection_string_fails_at_registration_not_at_first_resolve()
|
||||
{
|
||||
IConfiguration config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Secrets:SqlitePath"] = TempDb(),
|
||||
})
|
||||
.Build();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
|
||||
InvalidOperationException ex = Assert.Throws<InvalidOperationException>(
|
||||
() => services.AddZbSecretsSqlServerStore(config));
|
||||
|
||||
Assert.Contains("ConnectionString", ex.Message, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="ISecretReplicationHub"/> backed by a real SQLite store, standing in for the shared
|
||||
/// SQL-Server hub in offline convergence tests.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately backed by a real store rather than a dictionary: the behaviour under test is
|
||||
/// last-writer-wins convergence, which lives in <c>ApplyReplicatedAsync</c>. A hand-rolled fake would
|
||||
/// be re-implementing the very logic the tests exist to check, and would happily agree with a broken
|
||||
/// reconciler. The live SQL-Server tests then confirm the T-SQL store behaves the same way.
|
||||
/// </remarks>
|
||||
/// <param name="store">The SQLite store standing in for the hub database.</param>
|
||||
public sealed class SqliteBackedHub(SqliteSecretStore store) : ISecretReplicationHub
|
||||
{
|
||||
/// <summary>Number of times the sweep asked this hub for rows — asserts the fetch was skipped.</summary>
|
||||
public int GetManyCallCount { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
||||
store.GetManifestAsync(ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<StoredSecret>> GetManyAsync(
|
||||
IReadOnlyList<SecretName> names, CancellationToken ct)
|
||||
{
|
||||
GetManyCallCount++;
|
||||
|
||||
var rows = new List<StoredSecret>(names.Count);
|
||||
|
||||
foreach (SecretName name in names)
|
||||
{
|
||||
StoredSecret? row = await store.GetAsync(name, ct);
|
||||
|
||||
if (row is not null)
|
||||
{
|
||||
rows.Add(row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) =>
|
||||
store.ApplyReplicatedAsync(row, ct);
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
using Microsoft.Data.SqlClient;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
|
||||
|
||||
/// <summary>
|
||||
/// Shared connection details for the env-gated live SQL-Server suite.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Gated on <c>SECRETS_SQLSERVER_CONNSTR</c> and skips cleanly when it is unset, matching the family
|
||||
/// live-test idiom — the offline suite must stay runnable on any machine. Set it to a database the
|
||||
/// test is allowed to create and drop a schema in, for example:
|
||||
/// <c>Server=10.100.0.35,31433;Database=ZbSecretsTest;User Id=sa;Password=...;TrustServerCertificate=True</c>.
|
||||
/// </remarks>
|
||||
public static class LiveSqlServer
|
||||
{
|
||||
/// <summary>Environment variable holding the live connection string.</summary>
|
||||
public const string ConnectionStringVariable = "SECRETS_SQLSERVER_CONNSTR";
|
||||
|
||||
/// <summary>The configured connection string, or <see langword="null"/> when the suite is not enabled.</summary>
|
||||
public static string? ConnectionString =>
|
||||
Environment.GetEnvironmentVariable(ConnectionStringVariable);
|
||||
|
||||
/// <summary>Whether the live suite should run.</summary>
|
||||
public static bool IsEnabled => !string.IsNullOrWhiteSpace(ConnectionString);
|
||||
|
||||
/// <summary>Skip reason shown when the suite is not enabled.</summary>
|
||||
public const string SkipReason =
|
||||
"Live SQL-Server suite disabled; set SECRETS_SQLSERVER_CONNSTR to enable.";
|
||||
|
||||
/// <summary>
|
||||
/// Builds options against a throwaway schema so parallel runs and repeat runs cannot collide,
|
||||
/// and so a failed run leaves no trace in a shared database.
|
||||
/// </summary>
|
||||
/// <param name="schemaName">The unique schema name for this test.</param>
|
||||
/// <returns>Options targeting that schema.</returns>
|
||||
public static SqlServerSecretsOptions OptionsFor(string schemaName) => new()
|
||||
{
|
||||
ConnectionString = ConnectionString!,
|
||||
SchemaName = schemaName,
|
||||
};
|
||||
|
||||
/// <summary>Generates a unique, allow-list-legal schema name for one test class.</summary>
|
||||
/// <returns>A fresh schema name.</returns>
|
||||
public static string NewSchemaName() => $"zbtest_{Guid.NewGuid():N}"[..32];
|
||||
|
||||
/// <summary>Drops the throwaway schema and its tables.</summary>
|
||||
/// <param name="schemaName">The schema to drop.</param>
|
||||
/// <returns>A task that completes when the schema is gone.</returns>
|
||||
public static async Task DropSchemaAsync(string schemaName)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var connection = new SqlConnection(ConnectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using SqlCommand command = connection.CreateCommand();
|
||||
command.CommandText = $"""
|
||||
IF OBJECT_ID(N'[{schemaName}].[secret]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[secret];
|
||||
IF OBJECT_ID(N'[{schemaName}].[schema_version]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[schema_version];
|
||||
IF EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'{schemaName}') EXEC(N'DROP SCHEMA [{schemaName}]');
|
||||
""";
|
||||
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
|
||||
|
||||
/// <summary>
|
||||
/// The SQLite store's test suite, ported case-for-case onto the SQL-Server store.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Ported rather than newly written on purpose. The two stores must be behaviourally identical —
|
||||
/// a cluster can hold both at once — so the strongest available check is that the assertions
|
||||
/// written against SQLite pass unchanged here. Anywhere the T-SQL diverges in observable behaviour,
|
||||
/// one of these fails.
|
||||
/// </remarks>
|
||||
[Collection("live-sqlserver")]
|
||||
public sealed class SqlServerSecretStoreLiveTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _schema = LiveSqlServer.NewSchemaName();
|
||||
private SqlServerSecretStore _store = null!;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
if (!LiveSqlServer.IsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
||||
await new SqlServerSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
||||
_store = new SqlServerSecretStore(factory);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => LiveSqlServer.DropSchemaAsync(_schema);
|
||||
|
||||
private static StoredSecret MakeSecret(
|
||||
string name,
|
||||
long revision = 0,
|
||||
byte[]? ciphertext = null,
|
||||
DateTimeOffset? updatedUtc = null,
|
||||
DateTimeOffset? createdUtc = null,
|
||||
bool isDeleted = false,
|
||||
DateTimeOffset? deletedUtc = null,
|
||||
string? createdBy = "alice",
|
||||
string? updatedBy = "alice") => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
Description = "desc",
|
||||
ContentType = SecretContentType.ConnectionString,
|
||||
Ciphertext = ciphertext ?? [1, 2, 3],
|
||||
Nonce = [4, 5, 6],
|
||||
Tag = [7, 8, 9],
|
||||
WrappedDek = [10, 11, 12],
|
||||
WrapNonce = [13, 14, 15],
|
||||
WrapTag = [16, 17, 18],
|
||||
KekId = "kek-1",
|
||||
Revision = revision,
|
||||
IsDeleted = isDeleted,
|
||||
DeletedUtc = deletedUtc,
|
||||
CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
|
||||
UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
|
||||
CreatedBy = createdBy,
|
||||
UpdatedBy = updatedBy,
|
||||
};
|
||||
|
||||
[SkippableFact]
|
||||
public async Task Upsert_Then_Get_RoundTrips()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("app/db-conn"), CancellationToken.None);
|
||||
|
||||
StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
|
||||
|
||||
Assert.NotNull(got);
|
||||
Assert.Equal("app/db-conn", got!.Name.Value);
|
||||
Assert.Equal("desc", got.Description);
|
||||
Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
|
||||
Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
|
||||
Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
|
||||
Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
|
||||
Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
|
||||
Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
|
||||
Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
|
||||
Assert.Equal("kek-1", got.KekId);
|
||||
Assert.Equal(0, got.Revision);
|
||||
Assert.False(got.IsDeleted);
|
||||
Assert.Null(got.DeletedUtc);
|
||||
Assert.Equal("alice", got.CreatedBy);
|
||||
Assert.Equal("alice", got.UpdatedBy);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task Get_ReturnsNull_WhenAbsent()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
Assert.Null(await _store.GetAsync(new SecretName("nope"), CancellationToken.None));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(
|
||||
MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice"),
|
||||
CancellationToken.None);
|
||||
|
||||
StoredSecret afterFirst =
|
||||
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
||||
Assert.Equal(0, afterFirst.Revision);
|
||||
DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
|
||||
|
||||
await _store.UpsertAsync(
|
||||
MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob"),
|
||||
CancellationToken.None);
|
||||
|
||||
StoredSecret afterSecond =
|
||||
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
||||
Assert.Equal(1, afterSecond.Revision);
|
||||
Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
|
||||
Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
|
||||
Assert.Equal("alice", afterSecond.CreatedBy);
|
||||
Assert.Equal("bob", afterSecond.UpdatedBy);
|
||||
Assert.False(afterSecond.IsDeleted);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task List_ExcludesTombstoned_ByDefault()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
|
||||
await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
|
||||
await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
|
||||
|
||||
IReadOnlyList<SecretMetadata> visible =
|
||||
await _store.ListAsync(includeDeleted: false, CancellationToken.None);
|
||||
Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
|
||||
Assert.Contains(visible, m => m.Name.Value == "keep");
|
||||
|
||||
IReadOnlyList<SecretMetadata> all =
|
||||
await _store.ListAsync(includeDeleted: true, CancellationToken.None);
|
||||
Assert.Contains(all, m => m.Name.Value == "gone");
|
||||
Assert.Contains(all, m => m.Name.Value == "keep");
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
|
||||
|
||||
Assert.True(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
||||
|
||||
StoredSecret tombstoned =
|
||||
(await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
|
||||
Assert.True(tombstoned.IsDeleted);
|
||||
Assert.NotNull(tombstoned.DeletedUtc);
|
||||
Assert.Equal(1, tombstoned.Revision);
|
||||
Assert.Equal("carol", tombstoned.UpdatedBy);
|
||||
|
||||
Assert.False(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
||||
Assert.False(await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task GetManifest_ReturnsAllRows()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
|
||||
await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
|
||||
await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
|
||||
|
||||
IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, manifest.Count);
|
||||
SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
|
||||
SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
|
||||
Assert.False(a.IsDeleted);
|
||||
Assert.Equal(0, a.Revision);
|
||||
Assert.True(b.IsDeleted);
|
||||
Assert.Equal(1, b.Revision);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
||||
DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
|
||||
DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
await _store.ApplyReplicatedAsync(
|
||||
MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(5, (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!.Revision);
|
||||
|
||||
await _store.ApplyReplicatedAsync(
|
||||
MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1),
|
||||
CancellationToken.None);
|
||||
|
||||
StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
||||
Assert.Equal(6, afterNewer.Revision);
|
||||
Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
|
||||
Assert.Equal(t3, afterNewer.UpdatedUtc);
|
||||
|
||||
await _store.ApplyReplicatedAsync(
|
||||
MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1),
|
||||
CancellationToken.None);
|
||||
|
||||
StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
||||
Assert.Equal(6, afterStale.Revision);
|
||||
Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(
|
||||
MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)),
|
||||
CancellationToken.None);
|
||||
await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
|
||||
|
||||
StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
||||
Assert.Equal(1, before.Revision);
|
||||
|
||||
bool applied = await _store.ApplyRewrapAsync(
|
||||
before with
|
||||
{
|
||||
WrappedDek = [90, 91, 92],
|
||||
WrapNonce = [93, 94, 95],
|
||||
WrapTag = [96, 97, 98],
|
||||
KekId = "kek-2",
|
||||
},
|
||||
before.WrappedDek,
|
||||
CancellationToken.None);
|
||||
Assert.True(applied);
|
||||
|
||||
StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
||||
Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
|
||||
Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
|
||||
Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
|
||||
Assert.Equal("kek-2", after.KekId);
|
||||
Assert.Equal(before.Revision, after.Revision);
|
||||
Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
|
||||
Assert.Equal(before.CreatedUtc, after.CreatedUtc);
|
||||
Assert.Equal(before.UpdatedBy, after.UpdatedBy);
|
||||
Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
|
||||
Assert.Equal(before.Nonce, after.Nonce);
|
||||
Assert.Equal(before.Tag, after.Tag);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ApplyRewrap_RewrapsTombstonedRows()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
|
||||
await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
|
||||
|
||||
StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
||||
Assert.True(tomb.IsDeleted);
|
||||
|
||||
Assert.True(await _store.ApplyRewrapAsync(
|
||||
tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
||||
tomb.WrappedDek,
|
||||
CancellationToken.None));
|
||||
|
||||
StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
||||
Assert.Equal("kek-2", after.KekId);
|
||||
Assert.True(after.IsDeleted);
|
||||
Assert.Equal(tomb.Revision, after.Revision);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
|
||||
|
||||
Assert.False(await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("app/cas", ciphertext: [1, 2, 3]), CancellationToken.None);
|
||||
StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
||||
|
||||
await _store.UpsertAsync(
|
||||
MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
|
||||
CancellationToken.None);
|
||||
|
||||
bool applied = await _store.ApplyRewrapAsync(
|
||||
before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
||||
before.WrappedDek,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(applied);
|
||||
StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
||||
Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
|
||||
Assert.Equal("kek-1", after.KekId);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task GetMany_returns_only_the_requested_rows()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("a/one"), CancellationToken.None);
|
||||
await _store.UpsertAsync(MakeSecret("b/two"), CancellationToken.None);
|
||||
await _store.UpsertAsync(MakeSecret("c/three"), CancellationToken.None);
|
||||
|
||||
IReadOnlyList<StoredSecret> rows = await _store.GetManyAsync(
|
||||
[new SecretName("a/one"), new SecretName("c/three"), new SecretName("absent/name")],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.Contains(rows, r => r.Name.Value == "a/one");
|
||||
Assert.Contains(rows, r => r.Name.Value == "c/three");
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task GetMany_short_circuits_on_an_empty_request()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
// Guards the IN () clause: an empty list would otherwise produce invalid T-SQL.
|
||||
Assert.Empty(await _store.GetManyAsync([], CancellationToken.None));
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task Migrator_is_idempotent()
|
||||
{
|
||||
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
||||
|
||||
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
||||
var migrator = new SqlServerSecretsStoreMigrator(factory);
|
||||
|
||||
// Every node runs this at every startup — re-running must be a no-op, not an error.
|
||||
await migrator.MigrateAsync(CancellationToken.None);
|
||||
await migrator.MigrateAsync(CancellationToken.None);
|
||||
|
||||
await _store.UpsertAsync(MakeSecret("survives/migration"), CancellationToken.None);
|
||||
await migrator.MigrateAsync(CancellationToken.None);
|
||||
|
||||
Assert.NotNull(await _store.GetAsync(new SecretName("survives/migration"), CancellationToken.None));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serializes the live suite: the tests share one database server and each provisions its own
|
||||
/// schema, so running them in parallel would multiply connection load for no benefit.
|
||||
/// </summary>
|
||||
[CollectionDefinition("live-sqlserver", DisableParallelization = true)]
|
||||
public sealed class LiveSqlServerCollection;
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Convergence behaviour of the bidirectional hub sweep, exercised against real stores on both
|
||||
/// sides so the assertions are about actual persisted state rather than mock interactions.
|
||||
/// </summary>
|
||||
public sealed class SqlServerSecretSyncServiceTests : IDisposable
|
||||
{
|
||||
private readonly List<string> _dbPaths = [];
|
||||
private readonly SqliteSecretStore _local;
|
||||
private readonly SqliteSecretStore _hubStore;
|
||||
private readonly SqliteBackedHub _hub;
|
||||
|
||||
public SqlServerSecretSyncServiceTests()
|
||||
{
|
||||
_local = CreateStore();
|
||||
_hubStore = CreateStore();
|
||||
_hub = new SqliteBackedHub(_hubStore);
|
||||
}
|
||||
|
||||
private SqliteSecretStore CreateStore()
|
||||
{
|
||||
string path = Path.Combine(Path.GetTempPath(), $"zb-sync-{Guid.NewGuid():N}.db");
|
||||
_dbPaths.Add(path);
|
||||
var factory = new SecretsSqliteConnectionFactory(path);
|
||||
new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
|
||||
return new SqliteSecretStore(factory);
|
||||
}
|
||||
|
||||
private SqlServerSecretSyncService CreateService() => new(
|
||||
_local,
|
||||
_hub,
|
||||
cacheInvalidator: null,
|
||||
Options.Create(new SqlServerSecretsOptions { ConnectionString = "Server=unused;" }),
|
||||
NullLogger<SqlServerSecretSyncService>.Instance);
|
||||
|
||||
private static StoredSecret Row(string name, byte marker) => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
ContentType = SecretContentType.Text,
|
||||
Ciphertext = [marker],
|
||||
Nonce = [1],
|
||||
Tag = [2],
|
||||
WrappedDek = [3],
|
||||
WrapNonce = [4],
|
||||
WrapTag = [5],
|
||||
KekId = "sha256:test",
|
||||
Revision = 0,
|
||||
CreatedUtc = DateTimeOffset.UtcNow,
|
||||
UpdatedUtc = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_pulls_a_secret_the_node_has_never_seen()
|
||||
{
|
||||
await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None);
|
||||
|
||||
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, pulled);
|
||||
Assert.Equal(0, pushed);
|
||||
|
||||
StoredSecret? local = await _local.GetAsync(new SecretName("db/password"), CancellationToken.None);
|
||||
Assert.NotNull(local);
|
||||
Assert.Equal(0xAA, local!.Ciphertext[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_pushes_a_local_write_the_hub_never_received()
|
||||
{
|
||||
// The case that makes best-effort publishing safe: the publish failed (hub was down), so the
|
||||
// hub has never heard of this name and can never ask for it. Only a push sweep repairs this.
|
||||
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
|
||||
|
||||
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, pulled);
|
||||
Assert.Equal(1, pushed);
|
||||
|
||||
StoredSecret? hub = await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None);
|
||||
Assert.NotNull(hub);
|
||||
Assert.Equal(0xBB, hub!.Ciphertext[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_preserves_the_originating_revision_when_pushing()
|
||||
{
|
||||
// Pushed verbatim, not upserted: if the hub stamped its own revision the row would look newer
|
||||
// than the writer's copy and bounce straight back on the next sweep.
|
||||
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
|
||||
await _local.UpsertAsync(Row("api/key", 0xCC), CancellationToken.None);
|
||||
|
||||
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
|
||||
Assert.Equal(1, localRow.Revision);
|
||||
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
|
||||
Assert.Equal(localRow.Revision, hubRow.Revision);
|
||||
Assert.Equal(localRow.UpdatedUtc, hubRow.UpdatedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_converges_both_directions_in_one_pass()
|
||||
{
|
||||
await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None);
|
||||
await _hubStore.UpsertAsync(Row("only/hub", 0x22), CancellationToken.None);
|
||||
|
||||
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, pulled);
|
||||
Assert.Equal(1, pushed);
|
||||
|
||||
Assert.NotNull(await _local.GetAsync(new SecretName("only/hub"), CancellationToken.None));
|
||||
Assert.NotNull(await _hubStore.GetAsync(new SecretName("only/local"), CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_is_idempotent_so_a_converged_pair_stays_quiet()
|
||||
{
|
||||
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
|
||||
|
||||
SqlServerSecretSyncService service = CreateService();
|
||||
await service.SweepAsync(CancellationToken.None);
|
||||
|
||||
(int pulled, int pushed) = await service.SweepAsync(CancellationToken.None);
|
||||
|
||||
// A steady-state cluster must not write on every tick — that would churn the hub forever.
|
||||
Assert.Equal(0, pulled);
|
||||
Assert.Equal(0, pushed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_propagates_a_delete_as_a_tombstone()
|
||||
{
|
||||
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
await _local.DeleteAsync(new SecretName("api/key"), "tester", CancellationToken.None);
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
|
||||
Assert.True(hubRow.IsDeleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_does_not_resurrect_a_secret_deleted_on_the_hub()
|
||||
{
|
||||
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
await _hubStore.DeleteAsync(new SecretName("api/key"), "admin", CancellationToken.None);
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
|
||||
Assert.True(localRow.IsDeleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_skips_the_hub_fetch_when_nothing_is_newer()
|
||||
{
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, _hub.GetManyCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_lets_the_newer_side_win_a_conflicting_write()
|
||||
{
|
||||
await _local.UpsertAsync(Row("api/key", 0x11), CancellationToken.None);
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
// Both sides then write independently; the hub's write is later.
|
||||
await _local.UpsertAsync(Row("api/key", 0x22), CancellationToken.None);
|
||||
await Task.Delay(10);
|
||||
await _hubStore.UpsertAsync(Row("api/key", 0x33), CancellationToken.None);
|
||||
|
||||
await CreateService().SweepAsync(CancellationToken.None);
|
||||
|
||||
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
|
||||
Assert.Equal(0x33, localRow.Ciphertext[0]);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
|
||||
foreach (string db in _dbPaths)
|
||||
{
|
||||
foreach (string path in new[] { db, db + "-wal", db + "-shm" })
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The schema-name allow-list is the injection boundary for the one value that cannot be
|
||||
/// parameterized (a T-SQL object name), so it gets adversarial coverage rather than a happy path.
|
||||
/// </summary>
|
||||
public sealed class SqlServerSecretsOptionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("zbsecrets")]
|
||||
[InlineData("ZbSecrets")]
|
||||
[InlineData("_private")]
|
||||
[InlineData("schema_1")]
|
||||
public void SchemaName_accepts_plain_identifiers(string value)
|
||||
{
|
||||
var options = new SqlServerSecretsOptions { SchemaName = value };
|
||||
|
||||
Assert.Equal(value, options.SchemaName);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// The bracket-escape trick that would otherwise break out of the [quoting] at the use site.
|
||||
[InlineData("evil]]; DROP TABLE secret; --")]
|
||||
[InlineData("dbo.secret")]
|
||||
[InlineData("with space")]
|
||||
[InlineData("quote'name")]
|
||||
[InlineData("semi;colon")]
|
||||
[InlineData("1leading_digit")]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
public void SchemaName_rejects_anything_that_is_not_a_plain_identifier(string value)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => new SqlServerSecretsOptions { SchemaName = value });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SchemaName_rejects_an_over_long_identifier()
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
new SqlServerSecretsOptions { SchemaName = new string('a', 129) });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_rejects_a_missing_connection_string()
|
||||
{
|
||||
InvalidOperationException ex =
|
||||
Assert.Throws<InvalidOperationException>(() => new SqlServerSecretsOptions().Validate());
|
||||
|
||||
Assert.Contains("ConnectionString", ex.Message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_rejects_a_non_positive_sync_interval()
|
||||
{
|
||||
var options = new SqlServerSecretsOptions
|
||||
{
|
||||
ConnectionString = "Server=x;",
|
||||
SyncInterval = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
Assert.Throws<InvalidOperationException>(options.Validate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_rejects_a_non_positive_command_timeout()
|
||||
{
|
||||
var options = new SqlServerSecretsOptions
|
||||
{
|
||||
ConnectionString = "Server=x;",
|
||||
CommandTimeout = TimeSpan.FromSeconds(-1),
|
||||
};
|
||||
|
||||
Assert.Throws<InvalidOperationException>(options.Validate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_accepts_a_fully_configured_instance()
|
||||
{
|
||||
var options = new SqlServerSecretsOptions { ConnectionString = "Server=x;Database=y;" };
|
||||
|
||||
options.Validate();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateSchemaDdl_quotes_the_schema_name_everywhere_it_appears()
|
||||
{
|
||||
string ddl = SqlServerSecretsSchema.CreateSchemaDdl("zbsecrets");
|
||||
|
||||
Assert.Contains("[zbsecrets].[secret]", ddl, StringComparison.Ordinal);
|
||||
Assert.Contains("[zbsecrets].[schema_version]", ddl, StringComparison.Ordinal);
|
||||
// Every DDL statement is guarded, so re-running the migration on a provisioned database is
|
||||
// a no-op rather than an error.
|
||||
Assert.Contains("IF NOT EXISTS", ddl, StringComparison.Ordinal);
|
||||
Assert.Contains("IS NULL", ddl, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
<PackageReference Include="Xunit.SkippableFact" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\ZB.MOM.WW.Secrets.Replicator.SqlServer\ZB.MOM.WW.Secrets.Replicator.SqlServer.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,15 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Tests.Fakes;
|
||||
|
||||
/// <summary>Records every eviction so tests can assert the resolver cache was cleared.</summary>
|
||||
public sealed class RecordingCacheInvalidator : ISecretCacheInvalidator
|
||||
{
|
||||
private readonly List<SecretName> _invalidated = [];
|
||||
|
||||
/// <summary>The names evicted, in call order.</summary>
|
||||
public IReadOnlyList<SecretName> Invalidated => _invalidated;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(SecretName name) => _invalidated.Add(name);
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
using ZB.MOM.WW.Secrets.Tests.Fakes;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Tests.Replication;
|
||||
|
||||
/// <summary>
|
||||
/// Convergence rules for the transport-agnostic reconciler. These are the cases both replication
|
||||
/// packages inherit, so they are pinned here once against a real SQLite store rather than a fake —
|
||||
/// a reconciler that agrees with a mock but not with the store would be worthless.
|
||||
/// </summary>
|
||||
public sealed class SecretReplicationReconcilerTests : IDisposable
|
||||
{
|
||||
private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"zb-recon-{Guid.NewGuid():N}.db");
|
||||
private readonly SecretsSqliteConnectionFactory _factory;
|
||||
private readonly SqliteSecretStore _store;
|
||||
|
||||
public SecretReplicationReconcilerTests()
|
||||
{
|
||||
_factory = new SecretsSqliteConnectionFactory(_dbPath);
|
||||
new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
|
||||
_store = new SqliteSecretStore(_factory);
|
||||
}
|
||||
|
||||
private static readonly DateTimeOffset T0 = new(2026, 7, 18, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static SecretManifestEntry Entry(string name, long revision, DateTimeOffset updated, bool deleted = false) =>
|
||||
new() { Name = new SecretName(name), Revision = revision, UpdatedUtc = updated, IsDeleted = deleted };
|
||||
|
||||
private static StoredSecret Row(string name, long revision, DateTimeOffset updated, bool deleted = false) => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
ContentType = SecretContentType.Text,
|
||||
Ciphertext = [1, 2, 3],
|
||||
Nonce = [4],
|
||||
Tag = [5],
|
||||
WrappedDek = [6],
|
||||
WrapNonce = [7],
|
||||
WrapTag = [8],
|
||||
KekId = "sha256:test",
|
||||
Revision = revision,
|
||||
IsDeleted = deleted,
|
||||
DeletedUtc = deleted ? updated : null,
|
||||
CreatedUtc = T0,
|
||||
UpdatedUtc = updated,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_pulls_names_absent_locally()
|
||||
{
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [],
|
||||
remote: [Entry("db/password", 0, T0)]);
|
||||
|
||||
Assert.Equal(["db/password"], pull.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_pulls_a_newer_remote_row()
|
||||
{
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("api/key", 1, T0)],
|
||||
remote: [Entry("api/key", 2, T0.AddSeconds(1))]);
|
||||
|
||||
Assert.Equal(["api/key"], pull.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_breaks_a_timestamp_tie_on_revision()
|
||||
{
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("api/key", 1, T0)],
|
||||
remote: [Entry("api/key", 2, T0)]);
|
||||
|
||||
Assert.Equal(["api/key"], pull.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_ignores_an_identical_row_so_redelivery_is_free()
|
||||
{
|
||||
// At-least-once transports redeliver. A tie must not be "newer", or two nodes would
|
||||
// ping-pong the same row forever.
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("api/key", 2, T0)],
|
||||
remote: [Entry("api/key", 2, T0)]);
|
||||
|
||||
Assert.Empty(pull);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_ignores_a_peer_that_is_behind()
|
||||
{
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("api/key", 5, T0.AddMinutes(1))],
|
||||
remote: [Entry("api/key", 2, T0)]);
|
||||
|
||||
Assert.Empty(pull);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_pulls_a_tombstone_so_deletes_propagate()
|
||||
{
|
||||
// A delete is a newer row, not an absence — otherwise a deleted secret would be resurrected
|
||||
// by the next anti-entropy sweep from a node that still has it.
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("api/key", 1, T0)],
|
||||
remote: [Entry("api/key", 2, T0.AddSeconds(1), deleted: true)]);
|
||||
|
||||
Assert.Equal(["api/key"], pull.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputePullSet_does_not_pull_a_name_only_the_local_side_has()
|
||||
{
|
||||
// The local side only ever pulls. Pushing local-only names is the peer's own reconcile.
|
||||
IReadOnlyList<SecretName> pull = SecretReplicationReconciler.ComputePullSet(
|
||||
localManifest: [Entry("local/only", 0, T0)],
|
||||
remote: []);
|
||||
|
||||
Assert.Empty(pull);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_applies_only_the_rows_the_peer_holds_newer()
|
||||
{
|
||||
await _store.UpsertAsync(Row("shared/a", 0, T0), CancellationToken.None);
|
||||
|
||||
StoredSecret localA = (await _store.GetAsync(new SecretName("shared/a"), CancellationToken.None))!;
|
||||
|
||||
var reconciler = new SecretReplicationReconciler(_store);
|
||||
List<SecretName>? requested = null;
|
||||
|
||||
int applied = await reconciler.ReconcileAsync(
|
||||
remoteManifest:
|
||||
[
|
||||
// Behind the local copy — must not be fetched.
|
||||
Entry("shared/a", localA.Revision, localA.UpdatedUtc.AddMinutes(-1)),
|
||||
// Absent locally — must be fetched.
|
||||
Entry("shared/b", 3, T0.AddHours(1)),
|
||||
],
|
||||
fetchAsync: (names, _) =>
|
||||
{
|
||||
requested = [.. names];
|
||||
return Task.FromResult<IReadOnlyList<StoredSecret>>([Row("shared/b", 3, T0.AddHours(1))]);
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, applied);
|
||||
Assert.Equal(["shared/b"], requested!.Select(n => n.Value));
|
||||
|
||||
StoredSecret? b = await _store.GetAsync(new SecretName("shared/b"), CancellationToken.None);
|
||||
Assert.NotNull(b);
|
||||
// Applied VERBATIM — the peer's revision, not a locally bumped one.
|
||||
Assert.Equal(3, b!.Revision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_skips_the_fetch_entirely_when_nothing_is_newer()
|
||||
{
|
||||
var reconciler = new SecretReplicationReconciler(_store);
|
||||
bool fetched = false;
|
||||
|
||||
int applied = await reconciler.ReconcileAsync(
|
||||
remoteManifest: [],
|
||||
fetchAsync: (_, _) =>
|
||||
{
|
||||
fetched = true;
|
||||
return Task.FromResult<IReadOnlyList<StoredSecret>>([]);
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, applied);
|
||||
Assert.False(fetched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyAsync_evicts_the_resolver_cache_for_every_row()
|
||||
{
|
||||
var invalidator = new RecordingCacheInvalidator();
|
||||
var reconciler = new SecretReplicationReconciler(_store, invalidator);
|
||||
|
||||
await reconciler.ApplyAsync([Row("a/one", 1, T0), Row("b/two", 1, T0)], CancellationToken.None);
|
||||
|
||||
Assert.Equal(["a/one", "b/two"], invalidator.Invalidated.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyAsync_evicts_even_when_the_store_rejected_the_row_as_stale()
|
||||
{
|
||||
// Serving a stale plaintext for the rest of the TTL is worse than a redundant re-read.
|
||||
await _store.UpsertAsync(Row("api/key", 0, T0), CancellationToken.None);
|
||||
|
||||
var invalidator = new RecordingCacheInvalidator();
|
||||
var reconciler = new SecretReplicationReconciler(_store, invalidator);
|
||||
|
||||
await reconciler.ApplyAsync([Row("api/key", 0, T0.AddYears(-1))], CancellationToken.None);
|
||||
|
||||
Assert.Equal(["api/key"], invalidator.Invalidated.Select(n => n.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyAsync_skips_a_poison_row_and_keeps_applying_the_rest()
|
||||
{
|
||||
// A batch is not a transaction. Without per-row isolation, one row a peer cannot supply
|
||||
// cleanly abandons every row after it — and since the pull set is recomputed identically
|
||||
// each sweep, in the same manifest order, the same row poisons the same batch forever.
|
||||
var failures = new List<SecretName>();
|
||||
var reconciler = new SecretReplicationReconciler(
|
||||
_store, cacheInvalidator: null, onRowFailed: (name, _) => failures.Add(name));
|
||||
|
||||
// A null ciphertext cannot be persisted (NOT NULL column) — stands in for any bad row.
|
||||
StoredSecret poison = Row("bad/row", 1, T0) with { Ciphertext = null! };
|
||||
|
||||
int applied = await reconciler.ApplyAsync(
|
||||
[Row("good/first", 1, T0), poison, Row("good/second", 1, T0)],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, applied);
|
||||
Assert.Equal(["bad/row"], failures.Select(n => n.Value));
|
||||
Assert.NotNull(await _store.GetAsync(new SecretName("good/first"), CancellationToken.None));
|
||||
Assert.NotNull(await _store.GetAsync(new SecretName("good/second"), CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_fetches_in_bounded_batches()
|
||||
{
|
||||
// An unbounded fetch breaks on real backends: SQL Server caps a command at 2100 parameters,
|
||||
// so a cold-starting node pulling a large hub's entire inventory would fail identically
|
||||
// every interval and never receive a single row.
|
||||
const int total = SecretReplicationReconciler.FetchBatchSize + 20;
|
||||
|
||||
List<SecretManifestEntry> remote = [.. Enumerable.Range(0, total)
|
||||
.Select(i => Entry($"bulk/{i}", 1, T0))];
|
||||
|
||||
var batchSizes = new List<int>();
|
||||
var reconciler = new SecretReplicationReconciler(_store);
|
||||
|
||||
int applied = await reconciler.ReconcileAsync(
|
||||
remote,
|
||||
fetchAsync: (names, _) =>
|
||||
{
|
||||
batchSizes.Add(names.Count);
|
||||
return Task.FromResult<IReadOnlyList<StoredSecret>>(
|
||||
[.. names.Select(n => Row(n.Value, 1, T0))]);
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(total, applied);
|
||||
Assert.Equal(2, batchSizes.Count);
|
||||
Assert.All(batchSizes, size => Assert.True(size <= SecretReplicationReconciler.FetchBatchSize));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user