using Akka.Actor; using Akka.Cluster; using Akka.Configuration; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.Secrets.Abstractions; using ZB.MOM.WW.Secrets.Replication; using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests.Fakes; using ZB.MOM.WW.Secrets.Sqlite; namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests; /// /// Two real cluster nodes, two real SQLite stores, real remoting over loopback — the gate the G-7 /// design called for, run in-process so it belongs to the ordinary offline suite instead of needing /// a deployed rig. /// /// /// Nothing here is faked: each node runs its own with Akka.Remote on its /// own port, they form a cluster, and rows travel through the actual serializer and the actual /// distributed pub/sub mediator. A mock-based test of this behaviour would prove almost nothing — /// the failure modes worth catching (self-echo, sender loss across the mediator, tombstones that do /// not propagate, a partition that never re-converges) only appear when messages really cross nodes. /// public sealed class TwoNodeClusterReplicationTests : IAsyncLifetime { private readonly List _dbPaths = []; private ActorSystem _systemA = null!; private ActorSystem _systemB = null!; private SqliteSecretStore _storeA = null!; private SqliteSecretStore _storeB = null!; private ISecretStore _writableA = null!; private IActorRef _replicatorB = null!; // Short so anti-entropy assertions do not dominate the suite's runtime. private static readonly TimeSpan AnnounceInterval = TimeSpan.FromMilliseconds(300); private static readonly TimeSpan Patience = TimeSpan.FromSeconds(20); public async Task InitializeAsync() { _storeA = CreateStore(); _storeB = CreateStore(); // Port 0 lets the OS assign free ports, so parallel test runs cannot collide. _systemA = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(port: 0)); int portA = ClusterPort(_systemA); // Node A seeds itself; node B joins it. _systemA.Dispose(); _systemA = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(portA, seedPort: portA)); _systemB = ActorSystem.Create("zb-secrets-cluster", ClusterConfig(port: 0, seedPort: portA)); await AwaitClusterUpAsync(_systemA, expectedMembers: 2); await AwaitClusterUpAsync(_systemB, expectedMembers: 2); IActorRef replicatorA = _systemA.ActorOf( SecretReplicationActor.Props(_storeA, null, AnnounceInterval), "zb-secret-replication"); _replicatorB = _systemB.ActorOf( SecretReplicationActor.Props(_storeB, null, AnnounceInterval), "zb-secret-replication"); _writableA = new ReplicatingSecretStore( _storeA, new AkkaSecretReplicator(replicatorA), NullLogger.Instance); // The mediator gossips subscriptions between nodes; publishing before both sides are // registered would silently drop the message. await AwaitBothSubscribedAsync(); } private SqliteSecretStore CreateStore() { string path = Path.Combine(Path.GetTempPath(), $"zb-akka-{Guid.NewGuid():N}.db"); _dbPaths.Add(path); var factory = new SecretsSqliteConnectionFactory(path); new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult(); return new SqliteSecretStore(factory); } private static Config ClusterConfig(int port, int? seedPort = null) { string seeds = seedPort is null ? "[]" : $"[\"akka.tcp://zb-secrets-cluster@127.0.0.1:{seedPort}\"]"; return ConfigurationFactory.ParseString($$""" akka { loglevel = WARNING actor.provider = cluster remote.dot-netty.tcp { hostname = "127.0.0.1" public-hostname = "127.0.0.1" port = {{port}} } cluster { seed-nodes = {{seeds}} downing-provider-class = "Akka.Cluster.SBR.SplitBrainResolverProvider" } } """).WithFallback(AkkaSecretsReplication.SerializationConfig); } private static int ClusterPort(ActorSystem system) => Cluster.Get(system).SelfAddress.Port ?? throw new InvalidOperationException("Cluster address has no port."); private static async Task AwaitClusterUpAsync(ActorSystem system, int expectedMembers) { Cluster cluster = Cluster.Get(system); DateTime deadline = DateTime.UtcNow + Patience; while (DateTime.UtcNow < deadline) { if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers) { return; } await Task.Delay(100); } throw new TimeoutException( $"Cluster did not reach {expectedMembers} Up members within {Patience}."); } // Writes a throwaway secret on A and waits for it to land on B. That round trip is the only // reliable signal that pub/sub subscriptions have gossiped across both nodes. private async Task AwaitBothSubscribedAsync() { var probe = new SecretName("warmup/probe"); DateTime deadline = DateTime.UtcNow + Patience; while (DateTime.UtcNow < deadline) { await _writableA.UpsertAsync(Row("warmup/probe", 0x01), CancellationToken.None); if (await WaitForAsync(_storeB, probe, _ => true, TimeSpan.FromMilliseconds(500))) { await _storeA.DeleteAsync(probe, "warmup", CancellationToken.None); await _storeB.DeleteAsync(probe, "warmup", CancellationToken.None); return; } } throw new TimeoutException("Nodes did not establish pub/sub subscriptions in time."); } private static StoredSecret Row(string name, byte marker) => new() { Name = new SecretName(name), ContentType = SecretContentType.Text, Ciphertext = [marker], Nonce = [1], Tag = [2], WrappedDek = [3], WrapNonce = [4], WrapTag = [5], KekId = "sha256:shared-kek", Revision = 0, CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; private static async Task WaitForAsync( ISecretStore store, SecretName name, Func predicate, TimeSpan? timeout = null) { DateTime deadline = DateTime.UtcNow + (timeout ?? Patience); while (DateTime.UtcNow < deadline) { StoredSecret? row = await store.GetAsync(name, CancellationToken.None); if (row is not null && predicate(row)) { return true; } await Task.Delay(50); } return false; } [Fact] public async Task A_secret_written_on_node_A_becomes_resolvable_on_node_B() { var name = new SecretName("db/password"); await _writableA.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); Assert.True( await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0xAA), "Node B never received the secret written on node A."); } [Fact] public async Task The_ciphertext_arrives_byte_identical() { var name = new SecretName("app/exact"); await _writableA.UpsertAsync(Row("app/exact", 0x7F), CancellationToken.None); Assert.True(await WaitForAsync(_storeB, name, _ => true)); StoredSecret onA = (await _storeA.GetAsync(name, CancellationToken.None))!; StoredSecret onB = (await _storeB.GetAsync(name, CancellationToken.None))!; Assert.Equal(onA.Ciphertext, onB.Ciphertext); Assert.Equal(onA.WrappedDek, onB.WrappedDek); Assert.Equal(onA.KekId, onB.KekId); // Verbatim: the peer keeps the originating node's revision and timestamp, or the row would // look newer on B than on A and bounce back. Assert.Equal(onA.Revision, onB.Revision); Assert.Equal(onA.UpdatedUtc, onB.UpdatedUtc); } [Fact] public async Task A_delete_on_node_A_tombstones_the_row_on_node_B() { var name = new SecretName("app/doomed"); await _writableA.UpsertAsync(Row("app/doomed", 0xBB), CancellationToken.None); Assert.True(await WaitForAsync(_storeB, name, _ => true)); await _writableA.DeleteAsync(name, "operator", CancellationToken.None); Assert.True( await WaitForAsync(_storeB, name, r => r.IsDeleted), "The delete never propagated to node B."); } [Fact] public async Task An_update_overwrites_the_earlier_value_on_the_peer() { var name = new SecretName("app/rotating"); await _writableA.UpsertAsync(Row("app/rotating", 0x01), CancellationToken.None); Assert.True(await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x01)); await _writableA.UpsertAsync(Row("app/rotating", 0x02), CancellationToken.None); Assert.True( await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x02 && r.Revision == 1), "Node B did not converge on the updated value."); } [Fact] public async Task Anti_entropy_repairs_a_write_that_never_got_broadcast() { // Simulates a dropped publish — the write goes straight to node A's store, bypassing the // replicator entirely, so the ONLY route to node B is the periodic manifest exchange. This is // the case that makes best-effort publishing safe. var name = new SecretName("app/missed-broadcast"); await _storeA.UpsertAsync(Row("app/missed-broadcast", 0xCC), CancellationToken.None); Assert.True( await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0xCC), "Anti-entropy failed to repair a write that was never broadcast."); } [Fact] public async Task Anti_entropy_pulls_in_the_other_direction_too() { // The mirror case: node B holds a row node A has never heard of. A cannot request a name it // does not know exists, so this only converges if the exchange pushes as well as pulls. var name = new SecretName("app/born-on-b"); await _storeB.UpsertAsync(Row("app/born-on-b", 0xDD), CancellationToken.None); Assert.True( await WaitForAsync(_storeA, name, r => r.Ciphertext[0] == 0xDD), "Node A never learned about a secret that originated on node B."); } [Fact] public async Task Anti_entropy_still_works_when_the_store_is_genuinely_asynchronous() { // Regression guard for a bug the rest of this suite could not see. A local SQLite store // usually completes `await` synchronously, so continuations stayed on the actor's mailbox // thread and code that touched actor context after an await worked by accident. Wrapping the // store so every call really yields makes that illegal access deterministic — this test // fails outright if anti-entropy touches actor context across an await. SqliteSecretStore backing = CreateStore(); var slowStore = new GenuinelyAsyncSecretStore(backing); _systemB.ActorOf( SecretReplicationActor.Props(slowStore, null, AnnounceInterval), "slow-node"); await _writableA.UpsertAsync(Row("slow/inbound", 0x42), CancellationToken.None); Assert.True( await WaitForAsync(slowStore, new SecretName("slow/inbound"), r => r.Ciphertext[0] == 0x42), "A node whose store is genuinely async never received the secret."); // And the other direction, which can ONLY converge through the anti-entropy exchange. await backing.UpsertAsync(Row("slow/outbound", 0x43), CancellationToken.None); Assert.True( await WaitForAsync(_storeA, new SecretName("slow/outbound"), r => r.Ciphertext[0] == 0x43), "Node A never learned about a secret held by a node whose store is genuinely async."); } [Fact] public async Task A_node_that_joins_late_catches_up_on_everything() { // Partition-heal / cold-start: the whole reason to choose this transport over a shared store. await _writableA.UpsertAsync(Row("bulk/one", 0x11), CancellationToken.None); await _writableA.UpsertAsync(Row("bulk/two", 0x22), CancellationToken.None); await _writableA.UpsertAsync(Row("bulk/three", 0x33), CancellationToken.None); SqliteSecretStore lateStore = CreateStore(); _systemB.ActorOf( SecretReplicationActor.Props(lateStore, null, AnnounceInterval), "late-joiner"); Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/one"), r => r.Ciphertext[0] == 0x11)); Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/two"), r => r.Ciphertext[0] == 0x22)); Assert.True(await WaitForAsync(lateStore, new SecretName("bulk/three"), r => r.Ciphertext[0] == 0x33)); } [Fact] public async Task A_broadcast_round_trip_does_not_drift_the_originating_nodes_revision() { // Note on what this does and does NOT prove: the revision would also stay put if the self-echo // filter were removed, because re-applying an identical row ties on (updated_utc, revision) // and last-writer-wins rejects a tie. So this covers the drift symptom via defence in depth, // not the filter itself. The filter is what stops the echo being forwarded onward, which // needs three nodes to observe — worth adding if a third node ever joins this rig. var name = new SecretName("app/no-echo"); await _writableA.UpsertAsync(Row("app/no-echo", 0xEE), CancellationToken.None); Assert.True(await WaitForAsync(_storeB, name, _ => true)); await Task.Delay(AnnounceInterval * 4); StoredSecret onA = (await _storeA.GetAsync(name, CancellationToken.None))!; Assert.Equal(0, onA.Revision); } [Fact] public async Task A_malformed_row_from_a_peer_is_discarded_without_stalling_replication() { // Peer input is untrusted: a path-traversing name must be dropped, and — critically — the // actor must keep serving everything else afterwards. _replicatorB.Tell(new Protocol.SecretRowsMessage( [ new Protocol.SecretRowDto { Name = "../../etc/passwd", ContentType = "Text", Ciphertext = [1], Nonce = [1], Tag = [1], WrappedDek = [1], WrapNonce = [1], WrapTag = [1], KekId = "sha256:x", Revision = 0, CreatedUtc = DateTimeOffset.UtcNow.ToString("O"), UpdatedUtc = DateTimeOffset.UtcNow.ToString("O"), }, ])); var name = new SecretName("app/after-malformed"); await _writableA.UpsertAsync(Row("app/after-malformed", 0x5A), CancellationToken.None); Assert.True( await WaitForAsync(_storeB, name, r => r.Ciphertext[0] == 0x5A), "Replication stalled after a malformed row was received."); } public async Task DisposeAsync() { await CoordinatedShutdown.Get(_systemA).Run(CoordinatedShutdown.ClrExitReason.Instance); await CoordinatedShutdown.Get(_systemB).Run(CoordinatedShutdown.ClrExitReason.Instance); Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (string db in _dbPaths) { foreach (string path in new[] { db, db + "-wal", db + "-shm" }) { try { if (File.Exists(path)) { File.Delete(path); } } catch (IOException) { // Best-effort temp cleanup. } } } } }