using Akka.Actor; using Akka.Cluster; using Akka.Cluster.Tools.PublishSubscribe; using Akka.Configuration; using Akka.TestKit.Xunit2; using ZB.MOM.WW.Secrets.Abstractions; using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol; using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests.Fakes; using ZB.MOM.WW.Secrets.Sqlite; namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests; /// /// Secrets-011 (the periodic announce must carry the actor as sender even under a genuinely async /// store, and a failed manifest read must be logged) and Secrets-012 (anti-entropy payloads must be /// chunked under Akka.Remote's frame size), driven against one self-joined cluster node. /// public sealed class AnnounceAndChunkingTests : TestKit, IDisposable { private static readonly TimeSpan AnnounceInterval = TimeSpan.FromMilliseconds(200); private readonly List _dbPaths = []; public AnnounceAndChunkingTests() : base(ConfigurationFactory.ParseString(""" akka { loglevel = WARNING actor.provider = cluster remote.dot-netty.tcp { hostname = "127.0.0.1", public-hostname = "127.0.0.1", port = 0 } } """).WithFallback(AkkaSecretsReplication.SerializationConfig)) { Cluster cluster = Cluster.Get(Sys); cluster.Join(cluster.SelfAddress); AwaitCondition(() => cluster.State.Members.Any(m => m.Status == MemberStatus.Up), TimeSpan.FromSeconds(20)); } void IDisposable.Dispose() { Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (string path in _dbPaths) { try { File.Delete(path); } catch (IOException) { } } } private SqliteSecretStore CreateStore() { string path = Path.Combine(Path.GetTempPath(), $"zb-akka-chunk-{Guid.NewGuid():N}.db"); _dbPaths.Add(path); var factory = new SecretsSqliteConnectionFactory(path); new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult(); return new SqliteSecretStore(factory); } private static StoredSecret Row(string name, int ciphertextBytes) => new() { Name = new SecretName(name), ContentType = SecretContentType.Text, Ciphertext = new byte[ciphertextBytes], Nonce = new byte[12], Tag = new byte[16], WrappedDek = new byte[32], WrapNonce = new byte[12], WrapTag = new byte[16], KekId = "sha256:shared-kek", Revision = 0, CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; private int SerializedSize(object message) => new SecretReplicationSerializer((ExtendedActorSystem)Sys).ToBinary(message).Length; private void SubscribeTestActor() { DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(SecretReplicationActor.Topic, TestActor)); ExpectMsg(TimeSpan.FromSeconds(10)); } private sealed class FailingManifestStore(ISecretStore inner) : ISecretStore { public Task GetAsync(SecretName name, CancellationToken ct) => inner.GetAsync(name, ct); public Task UpsertAsync(StoredSecret row, CancellationToken ct) => inner.UpsertAsync(row, ct); public Task DeleteAsync(SecretName name, string? actor, CancellationToken ct) => inner.DeleteAsync(name, actor, ct); public Task> ListAsync(bool includeDeleted, CancellationToken ct) => inner.ListAsync(includeDeleted, ct); public async Task> GetManifestAsync(CancellationToken ct) { await Task.Delay(5, ct).ConfigureAwait(false); throw new InvalidOperationException("manifest-read-boom"); } public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) => inner.ApplyReplicatedAsync(row, ct); public Task ApplyRewrapAsync(StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) => inner.ApplyRewrapAsync(rewrappedRow, expectedCurrentWrappedDek, ct); } // ---------- Secrets-011 ---------- [Fact] public void An_announce_from_a_genuinely_async_store_carries_the_actor_as_sender() { SubscribeTestActor(); IActorRef node = Sys.ActorOf( SecretReplicationActor.Props(new GenuinelyAsyncSecretStore(CreateStore()), null, AnnounceInterval), "async-announcer"); ExpectMsg(TimeSpan.FromSeconds(10)); // A peer replies to the announce's sender; deadLetters here means every reply is lost. Assert.Equal(node, LastSender); } [Fact] public void A_failed_manifest_read_is_logged_not_dropped() { EventFilter.Warning(contains: "Could not read the local secret manifest").ExpectOne( TimeSpan.FromSeconds(10), () => Sys.ActorOf( SecretReplicationActor.Props(new FailingManifestStore(CreateStore()), null, AnnounceInterval), "failing-announcer")); } // ---------- Secrets-012 ---------- private async Task StoreWithAsync(int count, int ciphertextBytes) { SqliteSecretStore store = CreateStore(); for (int i = 0; i < count; i++) { await store.UpsertAsync(Row($"bulk/{i:D4}", ciphertextBytes), CancellationToken.None); } return store; } private List ReceiveRows(int expectedRows) { var chunks = new List(); int received = 0; while (received < expectedRows) { SecretRowsMessage chunk = ExpectMsg(TimeSpan.FromSeconds(10)); chunks.Add(chunk); received += chunk.Rows.Count; } return chunks; } [Fact] public async Task A_cold_peers_push_set_is_split_under_the_frame_size() { SqliteSecretStore store = await StoreWithAsync(300, ciphertextBytes: 600); IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(store, null, TimeSpan.FromHours(1)), "pusher"); // A cold peer announces an empty manifest: the node pushes its whole inventory. node.Tell(new SecretManifestAnnounce([]), TestActor); List chunks = ReceiveRows(300); Assert.True(chunks.Count > 1, "expected the push to be chunked"); Assert.Equal(300, chunks.Sum(c => c.Rows.Count)); Assert.All(chunks, c => Assert.True(SerializedSize(c) <= SecretReplicationChunks.MaxPayloadBytes)); Assert.Equal(300, chunks.SelectMany(c => c.Rows).Select(r => r.Name).Distinct().Count()); } [Fact] public async Task A_pull_reply_is_split_under_the_frame_size() { SqliteSecretStore store = await StoreWithAsync(300, ciphertextBytes: 600); IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(store, null, TimeSpan.FromHours(1)), "puller"); node.Tell(new SecretPullRequest([.. Enumerable.Range(0, 300).Select(i => $"bulk/{i:D4}")]), TestActor); List chunks = ReceiveRows(300); Assert.True(chunks.Count > 1, "expected the pull reply to be chunked"); Assert.All(chunks, c => Assert.True(SerializedSize(c) <= SecretReplicationChunks.MaxPayloadBytes)); } [Fact] public void A_pull_request_for_a_large_inventory_is_split_under_the_frame_size() { IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(CreateStore(), null, TimeSpan.FromHours(1)), "asker"); string longPrefix = new('n', 200); DateTimeOffset now = DateTimeOffset.UtcNow; // The peer holds 600 long names this (empty) node lacks: it must ask for them in pieces. node.Tell( new SecretManifestAnnounce( [ .. Enumerable.Range(0, 600).Select(i => SecretManifestEntryDto.FromEntry( new SecretManifestEntry { Name = new SecretName($"{longPrefix}/{i:D4}"), UpdatedUtc = now, Revision = 1 })), ]), TestActor); var requests = new List(); while (requests.Sum(r => r.Names.Count) < 600) { requests.Add(ExpectMsg(TimeSpan.FromSeconds(10))); } Assert.True(requests.Count > 1, "expected the pull request to be chunked"); Assert.All(requests, r => Assert.True(SerializedSize(r) <= SecretReplicationChunks.MaxPayloadBytes)); } [Fact] public void A_single_row_larger_than_the_budget_is_still_sent_alone() { var big = SecretRowDto.FromStoredSecret(Row("big/one", 80_000)); var small = SecretRowDto.FromStoredSecret(Row("small/one", 10)); List chunks = [.. SecretReplicationChunks.Rows([small, big, small with { Name = "small/two" }])]; Assert.Equal(3, chunks.Count); Assert.Equal("big/one", Assert.Single(chunks[1].Rows).Name); } }