using Akka.Actor;
using Akka.Cluster;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Configuration;
using Akka.TestKit.Xunit2;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests.Fakes;
using ZB.MOM.WW.Secrets.Sqlite;
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests;
///
/// Secrets-011 (the periodic announce must carry the actor as sender even under a genuinely async
/// store, and a failed manifest read must be logged) and Secrets-012 (anti-entropy payloads must be
/// chunked under Akka.Remote's frame size), driven against one self-joined cluster node.
///
public sealed class AnnounceAndChunkingTests : TestKit, IDisposable
{
private static readonly TimeSpan AnnounceInterval = TimeSpan.FromMilliseconds(200);
private readonly List _dbPaths = [];
public AnnounceAndChunkingTests()
: base(ConfigurationFactory.ParseString("""
akka {
loglevel = WARNING
actor.provider = cluster
remote.dot-netty.tcp { hostname = "127.0.0.1", public-hostname = "127.0.0.1", port = 0 }
}
""").WithFallback(AkkaSecretsReplication.SerializationConfig))
{
Cluster cluster = Cluster.Get(Sys);
cluster.Join(cluster.SelfAddress);
AwaitCondition(() => cluster.State.Members.Any(m => m.Status == MemberStatus.Up), TimeSpan.FromSeconds(20));
}
void IDisposable.Dispose()
{
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (string path in _dbPaths)
{
try { File.Delete(path); } catch (IOException) { }
}
}
private SqliteSecretStore CreateStore()
{
string path = Path.Combine(Path.GetTempPath(), $"zb-akka-chunk-{Guid.NewGuid():N}.db");
_dbPaths.Add(path);
var factory = new SecretsSqliteConnectionFactory(path);
new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
return new SqliteSecretStore(factory);
}
private static StoredSecret Row(string name, int ciphertextBytes) => new()
{
Name = new SecretName(name),
ContentType = SecretContentType.Text,
Ciphertext = new byte[ciphertextBytes],
Nonce = new byte[12],
Tag = new byte[16],
WrappedDek = new byte[32],
WrapNonce = new byte[12],
WrapTag = new byte[16],
KekId = "sha256:shared-kek",
Revision = 0,
CreatedUtc = DateTimeOffset.UtcNow,
UpdatedUtc = DateTimeOffset.UtcNow,
};
private int SerializedSize(object message) =>
new SecretReplicationSerializer((ExtendedActorSystem)Sys).ToBinary(message).Length;
private void SubscribeTestActor()
{
DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(SecretReplicationActor.Topic, TestActor));
ExpectMsg(TimeSpan.FromSeconds(10));
}
private sealed class FailingManifestStore(ISecretStore inner) : ISecretStore
{
public Task GetAsync(SecretName name, CancellationToken ct) => inner.GetAsync(name, ct);
public Task UpsertAsync(StoredSecret row, CancellationToken ct) => inner.UpsertAsync(row, ct);
public Task DeleteAsync(SecretName name, string? actor, CancellationToken ct) => inner.DeleteAsync(name, actor, ct);
public Task> ListAsync(bool includeDeleted, CancellationToken ct) => inner.ListAsync(includeDeleted, ct);
public async Task> GetManifestAsync(CancellationToken ct)
{
await Task.Delay(5, ct).ConfigureAwait(false);
throw new InvalidOperationException("manifest-read-boom");
}
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) => inner.ApplyReplicatedAsync(row, ct);
public Task ApplyRewrapAsync(StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
inner.ApplyRewrapAsync(rewrappedRow, expectedCurrentWrappedDek, ct);
}
// ---------- Secrets-011 ----------
[Fact]
public void An_announce_from_a_genuinely_async_store_carries_the_actor_as_sender()
{
SubscribeTestActor();
IActorRef node = Sys.ActorOf(
SecretReplicationActor.Props(new GenuinelyAsyncSecretStore(CreateStore()), null, AnnounceInterval),
"async-announcer");
ExpectMsg(TimeSpan.FromSeconds(10));
// A peer replies to the announce's sender; deadLetters here means every reply is lost.
Assert.Equal(node, LastSender);
}
[Fact]
public void A_failed_manifest_read_is_logged_not_dropped()
{
EventFilter.Warning(contains: "Could not read the local secret manifest").ExpectOne(
TimeSpan.FromSeconds(10),
() => Sys.ActorOf(
SecretReplicationActor.Props(new FailingManifestStore(CreateStore()), null, AnnounceInterval),
"failing-announcer"));
}
// ---------- Secrets-012 ----------
private async Task StoreWithAsync(int count, int ciphertextBytes)
{
SqliteSecretStore store = CreateStore();
for (int i = 0; i < count; i++)
{
await store.UpsertAsync(Row($"bulk/{i:D4}", ciphertextBytes), CancellationToken.None);
}
return store;
}
private List ReceiveRows(int expectedRows)
{
var chunks = new List();
int received = 0;
while (received < expectedRows)
{
SecretRowsMessage chunk = ExpectMsg(TimeSpan.FromSeconds(10));
chunks.Add(chunk);
received += chunk.Rows.Count;
}
return chunks;
}
[Fact]
public async Task A_cold_peers_push_set_is_split_under_the_frame_size()
{
SqliteSecretStore store = await StoreWithAsync(300, ciphertextBytes: 600);
IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(store, null, TimeSpan.FromHours(1)), "pusher");
// A cold peer announces an empty manifest: the node pushes its whole inventory.
node.Tell(new SecretManifestAnnounce([]), TestActor);
List chunks = ReceiveRows(300);
Assert.True(chunks.Count > 1, "expected the push to be chunked");
Assert.Equal(300, chunks.Sum(c => c.Rows.Count));
Assert.All(chunks, c => Assert.True(SerializedSize(c) <= SecretReplicationChunks.MaxPayloadBytes));
Assert.Equal(300, chunks.SelectMany(c => c.Rows).Select(r => r.Name).Distinct().Count());
}
[Fact]
public async Task A_pull_reply_is_split_under_the_frame_size()
{
SqliteSecretStore store = await StoreWithAsync(300, ciphertextBytes: 600);
IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(store, null, TimeSpan.FromHours(1)), "puller");
node.Tell(new SecretPullRequest([.. Enumerable.Range(0, 300).Select(i => $"bulk/{i:D4}")]), TestActor);
List chunks = ReceiveRows(300);
Assert.True(chunks.Count > 1, "expected the pull reply to be chunked");
Assert.All(chunks, c => Assert.True(SerializedSize(c) <= SecretReplicationChunks.MaxPayloadBytes));
}
[Fact]
public void A_pull_request_for_a_large_inventory_is_split_under_the_frame_size()
{
IActorRef node = Sys.ActorOf(SecretReplicationActor.Props(CreateStore(), null, TimeSpan.FromHours(1)), "asker");
string longPrefix = new('n', 200);
DateTimeOffset now = DateTimeOffset.UtcNow;
// The peer holds 600 long names this (empty) node lacks: it must ask for them in pieces.
node.Tell(
new SecretManifestAnnounce(
[
.. Enumerable.Range(0, 600).Select(i => SecretManifestEntryDto.FromEntry(
new SecretManifestEntry { Name = new SecretName($"{longPrefix}/{i:D4}"), UpdatedUtc = now, Revision = 1 })),
]),
TestActor);
var requests = new List();
while (requests.Sum(r => r.Names.Count) < 600)
{
requests.Add(ExpectMsg(TimeSpan.FromSeconds(10)));
}
Assert.True(requests.Count > 1, "expected the pull request to be chunked");
Assert.All(requests, r => Assert.True(SerializedSize(r) <= SecretReplicationChunks.MaxPayloadBytes));
}
[Fact]
public void A_single_row_larger_than_the_budget_is_still_sent_alone()
{
var big = SecretRowDto.FromStoredSecret(Row("big/one", 80_000));
var small = SecretRowDto.FromStoredSecret(Row("small/one", 10));
List chunks = [.. SecretReplicationChunks.Rows([small, big, small with { Name = "small/two" }])];
Assert.Equal(3, chunks.Count);
Assert.Equal("big/one", Assert.Single(chunks[1].Rows).Name);
}
}