fix(secrets-akka): announce from the mailbox with a sender; chunk anti-entropy payloads (scadaproj#45)

Secrets-011: HandleAnnounceTick pipes the manifest read back to the actor and publishes the
announce from the mailbox with Self as sender. Under a genuinely async store it used to publish
from a pool continuation with NoSender, so every peer replied to deadLetters; a faulted manifest
read was dropped silently and is now logged.

Secrets-012: the exchange's push rows, the pull reply and the pull request are split into
messages of at most 64 KB (measured with the wire encoding), half Akka.Remote's default 128000-byte
frame, so a cold node converges against a large store. A single over-budget row is still sent
alone; the manifest announce is not split (documented in clustered-secrets.md).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 15:16:57 -05:00
parent 5e2d3dd1bc
commit 849640f467
4 changed files with 327 additions and 22 deletions
@@ -0,0 +1,65 @@
using System.Text.Json;
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
/// <summary>
/// Splits anti-entropy payloads into messages that stay well inside Akka.Remote's frame size
/// (Secrets-012). A cold node's push or pull set is the peer's whole inventory; sent as one
/// <see cref="SecretRowsMessage"/> or <see cref="SecretPullRequest"/> it exceeds the default
/// <c>akka.remote.dot-netty.tcp.maximum-frame-size</c> (128000 bytes) at a few hundred secrets, is
/// dropped by the transport, and is rebuilt identically every announce — so the node never converges.
/// </summary>
/// <remarks>
/// Each message is sized by the serializer's own encoding (compact STJ JSON) to at most
/// <see cref="MaxPayloadBytes"/>, half the default frame, leaving room for the remoting envelope.
/// A single row larger than the budget (a very large secret) is still sent, alone — it cannot be
/// split, and raising the frame size is then the operator's lever. The manifest announce itself is
/// not chunked: a receiver computes its push set against the WHOLE remote manifest, so a partial one
/// would be wrong rather than merely smaller; an entry is a name plus timestamps (~100 bytes), so the
/// announce fits the default frame up to roughly a thousand secrets.
/// </remarks>
internal static class SecretReplicationChunks
{
/// <summary>Byte budget for one serialized message: half the default 128000-byte frame.</summary>
internal const int MaxPayloadBytes = 64_000;
// {"Rows":[ ... ]} / {"Names":[ ... ]} plus slack.
private const int EnvelopeBytes = 32;
/// <summary>Groups <paramref name="rows"/> into messages of at most <paramref name="budgetBytes"/>.</summary>
internal static IEnumerable<SecretRowsMessage> Rows(
IEnumerable<SecretRowDto> rows, int budgetBytes = MaxPayloadBytes) =>
Chunk(rows, row => JsonSerializer.SerializeToUtf8Bytes(row).Length, budgetBytes)
.Select(chunk => new SecretRowsMessage(chunk));
/// <summary>Groups <paramref name="names"/> into pull requests of at most <paramref name="budgetBytes"/>.</summary>
internal static IEnumerable<SecretPullRequest> PullRequests(
IEnumerable<string> names, int budgetBytes = MaxPayloadBytes) =>
Chunk(names, name => JsonSerializer.SerializeToUtf8Bytes(name).Length, budgetBytes)
.Select(chunk => new SecretPullRequest(chunk));
private static IEnumerable<IReadOnlyList<T>> Chunk<T>(IEnumerable<T> items, Func<T, int> size, int budgetBytes)
{
var current = new List<T>();
int used = EnvelopeBytes;
foreach (T item in items)
{
int itemBytes = size(item) + 1; // separating comma
if (current.Count > 0 && used + itemBytes > budgetBytes)
{
yield return current;
current = new List<T>();
used = EnvelopeBytes;
}
current.Add(item);
used += itemBytes;
}
if (current.Count > 0)
{
yield return current;
}
}
}
@@ -82,6 +82,9 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
Receive<ExchangeFailed>(f =>
_log.Warning(f.Cause, "Secret anti-entropy exchange with a peer failed; retrying next announce."));
Receive<ExchangeDone>(_ => { /* successful exchange, piped back to self — nothing to do. */ });
Receive<ManifestReady>(HandleManifestReady);
Receive<ManifestFailed>(f =>
_log.Warning(f.Cause, "Could not read the local secret manifest to announce; retrying next interval."));
}
/// <summary>Props for the replication actor.</summary>
@@ -181,9 +184,11 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
if (task.IsCompletedSuccessfully && task.Result.Count > 0)
{
peer.Tell(
new SecretRowsMessage([.. task.Result.Select(SecretRowDto.FromStoredSecret)]),
self);
foreach (SecretRowsMessage chunk in
SecretReplicationChunks.Rows(task.Result.Select(SecretRowDto.FromStoredSecret)))
{
peer.Tell(chunk, self);
}
}
},
CancellationToken.None,
@@ -191,25 +196,27 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
TaskScheduler.Default);
}
// The manifest read is piped back to the mailbox and published from there (Secrets-011). It used
// to be published from the read's continuation: under a store whose read really yields, that ran
// on a pool thread with no actor context, so the announce carried NoSender, every peer replied to
// deadLetters, and anti-entropy between two such nodes never converged. A faulted read used to be
// dropped unobserved; now it is logged.
private void HandleAnnounceTick()
{
_store.GetManifestAsync(CancellationToken.None)
.ContinueWith(
task =>
{
if (task.IsCompletedSuccessfully)
{
_mediator.Tell(new Publish(
Topic,
new SecretManifestAnnounce(
[.. task.Result.Select(SecretManifestEntryDto.FromEntry)])));
}
},
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
IActorRef self = Self;
_store.GetManifestAsync(CancellationToken.None).PipeTo(
self,
success: manifest => new ManifestReady(manifest),
failure: ex => new ManifestFailed(ex));
}
private void HandleManifestReady(ManifestReady ready) =>
_mediator.Tell(
new Publish(
Topic,
new SecretManifestAnnounce([.. ready.Manifest.Select(SecretManifestEntryDto.FromEntry)])),
Self);
// Both halves of an anti-entropy exchange, computed from one manifest: push what the peer is
// behind on, ask for what we are behind on.
//
@@ -237,18 +244,18 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
IReadOnlyList<StoredSecret> rows =
await _reconciler.ReadLocalAsync(push, CancellationToken.None).ConfigureAwait(false);
if (rows.Count > 0)
foreach (SecretRowsMessage chunk in SecretReplicationChunks.Rows(rows.Select(SecretRowDto.FromStoredSecret)))
{
peer.Tell(new SecretRowsMessage([.. rows.Select(SecretRowDto.FromStoredSecret)]), self);
peer.Tell(chunk, self);
}
}
IReadOnlyList<SecretName> pull =
SecretReplicationReconciler.ComputePullSet(localManifest, remote);
if (pull.Count > 0)
foreach (SecretPullRequest chunk in SecretReplicationChunks.PullRequests(pull.Select(n => n.Value)))
{
peer.Tell(new SecretPullRequest([.. pull.Select(n => n.Value)]), self);
peer.Tell(chunk, self);
}
return ExchangeDone.Instance;
@@ -309,6 +316,10 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
private sealed record ExchangeFailed(Exception Cause);
private sealed record ManifestReady(IReadOnlyList<SecretManifestEntry> Manifest);
private sealed record ManifestFailed(Exception Cause);
private sealed record ExchangeDone
{
public static readonly ExchangeDone Instance = new();