fix(secrets-akka): announce from the mailbox with a sender; chunk anti-entropy payloads (scadaproj#45)
Secrets-011: HandleAnnounceTick pipes the manifest read back to the actor and publishes the announce from the mailbox with Self as sender. Under a genuinely async store it used to publish from a pool continuation with NoSender, so every peer replied to deadLetters; a faulted manifest read was dropped silently and is now logged. Secrets-012: the exchange's push rows, the pull reply and the pull request are split into messages of at most 64 KB (measured with the wire encoding), half Akka.Remote's default 128000-byte frame, so a cold node converges against a large store. A single over-budget row is still sent alone; the manifest announce is not split (documented in clustered-secrets.md). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+65
@@ -0,0 +1,65 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// Splits anti-entropy payloads into messages that stay well inside Akka.Remote's frame size
|
||||
/// (Secrets-012). A cold node's push or pull set is the peer's whole inventory; sent as one
|
||||
/// <see cref="SecretRowsMessage"/> or <see cref="SecretPullRequest"/> it exceeds the default
|
||||
/// <c>akka.remote.dot-netty.tcp.maximum-frame-size</c> (128000 bytes) at a few hundred secrets, is
|
||||
/// dropped by the transport, and is rebuilt identically every announce — so the node never converges.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each message is sized by the serializer's own encoding (compact STJ JSON) to at most
|
||||
/// <see cref="MaxPayloadBytes"/>, half the default frame, leaving room for the remoting envelope.
|
||||
/// A single row larger than the budget (a very large secret) is still sent, alone — it cannot be
|
||||
/// split, and raising the frame size is then the operator's lever. The manifest announce itself is
|
||||
/// not chunked: a receiver computes its push set against the WHOLE remote manifest, so a partial one
|
||||
/// would be wrong rather than merely smaller; an entry is a name plus timestamps (~100 bytes), so the
|
||||
/// announce fits the default frame up to roughly a thousand secrets.
|
||||
/// </remarks>
|
||||
internal static class SecretReplicationChunks
|
||||
{
|
||||
/// <summary>Byte budget for one serialized message: half the default 128000-byte frame.</summary>
|
||||
internal const int MaxPayloadBytes = 64_000;
|
||||
|
||||
// {"Rows":[ ... ]} / {"Names":[ ... ]} plus slack.
|
||||
private const int EnvelopeBytes = 32;
|
||||
|
||||
/// <summary>Groups <paramref name="rows"/> into messages of at most <paramref name="budgetBytes"/>.</summary>
|
||||
internal static IEnumerable<SecretRowsMessage> Rows(
|
||||
IEnumerable<SecretRowDto> rows, int budgetBytes = MaxPayloadBytes) =>
|
||||
Chunk(rows, row => JsonSerializer.SerializeToUtf8Bytes(row).Length, budgetBytes)
|
||||
.Select(chunk => new SecretRowsMessage(chunk));
|
||||
|
||||
/// <summary>Groups <paramref name="names"/> into pull requests of at most <paramref name="budgetBytes"/>.</summary>
|
||||
internal static IEnumerable<SecretPullRequest> PullRequests(
|
||||
IEnumerable<string> names, int budgetBytes = MaxPayloadBytes) =>
|
||||
Chunk(names, name => JsonSerializer.SerializeToUtf8Bytes(name).Length, budgetBytes)
|
||||
.Select(chunk => new SecretPullRequest(chunk));
|
||||
|
||||
private static IEnumerable<IReadOnlyList<T>> Chunk<T>(IEnumerable<T> items, Func<T, int> size, int budgetBytes)
|
||||
{
|
||||
var current = new List<T>();
|
||||
int used = EnvelopeBytes;
|
||||
|
||||
foreach (T item in items)
|
||||
{
|
||||
int itemBytes = size(item) + 1; // separating comma
|
||||
if (current.Count > 0 && used + itemBytes > budgetBytes)
|
||||
{
|
||||
yield return current;
|
||||
current = new List<T>();
|
||||
used = EnvelopeBytes;
|
||||
}
|
||||
|
||||
current.Add(item);
|
||||
used += itemBytes;
|
||||
}
|
||||
|
||||
if (current.Count > 0)
|
||||
{
|
||||
yield return current;
|
||||
}
|
||||
}
|
||||
}
|
||||
+33
-22
@@ -82,6 +82,9 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
Receive<ExchangeFailed>(f =>
|
||||
_log.Warning(f.Cause, "Secret anti-entropy exchange with a peer failed; retrying next announce."));
|
||||
Receive<ExchangeDone>(_ => { /* successful exchange, piped back to self — nothing to do. */ });
|
||||
Receive<ManifestReady>(HandleManifestReady);
|
||||
Receive<ManifestFailed>(f =>
|
||||
_log.Warning(f.Cause, "Could not read the local secret manifest to announce; retrying next interval."));
|
||||
}
|
||||
|
||||
/// <summary>Props for the replication actor.</summary>
|
||||
@@ -181,9 +184,11 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (task.IsCompletedSuccessfully && task.Result.Count > 0)
|
||||
{
|
||||
peer.Tell(
|
||||
new SecretRowsMessage([.. task.Result.Select(SecretRowDto.FromStoredSecret)]),
|
||||
self);
|
||||
foreach (SecretRowsMessage chunk in
|
||||
SecretReplicationChunks.Rows(task.Result.Select(SecretRowDto.FromStoredSecret)))
|
||||
{
|
||||
peer.Tell(chunk, self);
|
||||
}
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
@@ -191,25 +196,27 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
// The manifest read is piped back to the mailbox and published from there (Secrets-011). It used
|
||||
// to be published from the read's continuation: under a store whose read really yields, that ran
|
||||
// on a pool thread with no actor context, so the announce carried NoSender, every peer replied to
|
||||
// deadLetters, and anti-entropy between two such nodes never converged. A faulted read used to be
|
||||
// dropped unobserved; now it is logged.
|
||||
private void HandleAnnounceTick()
|
||||
{
|
||||
_store.GetManifestAsync(CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task =>
|
||||
{
|
||||
if (task.IsCompletedSuccessfully)
|
||||
{
|
||||
_mediator.Tell(new Publish(
|
||||
Topic,
|
||||
new SecretManifestAnnounce(
|
||||
[.. task.Result.Select(SecretManifestEntryDto.FromEntry)])));
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
IActorRef self = Self;
|
||||
_store.GetManifestAsync(CancellationToken.None).PipeTo(
|
||||
self,
|
||||
success: manifest => new ManifestReady(manifest),
|
||||
failure: ex => new ManifestFailed(ex));
|
||||
}
|
||||
|
||||
private void HandleManifestReady(ManifestReady ready) =>
|
||||
_mediator.Tell(
|
||||
new Publish(
|
||||
Topic,
|
||||
new SecretManifestAnnounce([.. ready.Manifest.Select(SecretManifestEntryDto.FromEntry)])),
|
||||
Self);
|
||||
|
||||
// Both halves of an anti-entropy exchange, computed from one manifest: push what the peer is
|
||||
// behind on, ask for what we are behind on.
|
||||
//
|
||||
@@ -237,18 +244,18 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
IReadOnlyList<StoredSecret> rows =
|
||||
await _reconciler.ReadLocalAsync(push, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
if (rows.Count > 0)
|
||||
foreach (SecretRowsMessage chunk in SecretReplicationChunks.Rows(rows.Select(SecretRowDto.FromStoredSecret)))
|
||||
{
|
||||
peer.Tell(new SecretRowsMessage([.. rows.Select(SecretRowDto.FromStoredSecret)]), self);
|
||||
peer.Tell(chunk, self);
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyList<SecretName> pull =
|
||||
SecretReplicationReconciler.ComputePullSet(localManifest, remote);
|
||||
|
||||
if (pull.Count > 0)
|
||||
foreach (SecretPullRequest chunk in SecretReplicationChunks.PullRequests(pull.Select(n => n.Value)))
|
||||
{
|
||||
peer.Tell(new SecretPullRequest([.. pull.Select(n => n.Value)]), self);
|
||||
peer.Tell(chunk, self);
|
||||
}
|
||||
|
||||
return ExchangeDone.Instance;
|
||||
@@ -309,6 +316,10 @@ public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
|
||||
private sealed record ExchangeFailed(Exception Cause);
|
||||
|
||||
private sealed record ManifestReady(IReadOnlyList<SecretManifestEntry> Manifest);
|
||||
|
||||
private sealed record ManifestFailed(Exception Cause);
|
||||
|
||||
private sealed record ExchangeDone
|
||||
{
|
||||
public static readonly ExchangeDone Instance = new();
|
||||
|
||||
Reference in New Issue
Block a user