feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)
Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,33 @@
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes local secret writes to cluster peers by handing them to this node's
|
||||
/// <see cref="SecretReplicationActor"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The hand-off is a <c>Tell</c>, so <see cref="PublishAsync"/> completes as soon as the row is in
|
||||
/// the actor's mailbox rather than when peers have applied it. That is the correct contract here:
|
||||
/// the caller (<c>ReplicatingSecretStore</c>) has already durably written the row locally and treats
|
||||
/// publishing as best-effort, and blocking a write until every peer acknowledged would make each
|
||||
/// node's availability depend on all the others. Delivery is guaranteed by anti-entropy, not by
|
||||
/// this call.
|
||||
/// </remarks>
|
||||
/// <param name="replicationActor">This node's replication actor.</param>
|
||||
public sealed class AkkaSecretReplicator(IActorRef replicationActor) : ISecretReplicator
|
||||
{
|
||||
private readonly IActorRef _replicationActor =
|
||||
replicationActor ?? throw new ArgumentNullException(nameof(replicationActor));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task PublishAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
_replicationActor.Tell(new SecretReplicationActor.PublishLocalWrite(row));
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
using Akka.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Akka configuration this package contributes.
|
||||
/// </summary>
|
||||
public static class AkkaSecretsReplication
|
||||
{
|
||||
/// <summary>
|
||||
/// HOCON binding the secret-replication wire protocol to
|
||||
/// <see cref="SecretReplicationSerializer"/>. Merge it into the application's Akka
|
||||
/// configuration when creating the <c>ActorSystem</c>:
|
||||
/// <code>
|
||||
/// Config config = AkkaSecretsReplication.SerializationConfig
|
||||
/// .WithFallback(myAppConfig);
|
||||
/// ActorSystem.Create("my-system", config);
|
||||
/// </code>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Optional — the protocol DTOs round-trip under Akka's default JSON serializer without it — but
|
||||
/// recommended, because it makes the serializer carrying secret ciphertext an explicit choice
|
||||
/// rather than an inherited default, and pins the manifest strings that a future protocol
|
||||
/// version would bump.
|
||||
/// </remarks>
|
||||
public static Config SerializationConfig { get; } = ConfigurationFactory.ParseString($$"""
|
||||
akka.actor {
|
||||
serializers {
|
||||
zb-secrets-replication = "{{typeof(SecretReplicationSerializer).AssemblyQualifiedName}}"
|
||||
}
|
||||
serialization-bindings {
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretRowsMessage, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretManifestAnnounce, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretPullRequest, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
}
|
||||
serialization-identifiers {
|
||||
"{{typeof(SecretReplicationSerializer).AssemblyQualifiedName}}" = {{SecretReplicationSerializer.SerializerId}}
|
||||
}
|
||||
}
|
||||
""");
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for peer-to-peer secret replication over the Akka cluster.
|
||||
/// </summary>
|
||||
public sealed class AkkaSecretsReplicationOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// How often this node broadcasts its manifest for anti-entropy. Defaults to 30 seconds.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the convergence bound, not the propagation latency: a live write reaches peers in
|
||||
/// milliseconds, and this interval only governs how quickly a <em>dropped</em> write or a
|
||||
/// rejoining node is repaired. The traffic is one manifest per node per interval — names,
|
||||
/// revisions, and timestamps, no ciphertext — so shortening it is cheap for a small cluster and
|
||||
/// grows as O(nodes × secrets).
|
||||
/// </remarks>
|
||||
public TimeSpan AnnounceInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Name of the replication actor within the actor system. Defaults to <c>zb-secret-replication</c>.</summary>
|
||||
public string ActorName { get; set; } = "zb-secret-replication";
|
||||
|
||||
/// <summary>
|
||||
/// Throws if the options are unusable.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The announce interval is not positive, or the actor name is blank.</exception>
|
||||
public void Validate()
|
||||
{
|
||||
if (AnnounceInterval <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Secret replication AnnounceInterval must be positive (was {AnnounceInterval}).");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ActorName))
|
||||
{
|
||||
throw new InvalidOperationException("Secret replication ActorName must not be blank.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// Dependency-injection entry point for peer-to-peer secret replication over an Akka.NET cluster.
|
||||
/// </summary>
|
||||
public static class AkkaSecretsServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers secret replication over the cluster: each node keeps its own local store, local
|
||||
/// writes broadcast to peers, and a periodic manifest exchange repairs any divergence in both
|
||||
/// directions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Choose this transport when a node must keep resolving secrets while <b>partitioned</b> from
|
||||
/// the rest of the cluster — reads never leave the node, so an isolated site keeps running on its
|
||||
/// last-known-good state and re-converges when the partition heals. If every node can always
|
||||
/// reach a shared database, the SQL-Server package's shared-store mode is simpler and has no
|
||||
/// distributed failure modes at all; prefer it unless partition tolerance is a stated requirement.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Requirements.</b> The application must (1) register its <see cref="ActorSystem"/> in DI
|
||||
/// before the first secret resolve, and (2) give every node the <b>same KEK</b> — rows carry
|
||||
/// ciphertext only, so a node with a different master key fails closed on a <c>kek_id</c>
|
||||
/// mismatch. Merging <see cref="AkkaSecretsReplication.SerializationConfig"/> into the actor
|
||||
/// system's configuration is recommended.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>KEK rotation does not replicate.</b> A re-wrap deliberately leaves the revision untouched
|
||||
/// so it stays invisible to last-writer-wins; run <c>rewrap-all</c> against every node's local
|
||||
/// store.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to add to.</param>
|
||||
/// <param name="config">The application configuration.</param>
|
||||
/// <param name="secretsSectionPath">Configuration section holding the core secrets options.</param>
|
||||
/// <param name="replicationSectionPath">Configuration section holding the replication options.</param>
|
||||
/// <returns>The same <paramref name="services"/> instance, for chaining.</returns>
|
||||
public static IServiceCollection AddZbSecretsAkkaReplication(
|
||||
this IServiceCollection services,
|
||||
IConfiguration config,
|
||||
string secretsSectionPath = "Secrets",
|
||||
string replicationSectionPath = "Secrets:Replication")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(secretsSectionPath);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(replicationSectionPath);
|
||||
|
||||
IConfigurationSection section = config.GetSection(replicationSectionPath);
|
||||
services.Configure<AkkaSecretsReplicationOptions>(section);
|
||||
|
||||
// Validate eagerly so a bad interval fails at startup, not on the first announce tick.
|
||||
var eager = new AkkaSecretsReplicationOptions();
|
||||
section.Bind(eager);
|
||||
eager.Validate();
|
||||
|
||||
// The local store + migrator come from AddZbSecrets (SQLite by default).
|
||||
services.AddZbSecrets(config, secretsSectionPath);
|
||||
|
||||
// The actor is created lazily on first use rather than at registration: the ActorSystem is
|
||||
// typically registered by the application AFTER this call, and a cluster node may also not be
|
||||
// ready to join at the moment the container is built.
|
||||
services.TryAddSingleton<SecretReplicationActorProvider>(sp => new SecretReplicationActorProvider(
|
||||
sp.GetRequiredService<ActorSystem>(),
|
||||
sp.GetRequiredService<SqliteSecretStore>(),
|
||||
sp.GetService<ISecretCacheInvalidator>(),
|
||||
sp.GetRequiredService<IOptions<AkkaSecretsReplicationOptions>>().Value));
|
||||
|
||||
services.TryAddSingleton<ISecretReplicator>(sp =>
|
||||
new AkkaSecretReplicator(sp.GetRequiredService<SecretReplicationActorProvider>().ActorRef));
|
||||
|
||||
// Decorate the local store so every write publishes. Resolved from the concrete SQLite store,
|
||||
// not ISecretStore — asking the container for ISecretStore here would resolve this very
|
||||
// decorator and recurse forever.
|
||||
services.AddSingleton<ISecretStore>(sp => new ReplicatingSecretStore(
|
||||
sp.GetRequiredService<SqliteSecretStore>(),
|
||||
sp.GetRequiredService<ISecretReplicator>(),
|
||||
sp.GetRequiredService<ILogger<ReplicatingSecretStore>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates this node's <see cref="SecretReplicationActor"/> on first access and hands out its ref.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The indirection exists because the actor must be created lazily — the application usually
|
||||
/// registers its <see cref="ActorSystem"/> after <c>AddZbSecretsAkkaReplication</c>, so eagerly
|
||||
/// resolving one at registration time would fail. Creation is guarded by
|
||||
/// <see cref="Lazy{T}"/> with <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so
|
||||
/// concurrent first writes cannot spawn two actors on the same node.
|
||||
/// </remarks>
|
||||
/// <param name="system">The application's actor system.</param>
|
||||
/// <param name="store">The node's local secret store — the undecorated one.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam.</param>
|
||||
/// <param name="options">Replication options.</param>
|
||||
public sealed class SecretReplicationActorProvider(
|
||||
ActorSystem system,
|
||||
ISecretStore store,
|
||||
ISecretCacheInvalidator? cacheInvalidator,
|
||||
AkkaSecretsReplicationOptions options)
|
||||
{
|
||||
private readonly Lazy<IActorRef> _actorRef = new(
|
||||
() => system.ActorOf(
|
||||
SecretReplicationActor.Props(store, cacheInvalidator, options.AnnounceInterval),
|
||||
options.ActorName),
|
||||
LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
/// <summary>This node's replication actor, created on first access.</summary>
|
||||
public IActorRef ActorRef => _actorRef.Value;
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// The wire representation of one encrypted secret row.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is a deliberate hand-written mirror of <see cref="StoredSecret"/> rather than the type
|
||||
/// itself, and the duplication is the point. It makes the trust boundary <b>structural</b>: the wire
|
||||
/// contract is a flat set of primitives that a reader can check against "ciphertext only, never the
|
||||
/// KEK" in one glance, and a field added to <see cref="StoredSecret"/> in future cannot start
|
||||
/// crossing the network merely by existing — someone has to add it here on purpose.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// All fields are primitives (<see cref="string"/>, <see cref="byte"/> arrays, <see cref="long"/>,
|
||||
/// <see cref="bool"/>) so the DTO round-trips through any serializer, including Akka's default JSON
|
||||
/// one if an application never registers <see cref="SecretReplicationSerializer"/>. Timestamps are
|
||||
/// ISO-8601 <c>"O"</c> strings for the same reason the stores use them: last-writer-wins compares
|
||||
/// them for equality, so the representation has to be exact rather than merely close.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed record SecretRowDto
|
||||
{
|
||||
/// <summary>The normalized secret name.</summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>Optional human-readable description.</summary>
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>The <see cref="SecretContentType"/> enum name.</summary>
|
||||
public required string ContentType { get; init; }
|
||||
|
||||
/// <summary>AES-256-GCM ciphertext of the secret plaintext.</summary>
|
||||
public required byte[] Ciphertext { get; init; }
|
||||
|
||||
/// <summary>Nonce sealing <see cref="Ciphertext"/>.</summary>
|
||||
public required byte[] Nonce { get; init; }
|
||||
|
||||
/// <summary>Authentication tag for <see cref="Ciphertext"/>.</summary>
|
||||
public required byte[] Tag { get; init; }
|
||||
|
||||
/// <summary>The per-secret data key, wrapped by the KEK. The KEK itself never travels.</summary>
|
||||
public required byte[] WrappedDek { get; init; }
|
||||
|
||||
/// <summary>Nonce used to wrap <see cref="WrappedDek"/>.</summary>
|
||||
public required byte[] WrapNonce { get; init; }
|
||||
|
||||
/// <summary>Authentication tag for <see cref="WrappedDek"/>.</summary>
|
||||
public required byte[] WrapTag { get; init; }
|
||||
|
||||
/// <summary>Identifier of the KEK that wrapped the DEK — an identifier, not key material.</summary>
|
||||
public required string KekId { get; init; }
|
||||
|
||||
/// <summary>Monotonic revision.</summary>
|
||||
public required long Revision { get; init; }
|
||||
|
||||
/// <summary>Whether the row is a tombstone.</summary>
|
||||
public bool IsDeleted { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 deletion timestamp, or <see langword="null"/>.</summary>
|
||||
public string? DeletedUtc { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 creation timestamp.</summary>
|
||||
public required string CreatedUtc { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 last-update timestamp — half of the last-writer-wins ordering key.</summary>
|
||||
public required string UpdatedUtc { get; init; }
|
||||
|
||||
/// <summary>Principal that created the row, if known.</summary>
|
||||
public string? CreatedBy { get; init; }
|
||||
|
||||
/// <summary>Principal that last updated the row, if known.</summary>
|
||||
public string? UpdatedBy { get; init; }
|
||||
|
||||
/// <summary>Projects a stored row onto the wire contract.</summary>
|
||||
/// <param name="row">The encrypted row.</param>
|
||||
/// <returns>The wire representation.</returns>
|
||||
public static SecretRowDto FromStoredSecret(StoredSecret row)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
return new SecretRowDto
|
||||
{
|
||||
Name = row.Name.Value,
|
||||
Description = row.Description,
|
||||
ContentType = row.ContentType.ToString(),
|
||||
Ciphertext = row.Ciphertext,
|
||||
Nonce = row.Nonce,
|
||||
Tag = row.Tag,
|
||||
WrappedDek = row.WrappedDek,
|
||||
WrapNonce = row.WrapNonce,
|
||||
WrapTag = row.WrapTag,
|
||||
KekId = row.KekId,
|
||||
Revision = row.Revision,
|
||||
IsDeleted = row.IsDeleted,
|
||||
DeletedUtc = row.DeletedUtc?.ToString("O"),
|
||||
CreatedUtc = row.CreatedUtc.ToString("O"),
|
||||
UpdatedUtc = row.UpdatedUtc.ToString("O"),
|
||||
CreatedBy = row.CreatedBy,
|
||||
UpdatedBy = row.UpdatedBy,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Rebuilds the stored row from the wire contract.</summary>
|
||||
/// <returns>The encrypted row.</returns>
|
||||
/// <exception cref="ArgumentException">The name or content type is not valid.</exception>
|
||||
public StoredSecret ToStoredSecret() => new()
|
||||
{
|
||||
// SecretName re-validates on construction, so a malformed or path-traversing name from a
|
||||
// peer is rejected here rather than reaching the store.
|
||||
Name = new SecretName(Name),
|
||||
Description = Description,
|
||||
ContentType = ParseContentType(ContentType),
|
||||
// `required` on a byte[] means "present in the payload", NOT "non-null" — a peer sending an
|
||||
// explicit JSON null satisfies it and yields a null array, which would then blow up deep in
|
||||
// the store's parameter binding instead of being rejected at the boundary.
|
||||
Ciphertext = RequireBlob(Ciphertext, nameof(Ciphertext)),
|
||||
Nonce = RequireBlob(Nonce, nameof(Nonce)),
|
||||
Tag = RequireBlob(Tag, nameof(Tag)),
|
||||
WrappedDek = RequireBlob(WrappedDek, nameof(WrappedDek)),
|
||||
WrapNonce = RequireBlob(WrapNonce, nameof(WrapNonce)),
|
||||
WrapTag = RequireBlob(WrapTag, nameof(WrapTag)),
|
||||
KekId = KekId ?? throw new ArgumentException("Replicated secret row has a null KekId."),
|
||||
Revision = Revision,
|
||||
IsDeleted = IsDeleted,
|
||||
DeletedUtc = DeletedUtc is null ? null : ParseUtc(DeletedUtc),
|
||||
CreatedUtc = ParseUtc(CreatedUtc),
|
||||
UpdatedUtc = ParseUtc(UpdatedUtc),
|
||||
CreatedBy = CreatedBy,
|
||||
UpdatedBy = UpdatedBy,
|
||||
};
|
||||
|
||||
private static DateTimeOffset ParseUtc(string value) => DateTimeOffset.Parse(
|
||||
value,
|
||||
System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
|
||||
private static byte[] RequireBlob(byte[]? value, string field) =>
|
||||
value ?? throw new ArgumentException($"Replicated secret row has a null {field}.", field);
|
||||
|
||||
// Enum.Parse is wrong for untrusted input on two counts: a numeric string that overflows the
|
||||
// underlying type throws OverflowException (which the receiving actor's ArgumentException filter
|
||||
// would not catch, restarting it in a redelivery loop), and any in-range number is accepted even
|
||||
// if it names no member — persisting an undefined enum value that later readers assume is valid.
|
||||
private static SecretContentType ParseContentType(string value) =>
|
||||
Enum.TryParse(value, out SecretContentType parsed) && Enum.IsDefined(parsed)
|
||||
? parsed
|
||||
: throw new ArgumentException(
|
||||
$"Replicated secret row has an unrecognized content type '{value}'.", nameof(value));
|
||||
}
|
||||
|
||||
/// <summary>One manifest entry on the wire: enough to decide who is newer, with no ciphertext.</summary>
|
||||
public sealed record SecretManifestEntryDto
|
||||
{
|
||||
/// <summary>The normalized secret name.</summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>Monotonic revision.</summary>
|
||||
public required long Revision { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 last-update timestamp.</summary>
|
||||
public required string UpdatedUtc { get; init; }
|
||||
|
||||
/// <summary>Whether the row is a tombstone.</summary>
|
||||
public bool IsDeleted { get; init; }
|
||||
|
||||
/// <summary>Projects a manifest entry onto the wire contract.</summary>
|
||||
/// <param name="entry">The manifest entry.</param>
|
||||
/// <returns>The wire representation.</returns>
|
||||
public static SecretManifestEntryDto FromEntry(SecretManifestEntry entry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
return new SecretManifestEntryDto
|
||||
{
|
||||
Name = entry.Name.Value,
|
||||
Revision = entry.Revision,
|
||||
UpdatedUtc = entry.UpdatedUtc.ToString("O"),
|
||||
IsDeleted = entry.IsDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Rebuilds the manifest entry from the wire contract.</summary>
|
||||
/// <returns>The manifest entry.</returns>
|
||||
public SecretManifestEntry ToEntry() => new()
|
||||
{
|
||||
Name = new SecretName(Name),
|
||||
Revision = Revision,
|
||||
UpdatedUtc = DateTimeOffset.Parse(
|
||||
UpdatedUtc,
|
||||
System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind),
|
||||
IsDeleted = IsDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A node broadcasting its full inventory so peers can work out, in one round trip, both what they
|
||||
/// need from it and what it needs from them.
|
||||
/// </summary>
|
||||
/// <param name="Entries">The sender's manifest.</param>
|
||||
public sealed record SecretManifestAnnounce(IReadOnlyList<SecretManifestEntryDto> Entries);
|
||||
|
||||
/// <summary>A peer asking for the encrypted rows behind names it found it was missing or behind on.</summary>
|
||||
/// <param name="Names">The requested secret names.</param>
|
||||
public sealed record SecretPullRequest(IReadOnlyList<string> Names);
|
||||
|
||||
/// <summary>
|
||||
/// Encrypted rows delivered to a peer — the reply to a <see cref="SecretPullRequest"/>, the
|
||||
/// unsolicited push half of an anti-entropy exchange, and the live broadcast of a fresh local write.
|
||||
/// </summary>
|
||||
/// <param name="Rows">The encrypted rows.</param>
|
||||
public sealed record SecretRowsMessage(IReadOnlyList<SecretRowDto> Rows);
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// One per node. Owns this node's participation in secret replication: broadcasts local writes,
|
||||
/// periodically announces its manifest, and services peers' pulls.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The protocol is deliberately small — three messages, no leader, no singleton:
|
||||
/// </para>
|
||||
/// <list type="number">
|
||||
/// <item><description>
|
||||
/// <b>Live push.</b> A local write publishes a <see cref="SecretRowsMessage"/> to the topic. Peers
|
||||
/// apply it last-writer-wins. This is the fast path — propagation in milliseconds — and it is
|
||||
/// allowed to fail.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>Anti-entropy.</b> Every node periodically publishes a <see cref="SecretManifestAnnounce"/>.
|
||||
/// A peer receiving it computes <em>both</em> directions in one pass: it pushes back rows it holds
|
||||
/// newer, and asks (<see cref="SecretPullRequest"/>) for rows it is missing or behind on. This is
|
||||
/// the correctness path — it repairs anything the fast path dropped, in either direction.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>Pull service.</b> A <see cref="SecretPullRequest"/> is answered with the requested rows.
|
||||
/// </description></item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// There is no coordinator because there does not need to be one: every exchange is idempotent and
|
||||
/// commutative under last-writer-wins, so nodes converge regardless of message order, duplication,
|
||||
/// or which subset of peers is currently reachable. A node that was partitioned keeps serving its
|
||||
/// local store the whole time and re-converges on the first announce after the partition heals —
|
||||
/// which is the entire reason to choose this transport over the shared SQL-Server store.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
/// <summary>The distributed pub/sub topic secret replication traffic rides on.</summary>
|
||||
public const string Topic = "zb-mom-ww-secrets";
|
||||
|
||||
private readonly ISecretStore _store;
|
||||
private readonly SecretReplicationReconciler _reconciler;
|
||||
private readonly TimeSpan _announceInterval;
|
||||
private readonly IActorRef _mediator;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <inheritdoc />
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>Creates the replication actor.</summary>
|
||||
/// <param name="store">The node's local secret store.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam, evicted for every applied row.</param>
|
||||
/// <param name="announceInterval">How often this node announces its manifest.</param>
|
||||
public SecretReplicationActor(
|
||||
ISecretStore store,
|
||||
ISecretCacheInvalidator? cacheInvalidator,
|
||||
TimeSpan announceInterval)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_reconciler = new SecretReplicationReconciler(
|
||||
_store,
|
||||
cacheInvalidator,
|
||||
onRowFailed: (name, ex) => _log.Warning(
|
||||
ex, "Discarding replicated secret row {0} that could not be applied locally.", name.Value));
|
||||
_announceInterval = announceInterval;
|
||||
_mediator = DistributedPubSub.Get(Context.System).Mediator;
|
||||
|
||||
_mediator.Tell(new Subscribe(Topic, Self));
|
||||
|
||||
Receive<PublishLocalWrite>(HandlePublishLocalWrite);
|
||||
Receive<SecretRowsMessage>(HandleRows);
|
||||
Receive<SecretManifestAnnounce>(HandleManifestAnnounce);
|
||||
Receive<SecretPullRequest>(HandlePullRequest);
|
||||
Receive<AnnounceTick>(_ => HandleAnnounceTick());
|
||||
Receive<SubscribeAck>(_ => _log.Debug("Subscribed to secret replication topic {0}.", Topic));
|
||||
Receive<ExchangeFailed>(f =>
|
||||
_log.Warning(f.Cause, "Secret anti-entropy exchange with a peer failed; retrying next announce."));
|
||||
Receive<ExchangeDone>(_ => { /* successful exchange, piped back to self — nothing to do. */ });
|
||||
}
|
||||
|
||||
/// <summary>Props for the replication actor.</summary>
|
||||
/// <param name="store">The node's local secret store.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam.</param>
|
||||
/// <param name="announceInterval">How often this node announces its manifest.</param>
|
||||
/// <returns>Props creating the actor.</returns>
|
||||
public static Props Props(
|
||||
ISecretStore store, ISecretCacheInvalidator? cacheInvalidator, TimeSpan announceInterval) =>
|
||||
Akka.Actor.Props.Create(() =>
|
||||
new SecretReplicationActor(store, cacheInvalidator, announceInterval));
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart() =>
|
||||
Timers.StartPeriodicTimer(AnnounceTick.Instance, AnnounceTick.Instance, _announceInterval, _announceInterval);
|
||||
|
||||
// A local write, handed over by AkkaSecretReplicator. Published from inside the actor so the
|
||||
// message carries Self as sender — that is what lets a peer reply directly to this node instead
|
||||
// of broadcasting its answer to everyone.
|
||||
private void HandlePublishLocalWrite(PublishLocalWrite message)
|
||||
{
|
||||
_mediator.Tell(new Publish(
|
||||
Topic,
|
||||
new SecretRowsMessage([SecretRowDto.FromStoredSecret(message.Row)])));
|
||||
}
|
||||
|
||||
private void HandleRows(SecretRowsMessage message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IReadOnlyList<StoredSecret> rows = Materialize(message.Rows);
|
||||
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// PipeTo nothing: applying is fire-and-forget from the mailbox's point of view. Failures are
|
||||
// logged, and anti-entropy re-delivers anything that did not land.
|
||||
_reconciler.ApplyAsync(rows, CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task => _log.Warning(
|
||||
task.Exception,
|
||||
"Failed to apply {0} replicated secret row(s) from a peer; anti-entropy will retry.",
|
||||
rows.Count),
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private void HandleManifestAnnounce(SecretManifestAnnounce message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IActorRef peer = Sender;
|
||||
IReadOnlyList<SecretManifestEntry> remote = [.. message.Entries.Select(e => e.ToEntry())];
|
||||
|
||||
ReconcileWithPeerAsync(peer, remote).PipeTo(Self, failure: ex => new ExchangeFailed(ex));
|
||||
}
|
||||
|
||||
private void HandlePullRequest(SecretPullRequest message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IActorRef peer = Sender;
|
||||
// Captured before the continuation: Self is context-dependent and there is no guarantee the
|
||||
// actor context is still current when the continuation runs (Akka analyzer AK1005).
|
||||
IActorRef self = Self;
|
||||
IReadOnlyList<SecretName> names = ParseNames(message.Names);
|
||||
|
||||
_reconciler.ReadLocalAsync(names, CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task =>
|
||||
{
|
||||
if (task.IsFaulted)
|
||||
{
|
||||
// Logged rather than swallowed: the peer is waiting for a reply it will never
|
||||
// get, and without this the only symptom is a secret that quietly never
|
||||
// converges. The peer re-asks on its next announce.
|
||||
_log.Warning(
|
||||
task.Exception,
|
||||
"Failed to read {0} local secret row(s) for a peer's pull request.",
|
||||
names.Count);
|
||||
return;
|
||||
}
|
||||
|
||||
if (task.IsCompletedSuccessfully && task.Result.Count > 0)
|
||||
{
|
||||
peer.Tell(
|
||||
new SecretRowsMessage([.. task.Result.Select(SecretRowDto.FromStoredSecret)]),
|
||||
self);
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private void HandleAnnounceTick()
|
||||
{
|
||||
_store.GetManifestAsync(CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task =>
|
||||
{
|
||||
if (task.IsCompletedSuccessfully)
|
||||
{
|
||||
_mediator.Tell(new Publish(
|
||||
Topic,
|
||||
new SecretManifestAnnounce(
|
||||
[.. task.Result.Select(SecretManifestEntryDto.FromEntry)])));
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
// Both halves of an anti-entropy exchange, computed from one manifest: push what the peer is
|
||||
// behind on, ask for what we are behind on.
|
||||
private async Task<ExchangeDone> ReconcileWithPeerAsync(
|
||||
IActorRef peer, IReadOnlyList<SecretManifestEntry> remote)
|
||||
{
|
||||
IReadOnlyList<SecretManifestEntry> localManifest =
|
||||
await _store.GetManifestAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
IReadOnlyList<SecretName> push =
|
||||
SecretReplicationReconciler.ComputePushSet(localManifest, remote);
|
||||
|
||||
if (push.Count > 0)
|
||||
{
|
||||
IReadOnlyList<StoredSecret> rows =
|
||||
await _reconciler.ReadLocalAsync(push, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
peer.Tell(new SecretRowsMessage([.. rows.Select(SecretRowDto.FromStoredSecret)]), Self);
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyList<SecretName> pull =
|
||||
SecretReplicationReconciler.ComputePullSet(localManifest, remote);
|
||||
|
||||
if (pull.Count > 0)
|
||||
{
|
||||
peer.Tell(new SecretPullRequest([.. pull.Select(n => n.Value)]), Self);
|
||||
}
|
||||
|
||||
return ExchangeDone.Instance;
|
||||
}
|
||||
|
||||
// DistributedPubSub delivers a node's own publications back to it. Dropping them here is what
|
||||
// stops a broadcast from being re-applied locally and, worse, echoed onward.
|
||||
private bool IsFromSelf() => Sender.Equals(Self);
|
||||
|
||||
// A peer's rows are untrusted input: a malformed name or content type must drop that row, not
|
||||
// fault the actor and stall replication for every other secret.
|
||||
private IReadOnlyList<StoredSecret> Materialize(IReadOnlyList<SecretRowDto> dtos)
|
||||
{
|
||||
var rows = new List<StoredSecret>(dtos.Count);
|
||||
|
||||
foreach (SecretRowDto dto in dtos)
|
||||
{
|
||||
try
|
||||
{
|
||||
rows.Add(dto.ToStoredSecret());
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or FormatException)
|
||||
{
|
||||
_log.Warning(ex, "Discarding a malformed replicated secret row from a peer.");
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
private IReadOnlyList<SecretName> ParseNames(IReadOnlyList<string> names)
|
||||
{
|
||||
var parsed = new List<SecretName>(names.Count);
|
||||
|
||||
foreach (string name in names)
|
||||
{
|
||||
try
|
||||
{
|
||||
parsed.Add(new SecretName(name));
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
_log.Warning(ex, "Discarding a malformed secret name in a peer's pull request.");
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/// <summary>Instruction from <see cref="AkkaSecretReplicator"/> to broadcast a local write.</summary>
|
||||
/// <param name="Row">The encrypted row to broadcast.</param>
|
||||
public sealed record PublishLocalWrite(StoredSecret Row);
|
||||
|
||||
private sealed record AnnounceTick
|
||||
{
|
||||
public static readonly AnnounceTick Instance = new();
|
||||
}
|
||||
|
||||
private sealed record ExchangeFailed(Exception Cause);
|
||||
|
||||
private sealed record ExchangeDone
|
||||
{
|
||||
public static readonly ExchangeDone Instance = new();
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.Serialization;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Explicit System.Text.Json serializer for the secret-replication wire protocol.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Registering this is <b>optional but recommended</b>. The protocol DTOs are all primitives, so
|
||||
/// Akka's default JSON serializer round-trips them correctly if an application never wires this up —
|
||||
/// but relying on the default means secret ciphertext is serialized by whatever the cluster's
|
||||
/// fallback happens to be, which is not a decision worth leaving implicit for this payload. Binding
|
||||
/// it explicitly also pins the manifest strings below as the versioning boundary: a future protocol
|
||||
/// change adds a manifest, so a mixed-version cluster fails loudly on an unknown manifest instead of
|
||||
/// silently mis-deserializing a secret.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Wire it up by merging <see cref="AkkaSecretsReplication.SerializationConfig"/> into the
|
||||
/// application's Akka configuration — see that member for the HOCON.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="system">The actor system this serializer belongs to.</param>
|
||||
public sealed class SecretReplicationSerializer(ExtendedActorSystem system) : SerializerWithStringManifest(system)
|
||||
{
|
||||
/// <summary>Stable serializer id. Must not collide with other serializers in the cluster.</summary>
|
||||
public const int SerializerId = 7710;
|
||||
|
||||
private const string RowsManifest = "zbs-rows-v1";
|
||||
private const string ManifestAnnounceManifest = "zbs-manifest-v1";
|
||||
private const string PullRequestManifest = "zbs-pull-v1";
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
// Explicitly NOT indented and NOT camel-cased: the wire form should be compact and stable,
|
||||
// not pretty. Property names are the C# names, pinned by the manifest version.
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int Identifier => SerializerId;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string Manifest(object o) => o switch
|
||||
{
|
||||
SecretRowsMessage => RowsManifest,
|
||||
SecretManifestAnnounce => ManifestAnnounceManifest,
|
||||
SecretPullRequest => PullRequestManifest,
|
||||
_ => throw new ArgumentException(
|
||||
$"{nameof(SecretReplicationSerializer)} cannot serialize {o?.GetType().FullName ?? "null"}.",
|
||||
nameof(o)),
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override byte[] ToBinary(object obj) => obj switch
|
||||
{
|
||||
SecretRowsMessage rows => JsonSerializer.SerializeToUtf8Bytes(rows, JsonOptions),
|
||||
SecretManifestAnnounce announce => JsonSerializer.SerializeToUtf8Bytes(announce, JsonOptions),
|
||||
SecretPullRequest pull => JsonSerializer.SerializeToUtf8Bytes(pull, JsonOptions),
|
||||
_ => throw new ArgumentException(
|
||||
$"{nameof(SecretReplicationSerializer)} cannot serialize {obj?.GetType().FullName ?? "null"}.",
|
||||
nameof(obj)),
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override object FromBinary(byte[] bytes, string manifest) => manifest switch
|
||||
{
|
||||
RowsManifest => Deserialize<SecretRowsMessage>(bytes, manifest),
|
||||
ManifestAnnounceManifest => Deserialize<SecretManifestAnnounce>(bytes, manifest),
|
||||
PullRequestManifest => Deserialize<SecretPullRequest>(bytes, manifest),
|
||||
_ => throw new SerializationException(
|
||||
$"Unknown secret-replication manifest '{manifest}'. This usually means a peer is running a " +
|
||||
"newer protocol version than this node."),
|
||||
};
|
||||
|
||||
private static T Deserialize<T>(byte[] bytes, string manifest) =>
|
||||
JsonSerializer.Deserialize<T>(bytes, JsonOptions)
|
||||
?? throw new SerializationException(
|
||||
$"Secret-replication payload with manifest '{manifest}' deserialized to null.");
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>ZB.MOM.WW.Secrets.Replicator.AkkaDotNet</PackageId>
|
||||
<Authors>ZB.MOM.WW</Authors>
|
||||
<Description>Peer-to-peer secret replication over an Akka.NET cluster for the ZB.MOM.WW SCADA family — partition-tolerant, ciphertext-only.</Description>
|
||||
<PackageProjectUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</PackageProjectUrl>
|
||||
<RepositoryUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</RepositoryUrl>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets\ZB.MOM.WW.Secrets.csproj" />
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets.Abstractions\ZB.MOM.WW.Secrets.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka" />
|
||||
<PackageReference Include="Akka.Cluster" />
|
||||
<PackageReference Include="Akka.Cluster.Tools" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user