feat(secrets): cluster replication via SQL Server and Akka.NET (G-7, 0.2.0)
Secrets were per-node SQLite, so a secret written on one node was invisible to the rest of a cluster. G-7's design resolved the "shared SQL store vs Akka replicator" fork to build only the former; both are built here so the choice is a deployment decision (availability vs partition tolerance) rather than a library limitation. Two new packages — ZB.MOM.WW.Secrets.Replicator.SqlServer (shared store, plus a local-store-with-hub mode) and .Replicator.AkkaDotNet (peer-to-peer over distributed pub/sub). Core gains ISecretsStoreMigrator, one shared SecretLastWriterWins predicate so no two stores can disagree on a tie, the transport-agnostic reconciler, and ReplicatingSecretStore — which closes a real gap: nothing had ever called ISecretReplicator.PublishAsync, so the seam was inert and local writes would not have propagated at all. Verified 182 pass / 1 skip / 0 warnings, including 15 live tests against a real SQL Server 2022 (the SQLite suite ported case-for-case, so any behavioural divergence between the stores fails) and a 9-test in-process 2-node Akka cluster over real remoting. A post-build review caught six defects, all fixed and now covered: both replication modes could not resolve from the container (no test had built one), an unbounded fetch that broke past SQL Server's 2100-parameter cap, a poison row that aborted the rest of its batch forever, Enum.Parse on peer input that could restart the actor in a loop, null crypto blobs crossing the trust boundary, and a silently dropped pull-read failure. Packed at 0.2.0 and vulnerability-scanned clean; not yet published to the feed. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
@@ -2,7 +2,8 @@ namespace ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes an encrypted row to cluster peers. Core registers a no-op implementation;
|
||||
/// <c>ZB.MOM.WW.Secrets.Akka</c> (deferred) provides the real cluster implementation.
|
||||
/// <c>ZB.MOM.WW.Secrets.Replication.Akka</c> (peer-to-peer over the cluster) and
|
||||
/// <c>ZB.MOM.WW.Secrets.Replication.SqlServer</c> (via a shared hub database) provide the real ones.
|
||||
/// </summary>
|
||||
public interface ISecretReplicator
|
||||
{
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Provisions the schema backing an <see cref="ISecretStore"/>. Implementations are idempotent —
|
||||
/// running a migration repeatedly is safe — and refuse to run against a store whose on-disk schema
|
||||
/// version is <em>newer</em> than the build supports, rather than silently downgrading it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This seam exists so the startup migration hosted service and the CLI stay store-agnostic: the
|
||||
/// SQLite store, the shared SQL-Server store, and any future provider are all provisioned through
|
||||
/// the same call, and swapping the store never touches the migration wiring.
|
||||
/// </remarks>
|
||||
public interface ISecretsStoreMigrator
|
||||
{
|
||||
/// <summary>Creates or updates the secret-store schema to the version this build supports.</summary>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the schema is at the supported version.</returns>
|
||||
/// <exception cref="SecretStoreMigrationException">
|
||||
/// The store's schema version is newer than this build supports.
|
||||
/// </exception>
|
||||
Task MigrateAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// The single definition of the last-writer-wins (LWW) ordering used to reconcile a secret row
|
||||
/// against a peer's copy: newer <see cref="StoredSecret.UpdatedUtc"/> wins, and
|
||||
/// <see cref="StoredSecret.Revision"/> breaks a timestamp tie.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Every store's <see cref="ISecretStore.ApplyReplicatedAsync"/> and every replication transport
|
||||
/// reconciler routes its "is the incoming row newer?" decision through here. Keeping one definition
|
||||
/// is a correctness requirement, not a tidiness one: a cluster can mix stores (a node backed by the
|
||||
/// shared SQL-Server store alongside nodes on local SQLite), and two implementations that disagree
|
||||
/// about a tie would converge to <em>different</em> rows on different nodes — a silent split-brain
|
||||
/// that no single-node test can catch.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The comparison is deliberately <b>strict</b>: a row that merely ties the local one is not newer
|
||||
/// and is ignored. That makes replication idempotent — re-delivering a row a node already has is a
|
||||
/// no-op rather than a write — so at-least-once transports (pub/sub redelivery, an anti-entropy
|
||||
/// sweep racing a live broadcast) cost nothing and cannot flap a row back and forth.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class SecretLastWriterWins
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns <see langword="true"/> if a row at (<paramref name="incomingUpdatedUtc"/>,
|
||||
/// <paramref name="incomingRevision"/>) should overwrite a local row at
|
||||
/// (<paramref name="localUpdatedUtc"/>, <paramref name="localRevision"/>).
|
||||
/// </summary>
|
||||
/// <param name="incomingUpdatedUtc">The incoming row's last-updated timestamp.</param>
|
||||
/// <param name="incomingRevision">The incoming row's revision.</param>
|
||||
/// <param name="localUpdatedUtc">The local row's last-updated timestamp.</param>
|
||||
/// <param name="localRevision">The local row's revision.</param>
|
||||
/// <returns><see langword="true"/> if the incoming row is strictly newer.</returns>
|
||||
public static bool IsNewer(
|
||||
DateTimeOffset incomingUpdatedUtc,
|
||||
long incomingRevision,
|
||||
DateTimeOffset localUpdatedUtc,
|
||||
long localRevision) =>
|
||||
incomingUpdatedUtc > localUpdatedUtc ||
|
||||
(incomingUpdatedUtc == localUpdatedUtc && incomingRevision > localRevision);
|
||||
|
||||
/// <summary>
|
||||
/// Returns <see langword="true"/> if <paramref name="incoming"/> should overwrite the local row
|
||||
/// described by <paramref name="local"/>.
|
||||
/// </summary>
|
||||
/// <param name="incoming">The manifest entry received from a peer.</param>
|
||||
/// <param name="local">The corresponding local manifest entry.</param>
|
||||
/// <returns><see langword="true"/> if the incoming entry is strictly newer.</returns>
|
||||
public static bool IsNewer(SecretManifestEntry incoming, SecretManifestEntry local)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(incoming);
|
||||
ArgumentNullException.ThrowIfNull(local);
|
||||
|
||||
return IsNewer(incoming.UpdatedUtc, incoming.Revision, local.UpdatedUtc, local.Revision);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Cli;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.MasterKey;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
// Headless `secret` CLI: set / get / list / rm / rotate over the ZB.MOM.WW secrets store.
|
||||
// The heavy lifting lives in SecretCommands; Program only builds the host, runs the schema
|
||||
@@ -20,8 +18,9 @@ builder.Services.AddZbSecrets(builder.Configuration, "Secrets");
|
||||
|
||||
using IHost host = builder.Build();
|
||||
|
||||
// Ensure the schema exists before any store operation.
|
||||
await host.Services.GetRequiredService<SqliteSecretsStoreMigrator>()
|
||||
// Ensure the schema exists before any store operation. Resolved through the ISecretsStoreMigrator
|
||||
// seam so the CLI provisions whichever store the app configured (local SQLite or shared SQL Server).
|
||||
await host.Services.GetRequiredService<ISecretsStoreMigrator>()
|
||||
.MigrateAsync(CancellationToken.None);
|
||||
|
||||
var commands = new SecretCommands(
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes local secret writes to cluster peers by handing them to this node's
|
||||
/// <see cref="SecretReplicationActor"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The hand-off is a <c>Tell</c>, so <see cref="PublishAsync"/> completes as soon as the row is in
|
||||
/// the actor's mailbox rather than when peers have applied it. That is the correct contract here:
|
||||
/// the caller (<c>ReplicatingSecretStore</c>) has already durably written the row locally and treats
|
||||
/// publishing as best-effort, and blocking a write until every peer acknowledged would make each
|
||||
/// node's availability depend on all the others. Delivery is guaranteed by anti-entropy, not by
|
||||
/// this call.
|
||||
/// </remarks>
|
||||
/// <param name="replicationActor">This node's replication actor.</param>
|
||||
public sealed class AkkaSecretReplicator(IActorRef replicationActor) : ISecretReplicator
|
||||
{
|
||||
private readonly IActorRef _replicationActor =
|
||||
replicationActor ?? throw new ArgumentNullException(nameof(replicationActor));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task PublishAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
_replicationActor.Tell(new SecretReplicationActor.PublishLocalWrite(row));
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
using Akka.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Akka configuration this package contributes.
|
||||
/// </summary>
|
||||
public static class AkkaSecretsReplication
|
||||
{
|
||||
/// <summary>
|
||||
/// HOCON binding the secret-replication wire protocol to
|
||||
/// <see cref="SecretReplicationSerializer"/>. Merge it into the application's Akka
|
||||
/// configuration when creating the <c>ActorSystem</c>:
|
||||
/// <code>
|
||||
/// Config config = AkkaSecretsReplication.SerializationConfig
|
||||
/// .WithFallback(myAppConfig);
|
||||
/// ActorSystem.Create("my-system", config);
|
||||
/// </code>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Optional — the protocol DTOs round-trip under Akka's default JSON serializer without it — but
|
||||
/// recommended, because it makes the serializer carrying secret ciphertext an explicit choice
|
||||
/// rather than an inherited default, and pins the manifest strings that a future protocol
|
||||
/// version would bump.
|
||||
/// </remarks>
|
||||
public static Config SerializationConfig { get; } = ConfigurationFactory.ParseString($$"""
|
||||
akka.actor {
|
||||
serializers {
|
||||
zb-secrets-replication = "{{typeof(SecretReplicationSerializer).AssemblyQualifiedName}}"
|
||||
}
|
||||
serialization-bindings {
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretRowsMessage, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretManifestAnnounce, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
"ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol.SecretPullRequest, ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" = zb-secrets-replication
|
||||
}
|
||||
serialization-identifiers {
|
||||
"{{typeof(SecretReplicationSerializer).AssemblyQualifiedName}}" = {{SecretReplicationSerializer.SerializerId}}
|
||||
}
|
||||
}
|
||||
""");
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for peer-to-peer secret replication over the Akka cluster.
|
||||
/// </summary>
|
||||
public sealed class AkkaSecretsReplicationOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// How often this node broadcasts its manifest for anti-entropy. Defaults to 30 seconds.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the convergence bound, not the propagation latency: a live write reaches peers in
|
||||
/// milliseconds, and this interval only governs how quickly a <em>dropped</em> write or a
|
||||
/// rejoining node is repaired. The traffic is one manifest per node per interval — names,
|
||||
/// revisions, and timestamps, no ciphertext — so shortening it is cheap for a small cluster and
|
||||
/// grows as O(nodes × secrets).
|
||||
/// </remarks>
|
||||
public TimeSpan AnnounceInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Name of the replication actor within the actor system. Defaults to <c>zb-secret-replication</c>.</summary>
|
||||
public string ActorName { get; set; } = "zb-secret-replication";
|
||||
|
||||
/// <summary>
|
||||
/// Throws if the options are unusable.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The announce interval is not positive, or the actor name is blank.</exception>
|
||||
public void Validate()
|
||||
{
|
||||
if (AnnounceInterval <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Secret replication AnnounceInterval must be positive (was {AnnounceInterval}).");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ActorName))
|
||||
{
|
||||
throw new InvalidOperationException("Secret replication ActorName must not be blank.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// Dependency-injection entry point for peer-to-peer secret replication over an Akka.NET cluster.
|
||||
/// </summary>
|
||||
public static class AkkaSecretsServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers secret replication over the cluster: each node keeps its own local store, local
|
||||
/// writes broadcast to peers, and a periodic manifest exchange repairs any divergence in both
|
||||
/// directions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Choose this transport when a node must keep resolving secrets while <b>partitioned</b> from
|
||||
/// the rest of the cluster — reads never leave the node, so an isolated site keeps running on its
|
||||
/// last-known-good state and re-converges when the partition heals. If every node can always
|
||||
/// reach a shared database, the SQL-Server package's shared-store mode is simpler and has no
|
||||
/// distributed failure modes at all; prefer it unless partition tolerance is a stated requirement.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Requirements.</b> The application must (1) register its <see cref="ActorSystem"/> in DI
|
||||
/// before the first secret resolve, and (2) give every node the <b>same KEK</b> — rows carry
|
||||
/// ciphertext only, so a node with a different master key fails closed on a <c>kek_id</c>
|
||||
/// mismatch. Merging <see cref="AkkaSecretsReplication.SerializationConfig"/> into the actor
|
||||
/// system's configuration is recommended.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>KEK rotation does not replicate.</b> A re-wrap deliberately leaves the revision untouched
|
||||
/// so it stays invisible to last-writer-wins; run <c>rewrap-all</c> against every node's local
|
||||
/// store.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to add to.</param>
|
||||
/// <param name="config">The application configuration.</param>
|
||||
/// <param name="secretsSectionPath">Configuration section holding the core secrets options.</param>
|
||||
/// <param name="replicationSectionPath">Configuration section holding the replication options.</param>
|
||||
/// <returns>The same <paramref name="services"/> instance, for chaining.</returns>
|
||||
public static IServiceCollection AddZbSecretsAkkaReplication(
|
||||
this IServiceCollection services,
|
||||
IConfiguration config,
|
||||
string secretsSectionPath = "Secrets",
|
||||
string replicationSectionPath = "Secrets:Replication")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(secretsSectionPath);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(replicationSectionPath);
|
||||
|
||||
IConfigurationSection section = config.GetSection(replicationSectionPath);
|
||||
services.Configure<AkkaSecretsReplicationOptions>(section);
|
||||
|
||||
// Validate eagerly so a bad interval fails at startup, not on the first announce tick.
|
||||
var eager = new AkkaSecretsReplicationOptions();
|
||||
section.Bind(eager);
|
||||
eager.Validate();
|
||||
|
||||
// The local store + migrator come from AddZbSecrets (SQLite by default).
|
||||
services.AddZbSecrets(config, secretsSectionPath);
|
||||
|
||||
// The actor is created lazily on first use rather than at registration: the ActorSystem is
|
||||
// typically registered by the application AFTER this call, and a cluster node may also not be
|
||||
// ready to join at the moment the container is built.
|
||||
services.TryAddSingleton<SecretReplicationActorProvider>(sp => new SecretReplicationActorProvider(
|
||||
sp.GetRequiredService<ActorSystem>(),
|
||||
sp.GetRequiredService<SqliteSecretStore>(),
|
||||
sp.GetService<ISecretCacheInvalidator>(),
|
||||
sp.GetRequiredService<IOptions<AkkaSecretsReplicationOptions>>().Value));
|
||||
|
||||
services.TryAddSingleton<ISecretReplicator>(sp =>
|
||||
new AkkaSecretReplicator(sp.GetRequiredService<SecretReplicationActorProvider>().ActorRef));
|
||||
|
||||
// Decorate the local store so every write publishes. Resolved from the concrete SQLite store,
|
||||
// not ISecretStore — asking the container for ISecretStore here would resolve this very
|
||||
// decorator and recurse forever.
|
||||
services.AddSingleton<ISecretStore>(sp => new ReplicatingSecretStore(
|
||||
sp.GetRequiredService<SqliteSecretStore>(),
|
||||
sp.GetRequiredService<ISecretReplicator>(),
|
||||
sp.GetRequiredService<ILogger<ReplicatingSecretStore>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates this node's <see cref="SecretReplicationActor"/> on first access and hands out its ref.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The indirection exists because the actor must be created lazily — the application usually
|
||||
/// registers its <see cref="ActorSystem"/> after <c>AddZbSecretsAkkaReplication</c>, so eagerly
|
||||
/// resolving one at registration time would fail. Creation is guarded by
|
||||
/// <see cref="Lazy{T}"/> with <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so
|
||||
/// concurrent first writes cannot spawn two actors on the same node.
|
||||
/// </remarks>
|
||||
/// <param name="system">The application's actor system.</param>
|
||||
/// <param name="store">The node's local secret store — the undecorated one.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam.</param>
|
||||
/// <param name="options">Replication options.</param>
|
||||
public sealed class SecretReplicationActorProvider(
|
||||
ActorSystem system,
|
||||
ISecretStore store,
|
||||
ISecretCacheInvalidator? cacheInvalidator,
|
||||
AkkaSecretsReplicationOptions options)
|
||||
{
|
||||
private readonly Lazy<IActorRef> _actorRef = new(
|
||||
() => system.ActorOf(
|
||||
SecretReplicationActor.Props(store, cacheInvalidator, options.AnnounceInterval),
|
||||
options.ActorName),
|
||||
LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
/// <summary>This node's replication actor, created on first access.</summary>
|
||||
public IActorRef ActorRef => _actorRef.Value;
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// The wire representation of one encrypted secret row.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is a deliberate hand-written mirror of <see cref="StoredSecret"/> rather than the type
|
||||
/// itself, and the duplication is the point. It makes the trust boundary <b>structural</b>: the wire
|
||||
/// contract is a flat set of primitives that a reader can check against "ciphertext only, never the
|
||||
/// KEK" in one glance, and a field added to <see cref="StoredSecret"/> in future cannot start
|
||||
/// crossing the network merely by existing — someone has to add it here on purpose.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// All fields are primitives (<see cref="string"/>, <see cref="byte"/> arrays, <see cref="long"/>,
|
||||
/// <see cref="bool"/>) so the DTO round-trips through any serializer, including Akka's default JSON
|
||||
/// one if an application never registers <see cref="SecretReplicationSerializer"/>. Timestamps are
|
||||
/// ISO-8601 <c>"O"</c> strings for the same reason the stores use them: last-writer-wins compares
|
||||
/// them for equality, so the representation has to be exact rather than merely close.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed record SecretRowDto
|
||||
{
|
||||
/// <summary>The normalized secret name.</summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>Optional human-readable description.</summary>
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>The <see cref="SecretContentType"/> enum name.</summary>
|
||||
public required string ContentType { get; init; }
|
||||
|
||||
/// <summary>AES-256-GCM ciphertext of the secret plaintext.</summary>
|
||||
public required byte[] Ciphertext { get; init; }
|
||||
|
||||
/// <summary>Nonce sealing <see cref="Ciphertext"/>.</summary>
|
||||
public required byte[] Nonce { get; init; }
|
||||
|
||||
/// <summary>Authentication tag for <see cref="Ciphertext"/>.</summary>
|
||||
public required byte[] Tag { get; init; }
|
||||
|
||||
/// <summary>The per-secret data key, wrapped by the KEK. The KEK itself never travels.</summary>
|
||||
public required byte[] WrappedDek { get; init; }
|
||||
|
||||
/// <summary>Nonce used to wrap <see cref="WrappedDek"/>.</summary>
|
||||
public required byte[] WrapNonce { get; init; }
|
||||
|
||||
/// <summary>Authentication tag for <see cref="WrappedDek"/>.</summary>
|
||||
public required byte[] WrapTag { get; init; }
|
||||
|
||||
/// <summary>Identifier of the KEK that wrapped the DEK — an identifier, not key material.</summary>
|
||||
public required string KekId { get; init; }
|
||||
|
||||
/// <summary>Monotonic revision.</summary>
|
||||
public required long Revision { get; init; }
|
||||
|
||||
/// <summary>Whether the row is a tombstone.</summary>
|
||||
public bool IsDeleted { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 deletion timestamp, or <see langword="null"/>.</summary>
|
||||
public string? DeletedUtc { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 creation timestamp.</summary>
|
||||
public required string CreatedUtc { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 last-update timestamp — half of the last-writer-wins ordering key.</summary>
|
||||
public required string UpdatedUtc { get; init; }
|
||||
|
||||
/// <summary>Principal that created the row, if known.</summary>
|
||||
public string? CreatedBy { get; init; }
|
||||
|
||||
/// <summary>Principal that last updated the row, if known.</summary>
|
||||
public string? UpdatedBy { get; init; }
|
||||
|
||||
/// <summary>Projects a stored row onto the wire contract.</summary>
|
||||
/// <param name="row">The encrypted row.</param>
|
||||
/// <returns>The wire representation.</returns>
|
||||
public static SecretRowDto FromStoredSecret(StoredSecret row)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
return new SecretRowDto
|
||||
{
|
||||
Name = row.Name.Value,
|
||||
Description = row.Description,
|
||||
ContentType = row.ContentType.ToString(),
|
||||
Ciphertext = row.Ciphertext,
|
||||
Nonce = row.Nonce,
|
||||
Tag = row.Tag,
|
||||
WrappedDek = row.WrappedDek,
|
||||
WrapNonce = row.WrapNonce,
|
||||
WrapTag = row.WrapTag,
|
||||
KekId = row.KekId,
|
||||
Revision = row.Revision,
|
||||
IsDeleted = row.IsDeleted,
|
||||
DeletedUtc = row.DeletedUtc?.ToString("O"),
|
||||
CreatedUtc = row.CreatedUtc.ToString("O"),
|
||||
UpdatedUtc = row.UpdatedUtc.ToString("O"),
|
||||
CreatedBy = row.CreatedBy,
|
||||
UpdatedBy = row.UpdatedBy,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Rebuilds the stored row from the wire contract.</summary>
|
||||
/// <returns>The encrypted row.</returns>
|
||||
/// <exception cref="ArgumentException">The name or content type is not valid.</exception>
|
||||
public StoredSecret ToStoredSecret() => new()
|
||||
{
|
||||
// SecretName re-validates on construction, so a malformed or path-traversing name from a
|
||||
// peer is rejected here rather than reaching the store.
|
||||
Name = new SecretName(Name),
|
||||
Description = Description,
|
||||
ContentType = ParseContentType(ContentType),
|
||||
// `required` on a byte[] means "present in the payload", NOT "non-null" — a peer sending an
|
||||
// explicit JSON null satisfies it and yields a null array, which would then blow up deep in
|
||||
// the store's parameter binding instead of being rejected at the boundary.
|
||||
Ciphertext = RequireBlob(Ciphertext, nameof(Ciphertext)),
|
||||
Nonce = RequireBlob(Nonce, nameof(Nonce)),
|
||||
Tag = RequireBlob(Tag, nameof(Tag)),
|
||||
WrappedDek = RequireBlob(WrappedDek, nameof(WrappedDek)),
|
||||
WrapNonce = RequireBlob(WrapNonce, nameof(WrapNonce)),
|
||||
WrapTag = RequireBlob(WrapTag, nameof(WrapTag)),
|
||||
KekId = KekId ?? throw new ArgumentException("Replicated secret row has a null KekId."),
|
||||
Revision = Revision,
|
||||
IsDeleted = IsDeleted,
|
||||
DeletedUtc = DeletedUtc is null ? null : ParseUtc(DeletedUtc),
|
||||
CreatedUtc = ParseUtc(CreatedUtc),
|
||||
UpdatedUtc = ParseUtc(UpdatedUtc),
|
||||
CreatedBy = CreatedBy,
|
||||
UpdatedBy = UpdatedBy,
|
||||
};
|
||||
|
||||
private static DateTimeOffset ParseUtc(string value) => DateTimeOffset.Parse(
|
||||
value,
|
||||
System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
|
||||
private static byte[] RequireBlob(byte[]? value, string field) =>
|
||||
value ?? throw new ArgumentException($"Replicated secret row has a null {field}.", field);
|
||||
|
||||
// Enum.Parse is wrong for untrusted input on two counts: a numeric string that overflows the
|
||||
// underlying type throws OverflowException (which the receiving actor's ArgumentException filter
|
||||
// would not catch, restarting it in a redelivery loop), and any in-range number is accepted even
|
||||
// if it names no member — persisting an undefined enum value that later readers assume is valid.
|
||||
private static SecretContentType ParseContentType(string value) =>
|
||||
Enum.TryParse(value, out SecretContentType parsed) && Enum.IsDefined(parsed)
|
||||
? parsed
|
||||
: throw new ArgumentException(
|
||||
$"Replicated secret row has an unrecognized content type '{value}'.", nameof(value));
|
||||
}
|
||||
|
||||
/// <summary>One manifest entry on the wire: enough to decide who is newer, with no ciphertext.</summary>
|
||||
public sealed record SecretManifestEntryDto
|
||||
{
|
||||
/// <summary>The normalized secret name.</summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>Monotonic revision.</summary>
|
||||
public required long Revision { get; init; }
|
||||
|
||||
/// <summary>ISO-8601 last-update timestamp.</summary>
|
||||
public required string UpdatedUtc { get; init; }
|
||||
|
||||
/// <summary>Whether the row is a tombstone.</summary>
|
||||
public bool IsDeleted { get; init; }
|
||||
|
||||
/// <summary>Projects a manifest entry onto the wire contract.</summary>
|
||||
/// <param name="entry">The manifest entry.</param>
|
||||
/// <returns>The wire representation.</returns>
|
||||
public static SecretManifestEntryDto FromEntry(SecretManifestEntry entry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
return new SecretManifestEntryDto
|
||||
{
|
||||
Name = entry.Name.Value,
|
||||
Revision = entry.Revision,
|
||||
UpdatedUtc = entry.UpdatedUtc.ToString("O"),
|
||||
IsDeleted = entry.IsDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Rebuilds the manifest entry from the wire contract.</summary>
|
||||
/// <returns>The manifest entry.</returns>
|
||||
public SecretManifestEntry ToEntry() => new()
|
||||
{
|
||||
Name = new SecretName(Name),
|
||||
Revision = Revision,
|
||||
UpdatedUtc = DateTimeOffset.Parse(
|
||||
UpdatedUtc,
|
||||
System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind),
|
||||
IsDeleted = IsDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A node broadcasting its full inventory so peers can work out, in one round trip, both what they
|
||||
/// need from it and what it needs from them.
|
||||
/// </summary>
|
||||
/// <param name="Entries">The sender's manifest.</param>
|
||||
public sealed record SecretManifestAnnounce(IReadOnlyList<SecretManifestEntryDto> Entries);
|
||||
|
||||
/// <summary>A peer asking for the encrypted rows behind names it found it was missing or behind on.</summary>
|
||||
/// <param name="Names">The requested secret names.</param>
|
||||
public sealed record SecretPullRequest(IReadOnlyList<string> Names);
|
||||
|
||||
/// <summary>
|
||||
/// Encrypted rows delivered to a peer — the reply to a <see cref="SecretPullRequest"/>, the
|
||||
/// unsolicited push half of an anti-entropy exchange, and the live broadcast of a fresh local write.
|
||||
/// </summary>
|
||||
/// <param name="Rows">The encrypted rows.</param>
|
||||
public sealed record SecretRowsMessage(IReadOnlyList<SecretRowDto> Rows);
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// One per node. Owns this node's participation in secret replication: broadcasts local writes,
|
||||
/// periodically announces its manifest, and services peers' pulls.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The protocol is deliberately small — three messages, no leader, no singleton:
|
||||
/// </para>
|
||||
/// <list type="number">
|
||||
/// <item><description>
|
||||
/// <b>Live push.</b> A local write publishes a <see cref="SecretRowsMessage"/> to the topic. Peers
|
||||
/// apply it last-writer-wins. This is the fast path — propagation in milliseconds — and it is
|
||||
/// allowed to fail.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>Anti-entropy.</b> Every node periodically publishes a <see cref="SecretManifestAnnounce"/>.
|
||||
/// A peer receiving it computes <em>both</em> directions in one pass: it pushes back rows it holds
|
||||
/// newer, and asks (<see cref="SecretPullRequest"/>) for rows it is missing or behind on. This is
|
||||
/// the correctness path — it repairs anything the fast path dropped, in either direction.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>Pull service.</b> A <see cref="SecretPullRequest"/> is answered with the requested rows.
|
||||
/// </description></item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// There is no coordinator because there does not need to be one: every exchange is idempotent and
|
||||
/// commutative under last-writer-wins, so nodes converge regardless of message order, duplication,
|
||||
/// or which subset of peers is currently reachable. A node that was partitioned keeps serving its
|
||||
/// local store the whole time and re-converges on the first announce after the partition heals —
|
||||
/// which is the entire reason to choose this transport over the shared SQL-Server store.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SecretReplicationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
/// <summary>The distributed pub/sub topic secret replication traffic rides on.</summary>
|
||||
public const string Topic = "zb-mom-ww-secrets";
|
||||
|
||||
private readonly ISecretStore _store;
|
||||
private readonly SecretReplicationReconciler _reconciler;
|
||||
private readonly TimeSpan _announceInterval;
|
||||
private readonly IActorRef _mediator;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <inheritdoc />
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>Creates the replication actor.</summary>
|
||||
/// <param name="store">The node's local secret store.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam, evicted for every applied row.</param>
|
||||
/// <param name="announceInterval">How often this node announces its manifest.</param>
|
||||
public SecretReplicationActor(
|
||||
ISecretStore store,
|
||||
ISecretCacheInvalidator? cacheInvalidator,
|
||||
TimeSpan announceInterval)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_reconciler = new SecretReplicationReconciler(
|
||||
_store,
|
||||
cacheInvalidator,
|
||||
onRowFailed: (name, ex) => _log.Warning(
|
||||
ex, "Discarding replicated secret row {0} that could not be applied locally.", name.Value));
|
||||
_announceInterval = announceInterval;
|
||||
_mediator = DistributedPubSub.Get(Context.System).Mediator;
|
||||
|
||||
_mediator.Tell(new Subscribe(Topic, Self));
|
||||
|
||||
Receive<PublishLocalWrite>(HandlePublishLocalWrite);
|
||||
Receive<SecretRowsMessage>(HandleRows);
|
||||
Receive<SecretManifestAnnounce>(HandleManifestAnnounce);
|
||||
Receive<SecretPullRequest>(HandlePullRequest);
|
||||
Receive<AnnounceTick>(_ => HandleAnnounceTick());
|
||||
Receive<SubscribeAck>(_ => _log.Debug("Subscribed to secret replication topic {0}.", Topic));
|
||||
Receive<ExchangeFailed>(f =>
|
||||
_log.Warning(f.Cause, "Secret anti-entropy exchange with a peer failed; retrying next announce."));
|
||||
Receive<ExchangeDone>(_ => { /* successful exchange, piped back to self — nothing to do. */ });
|
||||
}
|
||||
|
||||
/// <summary>Props for the replication actor.</summary>
|
||||
/// <param name="store">The node's local secret store.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam.</param>
|
||||
/// <param name="announceInterval">How often this node announces its manifest.</param>
|
||||
/// <returns>Props creating the actor.</returns>
|
||||
public static Props Props(
|
||||
ISecretStore store, ISecretCacheInvalidator? cacheInvalidator, TimeSpan announceInterval) =>
|
||||
Akka.Actor.Props.Create(() =>
|
||||
new SecretReplicationActor(store, cacheInvalidator, announceInterval));
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart() =>
|
||||
Timers.StartPeriodicTimer(AnnounceTick.Instance, AnnounceTick.Instance, _announceInterval, _announceInterval);
|
||||
|
||||
// A local write, handed over by AkkaSecretReplicator. Published from inside the actor so the
|
||||
// message carries Self as sender — that is what lets a peer reply directly to this node instead
|
||||
// of broadcasting its answer to everyone.
|
||||
private void HandlePublishLocalWrite(PublishLocalWrite message)
|
||||
{
|
||||
_mediator.Tell(new Publish(
|
||||
Topic,
|
||||
new SecretRowsMessage([SecretRowDto.FromStoredSecret(message.Row)])));
|
||||
}
|
||||
|
||||
private void HandleRows(SecretRowsMessage message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IReadOnlyList<StoredSecret> rows = Materialize(message.Rows);
|
||||
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// PipeTo nothing: applying is fire-and-forget from the mailbox's point of view. Failures are
|
||||
// logged, and anti-entropy re-delivers anything that did not land.
|
||||
_reconciler.ApplyAsync(rows, CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task => _log.Warning(
|
||||
task.Exception,
|
||||
"Failed to apply {0} replicated secret row(s) from a peer; anti-entropy will retry.",
|
||||
rows.Count),
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private void HandleManifestAnnounce(SecretManifestAnnounce message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IActorRef peer = Sender;
|
||||
IReadOnlyList<SecretManifestEntry> remote = [.. message.Entries.Select(e => e.ToEntry())];
|
||||
|
||||
ReconcileWithPeerAsync(peer, remote).PipeTo(Self, failure: ex => new ExchangeFailed(ex));
|
||||
}
|
||||
|
||||
private void HandlePullRequest(SecretPullRequest message)
|
||||
{
|
||||
if (IsFromSelf())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
IActorRef peer = Sender;
|
||||
// Captured before the continuation: Self is context-dependent and there is no guarantee the
|
||||
// actor context is still current when the continuation runs (Akka analyzer AK1005).
|
||||
IActorRef self = Self;
|
||||
IReadOnlyList<SecretName> names = ParseNames(message.Names);
|
||||
|
||||
_reconciler.ReadLocalAsync(names, CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task =>
|
||||
{
|
||||
if (task.IsFaulted)
|
||||
{
|
||||
// Logged rather than swallowed: the peer is waiting for a reply it will never
|
||||
// get, and without this the only symptom is a secret that quietly never
|
||||
// converges. The peer re-asks on its next announce.
|
||||
_log.Warning(
|
||||
task.Exception,
|
||||
"Failed to read {0} local secret row(s) for a peer's pull request.",
|
||||
names.Count);
|
||||
return;
|
||||
}
|
||||
|
||||
if (task.IsCompletedSuccessfully && task.Result.Count > 0)
|
||||
{
|
||||
peer.Tell(
|
||||
new SecretRowsMessage([.. task.Result.Select(SecretRowDto.FromStoredSecret)]),
|
||||
self);
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private void HandleAnnounceTick()
|
||||
{
|
||||
_store.GetManifestAsync(CancellationToken.None)
|
||||
.ContinueWith(
|
||||
task =>
|
||||
{
|
||||
if (task.IsCompletedSuccessfully)
|
||||
{
|
||||
_mediator.Tell(new Publish(
|
||||
Topic,
|
||||
new SecretManifestAnnounce(
|
||||
[.. task.Result.Select(SecretManifestEntryDto.FromEntry)])));
|
||||
}
|
||||
},
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
// Both halves of an anti-entropy exchange, computed from one manifest: push what the peer is
|
||||
// behind on, ask for what we are behind on.
|
||||
private async Task<ExchangeDone> ReconcileWithPeerAsync(
|
||||
IActorRef peer, IReadOnlyList<SecretManifestEntry> remote)
|
||||
{
|
||||
IReadOnlyList<SecretManifestEntry> localManifest =
|
||||
await _store.GetManifestAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
IReadOnlyList<SecretName> push =
|
||||
SecretReplicationReconciler.ComputePushSet(localManifest, remote);
|
||||
|
||||
if (push.Count > 0)
|
||||
{
|
||||
IReadOnlyList<StoredSecret> rows =
|
||||
await _reconciler.ReadLocalAsync(push, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
peer.Tell(new SecretRowsMessage([.. rows.Select(SecretRowDto.FromStoredSecret)]), Self);
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyList<SecretName> pull =
|
||||
SecretReplicationReconciler.ComputePullSet(localManifest, remote);
|
||||
|
||||
if (pull.Count > 0)
|
||||
{
|
||||
peer.Tell(new SecretPullRequest([.. pull.Select(n => n.Value)]), Self);
|
||||
}
|
||||
|
||||
return ExchangeDone.Instance;
|
||||
}
|
||||
|
||||
// DistributedPubSub delivers a node's own publications back to it. Dropping them here is what
|
||||
// stops a broadcast from being re-applied locally and, worse, echoed onward.
|
||||
private bool IsFromSelf() => Sender.Equals(Self);
|
||||
|
||||
// A peer's rows are untrusted input: a malformed name or content type must drop that row, not
|
||||
// fault the actor and stall replication for every other secret.
|
||||
private IReadOnlyList<StoredSecret> Materialize(IReadOnlyList<SecretRowDto> dtos)
|
||||
{
|
||||
var rows = new List<StoredSecret>(dtos.Count);
|
||||
|
||||
foreach (SecretRowDto dto in dtos)
|
||||
{
|
||||
try
|
||||
{
|
||||
rows.Add(dto.ToStoredSecret());
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or FormatException)
|
||||
{
|
||||
_log.Warning(ex, "Discarding a malformed replicated secret row from a peer.");
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
private IReadOnlyList<SecretName> ParseNames(IReadOnlyList<string> names)
|
||||
{
|
||||
var parsed = new List<SecretName>(names.Count);
|
||||
|
||||
foreach (string name in names)
|
||||
{
|
||||
try
|
||||
{
|
||||
parsed.Add(new SecretName(name));
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
_log.Warning(ex, "Discarding a malformed secret name in a peer's pull request.");
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/// <summary>Instruction from <see cref="AkkaSecretReplicator"/> to broadcast a local write.</summary>
|
||||
/// <param name="Row">The encrypted row to broadcast.</param>
|
||||
public sealed record PublishLocalWrite(StoredSecret Row);
|
||||
|
||||
private sealed record AnnounceTick
|
||||
{
|
||||
public static readonly AnnounceTick Instance = new();
|
||||
}
|
||||
|
||||
private sealed record ExchangeFailed(Exception Cause);
|
||||
|
||||
private sealed record ExchangeDone
|
||||
{
|
||||
public static readonly ExchangeDone Instance = new();
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.Serialization;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
|
||||
/// <summary>
|
||||
/// Explicit System.Text.Json serializer for the secret-replication wire protocol.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Registering this is <b>optional but recommended</b>. The protocol DTOs are all primitives, so
|
||||
/// Akka's default JSON serializer round-trips them correctly if an application never wires this up —
|
||||
/// but relying on the default means secret ciphertext is serialized by whatever the cluster's
|
||||
/// fallback happens to be, which is not a decision worth leaving implicit for this payload. Binding
|
||||
/// it explicitly also pins the manifest strings below as the versioning boundary: a future protocol
|
||||
/// change adds a manifest, so a mixed-version cluster fails loudly on an unknown manifest instead of
|
||||
/// silently mis-deserializing a secret.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Wire it up by merging <see cref="AkkaSecretsReplication.SerializationConfig"/> into the
|
||||
/// application's Akka configuration — see that member for the HOCON.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="system">The actor system this serializer belongs to.</param>
|
||||
public sealed class SecretReplicationSerializer(ExtendedActorSystem system) : SerializerWithStringManifest(system)
|
||||
{
|
||||
/// <summary>Stable serializer id. Must not collide with other serializers in the cluster.</summary>
|
||||
public const int SerializerId = 7710;
|
||||
|
||||
private const string RowsManifest = "zbs-rows-v1";
|
||||
private const string ManifestAnnounceManifest = "zbs-manifest-v1";
|
||||
private const string PullRequestManifest = "zbs-pull-v1";
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
// Explicitly NOT indented and NOT camel-cased: the wire form should be compact and stable,
|
||||
// not pretty. Property names are the C# names, pinned by the manifest version.
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int Identifier => SerializerId;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string Manifest(object o) => o switch
|
||||
{
|
||||
SecretRowsMessage => RowsManifest,
|
||||
SecretManifestAnnounce => ManifestAnnounceManifest,
|
||||
SecretPullRequest => PullRequestManifest,
|
||||
_ => throw new ArgumentException(
|
||||
$"{nameof(SecretReplicationSerializer)} cannot serialize {o?.GetType().FullName ?? "null"}.",
|
||||
nameof(o)),
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override byte[] ToBinary(object obj) => obj switch
|
||||
{
|
||||
SecretRowsMessage rows => JsonSerializer.SerializeToUtf8Bytes(rows, JsonOptions),
|
||||
SecretManifestAnnounce announce => JsonSerializer.SerializeToUtf8Bytes(announce, JsonOptions),
|
||||
SecretPullRequest pull => JsonSerializer.SerializeToUtf8Bytes(pull, JsonOptions),
|
||||
_ => throw new ArgumentException(
|
||||
$"{nameof(SecretReplicationSerializer)} cannot serialize {obj?.GetType().FullName ?? "null"}.",
|
||||
nameof(obj)),
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public override object FromBinary(byte[] bytes, string manifest) => manifest switch
|
||||
{
|
||||
RowsManifest => Deserialize<SecretRowsMessage>(bytes, manifest),
|
||||
ManifestAnnounceManifest => Deserialize<SecretManifestAnnounce>(bytes, manifest),
|
||||
PullRequestManifest => Deserialize<SecretPullRequest>(bytes, manifest),
|
||||
_ => throw new SerializationException(
|
||||
$"Unknown secret-replication manifest '{manifest}'. This usually means a peer is running a " +
|
||||
"newer protocol version than this node."),
|
||||
};
|
||||
|
||||
private static T Deserialize<T>(byte[] bytes, string manifest) =>
|
||||
JsonSerializer.Deserialize<T>(bytes, JsonOptions)
|
||||
?? throw new SerializationException(
|
||||
$"Secret-replication payload with manifest '{manifest}' deserialized to null.");
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>ZB.MOM.WW.Secrets.Replicator.AkkaDotNet</PackageId>
|
||||
<Authors>ZB.MOM.WW</Authors>
|
||||
<Description>Peer-to-peer secret replication over an Akka.NET cluster for the ZB.MOM.WW SCADA family — partition-tolerant, ciphertext-only.</Description>
|
||||
<PackageProjectUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</PackageProjectUrl>
|
||||
<RepositoryUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</RepositoryUrl>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets\ZB.MOM.WW.Secrets.csproj" />
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets.Abstractions\ZB.MOM.WW.Secrets.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka" />
|
||||
<PackageReference Include="Akka.Cluster" />
|
||||
<PackageReference Include="Akka.Cluster.Tools" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// Dependency-injection entry points for backing a cluster's secrets with SQL Server, in either of
|
||||
/// the two supported topologies.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Both require the same KEK on every node.</b> The database stores ciphertext only, so a node
|
||||
/// whose master key differs cannot decrypt rows another node wrote — it fails closed on resolve with
|
||||
/// a <c>kek_id</c> mismatch, which reads like data corruption but is a deployment error. Mount the
|
||||
/// same key file or supply the same key environment variable everywhere.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class SqlServerSecretsServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// <b>Shared-store mode.</b> Points every node's <see cref="ISecretStore"/> at one shared
|
||||
/// SQL-Server database. There is exactly one copy of each row, so there is nothing to replicate
|
||||
/// and nothing to reconcile — no split-brain, no anti-entropy lag, no clock skew.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Prefer this unless a node must keep resolving secrets while <em>partitioned</em> from the
|
||||
/// shared database. The trade is availability: a node cut off from the database falls back only
|
||||
/// to the resolver's short in-memory cache. If that is unacceptable, use
|
||||
/// <see cref="AddZbSecretsSqlServerReplication"/> (local store + hub) or the Akka package
|
||||
/// (peer-to-peer, no shared database at all).
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to add to.</param>
|
||||
/// <param name="config">The application configuration.</param>
|
||||
/// <param name="secretsSectionPath">Configuration section holding the core secrets options.</param>
|
||||
/// <param name="sqlServerSectionPath">Configuration section holding the SQL-Server options.</param>
|
||||
/// <returns>The same <paramref name="services"/> instance, for chaining.</returns>
|
||||
public static IServiceCollection AddZbSecretsSqlServerStore(
|
||||
this IServiceCollection services,
|
||||
IConfiguration config,
|
||||
string secretsSectionPath = "Secrets",
|
||||
string sqlServerSectionPath = "Secrets:SqlServer")
|
||||
{
|
||||
RegisterSqlServerCore(services, config, sqlServerSectionPath);
|
||||
|
||||
// Registered BEFORE AddZbSecrets so its TryAdd calls find these already present and leave
|
||||
// them alone — that is the whole mechanism by which the SQL-Server store displaces SQLite.
|
||||
services.TryAddSingleton<ISecretStore>(sp => sp.GetRequiredService<SqlServerSecretStore>());
|
||||
services.TryAddSingleton<ISecretsStoreMigrator>(sp =>
|
||||
sp.GetRequiredService<SqlServerSecretsStoreMigrator>());
|
||||
|
||||
services.AddZbSecrets(config, secretsSectionPath);
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Hub-replication mode.</b> Each node keeps its own local store (SQLite by default) and
|
||||
/// stays converged with a shared SQL-Server hub: local writes publish to the hub immediately, and
|
||||
/// a background sweep reconciles both directions on an interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Choose this over <see cref="AddZbSecretsSqlServerStore"/> when a node must keep resolving
|
||||
/// secrets while cut off from the hub — reads are served from local state, so a hub outage
|
||||
/// degrades propagation rather than availability. The cost is real distributed-systems
|
||||
/// behaviour: rows converge eventually (bounded by the sweep interval), conflicting concurrent
|
||||
/// writes to the same secret resolve last-writer-wins, and KEK rotation must be run against the
|
||||
/// hub <em>and</em> each node's local store, because a re-wrap deliberately does not replicate.
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to add to.</param>
|
||||
/// <param name="config">The application configuration.</param>
|
||||
/// <param name="secretsSectionPath">Configuration section holding the core secrets options.</param>
|
||||
/// <param name="sqlServerSectionPath">Configuration section holding the SQL-Server options.</param>
|
||||
/// <returns>The same <paramref name="services"/> instance, for chaining.</returns>
|
||||
public static IServiceCollection AddZbSecretsSqlServerReplication(
|
||||
this IServiceCollection services,
|
||||
IConfiguration config,
|
||||
string secretsSectionPath = "Secrets",
|
||||
string sqlServerSectionPath = "Secrets:SqlServer")
|
||||
{
|
||||
RegisterSqlServerCore(services, config, sqlServerSectionPath);
|
||||
|
||||
services.TryAddSingleton<ISecretReplicationHub>(sp => sp.GetRequiredService<SqlServerSecretStore>());
|
||||
|
||||
services.TryAddSingleton<ISecretReplicator>(sp =>
|
||||
new SqlServerSecretReplicator(sp.GetRequiredService<ISecretReplicationHub>()));
|
||||
|
||||
// The local store + its migrator come from AddZbSecrets (SQLite by default).
|
||||
services.AddZbSecrets(config, secretsSectionPath);
|
||||
|
||||
// Decorate the local store so every write publishes. Resolved from the concrete SQLite store
|
||||
// rather than ISecretStore, because ISecretStore is what we are replacing — asking the
|
||||
// container for it here would resolve this very decorator and recurse forever.
|
||||
services.AddSingleton<ISecretStore>(sp => new ReplicatingSecretStore(
|
||||
sp.GetRequiredService<Secrets.Sqlite.SqliteSecretStore>(),
|
||||
sp.GetRequiredService<ISecretReplicator>(),
|
||||
sp.GetRequiredService<ILogger<ReplicatingSecretStore>>()));
|
||||
|
||||
// The hub's schema must exist too — the node's own migrator only provisions the local store.
|
||||
services.AddHostedService<SqlServerHubMigrationHostedService>();
|
||||
services.AddHostedService<SqlServerSecretSyncService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
// Registers the pieces both modes need: validated options, connection factory, hub store, and
|
||||
// hub migrator.
|
||||
private static void RegisterSqlServerCore(
|
||||
IServiceCollection services,
|
||||
IConfiguration config,
|
||||
string sqlServerSectionPath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sqlServerSectionPath);
|
||||
|
||||
IConfigurationSection section = config.GetSection(sqlServerSectionPath);
|
||||
services.Configure<SqlServerSecretsOptions>(section);
|
||||
|
||||
// Bind and validate NOW as well as through the options pipeline: a missing connection string
|
||||
// or a bad schema name should fail at startup with a clear message, not on the first resolve.
|
||||
var eager = new SqlServerSecretsOptions();
|
||||
section.Bind(eager);
|
||||
eager.Validate();
|
||||
|
||||
services.TryAddSingleton(sp =>
|
||||
new SecretsSqlServerConnectionFactory(
|
||||
sp.GetRequiredService<IOptions<SqlServerSecretsOptions>>().Value));
|
||||
|
||||
services.TryAddSingleton(sp =>
|
||||
new SqlServerSecretStore(sp.GetRequiredService<SecretsSqlServerConnectionFactory>()));
|
||||
|
||||
services.TryAddSingleton(sp =>
|
||||
new SqlServerSecretsStoreMigrator(sp.GetRequiredService<SecretsSqlServerConnectionFactory>()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// The subset of store operations hub replication needs from the shared database: compare
|
||||
/// inventories, fetch rows in bulk, and accept a row verbatim.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Narrower than <see cref="ISecretStore"/> on purpose. The hub is never resolved from, never
|
||||
/// written through <see cref="ISecretStore.UpsertAsync"/> (which would stamp the hub's own revision
|
||||
/// and bounce the row back), and never rewrapped by a syncing node — so exposing those operations
|
||||
/// would only create ways to use it wrongly. It also makes the sync sweep testable against an
|
||||
/// in-process fake instead of requiring a live SQL Server for every convergence case.
|
||||
/// </remarks>
|
||||
public interface ISecretReplicationHub
|
||||
{
|
||||
/// <summary>Returns the hub's manifest — names, revisions, timestamps, tombstone flags.</summary>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The hub's manifest entries.</returns>
|
||||
Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct);
|
||||
|
||||
/// <summary>Fetches several encrypted rows by name; names absent from the hub are omitted.</summary>
|
||||
/// <param name="names">The names to fetch.</param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The matching encrypted rows.</returns>
|
||||
Task<IReadOnlyList<StoredSecret>> GetManyAsync(IReadOnlyList<SecretName> names, CancellationToken ct);
|
||||
|
||||
/// <summary>Applies a row to the hub verbatim, last-writer-wins.</summary>
|
||||
/// <param name="row">The encrypted row.</param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the row has been applied or ignored.</returns>
|
||||
Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct);
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Data.SqlClient;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Opens connections to the shared SQL-Server secret database and carries the settings every
|
||||
/// command needs (schema name, command timeout).
|
||||
/// </summary>
|
||||
/// <param name="options">The validated SQL-Server store options.</param>
|
||||
public sealed class SecretsSqlServerConnectionFactory(SqlServerSecretsOptions options)
|
||||
{
|
||||
private readonly SqlServerSecretsOptions _options =
|
||||
options ?? throw new ArgumentNullException(nameof(options));
|
||||
|
||||
/// <summary>The bracket-quoted, allow-list-validated schema prefix for object names.</summary>
|
||||
public string SchemaName => _options.SchemaName;
|
||||
|
||||
/// <summary>Command timeout, in whole seconds, for every command issued against this store.</summary>
|
||||
public int CommandTimeoutSeconds => Math.Max(1, (int)_options.CommandTimeout.TotalSeconds);
|
||||
|
||||
/// <summary>Opens a connection to the shared secret database.</summary>
|
||||
/// <param name="cancellationToken">A token to cancel the connect.</param>
|
||||
/// <returns>An opened connection. The caller owns disposal.</returns>
|
||||
public async Task<SqlConnection> OpenConnectionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = new SqlConnection(_options.ConnectionString);
|
||||
try
|
||||
{
|
||||
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
return connection;
|
||||
}
|
||||
catch
|
||||
{
|
||||
await connection.DisposeAsync().ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a command on <paramref name="connection"/> with the configured timeout applied.
|
||||
/// </summary>
|
||||
/// <param name="connection">The open connection.</param>
|
||||
/// <param name="commandText">The command text.</param>
|
||||
/// <param name="transaction">The ambient transaction, if any.</param>
|
||||
/// <returns>The configured command. The caller owns disposal.</returns>
|
||||
public SqlCommand CreateCommand(
|
||||
SqlConnection connection, string commandText, SqlTransaction? transaction = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
|
||||
SqlCommand command = connection.CreateCommand();
|
||||
command.CommandText = commandText;
|
||||
command.CommandTimeout = CommandTimeoutSeconds;
|
||||
command.Transaction = transaction;
|
||||
return command;
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Provisions the shared hub schema at startup in hub-replication mode.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Separate from the core migration hosted service on purpose: in hub mode there are <em>two</em>
|
||||
/// stores to provision — the node's local one (handled by the core service through the
|
||||
/// <c>ISecretsStoreMigrator</c> seam) and the shared hub, which no node "owns". Every node runs this
|
||||
/// idempotently at startup, so the hub exists as soon as any node has started, with no separate
|
||||
/// provisioning step for an operator to forget.
|
||||
/// </remarks>
|
||||
/// <param name="migrator">The hub migrator.</param>
|
||||
/// <param name="options">Core secrets options, for the shared startup-migration switch.</param>
|
||||
internal sealed class SqlServerHubMigrationHostedService(
|
||||
SqlServerSecretsStoreMigrator migrator,
|
||||
IOptions<SecretsOptions> options) : IHostedService
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (options.Value.RunMigrationsOnStartup)
|
||||
{
|
||||
await migrator.MigrateAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes local writes to the shared SQL-Server hub, so a secret written on one node is
|
||||
/// immediately visible to every other node that syncs against the same hub.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The row is written through <see cref="ISecretStore.ApplyReplicatedAsync"/>, not
|
||||
/// <see cref="ISecretStore.UpsertAsync"/> — the hub must record the row <em>verbatim</em>, keeping
|
||||
/// the originating node's revision and <c>updated_utc</c>. An upsert would stamp the hub's own
|
||||
/// revision and timestamp, and the row would then look newer than the writer's own copy, so it
|
||||
/// would bounce straight back on the next sync.
|
||||
/// </remarks>
|
||||
/// <param name="hub">The shared hub store.</param>
|
||||
public sealed class SqlServerSecretReplicator(ISecretReplicationHub hub) : ISecretReplicator
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task PublishAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
return hub.ApplyReplicatedAsync(row, ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
using System.Data;
|
||||
using System.Globalization;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// SQL-Server-backed <see cref="ISecretStore"/>. Holds the same envelope-encrypted
|
||||
/// <see cref="StoredSecret"/> rows as the SQLite store, in a database every node of a cluster can
|
||||
/// reach — so one write is visible everywhere with no replication at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The database holds <b>ciphertext only</b>. The KEK stays per-node and out of the database, so a
|
||||
/// DBA, a backup tape, or a replica cannot decrypt anything. The corollary is a hard operational
|
||||
/// constraint: every node must resolve the <em>same</em> KEK, because a row whose <c>kek_id</c> does
|
||||
/// not match the local provider fails closed on resolve.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Semantics are deliberately identical to <c>SqliteSecretStore</c> — revision bump on upsert,
|
||||
/// created stamps preserved on overwrite, verbatim last-writer-wins apply, compare-and-swap rewrap.
|
||||
/// A cluster can run both stores at once (nodes on local SQLite replicating against a shared hub),
|
||||
/// so a divergence between the two would be a silent convergence bug.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="connectionFactory">Factory for connections to the shared secret database.</param>
|
||||
public sealed class SqlServerSecretStore(SecretsSqlServerConnectionFactory connectionFactory)
|
||||
: ISecretStore, ISecretReplicationHub
|
||||
{
|
||||
// All columns of the secret table, in schema order, for full-row reads.
|
||||
private const string AllColumns =
|
||||
"name, description, content_type, ciphertext, nonce, tag, wrapped_dek, wrap_nonce, wrap_tag, " +
|
||||
"kek_id, revision, is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by";
|
||||
|
||||
// The safe metadata projection — deliberately excludes every ciphertext / crypto BLOB column.
|
||||
private const string MetadataColumns =
|
||||
"name, description, content_type, kek_id, revision, is_deleted, created_utc, updated_utc, created_by, updated_by";
|
||||
|
||||
private string Secret => $"[{connectionFactory.SchemaName}].[{SqlServerSecretsSchema.SecretTable}]";
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct)
|
||||
{
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(
|
||||
connection, $"SELECT {AllColumns} FROM {Secret} WHERE name = @name;");
|
||||
command.Parameters.AddWithValue("@name", name.Value);
|
||||
|
||||
await using SqlDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
|
||||
return await reader.ReadAsync(ct).ConfigureAwait(false)
|
||||
? ReadStoredSecret(reader)
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
string now = DateTimeOffset.UtcNow.ToString("O");
|
||||
|
||||
// MERGE establishes revision 0 / created==updated==now on insert; on match it overwrites the
|
||||
// crypto material in place, bumps the revision, refreshes updated_utc/updated_by, and clears
|
||||
// any tombstone — but deliberately preserves created_utc / created_by (the SQLite store's
|
||||
// ON CONFLICT semantics, expressed in T-SQL).
|
||||
//
|
||||
// HOLDLOCK is not optional: without it MERGE takes only an update lock on the key range and
|
||||
// two concurrent nodes upserting the same new name can both miss the match and both INSERT,
|
||||
// producing a primary-key violation instead of one insert + one update.
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(connection, $"""
|
||||
MERGE {Secret} WITH (HOLDLOCK) AS target
|
||||
USING (SELECT @name AS name) AS source
|
||||
ON target.name = source.name
|
||||
WHEN MATCHED THEN UPDATE SET
|
||||
description = @description,
|
||||
content_type = @content_type,
|
||||
ciphertext = @ciphertext,
|
||||
nonce = @nonce,
|
||||
tag = @tag,
|
||||
wrapped_dek = @wrapped_dek,
|
||||
wrap_nonce = @wrap_nonce,
|
||||
wrap_tag = @wrap_tag,
|
||||
kek_id = @kek_id,
|
||||
revision = target.revision + 1,
|
||||
updated_utc = @now,
|
||||
updated_by = @updated_by,
|
||||
is_deleted = 0,
|
||||
deleted_utc = NULL
|
||||
WHEN NOT MATCHED THEN INSERT (
|
||||
name, description, content_type, ciphertext, nonce, tag,
|
||||
wrapped_dek, wrap_nonce, wrap_tag, kek_id, revision,
|
||||
is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by)
|
||||
VALUES (
|
||||
@name, @description, @content_type, @ciphertext, @nonce, @tag,
|
||||
@wrapped_dek, @wrap_nonce, @wrap_tag, @kek_id, 0,
|
||||
0, NULL, @now, @now, @created_by, @updated_by);
|
||||
""");
|
||||
BindCryptoColumns(command, row);
|
||||
command.Parameters.AddWithValue("@now", now);
|
||||
command.Parameters.AddWithValue("@created_by", (object?)row.CreatedBy ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("@updated_by", (object?)row.UpdatedBy ?? DBNull.Value);
|
||||
|
||||
await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
|
||||
{
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
string now = DateTimeOffset.UtcNow.ToString("O");
|
||||
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(connection, $"""
|
||||
UPDATE {Secret} SET
|
||||
is_deleted = 1,
|
||||
deleted_utc = @now,
|
||||
revision = revision + 1,
|
||||
updated_utc = @now,
|
||||
updated_by = @actor
|
||||
WHERE name = @name AND is_deleted = 0;
|
||||
""");
|
||||
command.Parameters.AddWithValue("@now", now);
|
||||
command.Parameters.AddWithValue("@actor", (object?)actor ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("@name", name.Value);
|
||||
|
||||
int rowsAffected = await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
return rowsAffected > 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct)
|
||||
{
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(
|
||||
connection,
|
||||
$"SELECT {MetadataColumns} FROM {Secret} WHERE (@include_deleted = 1 OR is_deleted = 0) ORDER BY name;");
|
||||
command.Parameters.AddWithValue("@include_deleted", includeDeleted ? 1 : 0);
|
||||
|
||||
var results = new List<SecretMetadata>();
|
||||
await using SqlDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(new SecretMetadata
|
||||
{
|
||||
Name = new SecretName(reader.GetString(0)),
|
||||
Description = reader.IsDBNull(1) ? null : reader.GetString(1),
|
||||
ContentType = Enum.Parse<SecretContentType>(reader.GetString(2)),
|
||||
KekId = reader.GetString(3),
|
||||
Revision = reader.GetInt64(4),
|
||||
IsDeleted = reader.GetBoolean(5),
|
||||
CreatedUtc = ParseUtc(reader.GetString(6)),
|
||||
UpdatedUtc = ParseUtc(reader.GetString(7)),
|
||||
CreatedBy = reader.IsDBNull(8) ? null : reader.GetString(8),
|
||||
UpdatedBy = reader.IsDBNull(9) ? null : reader.GetString(9),
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct)
|
||||
{
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(
|
||||
connection, $"SELECT name, revision, updated_utc, is_deleted FROM {Secret} ORDER BY name;");
|
||||
|
||||
var results = new List<SecretManifestEntry>();
|
||||
await using SqlDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(new SecretManifestEntry
|
||||
{
|
||||
Name = new SecretName(reader.GetString(0)),
|
||||
Revision = reader.GetInt64(1),
|
||||
UpdatedUtc = ParseUtc(reader.GetString(2)),
|
||||
IsDeleted = reader.GetBoolean(3),
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
await using SqlTransaction transaction = (SqlTransaction)
|
||||
await connection.BeginTransactionAsync(IsolationLevel.Serializable, ct).ConfigureAwait(false);
|
||||
|
||||
// Read the local (updated_utc, revision) so we can apply last-writer-wins. UPDLOCK/HOLDLOCK
|
||||
// keeps another node from slipping a newer row in between this read and the write below —
|
||||
// Serializable alone would not lock a key range that currently has no row.
|
||||
await using (SqlCommand read = connectionFactory.CreateCommand(
|
||||
connection,
|
||||
$"SELECT updated_utc, revision FROM {Secret} WITH (UPDLOCK, HOLDLOCK) WHERE name = @name;",
|
||||
transaction))
|
||||
{
|
||||
read.Parameters.AddWithValue("@name", row.Name.Value);
|
||||
|
||||
await using SqlDataReader reader = await read.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
if (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
DateTimeOffset localUpdated = ParseUtc(reader.GetString(0));
|
||||
long localRevision = reader.GetInt64(1);
|
||||
|
||||
// Routed through the shared predicate so every store agrees on ties.
|
||||
if (!SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, localUpdated, localRevision))
|
||||
{
|
||||
await reader.CloseAsync().ConfigureAwait(false);
|
||||
await transaction.CommitAsync(ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the incoming row VERBATIM — its own revision, timestamps, tombstone flag, and crypto
|
||||
// material — with no revision bump (this is the replication path, not a local write).
|
||||
await using (SqlCommand upsert = connectionFactory.CreateCommand(connection, $"""
|
||||
MERGE {Secret} WITH (HOLDLOCK) AS target
|
||||
USING (SELECT @name AS name) AS source
|
||||
ON target.name = source.name
|
||||
WHEN MATCHED THEN UPDATE SET
|
||||
description = @description,
|
||||
content_type = @content_type,
|
||||
ciphertext = @ciphertext,
|
||||
nonce = @nonce,
|
||||
tag = @tag,
|
||||
wrapped_dek = @wrapped_dek,
|
||||
wrap_nonce = @wrap_nonce,
|
||||
wrap_tag = @wrap_tag,
|
||||
kek_id = @kek_id,
|
||||
revision = @revision,
|
||||
is_deleted = @is_deleted,
|
||||
deleted_utc = @deleted_utc,
|
||||
created_utc = @created_utc,
|
||||
updated_utc = @updated_utc,
|
||||
created_by = @created_by,
|
||||
updated_by = @updated_by
|
||||
WHEN NOT MATCHED THEN INSERT (
|
||||
name, description, content_type, ciphertext, nonce, tag,
|
||||
wrapped_dek, wrap_nonce, wrap_tag, kek_id, revision,
|
||||
is_deleted, deleted_utc, created_utc, updated_utc, created_by, updated_by)
|
||||
VALUES (
|
||||
@name, @description, @content_type, @ciphertext, @nonce, @tag,
|
||||
@wrapped_dek, @wrap_nonce, @wrap_tag, @kek_id, @revision,
|
||||
@is_deleted, @deleted_utc, @created_utc, @updated_utc, @created_by, @updated_by);
|
||||
""", transaction))
|
||||
{
|
||||
BindCryptoColumns(upsert, row);
|
||||
upsert.Parameters.AddWithValue("@revision", row.Revision);
|
||||
upsert.Parameters.AddWithValue("@is_deleted", row.IsDeleted ? 1 : 0);
|
||||
upsert.Parameters.AddWithValue("@deleted_utc", (object?)row.DeletedUtc?.ToString("O") ?? DBNull.Value);
|
||||
upsert.Parameters.AddWithValue("@created_utc", row.CreatedUtc.ToString("O"));
|
||||
upsert.Parameters.AddWithValue("@updated_utc", row.UpdatedUtc.ToString("O"));
|
||||
upsert.Parameters.AddWithValue("@created_by", (object?)row.CreatedBy ?? DBNull.Value);
|
||||
upsert.Parameters.AddWithValue("@updated_by", (object?)row.UpdatedBy ?? DBNull.Value);
|
||||
|
||||
await upsert.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> ApplyRewrapAsync(
|
||||
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rewrappedRow);
|
||||
ArgumentNullException.ThrowIfNull(expectedCurrentWrappedDek);
|
||||
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// Overwrite ONLY the KEK-wrap envelope + kek_id. Revision, updated_utc/by, the sealed body,
|
||||
// and the tombstone state are deliberately untouched — a re-wrap is not a logical change, so
|
||||
// it must not bump the revision or updated timestamp (see ISecretStore.ApplyRewrapAsync).
|
||||
//
|
||||
// Compare-and-swap on wrapped_dek: a concurrent set/rotate generates a fresh (random) DEK
|
||||
// wrap, so this matches 0 rows instead of pairing this stale re-wrap with a newer body —
|
||||
// which would leave the row permanently undecryptable. The caller re-processes a 0-row miss.
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(connection, $"""
|
||||
UPDATE {Secret} SET
|
||||
wrapped_dek = @wrapped_dek,
|
||||
wrap_nonce = @wrap_nonce,
|
||||
wrap_tag = @wrap_tag,
|
||||
kek_id = @kek_id
|
||||
WHERE name = @name AND wrapped_dek = @expected_wrapped_dek;
|
||||
""");
|
||||
command.Parameters.AddWithValue("@wrapped_dek", rewrappedRow.WrappedDek);
|
||||
command.Parameters.AddWithValue("@wrap_nonce", rewrappedRow.WrapNonce);
|
||||
command.Parameters.AddWithValue("@wrap_tag", rewrappedRow.WrapTag);
|
||||
command.Parameters.AddWithValue("@kek_id", rewrappedRow.KekId);
|
||||
command.Parameters.AddWithValue("@name", rewrappedRow.Name.Value);
|
||||
command.Parameters.AddWithValue("@expected_wrapped_dek", expectedCurrentWrappedDek);
|
||||
|
||||
int rowsAffected = await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
return rowsAffected > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches several rows by name in one round trip — the replication fetch path. Names not
|
||||
/// present are simply absent from the result.
|
||||
/// </summary>
|
||||
/// <param name="names">The names to fetch.</param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The matching encrypted rows.</returns>
|
||||
public async Task<IReadOnlyList<StoredSecret>> GetManyAsync(
|
||||
IReadOnlyList<SecretName> names, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(names);
|
||||
|
||||
if (names.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// One parameter per name — still fully parameterized, no name text is interpolated.
|
||||
string[] placeholders = [.. names.Select((_, i) => $"@n{i.ToString(CultureInfo.InvariantCulture)}")];
|
||||
|
||||
await using SqlCommand command = connectionFactory.CreateCommand(
|
||||
connection,
|
||||
$"SELECT {AllColumns} FROM {Secret} WHERE name IN ({string.Join(", ", placeholders)});");
|
||||
|
||||
for (int i = 0; i < names.Count; i++)
|
||||
{
|
||||
command.Parameters.AddWithValue(placeholders[i], names[i].Value);
|
||||
}
|
||||
|
||||
var results = new List<StoredSecret>(names.Count);
|
||||
await using SqlDataReader reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(ReadStoredSecret(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Binds the identity, description, content-type, KEK id, and all six crypto BLOB columns
|
||||
// shared by every insert path.
|
||||
private static void BindCryptoColumns(SqlCommand command, StoredSecret row)
|
||||
{
|
||||
command.Parameters.AddWithValue("@name", row.Name.Value);
|
||||
command.Parameters.AddWithValue("@description", (object?)row.Description ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("@content_type", row.ContentType.ToString());
|
||||
command.Parameters.AddWithValue("@ciphertext", row.Ciphertext);
|
||||
command.Parameters.AddWithValue("@nonce", row.Nonce);
|
||||
command.Parameters.AddWithValue("@tag", row.Tag);
|
||||
command.Parameters.AddWithValue("@wrapped_dek", row.WrappedDek);
|
||||
command.Parameters.AddWithValue("@wrap_nonce", row.WrapNonce);
|
||||
command.Parameters.AddWithValue("@wrap_tag", row.WrapTag);
|
||||
command.Parameters.AddWithValue("@kek_id", row.KekId);
|
||||
}
|
||||
|
||||
private static StoredSecret ReadStoredSecret(SqlDataReader reader) => new()
|
||||
{
|
||||
Name = new SecretName(reader.GetString(0)),
|
||||
Description = reader.IsDBNull(1) ? null : reader.GetString(1),
|
||||
ContentType = Enum.Parse<SecretContentType>(reader.GetString(2)),
|
||||
Ciphertext = reader.GetFieldValue<byte[]>(3),
|
||||
Nonce = reader.GetFieldValue<byte[]>(4),
|
||||
Tag = reader.GetFieldValue<byte[]>(5),
|
||||
WrappedDek = reader.GetFieldValue<byte[]>(6),
|
||||
WrapNonce = reader.GetFieldValue<byte[]>(7),
|
||||
WrapTag = reader.GetFieldValue<byte[]>(8),
|
||||
KekId = reader.GetString(9),
|
||||
Revision = reader.GetInt64(10),
|
||||
IsDeleted = reader.GetBoolean(11),
|
||||
DeletedUtc = reader.IsDBNull(12) ? null : ParseUtc(reader.GetString(12)),
|
||||
CreatedUtc = ParseUtc(reader.GetString(13)),
|
||||
UpdatedUtc = ParseUtc(reader.GetString(14)),
|
||||
CreatedBy = reader.IsDBNull(15) ? null : reader.GetString(15),
|
||||
UpdatedBy = reader.IsDBNull(16) ? null : reader.GetString(16),
|
||||
};
|
||||
|
||||
private static DateTimeOffset ParseUtc(string value) =>
|
||||
DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Replication;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps a node's local secret store converged with the shared SQL-Server hub, in hub-replication
|
||||
/// mode. Runs a bidirectional anti-entropy sweep on an interval: pulls rows the hub holds newer,
|
||||
/// pushes rows the node holds newer.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the durability guarantee behind best-effort publishing. Live publishes make a write
|
||||
/// visible in milliseconds; this sweep is what makes it <em>certain</em>. A node that was offline
|
||||
/// while a peer wrote catches up on its next sweep, and a write made while the hub was unreachable
|
||||
/// is pushed on the next sweep instead of being lost.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The sweep never throws out of <see cref="ExecuteAsync"/>: a hub outage must degrade the node to
|
||||
/// "serving its last-known-good local secrets" rather than crash the host. Failures are logged and
|
||||
/// retried on the next tick.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed partial class SqlServerSecretSyncService : BackgroundService
|
||||
{
|
||||
private readonly SecretReplicationReconciler _reconciler;
|
||||
private readonly ISecretStore _local;
|
||||
private readonly ISecretReplicationHub _hub;
|
||||
private readonly SqlServerSecretsOptions _options;
|
||||
private readonly ILogger<SqlServerSecretSyncService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
/// <summary>Creates the sync service.</summary>
|
||||
/// <param name="local">The node's local store.</param>
|
||||
/// <param name="hub">The shared hub store.</param>
|
||||
/// <param name="cacheInvalidator">Resolver-cache seam, evicted for every applied row.</param>
|
||||
/// <param name="options">SQL-Server replication options.</param>
|
||||
/// <param name="logger">Logger for sweep outcomes and failures.</param>
|
||||
/// <param name="timeProvider">Clock, injectable so tests need not wait a real interval.</param>
|
||||
public SqlServerSecretSyncService(
|
||||
ISecretStore local,
|
||||
ISecretReplicationHub hub,
|
||||
ISecretCacheInvalidator? cacheInvalidator,
|
||||
IOptions<SqlServerSecretsOptions> options,
|
||||
ILogger<SqlServerSecretSyncService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
_local = local ?? throw new ArgumentNullException(nameof(local));
|
||||
_hub = hub ?? throw new ArgumentNullException(nameof(hub));
|
||||
_options = options.Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_reconciler = new SecretReplicationReconciler(
|
||||
_local,
|
||||
cacheInvalidator,
|
||||
onRowFailed: (name, ex) => LogRowDiscarded(ex, name.Value));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (_options.SyncOnStartup)
|
||||
{
|
||||
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
using PeriodicTimer timer = new(_options.SyncInterval, _timeProvider);
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one bidirectional reconcile against the hub.
|
||||
/// </summary>
|
||||
/// <param name="ct">A token to cancel the sweep.</param>
|
||||
/// <returns>How many rows were pulled from and pushed to the hub.</returns>
|
||||
internal async Task<(int Pulled, int Pushed)> SweepAsync(CancellationToken ct)
|
||||
{
|
||||
IReadOnlyList<SecretManifestEntry> hubManifest =
|
||||
await _hub.GetManifestAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// Pull first: adopting the hub's newer rows before computing the push set means a row the hub
|
||||
// already had newer is not then pushed straight back at it.
|
||||
int pulled = await _reconciler
|
||||
.ReconcileAsync(hubManifest, (names, token) => _hub.GetManyAsync(names, token), ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
IReadOnlyList<SecretManifestEntry> localManifest =
|
||||
await _local.GetManifestAsync(ct).ConfigureAwait(false);
|
||||
|
||||
IReadOnlyList<SecretName> push =
|
||||
SecretReplicationReconciler.ComputePushSet(localManifest, hubManifest);
|
||||
|
||||
int pushed = 0;
|
||||
|
||||
if (push.Count > 0)
|
||||
{
|
||||
IReadOnlyList<StoredSecret> rows =
|
||||
await _reconciler.ReadLocalAsync(push, ct).ConfigureAwait(false);
|
||||
|
||||
foreach (StoredSecret row in rows)
|
||||
{
|
||||
// Verbatim, like the replicator — the hub records the originating node's revision.
|
||||
await _hub.ApplyReplicatedAsync(row, ct).ConfigureAwait(false);
|
||||
pushed++;
|
||||
}
|
||||
}
|
||||
|
||||
return (pulled, pushed);
|
||||
}
|
||||
|
||||
private async Task SweepSafelyAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
(int pulled, int pushed) = await SweepAsync(ct).ConfigureAwait(false);
|
||||
|
||||
if (pulled > 0 || pushed > 0)
|
||||
{
|
||||
LogSweepConverged(pulled, pushed);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Host shutdown — not a failure.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogSweepFailed(ex);
|
||||
}
|
||||
}
|
||||
|
||||
[LoggerMessage(
|
||||
EventId = 1,
|
||||
Level = LogLevel.Information,
|
||||
Message = "Secret hub sync converged: pulled {Pulled} row(s) from the hub, pushed {Pushed} row(s) to it.")]
|
||||
private partial void LogSweepConverged(int pulled, int pushed);
|
||||
|
||||
[LoggerMessage(
|
||||
EventId = 2,
|
||||
Level = LogLevel.Warning,
|
||||
Message = "Secret hub sync failed; the node continues serving its local store and will retry on the next interval.")]
|
||||
private partial void LogSweepFailed(Exception exception);
|
||||
|
||||
[LoggerMessage(
|
||||
EventId = 3,
|
||||
Level = LogLevel.Warning,
|
||||
Message = "Discarding replicated secret row {SecretName} that could not be applied locally; the rest of the batch continues.")]
|
||||
private partial void LogRowDiscarded(Exception exception, string secretName);
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the shared SQL-Server secret store and its hub replication. Bound from a
|
||||
/// configuration section by the <c>AddZbSecretsSqlServer*</c> extensions.
|
||||
/// </summary>
|
||||
public sealed class SqlServerSecretsOptions
|
||||
{
|
||||
private string _schemaName = SqlServerSecretsSchema.DefaultSchemaName;
|
||||
|
||||
/// <summary>
|
||||
/// Connection string for the shared secret database. Required. Supply it through a
|
||||
/// <c>${secret:}</c> / <c>secret:</c> reference like every other connection string in the family
|
||||
/// — the shared store holds ciphertext only, but its credentials are still credentials.
|
||||
/// </summary>
|
||||
public string ConnectionString { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// SQL schema the secret tables live in. Defaults to <c>zbsecrets</c> so the secret tables never
|
||||
/// collide with an application's own <c>dbo</c> objects when they share a database.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">
|
||||
/// The value is not a plain identifier (letters, digits, and underscore, not starting with a
|
||||
/// digit, at most 128 characters).
|
||||
/// </exception>
|
||||
/// <remarks>
|
||||
/// Validated on assignment rather than at use: the schema name is interpolated into DDL and
|
||||
/// query text (T-SQL cannot parameterize an object name), so the allow-list is the injection
|
||||
/// boundary and it must be impossible to get an unvalidated value into the property at all.
|
||||
/// </remarks>
|
||||
public string SchemaName
|
||||
{
|
||||
get => _schemaName;
|
||||
set => _schemaName = ValidateIdentifier(value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// How often the sync service reconciles the local store against the hub, in hub-replication
|
||||
/// mode. Defaults to 30 seconds. Ignored in shared-store mode (there is nothing to reconcile).
|
||||
/// </summary>
|
||||
public TimeSpan SyncInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// When <see langword="true"/> (the default), hub-replication mode runs a reconcile at startup
|
||||
/// before the first interval elapses, so a node that was down while a peer wrote does not serve
|
||||
/// stale secrets for a whole interval after it comes back.
|
||||
/// </summary>
|
||||
public bool SyncOnStartup { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Command timeout applied to every store operation. Defaults to 30 seconds — long enough for a
|
||||
/// contended shared database, short enough that a hung hub surfaces as an error rather than
|
||||
/// stalling a resolve indefinitely.
|
||||
/// </summary>
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Throws if the options are unusable. Called by the DI extensions at registration time so a
|
||||
/// misconfigured node fails at startup rather than on the first secret resolve.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The connection string is missing, or an interval is not positive.</exception>
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(ConnectionString))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"SQL-Server secret store requires a ConnectionString (Secrets:SqlServer:ConnectionString).");
|
||||
}
|
||||
|
||||
if (SyncInterval <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"SQL-Server secret replication SyncInterval must be positive (was {SyncInterval}).");
|
||||
}
|
||||
|
||||
if (CommandTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"SQL-Server secret store CommandTimeout must be positive (was {CommandTimeout}).");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Strict allow-list for a SQL identifier that will be interpolated into DDL or query text.
|
||||
/// Rejects everything that is not <c>[A-Za-z_][A-Za-z0-9_]{0,127}</c> — no brackets, quotes,
|
||||
/// dots, or spaces — so no bracket-escape (<c>]]</c>) trick can break out of the quoting applied
|
||||
/// at the use site. Uses the ASCII-only character tests deliberately, so homoglyph and
|
||||
/// full-width look-alikes are rejected rather than accepted as "letters".
|
||||
/// </summary>
|
||||
/// <param name="value">The candidate identifier.</param>
|
||||
/// <returns>The same value, once validated.</returns>
|
||||
/// <exception cref="ArgumentException">The value is not a plain identifier.</exception>
|
||||
internal static string ValidateIdentifier(string value)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(value);
|
||||
|
||||
if (value.Length > 128)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"SQL schema name must be at most 128 characters (was {value.Length}).", nameof(value));
|
||||
}
|
||||
|
||||
if (!char.IsAsciiLetter(value[0]) && value[0] != '_')
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"SQL schema name must start with a letter or underscore (was '{value}').", nameof(value));
|
||||
}
|
||||
|
||||
foreach (char c in value)
|
||||
{
|
||||
if (!char.IsAsciiLetterOrDigit(c) && c != '_')
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"SQL schema name may contain only letters, digits, and underscore (was '{value}').",
|
||||
nameof(value));
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Schema constants and DDL for the shared SQL-Server secret store. Mirrors
|
||||
/// <c>ZB.MOM.WW.Secrets.Sqlite.SqliteSecretsSchema</c> column-for-column so a row means exactly the
|
||||
/// same thing in either store.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Timestamps are ISO-8601 <c>nvarchar</c>, not <c>datetimeoffset</c>.</b> This looks wrong for a
|
||||
/// SQL-Server schema and is deliberate: the same <see cref="ZB.MOM.WW.Secrets.Abstractions.StoredSecret"/>
|
||||
/// rows replicate between this store and SQLite (which stores <c>"O"</c>-format text), and
|
||||
/// last-writer-wins compares those timestamps for equality. <c>datetimeoffset</c> rounds to 100ns
|
||||
/// while .NET <c>DateTimeOffset</c> ticks are also 100ns but the round-trip through the driver is
|
||||
/// not guaranteed bit-identical across providers — a sub-tick difference would turn a tie into a
|
||||
/// spurious "newer", and two nodes would then overwrite each other forever. Storing the exact
|
||||
/// round-trippable text both providers already agree on removes the failure mode entirely.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The cost is that <c>ORDER BY updated_utc</c> is a lexicographic sort. That is still chronological
|
||||
/// for <c>"O"</c>-format UTC strings (fixed-width, zero-padded, always <c>+00:00</c>), which is what
|
||||
/// the store writes.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class SqlServerSecretsSchema
|
||||
{
|
||||
/// <summary>
|
||||
/// The schema version this build creates and supports. The migrator refuses a database whose
|
||||
/// recorded version is <em>newer</em> than this.
|
||||
/// </summary>
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
/// <summary>Default schema the secret tables live in.</summary>
|
||||
public const string DefaultSchemaName = "zbsecrets";
|
||||
|
||||
/// <summary>Name of the single-row table tracking the applied schema version.</summary>
|
||||
public const string SchemaVersionTable = "schema_version";
|
||||
|
||||
/// <summary>Name of the table storing envelope-encrypted secret records.</summary>
|
||||
public const string SecretTable = "secret";
|
||||
|
||||
/// <summary>
|
||||
/// Returns the idempotent DDL that provisions the schema, both tables, and the supporting index
|
||||
/// in <paramref name="schemaName"/>.
|
||||
/// </summary>
|
||||
/// <param name="schemaName">The already-validated SQL schema name to create the objects in.</param>
|
||||
/// <returns>The DDL batch.</returns>
|
||||
/// <remarks>
|
||||
/// <paramref name="schemaName"/> is interpolated, not parameterized — T-SQL has no parameter form
|
||||
/// for an object name. It is re-validated here against the same strict identifier allow-list
|
||||
/// <see cref="SqlServerSecretsOptions.SchemaName"/> enforces, and bracket-quoted below. The
|
||||
/// duplicate check is deliberate: this method is <c>public</c>, so it must defend itself rather
|
||||
/// than trust that every caller routed through the options type.
|
||||
/// </remarks>
|
||||
/// <exception cref="ArgumentException"><paramref name="schemaName"/> is not a plain identifier.</exception>
|
||||
public static string CreateSchemaDdl(string schemaName)
|
||||
{
|
||||
SqlServerSecretsOptions.ValidateIdentifier(schemaName);
|
||||
|
||||
return $"""
|
||||
IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'{schemaName}')
|
||||
EXEC(N'CREATE SCHEMA [{schemaName}]');
|
||||
|
||||
IF OBJECT_ID(N'[{schemaName}].[{SchemaVersionTable}]', N'U') IS NULL
|
||||
CREATE TABLE [{schemaName}].[{SchemaVersionTable}] (
|
||||
id INT NOT NULL PRIMARY KEY CHECK (id = 1),
|
||||
version INT NOT NULL,
|
||||
applied_utc NVARCHAR(33) NOT NULL
|
||||
);
|
||||
|
||||
IF OBJECT_ID(N'[{schemaName}].[{SecretTable}]', N'U') IS NULL
|
||||
CREATE TABLE [{schemaName}].[{SecretTable}] (
|
||||
name NVARCHAR(450) NOT NULL PRIMARY KEY,
|
||||
description NVARCHAR(MAX) NULL,
|
||||
content_type NVARCHAR(64) NOT NULL,
|
||||
ciphertext VARBINARY(MAX) NOT NULL,
|
||||
nonce VARBINARY(MAX) NOT NULL,
|
||||
tag VARBINARY(MAX) NOT NULL,
|
||||
wrapped_dek VARBINARY(MAX) NOT NULL,
|
||||
wrap_nonce VARBINARY(MAX) NOT NULL,
|
||||
wrap_tag VARBINARY(MAX) NOT NULL,
|
||||
kek_id NVARCHAR(256) NOT NULL,
|
||||
revision BIGINT NOT NULL CONSTRAINT DF_secret_revision DEFAULT 0,
|
||||
is_deleted BIT NOT NULL CONSTRAINT DF_secret_is_deleted DEFAULT 0,
|
||||
deleted_utc NVARCHAR(33) NULL,
|
||||
created_utc NVARCHAR(33) NOT NULL,
|
||||
updated_utc NVARCHAR(33) NOT NULL,
|
||||
created_by NVARCHAR(256) NULL,
|
||||
updated_by NVARCHAR(256) NULL
|
||||
);
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM sys.indexes
|
||||
WHERE name = N'ix_secret_updated_utc'
|
||||
AND object_id = OBJECT_ID(N'[{schemaName}].[{SecretTable}]'))
|
||||
CREATE INDEX ix_secret_updated_utc
|
||||
ON [{schemaName}].[{SecretTable}] (updated_utc);
|
||||
""";
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using System.Data;
|
||||
using System.Globalization;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Creates the shared SQL-Server secret schema and records the applied version. Idempotent: safe to
|
||||
/// run on every node at every startup, including concurrently. Refuses to run against a database
|
||||
/// whose recorded version is newer than this build supports.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for connections to the shared secret database.</param>
|
||||
public sealed class SqlServerSecretsStoreMigrator(SecretsSqlServerConnectionFactory connectionFactory)
|
||||
: ISecretsStoreMigrator
|
||||
{
|
||||
// Concurrent Serializable DDL against the system catalogs can pick a deadlock victim (error
|
||||
// 1205) even though the IF NOT EXISTS guards make the logic itself safe. Every node runs this at
|
||||
// every startup, so a virgin database being provisioned by N nodes at once is the NORMAL case,
|
||||
// not a rare race — and an unretried victim aborts that node's whole host at startup. Retrying
|
||||
// is correct precisely because the migration is idempotent.
|
||||
private const int MaxDeadlockRetries = 5;
|
||||
private const int SqlDeadlockVictimErrorNumber = 1205;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <exception cref="SecretStoreMigrationException">
|
||||
/// The recorded schema version is newer than <see cref="SqlServerSecretsSchema.CurrentVersion"/>.
|
||||
/// </exception>
|
||||
public async Task MigrateAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
for (int attempt = 1; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await MigrateOnceAsync(cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (SqlException ex)
|
||||
when (ex.Number == SqlDeadlockVictimErrorNumber && attempt < MaxDeadlockRetries)
|
||||
{
|
||||
// Staggered back-off so two victims do not retry in lockstep and deadlock again.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MigrateOnceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
string schema = connectionFactory.SchemaName;
|
||||
|
||||
await using SqlConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Serializable: unlike the single-writer SQLite store, EVERY node in the cluster runs this
|
||||
// migration at startup, so concurrent first-boots are the normal case rather than a rare
|
||||
// race. The isolation level plus the IF NOT EXISTS guards make a simultaneous provision by
|
||||
// two nodes converge on one schema instead of one of them failing with "object exists".
|
||||
await using SqlTransaction transaction = (SqlTransaction)
|
||||
await connection.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
int existingVersion =
|
||||
await ReadExistingSchemaVersionAsync(connection, transaction, schema, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (existingVersion > SqlServerSecretsSchema.CurrentVersion)
|
||||
{
|
||||
throw new SecretStoreMigrationException(
|
||||
$"Shared secret database schema version {existingVersion} is newer than supported version " +
|
||||
$"{SqlServerSecretsSchema.CurrentVersion}. Upgrade this node before it writes to the shared store.");
|
||||
}
|
||||
|
||||
await using (SqlCommand ddl = connectionFactory.CreateCommand(
|
||||
connection, SqlServerSecretsSchema.CreateSchemaDdl(schema), transaction))
|
||||
{
|
||||
await ddl.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await using (SqlCommand version = connectionFactory.CreateCommand(
|
||||
connection,
|
||||
$"""
|
||||
MERGE [{schema}].[{SqlServerSecretsSchema.SchemaVersionTable}] AS target
|
||||
USING (SELECT 1 AS id, @version AS version, @applied_utc AS applied_utc) AS source
|
||||
ON target.id = source.id
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET version = source.version, applied_utc = source.applied_utc
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (id, version, applied_utc) VALUES (1, source.version, source.applied_utc);
|
||||
""",
|
||||
transaction))
|
||||
{
|
||||
version.Parameters.AddWithValue("@version", SqlServerSecretsSchema.CurrentVersion);
|
||||
version.Parameters.AddWithValue("@applied_utc", DateTimeOffset.UtcNow.ToString("O"));
|
||||
await version.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<int> ReadExistingSchemaVersionAsync(
|
||||
SqlConnection connection,
|
||||
SqlTransaction transaction,
|
||||
string schema,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using SqlCommand command = connection.CreateCommand();
|
||||
command.Transaction = transaction;
|
||||
|
||||
// OBJECT_ID returns NULL for an absent table, so this reads 0 on a virgin database without
|
||||
// needing the table to exist first.
|
||||
command.CommandText = $"""
|
||||
IF OBJECT_ID(N'[{schema}].[{SqlServerSecretsSchema.SchemaVersionTable}]', N'U') IS NULL
|
||||
SELECT 0;
|
||||
ELSE
|
||||
SELECT ISNULL((SELECT version FROM [{schema}].[{SqlServerSecretsSchema.SchemaVersionTable}] WHERE id = 1), 0);
|
||||
""";
|
||||
|
||||
object? version = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return version is null || version == DBNull.Value
|
||||
? 0
|
||||
: Convert.ToInt32(version, CultureInfo.InvariantCulture);
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>ZB.MOM.WW.Secrets.Replicator.SqlServer</PackageId>
|
||||
<Authors>ZB.MOM.WW</Authors>
|
||||
<Description>Shared SQL-Server secret store and hub replication for the ZB.MOM.WW SCADA family — cluster-wide, ciphertext-only secrets.</Description>
|
||||
<PackageProjectUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</PackageProjectUrl>
|
||||
<RepositoryUrl>https://gitea.dohertylan.com/dohertj2/zb-mom-ww-secrets</RepositoryUrl>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets\ZB.MOM.WW.Secrets.csproj" />
|
||||
<ProjectReference Include="..\ZB.MOM.WW.Secrets.Abstractions\ZB.MOM.WW.Secrets.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -4,8 +4,9 @@ namespace ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// The single-node default <see cref="ISecretReplicator"/>: publishing a row is a no-op because
|
||||
/// there are no peers to broadcast to. The deferred <c>ZB.MOM.WW.Secrets.Akka</c> package replaces
|
||||
/// this registration with a real cluster replicator for clustered deployments.
|
||||
/// there are no peers to broadcast to. The <c>ZB.MOM.WW.Secrets.Replication.Akka</c> and
|
||||
/// <c>ZB.MOM.WW.Secrets.Replication.SqlServer</c> packages replace this registration with a real
|
||||
/// replicator for clustered deployments.
|
||||
/// </summary>
|
||||
public sealed class NoOpSecretReplicator : ISecretReplicator
|
||||
{
|
||||
|
||||
+8
-3
@@ -1,16 +1,21 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the secrets SQLite schema migration at application startup when
|
||||
/// Runs the secret-store schema migration at application startup when
|
||||
/// <see cref="SecretsOptions.RunMigrationsOnStartup"/> is <see langword="true"/>. The migration is
|
||||
/// idempotent, so repeated restarts are safe.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Depends on the <see cref="ISecretsStoreMigrator"/> seam rather than a concrete migrator, so the
|
||||
/// same hosted service provisions the local SQLite store or the shared SQL-Server store depending
|
||||
/// only on which migrator the application registered.
|
||||
/// </remarks>
|
||||
internal sealed class SecretsMigrationHostedService(
|
||||
SqliteSecretsStoreMigrator migrator,
|
||||
ISecretsStoreMigrator migrator,
|
||||
IOptions<SecretsOptions> options) : IHostedService
|
||||
{
|
||||
/// <inheritdoc />
|
||||
|
||||
+14
-2
@@ -51,9 +51,15 @@ public static class SecretsServiceCollectionExtensions
|
||||
services.TryAddSingleton<ISecretCipher>(sp =>
|
||||
new AesGcmEnvelopeCipher(sp.GetRequiredService<IMasterKeyProvider>()));
|
||||
|
||||
services.TryAddSingleton<ISecretStore>(sp =>
|
||||
// Registered BOTH concretely and behind the interface, forwarding to the one singleton.
|
||||
// The concrete registration is what lets a replication package resolve the *undecorated*
|
||||
// local store when it replaces ISecretStore with a decorator — asking the container for
|
||||
// ISecretStore there would resolve the decorator itself and recurse forever.
|
||||
services.TryAddSingleton(sp =>
|
||||
new SqliteSecretStore(sp.GetRequiredService<SecretsSqliteConnectionFactory>()));
|
||||
|
||||
services.TryAddSingleton<ISecretStore>(sp => sp.GetRequiredService<SqliteSecretStore>());
|
||||
|
||||
services.TryAddSingleton<ISecretReplicator, NoOpSecretReplicator>();
|
||||
|
||||
// ONE shared resolver instance backs both the read seam (ISecretResolver) and the
|
||||
@@ -81,10 +87,16 @@ public static class SecretsServiceCollectionExtensions
|
||||
services.TryAddSingleton<ISecretCacheInvalidator>(sp => sp.GetRequiredService<DefaultSecretResolver>());
|
||||
|
||||
// Migrator: singleton, constructed from the already-registered connection factory. Needed
|
||||
// before any store operations so the schema exists.
|
||||
// before any store operations so the schema exists. Registered BOTH concretely (for callers
|
||||
// that want the SQLite migrator specifically) and behind the ISecretsStoreMigrator seam that
|
||||
// the hosted service and CLI consume — a replication package that swaps the store registers
|
||||
// its own ISecretsStoreMigrator BEFORE calling AddZbSecrets, and TryAdd leaves it in place.
|
||||
services.TryAddSingleton(sp =>
|
||||
new SqliteSecretsStoreMigrator(sp.GetRequiredService<SecretsSqliteConnectionFactory>()));
|
||||
|
||||
services.TryAddSingleton<ISecretsStoreMigrator>(sp =>
|
||||
sp.GetRequiredService<SqliteSecretsStoreMigrator>());
|
||||
|
||||
// Hosted service that runs migrations on startup when SecretsOptions.RunMigrationsOnStartup.
|
||||
services.AddHostedService<SecretsMigrationHostedService>();
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replication;
|
||||
|
||||
/// <summary>
|
||||
/// Decorates a local <see cref="ISecretStore"/> so every successful local write is published to
|
||||
/// peers through <see cref="ISecretReplicator"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is what turns the <see cref="ISecretReplicator"/> seam from a declaration into behaviour.
|
||||
/// Decorating the store — rather than calling the replicator from each write site — means the CLI,
|
||||
/// the Blazor UI, and any future caller all replicate automatically, and no new write path can
|
||||
/// forget to.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Publishing is best-effort.</b> A transport failure is logged and swallowed: the local write is
|
||||
/// already durable, and refusing it because a peer was unreachable would make every node a single
|
||||
/// point of failure for every other. Propagation is then guaranteed by the transport's periodic
|
||||
/// anti-entropy sweep instead, which is why every transport in this library reconciles
|
||||
/// <em>bidirectionally</em> — a dropped publish is repaired by the next sweep rather than lost.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <see cref="ApplyReplicatedAsync"/> deliberately does not publish (that would echo a peer's row
|
||||
/// straight back and, across three or more nodes, never stop). Neither does
|
||||
/// <see cref="ApplyRewrapAsync"/>: a re-wrap does not bump the revision, so it is invisible to
|
||||
/// last-writer-wins by design and a peer would discard it — KEK rotation is run once per independent
|
||||
/// store instead.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="inner">The local store doing the actual persistence.</param>
|
||||
/// <param name="replicator">The transport that publishes rows to peers.</param>
|
||||
/// <param name="logger">Logger for publish failures.</param>
|
||||
public sealed class ReplicatingSecretStore(
|
||||
ISecretStore inner,
|
||||
ISecretReplicator replicator,
|
||||
ILogger<ReplicatingSecretStore> logger) : ISecretStore
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
|
||||
inner.GetAsync(name, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
|
||||
inner.ListAsync(includeDeleted, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
||||
inner.GetManifestAsync(ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) =>
|
||||
inner.ApplyReplicatedAsync(row, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<bool> ApplyRewrapAsync(
|
||||
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
||||
inner.ApplyRewrapAsync(rewrappedRow, expectedCurrentWrappedDek, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertAsync(StoredSecret row, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
await inner.UpsertAsync(row, ct).ConfigureAwait(false);
|
||||
await PublishCurrentAsync(row.Name, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
|
||||
{
|
||||
bool deleted = await inner.DeleteAsync(name, actor, ct).ConfigureAwait(false);
|
||||
|
||||
if (deleted)
|
||||
{
|
||||
// The tombstone is the thing being replicated — a delete propagates as a newer row, not
|
||||
// as an absence, so peers converge on "deleted" instead of resurrecting it.
|
||||
await PublishCurrentAsync(name, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// Re-reads the row the store actually persisted and publishes THAT, never the caller's input:
|
||||
// the store assigns the revision and updated_utc, and those two fields are exactly what peers
|
||||
// order by. Publishing the caller's copy would broadcast a row whose ordering keys are wrong.
|
||||
private async Task PublishCurrentAsync(SecretName name, CancellationToken ct)
|
||||
{
|
||||
StoredSecret? persisted = await inner.GetAsync(name, ct).ConfigureAwait(false);
|
||||
|
||||
if (persisted is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await replicator.PublishAsync(persisted, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Deliberately swallowed — see the class remarks. Anti-entropy repairs it.
|
||||
logger.LogWarning(
|
||||
ex,
|
||||
"Failed to publish secret {SecretName} to peers; the local write succeeded and " +
|
||||
"anti-entropy will propagate it on the next sweep.",
|
||||
name.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Replication;
|
||||
|
||||
/// <summary>
|
||||
/// Transport-agnostic anti-entropy: compares a peer's manifest against the local store, pulls only
|
||||
/// the rows the peer holds a strictly newer copy of, and applies them last-writer-wins.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the half of replication that is pure logic, deliberately separated from <em>how</em> the
|
||||
/// peer is reached. The SQL-Server package points it at a shared hub database; the Akka package
|
||||
/// points it at a cluster peer over pub/sub. Both get identical convergence behaviour, and the
|
||||
/// interesting edge cases (ties, tombstones, a peer that is behind) are testable offline without a
|
||||
/// database or a cluster.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Reconciliation is a pull, never a push: it only ever writes rows the peer says are newer, so
|
||||
/// running it against a peer that is behind is a no-op rather than a regression. It is also safe to
|
||||
/// run concurrently with live replication — <see cref="ISecretStore.ApplyReplicatedAsync"/> re-checks
|
||||
/// the same predicate under its own transaction, so a row that a broadcast delivered first is simply
|
||||
/// ignored the second time.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="local">The local store to bring up to date.</param>
|
||||
/// <param name="cacheInvalidator">
|
||||
/// Optional resolver-cache seam. When supplied, every applied name is evicted so a node picks up a
|
||||
/// peer's write within the reconcile interval instead of serving a stale plaintext for the rest of
|
||||
/// the cache TTL.
|
||||
/// </param>
|
||||
/// <param name="onRowFailed">
|
||||
/// Optional callback invoked when a single replicated row could not be applied. The row is skipped
|
||||
/// and the rest of the batch proceeds; without a callback the failure is silent, so transports
|
||||
/// should supply one that logs.
|
||||
/// </param>
|
||||
public sealed class SecretReplicationReconciler(
|
||||
ISecretStore local,
|
||||
ISecretCacheInvalidator? cacheInvalidator = null,
|
||||
Action<SecretName, Exception>? onRowFailed = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns the names for which <paramref name="remote"/> holds a strictly newer row than
|
||||
/// <paramref name="localManifest"/> — including names absent locally, and including tombstones
|
||||
/// (a delete propagates as a newer row, not as an absence).
|
||||
/// </summary>
|
||||
/// <param name="localManifest">The local store's manifest.</param>
|
||||
/// <param name="remote">The peer's manifest.</param>
|
||||
/// <returns>The names to pull from the peer, in manifest order.</returns>
|
||||
public static IReadOnlyList<SecretName> ComputePullSet(
|
||||
IReadOnlyList<SecretManifestEntry> localManifest,
|
||||
IReadOnlyList<SecretManifestEntry> remote)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(localManifest);
|
||||
ArgumentNullException.ThrowIfNull(remote);
|
||||
|
||||
Dictionary<string, SecretManifestEntry> localByName =
|
||||
localManifest.ToDictionary(entry => entry.Name.Value, StringComparer.Ordinal);
|
||||
|
||||
var pull = new List<SecretName>();
|
||||
|
||||
foreach (SecretManifestEntry remoteEntry in remote)
|
||||
{
|
||||
// Absent locally, or the peer's copy wins LWW. A name we hold and the peer does not is
|
||||
// deliberately NOT handled here: this side pulls, the peer's own reconcile pushes.
|
||||
if (!localByName.TryGetValue(remoteEntry.Name.Value, out SecretManifestEntry? localEntry) ||
|
||||
SecretLastWriterWins.IsNewer(remoteEntry, localEntry))
|
||||
{
|
||||
pull.Add(remoteEntry.Name);
|
||||
}
|
||||
}
|
||||
|
||||
return pull;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the names the local store holds a strictly newer row for than <paramref name="remote"/>
|
||||
/// — the mirror of <see cref="ComputePullSet"/>, used to push local writes a peer is missing.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every transport here reconciles in <b>both</b> directions, and this is why: publishing a write
|
||||
/// is best-effort (see <c>ReplicatingSecretStore</c>), so without a push sweep a write made while
|
||||
/// a peer was unreachable would sit on one node forever — the peer never learns the name exists,
|
||||
/// so its own pull can never ask for it.
|
||||
/// </remarks>
|
||||
/// <param name="localManifest">The local store's manifest.</param>
|
||||
/// <param name="remote">The peer's manifest.</param>
|
||||
/// <returns>The names to push to the peer.</returns>
|
||||
public static IReadOnlyList<SecretName> ComputePushSet(
|
||||
IReadOnlyList<SecretManifestEntry> localManifest,
|
||||
IReadOnlyList<SecretManifestEntry> remote) =>
|
||||
// Exactly the pull computation with the two sides swapped: "what does the peer need from me"
|
||||
// is "what would I need, if I were the peer".
|
||||
ComputePullSet(remote, localManifest);
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles the local store against <paramref name="remoteManifest"/>, fetching and applying
|
||||
/// every row the peer holds a newer copy of.
|
||||
/// </summary>
|
||||
/// <param name="remoteManifest">The peer's manifest.</param>
|
||||
/// <param name="fetchAsync">
|
||||
/// Fetches the encrypted rows for the requested names from the peer. May return fewer rows than
|
||||
/// requested (a row deleted between manifest and fetch); extra rows are applied normally.
|
||||
/// </param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The number of rows applied locally.</returns>
|
||||
public async Task<int> ReconcileAsync(
|
||||
IReadOnlyList<SecretManifestEntry> remoteManifest,
|
||||
Func<IReadOnlyList<SecretName>, CancellationToken, Task<IReadOnlyList<StoredSecret>>> fetchAsync,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(remoteManifest);
|
||||
ArgumentNullException.ThrowIfNull(fetchAsync);
|
||||
|
||||
IReadOnlyList<SecretManifestEntry> localManifest =
|
||||
await local.GetManifestAsync(ct).ConfigureAwait(false);
|
||||
|
||||
IReadOnlyList<SecretName> pull = ComputePullSet(localManifest, remoteManifest);
|
||||
|
||||
if (pull.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int applied = 0;
|
||||
|
||||
// Chunked, not fetched in one shot. A cold-starting node's pull set is the peer's ENTIRE
|
||||
// inventory, and an unbounded fetch breaks on real backends — SQL Server caps a command at
|
||||
// 2100 parameters, so a hub with more than that many secrets would fail the same way every
|
||||
// interval and the node would never receive a single row. Chunking also bounds the size of
|
||||
// one replication message on the wire.
|
||||
foreach (IReadOnlyList<SecretName> batch in Chunk(pull, FetchBatchSize))
|
||||
{
|
||||
IReadOnlyList<StoredSecret> rows = await fetchAsync(batch, ct).ConfigureAwait(false);
|
||||
applied += await ApplyAsync(rows, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return applied;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Names fetched per round trip. Well under SQL Server's 2100-parameter command limit, and small
|
||||
/// enough that one replication message stays a sane size.
|
||||
/// </summary>
|
||||
internal const int FetchBatchSize = 500;
|
||||
|
||||
private static IEnumerable<IReadOnlyList<SecretName>> Chunk(
|
||||
IReadOnlyList<SecretName> names, int size)
|
||||
{
|
||||
for (int offset = 0; offset < names.Count; offset += size)
|
||||
{
|
||||
yield return [.. names.Skip(offset).Take(size)];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the local rows for <paramref name="names"/>, skipping any that vanished between the
|
||||
/// manifest and the read. Used by transports to assemble a push batch.
|
||||
/// </summary>
|
||||
/// <param name="names">The names to read.</param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The encrypted rows that still exist locally.</returns>
|
||||
public async Task<IReadOnlyList<StoredSecret>> ReadLocalAsync(
|
||||
IReadOnlyList<SecretName> names, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(names);
|
||||
|
||||
var rows = new List<StoredSecret>(names.Count);
|
||||
|
||||
foreach (SecretName name in names)
|
||||
{
|
||||
StoredSecret? row = await local.GetAsync(name, ct).ConfigureAwait(false);
|
||||
|
||||
if (row is not null)
|
||||
{
|
||||
rows.Add(row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies encrypted rows received from a peer, last-writer-wins, evicting the resolver cache for
|
||||
/// each. Used both by <see cref="ReconcileAsync"/> and by transports that receive live broadcasts.
|
||||
/// </summary>
|
||||
/// <param name="rows">The encrypted rows to apply.</param>
|
||||
/// <param name="ct">A token to cancel the operation.</param>
|
||||
/// <returns>The number of rows passed to the store.</returns>
|
||||
public async Task<int> ApplyAsync(IReadOnlyList<StoredSecret> rows, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
|
||||
int applied = 0;
|
||||
|
||||
foreach (StoredSecret row in rows)
|
||||
{
|
||||
try
|
||||
{
|
||||
await local.ApplyReplicatedAsync(row, ct).ConfigureAwait(false);
|
||||
applied++;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Isolated per row, because a batch is not a transaction and must not be all-or-
|
||||
// nothing. One row a peer cannot supply cleanly (a hostile or buggy peer, a
|
||||
// constraint violation) would otherwise abandon every row after it — and since the
|
||||
// pull set is recomputed identically each sweep, in the same manifest order, the
|
||||
// SAME row would poison the SAME batch forever and every name behind it would never
|
||||
// converge. Drop the row, keep the batch.
|
||||
onRowFailed?.Invoke(row.Name, ex);
|
||||
continue;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Evicted unconditionally: the store may have ignored the row as stale, but an
|
||||
// eviction only costs the next resolve a re-read, whereas skipping one when the row
|
||||
// DID land serves a stale plaintext until the TTL expires.
|
||||
cacheInvalidator?.Invalidate(row.Name);
|
||||
}
|
||||
}
|
||||
|
||||
return applied;
|
||||
}
|
||||
}
|
||||
@@ -200,12 +200,9 @@ public sealed class SqliteSecretStore(SecretsSqliteConnectionFactory connectionF
|
||||
DateTimeOffset localUpdated = ParseUtc(reader.GetString(0));
|
||||
long localRevision = reader.GetInt64(1);
|
||||
|
||||
// Incoming wins only if strictly newer by (updated_utc, then revision).
|
||||
bool incomingIsNewer =
|
||||
row.UpdatedUtc > localUpdated ||
|
||||
(row.UpdatedUtc == localUpdated && row.Revision > localRevision);
|
||||
|
||||
if (!incomingIsNewer)
|
||||
// Incoming wins only if strictly newer by (updated_utc, then revision). Routed through
|
||||
// the shared predicate so every store agrees on ties — see SecretLastWriterWins.
|
||||
if (!SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, localUpdated, localRevision))
|
||||
{
|
||||
await transaction.CommitAsync(ct).ConfigureAwait(false);
|
||||
return;
|
||||
|
||||
@@ -10,6 +10,7 @@ namespace ZB.MOM.WW.Secrets.Sqlite;
|
||||
/// newer than this build supports.
|
||||
/// </summary>
|
||||
public sealed class SqliteSecretsStoreMigrator(SecretsSqliteConnectionFactory connectionFactory)
|
||||
: ISecretsStoreMigrator
|
||||
{
|
||||
/// <summary>Applies the schema migration to the secret store.</summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="ZB.MOM.WW.Audit" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user