|
|
|
@@ -0,0 +1,284 @@
|
|
|
|
|
using Microsoft.Extensions.Hosting;
|
|
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
|
using Microsoft.Extensions.Options;
|
|
|
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
|
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
|
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
|
|
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The outcome of one <see cref="DeployedSecretsMirror.ReconcileAsync"/> pass: the secret
|
|
|
|
|
/// names applied (inserted or updated in this node's own secret store), the names removed
|
|
|
|
|
/// (deleted because central tombstoned them and this node had previously mirrored them), and
|
|
|
|
|
/// a human-readable failure message per live secret that did not decrypt on this node.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="Applied">Names applied (inserted or updated) this pass.</param>
|
|
|
|
|
/// <param name="Removed">Names deleted this pass because they were previously mirrored and are now tombstoned upstream.</param>
|
|
|
|
|
/// <param name="Failed">One message per live secret that failed to decrypt on this node — never includes secret values.</param>
|
|
|
|
|
public sealed record MirrorResult(
|
|
|
|
|
IReadOnlyList<string> Applied, IReadOnlyList<string> Removed, IReadOnlyList<string> Failed)
|
|
|
|
|
{
|
|
|
|
|
/// <summary>Whether this pass applied or removed anything (used to decide the summary log level).</summary>
|
|
|
|
|
public bool Changed => Applied.Count > 0 || Removed.Count > 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Per-node reconcile of the site-wide, LocalDb-replicated <c>deployed_secrets</c> table into
|
|
|
|
|
/// THIS node's own <c>ZB.MOM.WW.Secrets</c> store. Design §3.5 step 2.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <remarks>
|
|
|
|
|
/// <para>
|
|
|
|
|
/// Runs on EVERY site node — not a singleton — because <c>deployed_secrets</c> row content
|
|
|
|
|
/// (ciphertext) is site-wide (replicated by LocalDb), but which names THIS node has actually
|
|
|
|
|
/// applied into its own local secret store is node-local fact: <c>mirrored_secrets</c> is
|
|
|
|
|
/// deliberately NOT one of LocalDb's replicated tables (see
|
|
|
|
|
/// <see cref="SiteStorageService.GetMirroredSecretsAsync"/>). Each node reconciles for
|
|
|
|
|
/// itself, and — because the two nodes' secret stores are otherwise independent — each node
|
|
|
|
|
/// also independently verifies that the deployed ciphertext actually decrypts under ITS OWN
|
|
|
|
|
/// master key.
|
|
|
|
|
/// </para>
|
|
|
|
|
/// <para>
|
|
|
|
|
/// <b>Apply semantics (<see cref="ISecretStore.ApplyReplicatedAsync"/>).</b> The store applies
|
|
|
|
|
/// last-writer-wins on (<c>UpdatedUtc</c>, <c>Revision</c>) — see
|
|
|
|
|
/// <c>ZB.MOM.WW.Secrets.Abstractions.SecretLastWriterWins</c> — and a row that is not
|
|
|
|
|
/// <em>strictly</em> newer than the local row is silently ignored. Consequence worth knowing:
|
|
|
|
|
/// if an operator ever creates an out-of-band secret directly in a site node's local store
|
|
|
|
|
/// under the SAME name as a deployed secret, and stamps it with a later <c>UpdatedUtc</c> (or
|
|
|
|
|
/// an equal <c>UpdatedUtc</c> and a higher <c>Revision</c>) than the deployed row central last
|
|
|
|
|
/// shipped, that out-of-band secret WINS locally — every subsequent reconcile pass's
|
|
|
|
|
/// <c>ApplyReplicatedAsync</c> call for that name is a silent no-op until central ships a
|
|
|
|
|
/// still-newer revision. This mirror does not attempt to detect or warn about that case (it
|
|
|
|
|
/// has no way to distinguish "already applied, unchanged" from "out-of-band override in
|
|
|
|
|
/// place" — both look like "nothing to do" from here); deliberately not adding that
|
|
|
|
|
/// complexity per the design's scope.
|
|
|
|
|
/// </para>
|
|
|
|
|
/// <para>
|
|
|
|
|
/// <b>Resolve semantics and the resolver cache.</b> The trial-resolve verify step goes through
|
|
|
|
|
/// <see cref="ISecretResolver.GetAsync"/>, which the concrete resolver caches for a TTL
|
|
|
|
|
/// (<c>ResolveCacheTtl</c>, 30 s in the shipped implementation). If this node's master key
|
|
|
|
|
/// changes (e.g. a KEK rotation) between an earlier successful resolve and a reconcile pass
|
|
|
|
|
/// run within the cache TTL, the verify can return the STALE cached plaintext rather than
|
|
|
|
|
/// re-decrypting — that is still a successful decrypt of <em>a</em> value under the key that
|
|
|
|
|
/// produced the cache entry, which is all this check is trying to prove (this node's master
|
|
|
|
|
/// key can decrypt what central most recently shipped), so a cache hit is an acceptable false
|
|
|
|
|
/// negative for the "did the key change" question and not a bug in this mirror.
|
|
|
|
|
/// </para>
|
|
|
|
|
/// <para>
|
|
|
|
|
/// <b>Error-log de-spam.</b> A secret that keeps failing to decrypt would otherwise log an
|
|
|
|
|
/// Error line every reconcile pass forever. This mirror logs Error only the FIRST time a given
|
|
|
|
|
/// (name, revision, kekId) tuple fails on this node; identical repeats log at Debug instead.
|
|
|
|
|
/// The moment that same name resolves successfully (a fixed master key, or a newer revision
|
|
|
|
|
/// central re-shipped), the remembered tuple is cleared, so a LATER failure — even for the
|
|
|
|
|
/// exact same revision/kekId, e.g. a flapping key — logs at Error again. This state is
|
|
|
|
|
/// in-memory only (per node process), which matches the ledger's own node-local scope.
|
|
|
|
|
/// </para>
|
|
|
|
|
/// </remarks>
|
|
|
|
|
public sealed class DeployedSecretsMirror : BackgroundService
|
|
|
|
|
{
|
|
|
|
|
private readonly SiteStorageService _storage;
|
|
|
|
|
private readonly ISecretStore _store;
|
|
|
|
|
private readonly ISecretResolver _resolver;
|
|
|
|
|
private readonly IOptions<SiteRuntimeOptions> _options;
|
|
|
|
|
private readonly ILogger<DeployedSecretsMirror> _logger;
|
|
|
|
|
private readonly TimeProvider _time;
|
|
|
|
|
|
|
|
|
|
private readonly SemaphoreSlim _gate = new(1, 1);
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Per-secret-name record of the (Revision, KekId) tuple whose decrypt failure this node
|
|
|
|
|
/// has already logged at Error. Cleared for a name the moment it resolves successfully, so
|
|
|
|
|
/// a later failure — even an identical one — logs at Error again. In-memory only; not a
|
|
|
|
|
/// durable ledger, and deliberately so (it exists purely to de-spam this process's logs).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private readonly Dictionary<string, (long Revision, string KekId)> _lastLoggedFailure = new(StringComparer.Ordinal);
|
|
|
|
|
|
|
|
|
|
/// <summary>Initializes a new instance of <see cref="DeployedSecretsMirror"/>.</summary>
|
|
|
|
|
/// <param name="storage">Site-local storage exposing <c>deployed_secrets</c> and the <c>mirrored_secrets</c> ledger.</param>
|
|
|
|
|
/// <param name="store">This node's own secret store.</param>
|
|
|
|
|
/// <param name="resolver">This node's own secret resolver, used to trial-decrypt each deployed secret.</param>
|
|
|
|
|
/// <param name="options">Supplies <see cref="SiteRuntimeOptions.DeployedSecretsMirrorInterval"/>.</param>
|
|
|
|
|
/// <param name="logger">Logger instance. Never logs secret values — names, revisions, and KEK ids only.</param>
|
|
|
|
|
/// <param name="time">Clock driving the reconcile loop's <see cref="PeriodicTimer"/>.</param>
|
|
|
|
|
public DeployedSecretsMirror(
|
|
|
|
|
SiteStorageService storage,
|
|
|
|
|
ISecretStore store,
|
|
|
|
|
ISecretResolver resolver,
|
|
|
|
|
IOptions<SiteRuntimeOptions> options,
|
|
|
|
|
ILogger<DeployedSecretsMirror> logger,
|
|
|
|
|
TimeProvider time)
|
|
|
|
|
{
|
|
|
|
|
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
|
|
|
|
_store = store ?? throw new ArgumentNullException(nameof(store));
|
|
|
|
|
_resolver = resolver ?? throw new ArgumentNullException(nameof(resolver));
|
|
|
|
|
_options = options ?? throw new ArgumentNullException(nameof(options));
|
|
|
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
|
|
|
_time = time ?? throw new ArgumentNullException(nameof(time));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Runs one reconcile pass: applies every live <c>deployed_secrets</c> row into this
|
|
|
|
|
/// node's own secret store, deletes rows this node previously mirrored that are now
|
|
|
|
|
/// tombstoned upstream, and trial-decrypts every live row to verify it resolves under
|
|
|
|
|
/// this node's own master key. Safe to call concurrently with itself (and with the
|
|
|
|
|
/// hosted loop) — a single in-flight pass is enforced internally.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="ct">A token to cancel the operation.</param>
|
|
|
|
|
/// <returns>The pass's <see cref="MirrorResult"/>.</returns>
|
|
|
|
|
public async Task<MirrorResult> ReconcileAsync(CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
await _gate.WaitAsync(ct).ConfigureAwait(false);
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var rows = await _storage.GetDeployedSecretsAsync().ConfigureAwait(false);
|
|
|
|
|
var mirrored = (await _storage.GetMirroredSecretsAsync().ConfigureAwait(false))
|
|
|
|
|
.ToDictionary(m => m.Name, m => m.Revision, StringComparer.Ordinal);
|
|
|
|
|
|
|
|
|
|
List<string> applied = [];
|
|
|
|
|
List<string> removed = [];
|
|
|
|
|
List<string> failed = [];
|
|
|
|
|
|
|
|
|
|
foreach (var row in rows)
|
|
|
|
|
{
|
|
|
|
|
if (row.IsRemoved)
|
|
|
|
|
{
|
|
|
|
|
if (!mirrored.ContainsKey(row.Name))
|
|
|
|
|
{
|
|
|
|
|
// Never mirrored by this node (or already un-mirrored) — an
|
|
|
|
|
// out-of-band, same-named local secret is left alone.
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await _store.DeleteAsync(new SecretName(row.Name), "deployed-secrets-mirror", ct)
|
|
|
|
|
.ConfigureAwait(false);
|
|
|
|
|
await _storage.RemoveMirroredSecretAsync(row.Name).ConfigureAwait(false);
|
|
|
|
|
removed.Add(row.Name);
|
|
|
|
|
_lastLoggedFailure.Remove(row.Name);
|
|
|
|
|
_logger.LogInformation(
|
|
|
|
|
"Removed deployed secret {SecretName} (no longer referenced by this site)",
|
|
|
|
|
row.Name);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!mirrored.TryGetValue(row.Name, out var mirroredRevision)
|
|
|
|
|
|| mirroredRevision != row.Secret.Revision)
|
|
|
|
|
{
|
|
|
|
|
await _store.ApplyReplicatedAsync(ToStored(row.Secret), ct).ConfigureAwait(false);
|
|
|
|
|
await _storage.SetMirroredSecretAsync(row.Name, row.Secret.Revision).ConfigureAwait(false);
|
|
|
|
|
applied.Add(row.Name);
|
|
|
|
|
_logger.LogInformation(
|
|
|
|
|
"Applied deployed secret {SecretName} r{Revision} (kek {KekId})",
|
|
|
|
|
row.Name, row.Secret.Revision, row.Secret.KekId);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await VerifyAsync(row, failed, ct).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var result = new MirrorResult(applied, removed, failed);
|
|
|
|
|
|
|
|
|
|
if (result.Changed)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogInformation(
|
|
|
|
|
"Deployed secrets mirror: {Applied} applied, {Removed} removed, {Failed} failed",
|
|
|
|
|
applied.Count, removed.Count, failed.Count);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
_logger.LogDebug(
|
|
|
|
|
"Deployed secrets mirror in sync ({Count} secret(s), {FailedCount} failing to decrypt)",
|
|
|
|
|
rows.Count(r => !r.IsRemoved), failed.Count);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
|
_gate.Release();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Trial-resolves <paramref name="row"/>'s secret through <see cref="ISecretResolver"/> to
|
|
|
|
|
/// verify it decrypts on this node. On failure, appends a message to
|
|
|
|
|
/// <paramref name="failed"/> and logs — Error the first time this exact (name, revision,
|
|
|
|
|
/// kekId) has failed since it last resolved successfully on this node, Debug on every
|
|
|
|
|
/// repeat (see the class remarks on error-log de-spam).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task VerifyAsync(DeployedSecretRow row, List<string> failed, CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
bool resolved;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
resolved = await _resolver.GetAsync(new SecretName(row.Name), ct).ConfigureAwait(false) is not null;
|
|
|
|
|
}
|
|
|
|
|
catch (OperationCanceledException)
|
|
|
|
|
{
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
catch (Exception)
|
|
|
|
|
{
|
|
|
|
|
resolved = false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (resolved)
|
|
|
|
|
{
|
|
|
|
|
_lastLoggedFailure.Remove(row.Name);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var message =
|
|
|
|
|
$"Secret '{row.Name}' (kek {row.Secret.KekId}) did not decrypt on this node — " +
|
|
|
|
|
"the node's master key differs from central's";
|
|
|
|
|
failed.Add(message);
|
|
|
|
|
|
|
|
|
|
var current = (row.Secret.Revision, row.Secret.KekId);
|
|
|
|
|
if (_lastLoggedFailure.TryGetValue(row.Name, out var previous) && previous == current)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogDebug("{Failure}", message);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
_lastLoggedFailure[row.Name] = current;
|
|
|
|
|
_logger.LogError("{Failure}", message);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
|
|
|
{
|
|
|
|
|
using var timer = new PeriodicTimer(_options.Value.DeployedSecretsMirrorInterval, _time);
|
|
|
|
|
do
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
await ReconcileAsync(stoppingToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
catch (OperationCanceledException)
|
|
|
|
|
{
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogWarning(ex, "Deployed secrets mirror pass failed; retrying next interval.");
|
|
|
|
|
}
|
|
|
|
|
} while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Maps a wire <see cref="SecretArtifact"/> row back to the store's <see cref="StoredSecret"/> shape.</summary>
|
|
|
|
|
private static StoredSecret ToStored(SecretArtifact secret) => new()
|
|
|
|
|
{
|
|
|
|
|
Name = new SecretName(secret.Name),
|
|
|
|
|
Description = secret.Description,
|
|
|
|
|
ContentType = Enum.Parse<SecretContentType>(secret.ContentType, ignoreCase: false),
|
|
|
|
|
Ciphertext = secret.Ciphertext,
|
|
|
|
|
Nonce = secret.Nonce,
|
|
|
|
|
Tag = secret.Tag,
|
|
|
|
|
WrappedDek = secret.WrappedDek,
|
|
|
|
|
WrapNonce = secret.WrapNonce,
|
|
|
|
|
WrapTag = secret.WrapTag,
|
|
|
|
|
KekId = secret.KekId,
|
|
|
|
|
Revision = secret.Revision,
|
|
|
|
|
CreatedUtc = secret.CreatedUtc,
|
|
|
|
|
UpdatedUtc = secret.UpdatedUtc,
|
|
|
|
|
};
|
|
|
|
|
}
|
Joseph Doherty