using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
///
/// The outcome of one pass: the secret
/// names applied (inserted or updated in this node's own secret store), the names removed
/// (deleted because central tombstoned them and this node had previously mirrored them), and
/// a human-readable failure message per live secret that could not be applied and/or verified
/// on this node.
///
/// Names applied (inserted or updated) this pass.
/// Names deleted this pass because they were previously mirrored and are now tombstoned upstream.
/// One message per live secret that failed to apply and/or decrypt on this node — never includes secret values.
///
/// The secret name behind each entry, in the SAME order (index-for-
/// index — each pass reports at most one failure per row, so the two lists always stay in
/// lockstep). Added for DeploymentManagerActor.HandleDeployArtifacts (Task 10): this
/// mirror reconciles the WHOLE site-wide deployed_secrets table every pass, not only the
/// names a particular deploy just shipped, so a caller that needs to know whether a failure
/// belongs to ITS OWN shipped names — as opposed to some other, previously-deployed secret, or a
/// removal of a now-unreferenced one — cannot recover that from the message strings alone.
///
public sealed record MirrorResult(
IReadOnlyList Applied,
IReadOnlyList Removed,
IReadOnlyList Failed,
IReadOnlyList FailedNames)
{
/// Whether this pass applied or removed anything (used to decide the summary log level).
public bool Changed => Applied.Count > 0 || Removed.Count > 0;
}
///
/// Per-node reconcile of the site-wide, LocalDb-replicated deployed_secrets table into
/// THIS node's own ZB.MOM.WW.Secrets store. Design §3.5 step 2.
///
///
///
/// Runs on EVERY site node — not a singleton — because deployed_secrets row content
/// (ciphertext) is site-wide (replicated by LocalDb), but which names THIS node has actually
/// applied into its own local secret store is node-local fact: mirrored_secrets is
/// deliberately NOT one of LocalDb's replicated tables (see
/// ). Each node reconciles for
/// itself, and — because the two nodes' secret stores are otherwise independent — each node
/// also independently verifies that the deployed ciphertext actually decrypts under ITS OWN
/// master key.
///
///
/// Apply semantics and the confirm-then-ledger fix (fix round 1, review Important #1).
/// applies last-writer-wins strictly on
/// (UpdatedUtc, Revision) — see
/// — and a row that is not
/// strictly newer than the local row is silently ignored, so this mirror never trusts the call
/// blindly: every apply is followed by an read-back, and
/// mirrored_secrets is ledgered only when the local row is confirmed LIVE and its
/// ciphertext (Ciphertext + WrappedDek) matches what was just shipped. A
/// non-ledgered outcome is reported as a entry (never as the
/// decrypt-failure message, which would misdiagnose either case below) and never overwritten a
/// second way. Which of the two outcomes applies is decided by ledger ownership — whether
/// mirrored_secrets already has ANY entry for the name, at any revision — not merely by
/// whether the local row happens to be a tombstone:
///
/// -
/// This node's own ledger already owns the name (a tombstone this node wrote, or a LIVE
/// row with different ciphertext than what was just shipped — the latter can happen under
/// site/central clock skew, or when an earlier tombstone-recovery's bumped stamp lands ahead of
/// a since-shipped central UpdatedUtc — fix round 2, review "clock skew" finding).
/// Re-stamping is safe either way: neither UpdatedUtc nor Revision is bound into
/// the AEAD associated data — confirmed against
/// ZB.MOM.WW.Secrets.Crypto.AesGcmEnvelopeCipher, whose body layer binds only
/// SecretName.Value and whose wrap layer binds only KekId — so the mirror
/// re-applies the exact shipped ciphertext with UpdatedUtc = max(now, local.UpdatedUtc + 1
/// tick) and Revision = local.Revision + 1, which beats the current local row under
/// LWW without touching anything the cipher verifies. If that confirmed re-apply still does not
/// read back live and matching (a concurrent conflict), the row is reported failed instead of
/// silently left broken. A real-cipher integration test
/// (DeployedSecretsMirrorRealStackTests, fix round 2) pins that a bumped-and-recovered row
/// still decrypts to the original plaintext against the real SqliteSecretStore +
/// AesGcmEnvelopeCipher + DefaultSecretResolver stack, not only by source reading.
///
/// -
/// The name is NOT in this node's ledger and the local row is LIVE with different
/// ciphertext — a genuinely out-of-band secret created directly on this node under the same
/// name, stamped strictly newer than what central shipped. It wins LWW legitimately. This mirror
/// does NOT overwrite it, does NOT ledger it (so a later central tombstone of the name can never
/// delete someone's out-of-band secret via this mirror), and does NOT run the decrypt-verify
/// against it (verifying would trivially "succeed" against the operator's own value and mask
/// that the deployed revision never took effect).
///
///
/// Drift at the same revision (final review I1). A name already ledgered at the shipped
/// revision is still read back every pass: if an operator deleted it locally, or edited its value,
/// the local row no longer matches the shipped ciphertext and is healed exactly like the owned
/// re-stamp case above (bumped stamp, apply, read-back, cache invalidation) before the verify —
/// so a local delete is repaired rather than misreported as a master-key mismatch, and a local
/// edit is reverted to the deployed value.
/// Ledgering always uses the SHIPPED artifact's own revision (not any bumped local revision) so
/// a later pass's "is this ledgered at the shipped revision" check stays correct.
///
///
/// Per-row isolation (review Important #2). Each row's apply/verify body runs in its own
/// try/catch; an exception — an unrecognised ContentType from a newer central, a
/// per-row store fault, a malformed name — is caught, reported as a
/// entry naming the secret and the exception type/message (never a value), logged with the same
/// de-spam rule, and does not stop the rest of the pass. Only an exception attributable to the
/// caller's own cancellation (ct.IsCancellationRequested) propagates.
///
///
/// Cache invalidation (review Important #3). The concrete resolver
/// (DefaultSecretResolver) caches decrypted plaintext for a TTL and also implements
/// ; every ZB.MOM.WW.Secrets write path invalidates after a
/// write for exactly this reason. This mirror does the same: after every successful apply
/// (including a tombstone re-materialization) or delete, and before the verify step, it calls
/// (a null invalidator is tolerated for
/// direct construction; under DI AddZbSecrets always registers one) so neither the verify nor a script running immediately afterward
/// can observe a stale pre-rotation plaintext for up to the cache TTL.
///
///
/// Node-local store assumption (review Minor #5; Task 11 removed every replication mode).
/// This mirror assumes the site's is always plain, node-local, local
/// SQLite — the ONLY shape a Site node's store can be after Task 11 retired both the pull-only
/// gRPC secrets hub and the SQL-Server ReplicatingSecretStore mode that preceded it (see
/// SecretsRegistration). It would NOT have been safe to run this mirror against a store
/// decorated by either retired replication mode: DeleteAsync would have published the
/// tombstone to every other node sharing the decorator, so a site merely un-referencing a secret
/// would have deleted it fleet-wide. StartupValidator now rejects
/// Secrets:Replication:Enabled=true outright on either role, so that configuration can no
/// longer reach this mirror at all.
///
///
/// Resolve semantics and the resolver cache. The trial-resolve verify step goes through
/// , which the concrete resolver caches for a TTL
/// (ResolveCacheTtl, 30 s in the shipped implementation). Because every successful apply
/// or delete now invalidates that cache first (see above), the remaining exposure is narrow: a
/// KEK rotation on this node's OWN master key (not a secret rotation at central) landing inside
/// the cache window of an unrelated resolve. That residual case is still an acceptable false
/// negative for the "did the key change" question — the cache hit still proves this node once
/// decrypted a value with a key that produced that cache entry — and is not a bug in this
/// mirror.
///
///
/// Error-log de-spam. A secret that keeps failing (to apply OR to decrypt) would
/// otherwise log an Error line every reconcile pass forever. This mirror logs Error only the
/// FIRST time a given failure fingerprint (kind + revision + relevant detail, e.g. KekId for a
/// decrypt failure) occurs for a name on this node; identical repeats log at Debug instead. The
/// moment that same name applies/resolves successfully, the remembered fingerprint is cleared,
/// so a LATER failure — even one with an identical fingerprint — logs at Error again. This state
/// is in-memory only (per node process), which matches the ledger's own node-local scope.
///
///
/// Status row for central (review finding M1). A failure on the STANDBY node used to be
/// visible only in that node's own log — site health reports are sent by the active node only.
/// After each pass this mirror records its node's current failure set (secret name + message,
/// never a value) in the LocalDb-replicated secret_mirror_status table, keyed by node
/// name; SecretMirrorStatusReporter on the active node reads every node's row onto the
/// health report. Because every write replicates via CDC, the row is written only when the
/// failure set (compared as sorted name+message pairs) differs from what this process last
/// wrote — plus once on the first pass after start-up, so a restart refreshes a stale row. The
/// write is best-effort: a failure logs a Warning (repeats at Debug) and is retried on the next
/// pass, and never affects the pass's own .
///
///
public sealed class DeployedSecretsMirror : BackgroundService
{
private const string DeleteActor = "deployed-secrets-mirror";
private readonly SiteStorageService _storage;
private readonly ISecretStore _store;
private readonly ISecretResolver _resolver;
private readonly ISecretCacheInvalidator? _invalidator;
private readonly IOptions _options;
private readonly ILogger _logger;
private readonly TimeProvider _time;
private readonly string _nodeName;
///
/// The failure set (sorted name+message pairs) this process last wrote successfully to
/// secret_mirror_status, or before the first successful write
/// — which is what forces the once-per-start-up write (review finding M1).
///
private IReadOnlyList? _lastWrittenStatus;
/// Whether a failed status write has already been logged at Warning (de-spam).
private bool _statusWriteFailureLogged;
private readonly SemaphoreSlim _gate = new(1, 1);
private bool _disposed;
///
/// Per-secret-name record of the failure fingerprint this node has already logged at Error
/// (an apply-conflict or a decrypt failure — see the class remarks). Cleared for a name the
/// moment it applies/resolves successfully, so a later failure — even an identical one —
/// logs at Error again. In-memory only; not a durable ledger, and deliberately so (it exists
/// purely to de-spam this process's logs).
///
private readonly Dictionary _lastLoggedFailure = new(StringComparer.Ordinal);
///
/// Names whose same-revision local drift (final review I1) has already been logged at
/// Warning and not yet healed — a heal that keeps failing logs its drift at Debug on repeats,
/// matching the de-spam rule. Cleared on a successful apply.
///
private readonly HashSet _driftLogged = new(StringComparer.Ordinal);
/// Initializes a new instance of .
/// Site-local storage exposing deployed_secrets and the mirrored_secrets ledger.
/// This node's own secret store.
/// This node's own secret resolver, used to trial-decrypt each deployed secret.
///
/// Cache invalidator for (the concrete
/// DefaultSecretResolver implements this itself). The parameter has no default, so
/// under MS DI activation fails unless an is registered
/// — AddZbSecrets always registers one. A caller constructing the mirror directly may
/// pass , in which case no invalidation is attempted.
///
/// Supplies .
/// Logger instance. Never logs secret values — names, revisions, and KEK ids only.
/// Clock driving the reconcile loop's and the tombstone-recovery stamp.
///
/// Supplies this node's name (node-a/node-b), the key of its
/// secret_mirror_status row. Optional: when absent or unconfigured, the machine name
/// is used instead.
///
public DeployedSecretsMirror(
SiteStorageService storage,
ISecretStore store,
ISecretResolver resolver,
ISecretCacheInvalidator? invalidator,
IOptions options,
ILogger logger,
TimeProvider time,
INodeIdentityProvider? nodeIdentity = null)
{
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
_store = store ?? throw new ArgumentNullException(nameof(store));
_resolver = resolver ?? throw new ArgumentNullException(nameof(resolver));
_invalidator = invalidator;
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_time = time ?? throw new ArgumentNullException(nameof(time));
_nodeName = nodeIdentity?.NodeName ?? Environment.MachineName;
}
///
/// Runs one reconcile pass: applies every live deployed_secrets row into this
/// node's own secret store (confirmed by read-back before ledgering), deletes rows this
/// node previously mirrored that are now tombstoned upstream, and trial-decrypts every
/// live, confirmed-applied row to verify it resolves under this node's own master key.
/// Safe to call concurrently with itself (and with the hosted loop) — a single in-flight
/// pass is enforced internally.
///
/// A token to cancel the operation.
/// The pass's .
public async Task ReconcileAsync(CancellationToken ct)
{
await _gate.WaitAsync(ct).ConfigureAwait(false);
try
{
// Tolerant read (final review M4): a row whose row_json cannot be parsed is handed
// back separately instead of throwing out of the whole pass, so it is reported
// against its own name like any other per-row failure.
var (rows, unreadable) = await _storage.GetDeployedSecretsTolerantAsync().ConfigureAwait(false);
var mirrored = (await _storage.GetMirroredSecretsAsync().ConfigureAwait(false))
.ToDictionary(m => m.Name, m => m.Revision, StringComparer.Ordinal);
List applied = [];
List removed = [];
List failed = [];
List failedNames = [];
foreach (var row in rows)
{
try
{
if (row.IsRemoved)
{
await ProcessRemovedRowAsync(row.Name, mirrored, removed, ct).ConfigureAwait(false);
}
else
{
await ProcessLiveRowAsync(row, mirrored, applied, failed, failedNames, ct).ConfigureAwait(false);
}
}
catch (Exception ex) when (!ct.IsCancellationRequested)
{
ReportFailure(
row.Name,
$"error:{row.Secret.Revision}:{ex.GetType().Name}:{ex.Message}",
$"Secret '{row.Name}' failed to process on this node ({ex.GetType().Name}: {ex.Message})",
failed,
failedNames);
}
}
foreach (var bad in unreadable)
{
try
{
if (bad.IsRemoved)
{
// A removal needs only the name, so an unreadable tombstone is still
// honoured for a name this node previously mirrored.
await ProcessRemovedRowAsync(bad.Name, mirrored, removed, ct).ConfigureAwait(false);
}
else
{
ReportFailure(
bad.Name,
$"unreadable:{bad.Error}",
$"Secret '{bad.Name}' could not be read from deployed_secrets on this node ({bad.Error})",
failed,
failedNames);
}
}
catch (Exception ex) when (!ct.IsCancellationRequested)
{
ReportFailure(
bad.Name,
$"error:unreadable:{ex.GetType().Name}:{ex.Message}",
$"Secret '{bad.Name}' failed to process on this node ({ex.GetType().Name}: {ex.Message})",
failed,
failedNames);
}
}
var result = new MirrorResult(applied, removed, failed, failedNames);
if (result.Changed)
{
_logger.LogInformation(
"Deployed secrets mirror: {Applied} applied, {Removed} removed, {Failed} failed",
applied.Count, removed.Count, failed.Count);
}
else
{
_logger.LogDebug(
"Deployed secrets mirror in sync ({Count} secret(s), {FailedCount} failing)",
rows.Count(r => !r.IsRemoved) + unreadable.Count(u => !u.IsRemoved), failed.Count);
}
await RecordStatusIfChangedAsync(failedNames, failed).ConfigureAwait(false);
return result;
}
finally
{
_gate.Release();
}
}
///
/// Writes this node's secret_mirror_status row when the pass's failure set differs
/// from the last one this process wrote (or nothing has been written yet since start-up).
/// Best-effort: never throws, so the pass result is unaffected by a status-write fault.
/// See the class remarks (review finding M1).
///
private async Task RecordStatusIfChangedAsync(IReadOnlyList failedNames, IReadOnlyList failed)
{
var current = failedNames
.Select((name, i) => new SecretMirrorFailure(name, failed[i]))
.OrderBy(f => f.SecretName, StringComparer.Ordinal)
.ThenBy(f => f.Message, StringComparer.Ordinal)
.ToList();
if (_lastWrittenStatus is not null && _lastWrittenStatus.SequenceEqual(current))
{
return;
}
try
{
await _storage.SetSecretMirrorStatusAsync(_nodeName, current, _time.GetUtcNow()).ConfigureAwait(false);
_lastWrittenStatus = current;
_statusWriteFailureLogged = false;
}
catch (Exception ex)
{
_logger.Log(
_statusWriteFailureLogged ? LogLevel.Debug : LogLevel.Warning,
ex,
"Failed to record the deployed secrets mirror status for node {NodeName}; retrying next pass",
_nodeName);
_statusWriteFailureLogged = true;
}
}
///
/// Handles a tombstoned deployed_secrets row: deletes it from this node's store, but
/// only if this node had previously ledgered it as mirrored — an out-of-band, same-named
/// local secret this mirror never applied is left alone.
///
private async Task ProcessRemovedRowAsync(
string rowName, Dictionary mirrored, List removed, CancellationToken ct)
{
if (!mirrored.ContainsKey(rowName))
{
return;
}
await _store.DeleteAsync(new SecretName(rowName), DeleteActor, ct).ConfigureAwait(false);
_invalidator?.Invalidate(new SecretName(rowName));
await _storage.RemoveMirroredSecretAsync(rowName).ConfigureAwait(false);
removed.Add(rowName);
_lastLoggedFailure.Remove(rowName);
_logger.LogInformation(
"Removed deployed secret {SecretName} (no longer referenced by this site)",
rowName);
}
///
/// Handles a live deployed_secrets row: applies it (confirmed by read-back) when not
/// already ledgered at the shipped revision, or when it is ledgered but the local row has
/// since gone missing, been tombstoned or drifted to different ciphertext (healed via the
/// mirror-owned re-stamp path), then verifies it decrypts — unless the apply
/// outcome makes a verify meaningless (an out-of-band local secret won LWW), in which case a
/// single, distinct failure is reported instead.
///
private async Task ProcessLiveRowAsync(
DeployedSecretRow row,
Dictionary mirrored,
List applied,
List failed,
List failedNames,
CancellationToken ct)
{
var name = new SecretName(row.Name);
// "Owned by this mirror" is ANY ledger entry for the name, at whatever revision — not
// only a match on the shipped revision. Ownership, not revision equality, is what
// decides whether a live-but-mismatched local row is OUR stale stamp (safe to bump past)
// or someone else's out-of-band value (never touched) — see ApplyAndConfirmAsync and
// the class remarks (fix round 2, review Minor "clock skew").
var ownedByMirror = mirrored.TryGetValue(row.Name, out var mirroredRevision);
var alreadyLedgered = ownedByMirror && mirroredRevision == row.Secret.Revision;
// Ledgered at the shipped revision is NOT proof the local row still holds it (final
// review I1): an operator may have deleted or edited this mirror-owned name on the node
// since. Read it back; only a live row whose ciphertext matches the shipped artifact goes
// straight to verify. A missing, tombstoned or drifted row is healed through the same
// mirror-owned re-stamp path as any other owned mismatch — central is authoritative for
// every name it deploys — instead of being misreported as a master-key mismatch.
var needsApply = !alreadyLedgered;
if (alreadyLedgered)
{
var local = await _store.GetAsync(name, ct).ConfigureAwait(false);
if (local is null || local.IsDeleted || !MatchesShippedCiphertext(local, ToStored(row.Secret)))
{
needsApply = true;
_logger.Log(
_driftLogged.Add(row.Name) ? LogLevel.Warning : LogLevel.Debug,
"Deployed secret {SecretName} r{Revision} was {Drift} on this node; re-applying the deployed revision",
row.Name,
row.Secret.Revision,
local is null ? "missing" : local.IsDeleted ? "deleted" : "changed");
}
}
if (needsApply)
{
var outcome = await ApplyAndConfirmAsync(row, name, ownedByMirror, ct).ConfigureAwait(false);
switch (outcome)
{
case ApplyOutcome.Applied:
await _storage.SetMirroredSecretAsync(row.Name, row.Secret.Revision).ConfigureAwait(false);
_invalidator?.Invalidate(name);
applied.Add(row.Name);
_lastLoggedFailure.Remove(row.Name);
_driftLogged.Remove(row.Name);
_logger.LogInformation(
"Applied deployed secret {SecretName} r{Revision} (kek {KekId})",
row.Name, row.Secret.Revision, row.Secret.KekId);
break;
case ApplyOutcome.LocalNewer:
ReportFailure(
row.Name,
$"outofband:{row.Secret.Revision}",
$"Secret '{row.Name}' exists on this node with a newer local value; " +
"the deployed revision was not applied",
failed,
failedNames);
return; // Do not verify — that would decrypt the operator's own value.
case ApplyOutcome.Unconfirmed:
ReportFailure(
row.Name,
$"unconfirmed:{row.Secret.Revision}",
$"Secret '{row.Name}' could not be confirmed applied on this node " +
"after a re-apply attempt",
failed,
failedNames);
return;
}
}
await VerifyAsync(row, failed, failedNames, ct).ConfigureAwait(false);
}
/// The confirmed outcome of applying one shipped row into this node's store. See the class remarks.
private enum ApplyOutcome
{
/// The local row is now live and its ciphertext matches what was shipped.
Applied,
/// The local row is live with DIFFERENT ciphertext — an out-of-band secret that legitimately won LWW.
LocalNewer,
/// Neither of the above — a tombstone re-apply attempt did not read back as applied.
Unconfirmed,
}
///
/// Calls and confirms the outcome by reading
/// the row back — see the class remarks for why this mirror never trusts the call blindly.
///
/// The shipped row to apply.
/// The row's .
///
/// Whether this node's ledger already has ANY entry for this name (at any revision) — i.e.
/// this mirror, not an operator, is the one that put the current local row there. A
/// live-but-mismatched local row is only ever re-stamped (never reported as out-of-band)
/// when this is .
///
/// A token to cancel the operation.
private async Task ApplyAndConfirmAsync(
DeployedSecretRow row, SecretName name, bool ownedByMirror, CancellationToken ct)
{
var shipped = ToStored(row.Secret);
await _store.ApplyReplicatedAsync(shipped, ct).ConfigureAwait(false);
var local = await _store.GetAsync(name, ct).ConfigureAwait(false);
if (local is not null && !local.IsDeleted && MatchesShippedCiphertext(local, shipped))
{
return ApplyOutcome.Applied;
}
if (local is not null && (local.IsDeleted || ownedByMirror))
{
// Either a tombstone (this node deleted it, or some earlier apply left it that way),
// or a LIVE row that is nonetheless OURS — the ledger already has an entry for this
// name, so the mismatch is this mirror's own earlier stamp (e.g. a prior tombstone
// recovery's bumped UpdatedUtc landing ahead of a central UpdatedUtc that arrives
// later, under site/central clock skew — review "clock skew" finding), not an
// operator's out-of-band value. Both are safe to re-stamp: neither UpdatedUtc nor
// Revision is bound into the AEAD associated data (confirmed against
// AesGcmEnvelopeCipher — only SecretName and KekId are AAD), so re-stamping strictly
// past the current local row changes nothing the cipher verifies.
var nowUtc = _time.GetUtcNow();
var localUtcPlusOneTick = local.UpdatedUtc.AddTicks(1);
var bumpedUpdatedUtc = nowUtc > localUtcPlusOneTick ? nowUtc : localUtcPlusOneTick;
var bumped = shipped with { UpdatedUtc = bumpedUpdatedUtc, Revision = local.Revision + 1 };
await _store.ApplyReplicatedAsync(bumped, ct).ConfigureAwait(false);
var reconfirmed = await _store.GetAsync(name, ct).ConfigureAwait(false);
return reconfirmed is not null && !reconfirmed.IsDeleted && MatchesShippedCiphertext(reconfirmed, shipped)
? ApplyOutcome.Applied
: ApplyOutcome.Unconfirmed;
}
// local is live, NOT ours (never ledgered), and its ciphertext differs from what was
// shipped: a genuine out-of-band value with a strictly newer (UpdatedUtc, Revision) won
// LWW legitimately. (local is null only in a pathological concurrent-delete race;
// Unconfirmed covers that too.)
return local is null ? ApplyOutcome.Unconfirmed : ApplyOutcome.LocalNewer;
}
private static bool MatchesShippedCiphertext(StoredSecret local, StoredSecret shipped) =>
local.Ciphertext.AsSpan().SequenceEqual(shipped.Ciphertext)
&& local.WrappedDek.AsSpan().SequenceEqual(shipped.WrappedDek);
///
/// Trial-resolves 's secret through to
/// verify it decrypts on this node. On failure, appends a message to
/// and logs with the shared de-spam rule (see the class remarks).
///
private async Task VerifyAsync(
DeployedSecretRow row, List failed, List failedNames, CancellationToken ct)
{
bool resolved;
try
{
resolved = await _resolver.GetAsync(new SecretName(row.Name), ct).ConfigureAwait(false) is not null;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch (Exception)
{
resolved = false;
}
if (resolved)
{
_lastLoggedFailure.Remove(row.Name);
return;
}
var message =
$"Secret '{row.Name}' (kek {row.Secret.KekId}) did not decrypt on this node — " +
"the node's master key differs from central's";
ReportFailure(row.Name, $"decrypt:{row.Secret.Revision}:{row.Secret.KekId}", message, failed, failedNames);
}
///
/// Appends to (and
/// to , in lockstep) and logs it — Error the first time
/// is seen for since it last
/// succeeded, Debug on every identical repeat.
///
private void ReportFailure(
string name, string fingerprint, string message, List failed, List failedNames)
{
failed.Add(message);
failedNames.Add(name);
if (_lastLoggedFailure.TryGetValue(name, out var previous) && previous == fingerprint)
{
_logger.LogDebug("{Failure}", message);
}
else
{
_lastLoggedFailure[name] = fingerprint;
_logger.LogError("{Failure}", message);
}
}
///
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var timer = new PeriodicTimer(_options.Value.DeployedSecretsMirrorInterval, _time);
do
{
try
{
await ReconcileAsync(stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
// Covers every other failure, including an OperationCanceledException NOT
// attributable to this service's own shutdown token (e.g. a store/command
// timeout) — that must not silently end the loop (review Minor #4).
_logger.LogWarning(ex, "Deployed secrets mirror pass failed; retrying next interval.");
}
} while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
/// Maps a wire row back to the store's shape.
private static StoredSecret ToStored(SecretArtifact secret) => new()
{
Name = new SecretName(secret.Name),
Description = secret.Description,
ContentType = Enum.Parse(secret.ContentType, ignoreCase: false),
Ciphertext = secret.Ciphertext,
Nonce = secret.Nonce,
Tag = secret.Tag,
WrappedDek = secret.WrappedDek,
WrapNonce = secret.WrapNonce,
WrapTag = secret.WrapTag,
KekId = secret.KekId,
Revision = secret.Revision,
CreatedUtc = secret.CreatedUtc,
UpdatedUtc = secret.UpdatedUtc,
};
///
public override void Dispose()
{
if (!_disposed)
{
_disposed = true;
_gate.Dispose();
}
base.Dispose();
}
}