From 25c25ccef9df13fd5343f78661b4dd19f3c9b590 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 28 Sep 2026 18:52:44 -0400 Subject: [PATCH] =?UTF-8?q?feat(site):=20DeployedSecretsMirror=20=E2=80=94?= =?UTF-8?q?=20per-node=20apply=20+=20decrypt-verify=20of=20deployed=20secr?= =?UTF-8?q?ets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Secrets/DeployedSecretsMirror.cs | 284 +++++++++++ .../ServiceCollectionExtensions.cs | 17 + .../SiteRuntimeOptions.cs | 11 + .../SiteRuntimeOptionsValidator.cs | 5 + .../ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj | 1 + .../Secrets/DeployedSecretsMirrorTests.cs | 446 ++++++++++++++++++ 6 files changed, 764 insertions(+) create mode 100644 src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs new file mode 100644 index 00000000..7803869b --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs @@ -0,0 +1,284 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts; +using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets; + +/// +/// The outcome of one pass: the secret +/// names applied (inserted or updated in this node's own secret store), the names removed +/// (deleted because central tombstoned them and this node had previously mirrored them), and +/// a human-readable failure message per live secret that did not decrypt on this node. +/// +/// Names applied (inserted or updated) this pass. +/// Names deleted this pass because they were previously mirrored and are now tombstoned upstream. +/// One message per live secret that failed to decrypt on this node — never includes secret values. +public sealed record MirrorResult( + IReadOnlyList Applied, IReadOnlyList Removed, IReadOnlyList Failed) +{ + /// Whether this pass applied or removed anything (used to decide the summary log level). + public bool Changed => Applied.Count > 0 || Removed.Count > 0; +} + +/// +/// Per-node reconcile of the site-wide, LocalDb-replicated deployed_secrets table into +/// THIS node's own ZB.MOM.WW.Secrets store. Design §3.5 step 2. +/// +/// +/// +/// Runs on EVERY site node — not a singleton — because deployed_secrets row content +/// (ciphertext) is site-wide (replicated by LocalDb), but which names THIS node has actually +/// applied into its own local secret store is node-local fact: mirrored_secrets is +/// deliberately NOT one of LocalDb's replicated tables (see +/// ). Each node reconciles for +/// itself, and — because the two nodes' secret stores are otherwise independent — each node +/// also independently verifies that the deployed ciphertext actually decrypts under ITS OWN +/// master key. +/// +/// +/// Apply semantics (). The store applies +/// last-writer-wins on (UpdatedUtc, Revision) — see +/// ZB.MOM.WW.Secrets.Abstractions.SecretLastWriterWins — and a row that is not +/// strictly newer than the local row is silently ignored. Consequence worth knowing: +/// if an operator ever creates an out-of-band secret directly in a site node's local store +/// under the SAME name as a deployed secret, and stamps it with a later UpdatedUtc (or +/// an equal UpdatedUtc and a higher Revision) than the deployed row central last +/// shipped, that out-of-band secret WINS locally — every subsequent reconcile pass's +/// ApplyReplicatedAsync call for that name is a silent no-op until central ships a +/// still-newer revision. This mirror does not attempt to detect or warn about that case (it +/// has no way to distinguish "already applied, unchanged" from "out-of-band override in +/// place" — both look like "nothing to do" from here); deliberately not adding that +/// complexity per the design's scope. +/// +/// +/// Resolve semantics and the resolver cache. The trial-resolve verify step goes through +/// , which the concrete resolver caches for a TTL +/// (ResolveCacheTtl, 30 s in the shipped implementation). If this node's master key +/// changes (e.g. a KEK rotation) between an earlier successful resolve and a reconcile pass +/// run within the cache TTL, the verify can return the STALE cached plaintext rather than +/// re-decrypting — that is still a successful decrypt of a value under the key that +/// produced the cache entry, which is all this check is trying to prove (this node's master +/// key can decrypt what central most recently shipped), so a cache hit is an acceptable false +/// negative for the "did the key change" question and not a bug in this mirror. +/// +/// +/// Error-log de-spam. A secret that keeps failing to decrypt would otherwise log an +/// Error line every reconcile pass forever. This mirror logs Error only the FIRST time a given +/// (name, revision, kekId) tuple fails on this node; identical repeats log at Debug instead. +/// The moment that same name resolves successfully (a fixed master key, or a newer revision +/// central re-shipped), the remembered tuple is cleared, so a LATER failure — even for the +/// exact same revision/kekId, e.g. a flapping key — logs at Error again. This state is +/// in-memory only (per node process), which matches the ledger's own node-local scope. +/// +/// +public sealed class DeployedSecretsMirror : BackgroundService +{ + private readonly SiteStorageService _storage; + private readonly ISecretStore _store; + private readonly ISecretResolver _resolver; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _time; + + private readonly SemaphoreSlim _gate = new(1, 1); + + /// + /// Per-secret-name record of the (Revision, KekId) tuple whose decrypt failure this node + /// has already logged at Error. Cleared for a name the moment it resolves successfully, so + /// a later failure — even an identical one — logs at Error again. In-memory only; not a + /// durable ledger, and deliberately so (it exists purely to de-spam this process's logs). + /// + private readonly Dictionary _lastLoggedFailure = new(StringComparer.Ordinal); + + /// Initializes a new instance of . + /// Site-local storage exposing deployed_secrets and the mirrored_secrets ledger. + /// This node's own secret store. + /// This node's own secret resolver, used to trial-decrypt each deployed secret. + /// Supplies . + /// Logger instance. Never logs secret values — names, revisions, and KEK ids only. + /// Clock driving the reconcile loop's . + public DeployedSecretsMirror( + SiteStorageService storage, + ISecretStore store, + ISecretResolver resolver, + IOptions options, + ILogger logger, + TimeProvider time) + { + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _time = time ?? throw new ArgumentNullException(nameof(time)); + } + + /// + /// Runs one reconcile pass: applies every live deployed_secrets row into this + /// node's own secret store, deletes rows this node previously mirrored that are now + /// tombstoned upstream, and trial-decrypts every live row to verify it resolves under + /// this node's own master key. Safe to call concurrently with itself (and with the + /// hosted loop) — a single in-flight pass is enforced internally. + /// + /// A token to cancel the operation. + /// The pass's . + public async Task ReconcileAsync(CancellationToken ct) + { + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + var rows = await _storage.GetDeployedSecretsAsync().ConfigureAwait(false); + var mirrored = (await _storage.GetMirroredSecretsAsync().ConfigureAwait(false)) + .ToDictionary(m => m.Name, m => m.Revision, StringComparer.Ordinal); + + List applied = []; + List removed = []; + List failed = []; + + foreach (var row in rows) + { + if (row.IsRemoved) + { + if (!mirrored.ContainsKey(row.Name)) + { + // Never mirrored by this node (or already un-mirrored) — an + // out-of-band, same-named local secret is left alone. + continue; + } + + await _store.DeleteAsync(new SecretName(row.Name), "deployed-secrets-mirror", ct) + .ConfigureAwait(false); + await _storage.RemoveMirroredSecretAsync(row.Name).ConfigureAwait(false); + removed.Add(row.Name); + _lastLoggedFailure.Remove(row.Name); + _logger.LogInformation( + "Removed deployed secret {SecretName} (no longer referenced by this site)", + row.Name); + continue; + } + + if (!mirrored.TryGetValue(row.Name, out var mirroredRevision) + || mirroredRevision != row.Secret.Revision) + { + await _store.ApplyReplicatedAsync(ToStored(row.Secret), ct).ConfigureAwait(false); + await _storage.SetMirroredSecretAsync(row.Name, row.Secret.Revision).ConfigureAwait(false); + applied.Add(row.Name); + _logger.LogInformation( + "Applied deployed secret {SecretName} r{Revision} (kek {KekId})", + row.Name, row.Secret.Revision, row.Secret.KekId); + } + + await VerifyAsync(row, failed, ct).ConfigureAwait(false); + } + + var result = new MirrorResult(applied, removed, failed); + + if (result.Changed) + { + _logger.LogInformation( + "Deployed secrets mirror: {Applied} applied, {Removed} removed, {Failed} failed", + applied.Count, removed.Count, failed.Count); + } + else + { + _logger.LogDebug( + "Deployed secrets mirror in sync ({Count} secret(s), {FailedCount} failing to decrypt)", + rows.Count(r => !r.IsRemoved), failed.Count); + } + + return result; + } + finally + { + _gate.Release(); + } + } + + /// + /// Trial-resolves 's secret through to + /// verify it decrypts on this node. On failure, appends a message to + /// and logs — Error the first time this exact (name, revision, + /// kekId) has failed since it last resolved successfully on this node, Debug on every + /// repeat (see the class remarks on error-log de-spam). + /// + private async Task VerifyAsync(DeployedSecretRow row, List failed, CancellationToken ct) + { + bool resolved; + try + { + resolved = await _resolver.GetAsync(new SecretName(row.Name), ct).ConfigureAwait(false) is not null; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception) + { + resolved = false; + } + + if (resolved) + { + _lastLoggedFailure.Remove(row.Name); + return; + } + + var message = + $"Secret '{row.Name}' (kek {row.Secret.KekId}) did not decrypt on this node — " + + "the node's master key differs from central's"; + failed.Add(message); + + var current = (row.Secret.Revision, row.Secret.KekId); + if (_lastLoggedFailure.TryGetValue(row.Name, out var previous) && previous == current) + { + _logger.LogDebug("{Failure}", message); + } + else + { + _lastLoggedFailure[row.Name] = current; + _logger.LogError("{Failure}", message); + } + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_options.Value.DeployedSecretsMirrorInterval, _time); + do + { + try + { + await ReconcileAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Deployed secrets mirror pass failed; retrying next interval."); + } + } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + } + + /// Maps a wire row back to the store's shape. + private static StoredSecret ToStored(SecretArtifact secret) => new() + { + Name = new SecretName(secret.Name), + Description = secret.Description, + ContentType = Enum.Parse(secret.ContentType, ignoreCase: false), + Ciphertext = secret.Ciphertext, + Nonce = secret.Nonce, + Tag = secret.Tag, + WrappedDek = secret.WrappedDek, + WrapNonce = secret.WrapNonce, + WrapTag = secret.WrapTag, + KekId = secret.KekId, + Revision = secret.Revision, + CreatedUtc = secret.CreatedUtc, + UpdatedUtc = secret.UpdatedUtc, + }; +} diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs index 49c8141f..bbb8b139 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.LocalDb; @@ -9,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; +using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime; @@ -123,6 +125,21 @@ public static class ServiceCollectionExtensions sp.GetRequiredService>(), sp.GetRequiredService>())); + // Site-secrets artifact deploy (Task 9): per-node reconcile of the site-wide, + // LocalDb-replicated deployed_secrets table into THIS node's own ZB.MOM.WW.Secrets + // store. Runs on every site node — not a singleton, since mirrored_secrets (which + // names this node has already applied) is deliberately node-local, not replicated. + // Registered as a singleton (not just a hosted service) so ReconcileAsync can also + // be invoked synchronously — by tests, and by any future on-demand trigger (e.g. + // right after an artifact deploy lands secrets) — without waiting for the next tick. + // ISecretStore/ISecretResolver are registered by the Host's AddScadaBridgeSecrets + // (SecretsNodeRole.Site) — DI registration order does not matter, only that both + // calls happen before the provider is built, which SiteServiceRegistration.Configure + // guarantees. + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + return services; } diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs index 2321308d..cd20be95 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs @@ -263,4 +263,15 @@ public class SiteRuntimeOptions /// allow-listed roots and per-read size cap. Off by default (no roots). /// public Scripts.ScriptFileAccessOptions ScriptFileAccess { get; set; } = new(); + + /// + /// Interval at which reconciles the + /// site-wide, LocalDb-replicated deployed_secrets table into THIS node's own + /// ZB.MOM.WW.Secrets store — applying live rows, deleting rows this node + /// previously mirrored that are now tombstoned, and trial-decrypting every live row to + /// verify it resolves under this node's own master key. The reconcile also runs once + /// immediately at hosted-service start, so this interval only governs the steady-state + /// cadence. Default: 15 seconds. + /// + public TimeSpan DeployedSecretsMirrorInterval { get; set; } = TimeSpan.FromSeconds(15); } diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs index 09a0a292..969dfa67 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs @@ -131,5 +131,10 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase TimeSpan.Zero, + $"ScadaBridge:SiteRuntime:DeployedSecretsMirrorInterval must be a positive duration " + + $"(was {options.DeployedSecretsMirrorInterval}); it is the DeployedSecretsMirror " + + "reconcile PeriodicTimer period."); } } diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj index 577886f8..66a65bae 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj @@ -24,6 +24,7 @@ + diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs new file mode 100644 index 00000000..824ef930 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs @@ -0,0 +1,446 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts; +using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; +using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets; +using ZB.MOM.WW.ScadaBridge.TestSupport; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets; + +/// +/// Task 9, site-secrets artifact deploy: — per-node apply +/// + decrypt-verify of deployed_secrets into this node's own secret store. +/// +public class DeployedSecretsMirrorTests : IDisposable +{ + private readonly TestLocalDb _localDb; + + public DeployedSecretsMirrorTests() + { + _localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror"); + } + + public void Dispose() + { + var path = _localDb.Path; + _localDb.Dispose(); + TestLocalDb.DeleteFiles(path); + } + + // ── Apply / remove / out-of-band ── + + [Fact] + public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored() + { + var (mirror, storage, store, _) = await CreateAsync(); + store.Seed("out-of-band"); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1"); + var r1 = await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal)); + Assert.Empty(r1.Failed); + + await storage.UpsertDeployedSecretsAsync([], "d2"); + // Simulate: OUT-OF-BAND was never actually mirrored by this node (an operator secret + // that happens to share the name) by clearing its ledger row directly. + await storage.RemoveMirroredSecretAsync("out-of-band"); + + var r2 = await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Equal(["a"], r2.Removed); + Assert.True(store.Contains("out-of-band")); + Assert.False(store.Contains("a")); + } + + [Fact] + public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges() + { + var (mirror, storage, store, _) = await CreateAsync(); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + var r1 = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Equal(["a"], r1.Applied); + + var r2 = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Empty(r2.Applied); + Assert.False(r2.Changed); + + await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2"); + var r3 = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Equal(["a"], r3.Applied); + Assert.Equal(2, store.LastApplied["a"].Revision); + } + + [Fact] + public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact() + { + var (mirror, storage, store, _) = await CreateAsync(); + await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1"); + + await mirror.ReconcileAsync(CancellationToken.None); + + var applied = store.LastApplied["a"]; + Assert.Equal("kek-central", applied.KekId); + Assert.Equal(1, applied.Revision); + Assert.Equal(SecretContentType.Text, applied.ContentType); + } + + [Theory] + [InlineData(SecretContentType.Text)] + [InlineData(SecretContentType.ConnectionString)] + [InlineData(SecretContentType.Json)] + [InlineData(SecretContentType.BinaryBase64)] + public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType) + { + var (mirror, storage, store, _) = await CreateAsync(); + await storage.UpsertDeployedSecretsAsync( + [Art("a", 1, contentType: contentType.ToString())], "d1"); + + await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Equal(contentType, store.LastApplied["a"].ContentType); + } + + // ── Decrypt verification ── + + [Fact] + public async Task Reconcile_DecryptFailure_IsReportedWithKekId() + { + var (mirror, storage, _, resolver) = await CreateAsync(); + resolver.FailFor("a"); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1"); + var r = await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Contains("kek-central", Assert.Single(r.Failed)); + Assert.Contains("'a'", Assert.Single(r.Failed)); + } + + [Fact] + public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported() + { + var (mirror, storage, _, resolver) = await CreateAsync(); + resolver.ReturnNullFor("a"); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + var r = await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Single(r.Failed); + } + + [Fact] + public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsAlsoReported() + { + var (mirror, storage, _, resolver) = await CreateAsync(); + resolver.ThrowFor("a"); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + var r = await mirror.ReconcileAsync(CancellationToken.None); + + Assert.Single(r.Failed); + } + + [Fact] + public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied() + { + var (mirror, storage, _, resolver) = await CreateAsync(); + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + + var r1 = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Empty(r1.Failed); + + // Master key "changes" (or a KEK rotation invalidates the local unwrap) between + // passes — even though nothing about deployed_secrets changed, the next reconcile + // must still catch the failure because it re-verifies unconditionally. + resolver.FailFor("a"); + var r2 = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Single(r2.Failed); + Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified + } + + // ── Error-log de-spam (ruling 1) ── + + [Fact] + public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess() + { + var logger = new CapturingLogger(); + var (mirror, storage, _, resolver) = await CreateAsync(logger); + resolver.FailFor("a"); + await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1"); + + await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error + await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug + await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug + + var failureEntries = logger.Entries + .Where(e => e.Message.Contains("did not decrypt")) + .ToList(); + Assert.Equal(3, failureEntries.Count); + Assert.Equal(LogLevel.Error, failureEntries[0].Level); + Assert.Equal(LogLevel.Debug, failureEntries[1].Level); + Assert.Equal(LogLevel.Debug, failureEntries[2].Level); + + // Now the secret resolves (e.g. the key issue is fixed). + resolver.ClearFailures(); + var recovered = await mirror.ReconcileAsync(CancellationToken.None); + Assert.Empty(recovered.Failed); + + // Fails again — must log at Error again, not Debug, since it succeeded in between. + resolver.FailFor("a"); + await mirror.ReconcileAsync(CancellationToken.None); + + var failureEntriesAfterRecovery = logger.Entries + .Where(e => e.Message.Contains("did not decrypt")) + .ToList(); + Assert.Equal(4, failureEntriesAfterRecovery.Count); + Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level); + } + + [Fact] + public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween() + { + var logger = new CapturingLogger(); + var (mirror, storage, _, resolver) = await CreateAsync(logger); + resolver.FailFor("a"); + + await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1"); + await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1) + await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat) + + // Central ships a new revision of the same (still-failing) secret. + await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2"); + await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new) + + var levels = logger.Entries + .Where(e => e.Message.Contains("did not decrypt")) + .Select(e => e.Level) + .ToList(); + Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels); + } + + // ── Hosted loop ── + + [Fact] + public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval() + { + var (mirror, storage, store, resolver) = await CreateAsync( + interval: TimeSpan.FromMilliseconds(30)); + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + + await mirror.StartAsync(CancellationToken.None); + try + { + await WaitUntilAsync(() => store.Contains("a")); + Assert.True(store.Contains("a")); // ran once immediately at start + + // Prove the loop keeps ticking (not just the immediate start-of-day pass) by + // observing further resolver invocations over time. + var before = resolver.CallCount; + await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000); + Assert.True(resolver.CallCount > before + 1); + } + finally + { + await mirror.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task ExecuteAsync_ReconcileExceptionIsLoggedAndLoopContinues() + { + var logger = new CapturingLogger(); + var (mirror, storage, store, _) = await CreateAsync( + logger, interval: TimeSpan.FromMilliseconds(30)); + await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); + store.ThrowOnApply = true; + + await mirror.StartAsync(CancellationToken.None); + try + { + await WaitUntilAsync(() => logger.Entries.Any( + e => e.Level == LogLevel.Warning && e.Message.Contains("failed"))); + + // The loop must still be alive after the failure — force success and confirm a + // later pass applies the secret. + store.ThrowOnApply = false; + await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000); + Assert.True(store.Contains("a")); + } + finally + { + await mirror.StopAsync(CancellationToken.None); + } + } + + // ── Harness ── + + private async Task<(DeployedSecretsMirror Mirror, SiteStorageService Storage, FakeSecretStore Store, FakeSecretResolver Resolver)> CreateAsync( + CapturingLogger? logger = null, + TimeSpan? interval = null) + { + var storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); + await storage.InitializeAsync(); + + var store = new FakeSecretStore(); + var resolver = new FakeSecretResolver(); + var options = Options.Create(new SiteRuntimeOptions + { + DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15), + }); + + var mirror = new DeployedSecretsMirror( + storage, + store, + resolver, + options, + (ILogger?)logger ?? NullLogger.Instance, + TimeProvider.System); + + return (mirror, storage, store, resolver); + } + + private static SecretArtifact Art( + string name, long revision, string kek = "kek-1", string contentType = "Text") => new( + Name: name, + ContentType: contentType, + Ciphertext: [1, 2, 3, 4, 5], + Nonce: [10, 11, 12], + Tag: [20, 21, 22, 23], + WrappedDek: [30, 31, 32, 33, 34, 35], + WrapNonce: [40, 41, 42], + WrapTag: [50, 51, 52, 53], + KekId: kek, + Revision: revision, + CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero), + UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero), + Description: "test secret"); + + private static async Task WaitUntilAsync(Func condition, int timeoutMs = 5000) + { + var elapsed = 0; + const int step = 20; + while (!condition() && elapsed < timeoutMs) + { + await Task.Delay(step); + elapsed += step; + } + Assert.True(condition(), "condition not met within timeout"); + } + + /// Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests. + private sealed class CapturingLogger : ILogger + { + public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new(); + + IDisposable? ILogger.BeginScope(TState state) => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + => Entries.Enqueue((logLevel, formatter(state, exception))); + } + + /// + /// In-memory recording / + /// calls, for asserting what the mirror applied/removed without + /// a real ZB.MOM.WW.Secrets store. + /// + private sealed class FakeSecretStore : ISecretStore + { + private readonly Dictionary _rows = new(StringComparer.Ordinal); + + public IReadOnlyDictionary LastApplied => _rows; + + /// Set to make the next call throw, simulating a store fault. + public bool ThrowOnApply { get; set; } + + /// Seeds a row directly (bypassing ApplyReplicatedAsync) — an out-of-band secret this node never mirrored itself. + public void Seed(string name) => _rows[name] = new StoredSecret + { + Name = new SecretName(name), + ContentType = SecretContentType.Text, + Ciphertext = [9], + Nonce = [9], + Tag = [9], + WrappedDek = [9], + WrapNonce = [9], + WrapTag = [9], + KekId = "kek-out-of-band", + Revision = 1, + CreatedUtc = DateTimeOffset.UtcNow, + UpdatedUtc = DateTimeOffset.UtcNow, + }; + + public bool Contains(string name) => _rows.ContainsKey(name); + + public Task GetAsync(SecretName name, CancellationToken ct) => + Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null); + + public Task UpsertAsync(StoredSecret row, CancellationToken ct) + { + _rows[row.Name.Value] = row; + return Task.CompletedTask; + } + + public Task DeleteAsync(SecretName name, string? actor, CancellationToken ct) => + Task.FromResult(_rows.Remove(name.Value)); + + public Task> ListAsync(bool includeDeleted, CancellationToken ct) => + Task.FromResult>([]); + + public Task> GetManifestAsync(CancellationToken ct) => + Task.FromResult>([]); + + public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) + { + if (ThrowOnApply) + { + throw new InvalidOperationException("simulated store fault"); + } + + // Test double only — real store applies last-writer-wins; here every apply + // just overwrites, which is fine since these tests never seed a conflicting + // higher local revision through this path. + _rows[row.Name.Value] = row; + return Task.CompletedTask; + } + + public Task ApplyRewrapAsync( + StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) => + Task.FromResult(false); + } + + /// In-memory whose per-name resolution outcome tests control. + private sealed class FakeSecretResolver : ISecretResolver + { + private readonly HashSet _throwing = new(StringComparer.Ordinal); + private readonly HashSet _nullReturning = new(StringComparer.Ordinal); + + public int CallCount; + + public void FailFor(string name) => _nullReturning.Add(name); + public void ReturnNullFor(string name) => _nullReturning.Add(name); + public void ThrowFor(string name) => _throwing.Add(name); + public void ClearFailures() + { + _nullReturning.Clear(); + _throwing.Clear(); + } + + public Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref CallCount); + if (_throwing.Contains(name.Value)) + { + throw new InvalidOperationException("simulated decrypt failure"); + } + + return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value"); + } + } +}