feat(site): DeployedSecretsMirror — per-node apply + decrypt-verify of deployed secrets

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs

@@ -0,0 +1,284 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
+
+/// <summary>
+/// The outcome of one <see cref="DeployedSecretsMirror.ReconcileAsync"/> pass: the secret
+/// names applied (inserted or updated in this node's own secret store), the names removed
+/// (deleted because central tombstoned them and this node had previously mirrored them), and
+/// a human-readable failure message per live secret that did not decrypt on this node.
+/// </summary>
+/// <param name="Applied">Names applied (inserted or updated) this pass.</param>
+/// <param name="Removed">Names deleted this pass because they were previously mirrored and are now tombstoned upstream.</param>
+/// <param name="Failed">One message per live secret that failed to decrypt on this node — never includes secret values.</param>
+public sealed record MirrorResult(
+    IReadOnlyList<string> Applied, IReadOnlyList<string> Removed, IReadOnlyList<string> Failed)
+{
+    /// <summary>Whether this pass applied or removed anything (used to decide the summary log level).</summary>
+    public bool Changed => Applied.Count > 0 || Removed.Count > 0;
+}
+
+/// <summary>
+/// Per-node reconcile of the site-wide, LocalDb-replicated <c>deployed_secrets</c> table into
+/// THIS node's own <c>ZB.MOM.WW.Secrets</c> store. Design §3.5 step 2.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Runs on EVERY site node — not a singleton — because <c>deployed_secrets</c> row content
+/// (ciphertext) is site-wide (replicated by LocalDb), but which names THIS node has actually
+/// applied into its own local secret store is node-local fact: <c>mirrored_secrets</c> is
+/// deliberately NOT one of LocalDb's replicated tables (see
+/// <see cref="SiteStorageService.GetMirroredSecretsAsync"/>). Each node reconciles for
+/// itself, and — because the two nodes' secret stores are otherwise independent — each node
+/// also independently verifies that the deployed ciphertext actually decrypts under ITS OWN
+/// master key.
+/// </para>
+/// <para>
+/// <b>Apply semantics (<see cref="ISecretStore.ApplyReplicatedAsync"/>).</b> The store applies
+/// last-writer-wins on (<c>UpdatedUtc</c>, <c>Revision</c>) — see
+/// <c>ZB.MOM.WW.Secrets.Abstractions.SecretLastWriterWins</c> — and a row that is not
+/// <em>strictly</em> newer than the local row is silently ignored. Consequence worth knowing:
+/// if an operator ever creates an out-of-band secret directly in a site node's local store
+/// under the SAME name as a deployed secret, and stamps it with a later <c>UpdatedUtc</c> (or
+/// an equal <c>UpdatedUtc</c> and a higher <c>Revision</c>) than the deployed row central last
+/// shipped, that out-of-band secret WINS locally — every subsequent reconcile pass's
+/// <c>ApplyReplicatedAsync</c> call for that name is a silent no-op until central ships a
+/// still-newer revision. This mirror does not attempt to detect or warn about that case (it
+/// has no way to distinguish "already applied, unchanged" from "out-of-band override in
+/// place" — both look like "nothing to do" from here); deliberately not adding that
+/// complexity per the design's scope.
+/// </para>
+/// <para>
+/// <b>Resolve semantics and the resolver cache.</b> The trial-resolve verify step goes through
+/// <see cref="ISecretResolver.GetAsync"/>, which the concrete resolver caches for a TTL
+/// (<c>ResolveCacheTtl</c>, 30 s in the shipped implementation). If this node's master key
+/// changes (e.g. a KEK rotation) between an earlier successful resolve and a reconcile pass
+/// run within the cache TTL, the verify can return the STALE cached plaintext rather than
+/// re-decrypting — that is still a successful decrypt of <em>a</em> value under the key that
+/// produced the cache entry, which is all this check is trying to prove (this node's master
+/// key can decrypt what central most recently shipped), so a cache hit is an acceptable false
+/// negative for the "did the key change" question and not a bug in this mirror.
+/// </para>
+/// <para>
+/// <b>Error-log de-spam.</b> A secret that keeps failing to decrypt would otherwise log an
+/// Error line every reconcile pass forever. This mirror logs Error only the FIRST time a given
+/// (name, revision, kekId) tuple fails on this node; identical repeats log at Debug instead.
+/// The moment that same name resolves successfully (a fixed master key, or a newer revision
+/// central re-shipped), the remembered tuple is cleared, so a LATER failure — even for the
+/// exact same revision/kekId, e.g. a flapping key — logs at Error again. This state is
+/// in-memory only (per node process), which matches the ledger's own node-local scope.
+/// </para>
+/// </remarks>
+public sealed class DeployedSecretsMirror : BackgroundService
+{
+    private readonly SiteStorageService _storage;
+    private readonly ISecretStore _store;
+    private readonly ISecretResolver _resolver;
+    private readonly IOptions<SiteRuntimeOptions> _options;
+    private readonly ILogger<DeployedSecretsMirror> _logger;
+    private readonly TimeProvider _time;
+
+    private readonly SemaphoreSlim _gate = new(1, 1);
+
+    /// <summary>
+    /// Per-secret-name record of the (Revision, KekId) tuple whose decrypt failure this node
+    /// has already logged at Error. Cleared for a name the moment it resolves successfully, so
+    /// a later failure — even an identical one — logs at Error again. In-memory only; not a
+    /// durable ledger, and deliberately so (it exists purely to de-spam this process's logs).
+    /// </summary>
+    private readonly Dictionary<string, (long Revision, string KekId)> _lastLoggedFailure = new(StringComparer.Ordinal);
+
+    /// <summary>Initializes a new instance of <see cref="DeployedSecretsMirror"/>.</summary>
+    /// <param name="storage">Site-local storage exposing <c>deployed_secrets</c> and the <c>mirrored_secrets</c> ledger.</param>
+    /// <param name="store">This node's own secret store.</param>
+    /// <param name="resolver">This node's own secret resolver, used to trial-decrypt each deployed secret.</param>
+    /// <param name="options">Supplies <see cref="SiteRuntimeOptions.DeployedSecretsMirrorInterval"/>.</param>
+    /// <param name="logger">Logger instance. Never logs secret values — names, revisions, and KEK ids only.</param>
+    /// <param name="time">Clock driving the reconcile loop's <see cref="PeriodicTimer"/>.</param>
+    public DeployedSecretsMirror(
+        SiteStorageService storage,
+        ISecretStore store,
+        ISecretResolver resolver,
+        IOptions<SiteRuntimeOptions> options,
+        ILogger<DeployedSecretsMirror> logger,
+        TimeProvider time)
+    {
+        _storage = storage ?? throw new ArgumentNullException(nameof(storage));
+        _store = store ?? throw new ArgumentNullException(nameof(store));
+        _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver));
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _time = time ?? throw new ArgumentNullException(nameof(time));
+    }
+
+    /// <summary>
+    /// Runs one reconcile pass: applies every live <c>deployed_secrets</c> row into this
+    /// node's own secret store, deletes rows this node previously mirrored that are now
+    /// tombstoned upstream, and trial-decrypts every live row to verify it resolves under
+    /// this node's own master key. Safe to call concurrently with itself (and with the
+    /// hosted loop) — a single in-flight pass is enforced internally.
+    /// </summary>
+    /// <param name="ct">A token to cancel the operation.</param>
+    /// <returns>The pass's <see cref="MirrorResult"/>.</returns>
+    public async Task<MirrorResult> ReconcileAsync(CancellationToken ct)
+    {
+        await _gate.WaitAsync(ct).ConfigureAwait(false);
+        try
+        {
+            var rows = await _storage.GetDeployedSecretsAsync().ConfigureAwait(false);
+            var mirrored = (await _storage.GetMirroredSecretsAsync().ConfigureAwait(false))
+                .ToDictionary(m => m.Name, m => m.Revision, StringComparer.Ordinal);
+
+            List<string> applied = [];
+            List<string> removed = [];
+            List<string> failed = [];
+
+            foreach (var row in rows)
+            {
+                if (row.IsRemoved)
+                {
+                    if (!mirrored.ContainsKey(row.Name))
+                    {
+                        // Never mirrored by this node (or already un-mirrored) — an
+                        // out-of-band, same-named local secret is left alone.
+                        continue;
+                    }
+
+                    await _store.DeleteAsync(new SecretName(row.Name), "deployed-secrets-mirror", ct)
+                        .ConfigureAwait(false);
+                    await _storage.RemoveMirroredSecretAsync(row.Name).ConfigureAwait(false);
+                    removed.Add(row.Name);
+                    _lastLoggedFailure.Remove(row.Name);
+                    _logger.LogInformation(
+                        "Removed deployed secret {SecretName} (no longer referenced by this site)",
+                        row.Name);
+                    continue;
+                }
+
+                if (!mirrored.TryGetValue(row.Name, out var mirroredRevision)
+                    || mirroredRevision != row.Secret.Revision)
+                {
+                    await _store.ApplyReplicatedAsync(ToStored(row.Secret), ct).ConfigureAwait(false);
+                    await _storage.SetMirroredSecretAsync(row.Name, row.Secret.Revision).ConfigureAwait(false);
+                    applied.Add(row.Name);
+                    _logger.LogInformation(
+                        "Applied deployed secret {SecretName} r{Revision} (kek {KekId})",
+                        row.Name, row.Secret.Revision, row.Secret.KekId);
+                }
+
+                await VerifyAsync(row, failed, ct).ConfigureAwait(false);
+            }
+
+            var result = new MirrorResult(applied, removed, failed);
+
+            if (result.Changed)
+            {
+                _logger.LogInformation(
+                    "Deployed secrets mirror: {Applied} applied, {Removed} removed, {Failed} failed",
+                    applied.Count, removed.Count, failed.Count);
+            }
+            else
+            {
+                _logger.LogDebug(
+                    "Deployed secrets mirror in sync ({Count} secret(s), {FailedCount} failing to decrypt)",
+                    rows.Count(r => !r.IsRemoved), failed.Count);
+            }
+
+            return result;
+        }
+        finally
+        {
+            _gate.Release();
+        }
+    }
+
+    /// <summary>
+    /// Trial-resolves <paramref name="row"/>'s secret through <see cref="ISecretResolver"/> to
+    /// verify it decrypts on this node. On failure, appends a message to
+    /// <paramref name="failed"/> and logs — Error the first time this exact (name, revision,
+    /// kekId) has failed since it last resolved successfully on this node, Debug on every
+    /// repeat (see the class remarks on error-log de-spam).
+    /// </summary>
+    private async Task VerifyAsync(DeployedSecretRow row, List<string> failed, CancellationToken ct)
+    {
+        bool resolved;
+        try
+        {
+            resolved = await _resolver.GetAsync(new SecretName(row.Name), ct).ConfigureAwait(false) is not null;
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception)
+        {
+            resolved = false;
+        }
+
+        if (resolved)
+        {
+            _lastLoggedFailure.Remove(row.Name);
+            return;
+        }
+
+        var message =
+            $"Secret '{row.Name}' (kek {row.Secret.KekId}) did not decrypt on this node — " +
+            "the node's master key differs from central's";
+        failed.Add(message);
+
+        var current = (row.Secret.Revision, row.Secret.KekId);
+        if (_lastLoggedFailure.TryGetValue(row.Name, out var previous) && previous == current)
+        {
+            _logger.LogDebug("{Failure}", message);
+        }
+        else
+        {
+            _lastLoggedFailure[row.Name] = current;
+            _logger.LogError("{Failure}", message);
+        }
+    }
+
+    /// <inheritdoc />
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        using var timer = new PeriodicTimer(_options.Value.DeployedSecretsMirrorInterval, _time);
+        do
+        {
+            try
+            {
+                await ReconcileAsync(stoppingToken).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Deployed secrets mirror pass failed; retrying next interval.");
+            }
+        } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
+    }
+
+    /// <summary>Maps a wire <see cref="SecretArtifact"/> row back to the store's <see cref="StoredSecret"/> shape.</summary>
+    private static StoredSecret ToStored(SecretArtifact secret) => new()
+    {
+        Name = new SecretName(secret.Name),
+        Description = secret.Description,
+        ContentType = Enum.Parse<SecretContentType>(secret.ContentType, ignoreCase: false),
+        Ciphertext = secret.Ciphertext,
+        Nonce = secret.Nonce,
+        Tag = secret.Tag,
+        WrappedDek = secret.WrappedDek,
+        WrapNonce = secret.WrapNonce,
+        WrapTag = secret.WrapTag,
+        KekId = secret.KekId,
+        Revision = secret.Revision,
+        CreatedUtc = secret.CreatedUtc,
+        UpdatedUtc = secret.UpdatedUtc,
+    };
+}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.LocalDb;
@@ -9,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
 
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime;
@@ -123,6 +125,21 @@ public static class ServiceCollectionExtensions
             sp.GetRequiredService<IOptionsMonitor<SiteRuntimeOptions>>(),
             sp.GetRequiredService<ILogger<ScriptRunSummaryFlushService>>()));
 
+        // Site-secrets artifact deploy (Task 9): per-node reconcile of the site-wide,
+        // LocalDb-replicated deployed_secrets table into THIS node's own ZB.MOM.WW.Secrets
+        // store. Runs on every site node — not a singleton, since mirrored_secrets (which
+        // names this node has already applied) is deliberately node-local, not replicated.
+        // Registered as a singleton (not just a hosted service) so ReconcileAsync can also
+        // be invoked synchronously — by tests, and by any future on-demand trigger (e.g.
+        // right after an artifact deploy lands secrets) — without waiting for the next tick.
+        // ISecretStore/ISecretResolver are registered by the Host's AddScadaBridgeSecrets
+        // (SecretsNodeRole.Site) — DI registration order does not matter, only that both
+        // calls happen before the provider is built, which SiteServiceRegistration.Configure
+        // guarantees.
+        services.TryAddSingleton(TimeProvider.System);
+        services.AddSingleton<DeployedSecretsMirror>();
+        services.AddHostedService(sp => sp.GetRequiredService<DeployedSecretsMirror>());
+
         return services;
     }
 

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs

@@ -263,4 +263,15 @@ public class SiteRuntimeOptions
     /// allow-listed roots and per-read size cap. Off by default (no roots).
     /// </summary>
     public Scripts.ScriptFileAccessOptions ScriptFileAccess { get; set; } = new();
+
+    /// <summary>
+    /// Interval at which <see cref="Secrets.DeployedSecretsMirror"/> reconciles the
+    /// site-wide, LocalDb-replicated <c>deployed_secrets</c> table into THIS node's own
+    /// <c>ZB.MOM.WW.Secrets</c> store — applying live rows, deleting rows this node
+    /// previously mirrored that are now tombstoned, and trial-decrypting every live row to
+    /// verify it resolves under this node's own master key. The reconcile also runs once
+    /// immediately at hosted-service start, so this interval only governs the steady-state
+    /// cadence. Default: 15 seconds.
+    /// </summary>
+    public TimeSpan DeployedSecretsMirrorInterval { get; set; } = TimeSpan.FromSeconds(15);
 }

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs

@@ -131,5 +131,10 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
                 $"ScadaBridge:SiteRuntime:ScriptFileAccess:AllowedRoots:{i} must be an absolute directory path " +
                 $"(was '{root}'); a relative root would resolve against the service's working directory.");
         }
+
+        builder.RequireThat(options.DeployedSecretsMirrorInterval > TimeSpan.Zero,
+            $"ScadaBridge:SiteRuntime:DeployedSecretsMirrorInterval must be a positive duration " +
+            $"(was {options.DeployedSecretsMirrorInterval}); it is the DeployedSecretsMirror " +
+            "reconcile PeriodicTimer period.");
     }
 }

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj

@@ -24,6 +24,7 @@
     <PackageReference Include="Microsoft.Extensions.Options" />
     <PackageReference Include="ZB.MOM.WW.Configuration" />
     <PackageReference Include="ZB.MOM.WW.LocalDb" />
+    <PackageReference Include="ZB.MOM.WW.Secrets.Abstractions" />
   </ItemGroup>
 
   <ItemGroup>

tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Secrets/DeployedSecretsMirrorTests.cs

@@ -0,0 +1,446 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
+using ZB.MOM.WW.ScadaBridge.TestSupport;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
+
+/// <summary>
+/// Task 9, site-secrets artifact deploy: <see cref="DeployedSecretsMirror"/> — per-node apply
+/// + decrypt-verify of <c>deployed_secrets</c> into this node's own secret store.
+/// </summary>
+public class DeployedSecretsMirrorTests : IDisposable
+{
+    private readonly TestLocalDb _localDb;
+
+    public DeployedSecretsMirrorTests()
+    {
+        _localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror");
+    }
+
+    public void Dispose()
+    {
+        var path = _localDb.Path;
+        _localDb.Dispose();
+        TestLocalDb.DeleteFiles(path);
+    }
+
+    // ── Apply / remove / out-of-band ──
+
+    [Fact]
+    public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored()
+    {
+        var (mirror, storage, store, _) = await CreateAsync();
+        store.Seed("out-of-band");
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1");
+        var r1 = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal));
+        Assert.Empty(r1.Failed);
+
+        await storage.UpsertDeployedSecretsAsync([], "d2");
+        // Simulate: OUT-OF-BAND was never actually mirrored by this node (an operator secret
+        // that happens to share the name) by clearing its ledger row directly.
+        await storage.RemoveMirroredSecretAsync("out-of-band");
+
+        var r2 = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Equal(["a"], r2.Removed);
+        Assert.True(store.Contains("out-of-band"));
+        Assert.False(store.Contains("a"));
+    }
+
+    [Fact]
+    public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges()
+    {
+        var (mirror, storage, store, _) = await CreateAsync();
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+        var r1 = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Equal(["a"], r1.Applied);
+
+        var r2 = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Empty(r2.Applied);
+        Assert.False(r2.Changed);
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2");
+        var r3 = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Equal(["a"], r3.Applied);
+        Assert.Equal(2, store.LastApplied["a"].Revision);
+    }
+
+    [Fact]
+    public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
+    {
+        var (mirror, storage, store, _) = await CreateAsync();
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        var applied = store.LastApplied["a"];
+        Assert.Equal("kek-central", applied.KekId);
+        Assert.Equal(1, applied.Revision);
+        Assert.Equal(SecretContentType.Text, applied.ContentType);
+    }
+
+    [Theory]
+    [InlineData(SecretContentType.Text)]
+    [InlineData(SecretContentType.ConnectionString)]
+    [InlineData(SecretContentType.Json)]
+    [InlineData(SecretContentType.BinaryBase64)]
+    public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType)
+    {
+        var (mirror, storage, store, _) = await CreateAsync();
+        await storage.UpsertDeployedSecretsAsync(
+            [Art("a", 1, contentType: contentType.ToString())], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Equal(contentType, store.LastApplied["a"].ContentType);
+    }
+
+    // ── Decrypt verification ──
+
+    [Fact]
+    public async Task Reconcile_DecryptFailure_IsReportedWithKekId()
+    {
+        var (mirror, storage, _, resolver) = await CreateAsync();
+        resolver.FailFor("a");
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
+        var r = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Contains("kek-central", Assert.Single(r.Failed));
+        Assert.Contains("'a'", Assert.Single(r.Failed));
+    }
+
+    [Fact]
+    public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported()
+    {
+        var (mirror, storage, _, resolver) = await CreateAsync();
+        resolver.ReturnNullFor("a");
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+        var r = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Single(r.Failed);
+    }
+
+    [Fact]
+    public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsAlsoReported()
+    {
+        var (mirror, storage, _, resolver) = await CreateAsync();
+        resolver.ThrowFor("a");
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+        var r = await mirror.ReconcileAsync(CancellationToken.None);
+
+        Assert.Single(r.Failed);
+    }
+
+    [Fact]
+    public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied()
+    {
+        var (mirror, storage, _, resolver) = await CreateAsync();
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+
+        var r1 = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Empty(r1.Failed);
+
+        // Master key "changes" (or a KEK rotation invalidates the local unwrap) between
+        // passes — even though nothing about deployed_secrets changed, the next reconcile
+        // must still catch the failure because it re-verifies unconditionally.
+        resolver.FailFor("a");
+        var r2 = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Single(r2.Failed);
+        Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified
+    }
+
+    // ── Error-log de-spam (ruling 1) ──
+
+    [Fact]
+    public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess()
+    {
+        var logger = new CapturingLogger<DeployedSecretsMirror>();
+        var (mirror, storage, _, resolver) = await CreateAsync(logger);
+        resolver.FailFor("a");
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
+
+        await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error
+        await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug
+        await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug
+
+        var failureEntries = logger.Entries
+            .Where(e => e.Message.Contains("did not decrypt"))
+            .ToList();
+        Assert.Equal(3, failureEntries.Count);
+        Assert.Equal(LogLevel.Error, failureEntries[0].Level);
+        Assert.Equal(LogLevel.Debug, failureEntries[1].Level);
+        Assert.Equal(LogLevel.Debug, failureEntries[2].Level);
+
+        // Now the secret resolves (e.g. the key issue is fixed).
+        resolver.ClearFailures();
+        var recovered = await mirror.ReconcileAsync(CancellationToken.None);
+        Assert.Empty(recovered.Failed);
+
+        // Fails again — must log at Error again, not Debug, since it succeeded in between.
+        resolver.FailFor("a");
+        await mirror.ReconcileAsync(CancellationToken.None);
+
+        var failureEntriesAfterRecovery = logger.Entries
+            .Where(e => e.Message.Contains("did not decrypt"))
+            .ToList();
+        Assert.Equal(4, failureEntriesAfterRecovery.Count);
+        Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level);
+    }
+
+    [Fact]
+    public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween()
+    {
+        var logger = new CapturingLogger<DeployedSecretsMirror>();
+        var (mirror, storage, _, resolver) = await CreateAsync(logger);
+        resolver.FailFor("a");
+
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
+        await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1)
+        await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat)
+
+        // Central ships a new revision of the same (still-failing) secret.
+        await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2");
+        await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new)
+
+        var levels = logger.Entries
+            .Where(e => e.Message.Contains("did not decrypt"))
+            .Select(e => e.Level)
+            .ToList();
+        Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels);
+    }
+
+    // ── Hosted loop ──
+
+    [Fact]
+    public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval()
+    {
+        var (mirror, storage, store, resolver) = await CreateAsync(
+            interval: TimeSpan.FromMilliseconds(30));
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+
+        await mirror.StartAsync(CancellationToken.None);
+        try
+        {
+            await WaitUntilAsync(() => store.Contains("a"));
+            Assert.True(store.Contains("a")); // ran once immediately at start
+
+            // Prove the loop keeps ticking (not just the immediate start-of-day pass) by
+            // observing further resolver invocations over time.
+            var before = resolver.CallCount;
+            await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000);
+            Assert.True(resolver.CallCount > before + 1);
+        }
+        finally
+        {
+            await mirror.StopAsync(CancellationToken.None);
+        }
+    }
+
+    [Fact]
+    public async Task ExecuteAsync_ReconcileExceptionIsLoggedAndLoopContinues()
+    {
+        var logger = new CapturingLogger<DeployedSecretsMirror>();
+        var (mirror, storage, store, _) = await CreateAsync(
+            logger, interval: TimeSpan.FromMilliseconds(30));
+        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
+        store.ThrowOnApply = true;
+
+        await mirror.StartAsync(CancellationToken.None);
+        try
+        {
+            await WaitUntilAsync(() => logger.Entries.Any(
+                e => e.Level == LogLevel.Warning && e.Message.Contains("failed")));
+
+            // The loop must still be alive after the failure — force success and confirm a
+            // later pass applies the secret.
+            store.ThrowOnApply = false;
+            await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000);
+            Assert.True(store.Contains("a"));
+        }
+        finally
+        {
+            await mirror.StopAsync(CancellationToken.None);
+        }
+    }
+
+    // ── Harness ──
+
+    private async Task<(DeployedSecretsMirror Mirror, SiteStorageService Storage, FakeSecretStore Store, FakeSecretResolver Resolver)> CreateAsync(
+        CapturingLogger<DeployedSecretsMirror>? logger = null,
+        TimeSpan? interval = null)
+    {
+        var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
+        await storage.InitializeAsync();
+
+        var store = new FakeSecretStore();
+        var resolver = new FakeSecretResolver();
+        var options = Options.Create(new SiteRuntimeOptions
+        {
+            DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15),
+        });
+
+        var mirror = new DeployedSecretsMirror(
+            storage,
+            store,
+            resolver,
+            options,
+            (ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
+            TimeProvider.System);
+
+        return (mirror, storage, store, resolver);
+    }
+
+    private static SecretArtifact Art(
+        string name, long revision, string kek = "kek-1", string contentType = "Text") => new(
+        Name: name,
+        ContentType: contentType,
+        Ciphertext: [1, 2, 3, 4, 5],
+        Nonce: [10, 11, 12],
+        Tag: [20, 21, 22, 23],
+        WrappedDek: [30, 31, 32, 33, 34, 35],
+        WrapNonce: [40, 41, 42],
+        WrapTag: [50, 51, 52, 53],
+        KekId: kek,
+        Revision: revision,
+        CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
+        UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
+        Description: "test secret");
+
+    private static async Task WaitUntilAsync(Func<bool> condition, int timeoutMs = 5000)
+    {
+        var elapsed = 0;
+        const int step = 20;
+        while (!condition() && elapsed < timeoutMs)
+        {
+            await Task.Delay(step);
+            elapsed += step;
+        }
+        Assert.True(condition(), "condition not met within timeout");
+    }
+
+    /// <summary>Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests.</summary>
+    private sealed class CapturingLogger<T> : ILogger<T>
+    {
+        public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new();
+
+        IDisposable? ILogger.BeginScope<TState>(TState state) => null;
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(
+            LogLevel logLevel, EventId eventId, TState state, Exception? exception,
+            Func<TState, Exception?, string> formatter)
+            => Entries.Enqueue((logLevel, formatter(state, exception)));
+    }
+
+    /// <summary>
+    /// In-memory <see cref="ISecretStore"/> recording <see cref="ApplyReplicatedAsync"/> /
+    /// <see cref="DeleteAsync"/> calls, for asserting what the mirror applied/removed without
+    /// a real ZB.MOM.WW.Secrets store.
+    /// </summary>
+    private sealed class FakeSecretStore : ISecretStore
+    {
+        private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);
+
+        public IReadOnlyDictionary<string, StoredSecret> LastApplied => _rows;
+
+        /// <summary>Set to make the next <see cref="ApplyReplicatedAsync"/> call throw, simulating a store fault.</summary>
+        public bool ThrowOnApply { get; set; }
+
+        /// <summary>Seeds a row directly (bypassing ApplyReplicatedAsync) — an out-of-band secret this node never mirrored itself.</summary>
+        public void Seed(string name) => _rows[name] = new StoredSecret
+        {
+            Name = new SecretName(name),
+            ContentType = SecretContentType.Text,
+            Ciphertext = [9],
+            Nonce = [9],
+            Tag = [9],
+            WrappedDek = [9],
+            WrapNonce = [9],
+            WrapTag = [9],
+            KekId = "kek-out-of-band",
+            Revision = 1,
+            CreatedUtc = DateTimeOffset.UtcNow,
+            UpdatedUtc = DateTimeOffset.UtcNow,
+        };
+
+        public bool Contains(string name) => _rows.ContainsKey(name);
+
+        public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
+            Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);
+
+        public Task UpsertAsync(StoredSecret row, CancellationToken ct)
+        {
+            _rows[row.Name.Value] = row;
+            return Task.CompletedTask;
+        }
+
+        public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct) =>
+            Task.FromResult(_rows.Remove(name.Value));
+
+        public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
+            Task.FromResult<IReadOnlyList<SecretMetadata>>([]);
+
+        public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
+            Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
+
+        public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
+        {
+            if (ThrowOnApply)
+            {
+                throw new InvalidOperationException("simulated store fault");
+            }
+
+            // Test double only — real store applies last-writer-wins; here every apply
+            // just overwrites, which is fine since these tests never seed a conflicting
+            // higher local revision through this path.
+            _rows[row.Name.Value] = row;
+            return Task.CompletedTask;
+        }
+
+        public Task<bool> ApplyRewrapAsync(
+            StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
+            Task.FromResult(false);
+    }
+
+    /// <summary>In-memory <see cref="ISecretResolver"/> whose per-name resolution outcome tests control.</summary>
+    private sealed class FakeSecretResolver : ISecretResolver
+    {
+        private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);
+        private readonly HashSet<string> _nullReturning = new(StringComparer.Ordinal);
+
+        public int CallCount;
+
+        public void FailFor(string name) => _nullReturning.Add(name);
+        public void ReturnNullFor(string name) => _nullReturning.Add(name);
+        public void ThrowFor(string name) => _throwing.Add(name);
+        public void ClearFailures()
+        {
+            _nullReturning.Clear();
+            _throwing.Clear();
+        }
+
+        public Task<string?> GetAsync(SecretName name, CancellationToken ct)
+        {
+            Interlocked.Increment(ref CallCount);
+            if (_throwing.Contains(name.Value))
+            {
+                throw new InvalidOperationException("simulated decrypt failure");
+            }
+
+            return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
+        }
+    }
+}