using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using ZB.MOM.WW.Secrets.Abstractions;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;

/// <summary>
/// Task 9, site-secrets artifact deploy: <see cref="DeployedSecretsMirror"/> — per-node apply
/// + decrypt-verify of <c>deployed_secrets</c> into this node's own secret store.
/// </summary>
/// <remarks>
/// Fix round 1 (review <c>task-9-review.md</c>): <see cref="FakeSecretStore"/> now honours the
/// real <c>SqliteSecretStore</c> semantics — <c>ApplyReplicatedAsync</c> is strict
/// last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/>, and <c>DeleteAsync</c>
/// tombstones rather than hard-deleting — so tests exercise the same conflict shapes the real
/// store can produce, not an idealized "always overwrite" double.
/// </remarks>
public class DeployedSecretsMirrorTests : IDisposable
{
    private readonly TestLocalDb _localDb;

    public DeployedSecretsMirrorTests()
    {
        _localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror");
    }

    public void Dispose()
    {
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    // ── Apply / remove / out-of-band ──

    [Fact]
    public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        // Seeded OLDER than what will be shipped, so the shipped row legitimately wins LWW
        // and is applied — the strictly-NEWER out-of-band case is covered separately by
        // Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval.
        store.Seed("out-of-band", updatedUtc: new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero));

        await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1");
        var r1 = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal));
        Assert.Empty(r1.Failed);

        await storage.UpsertDeployedSecretsAsync([], "d2");
        // Simulate: "out-of-band" was never actually mirrored by this node (an operator secret
        // that happens to share the name) by clearing its ledger row directly — a defensive
        // check independent of the natural not-ledgered path exercised below.
        await storage.RemoveMirroredSecretAsync("out-of-band");

        var r2 = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["a"], r2.Removed);
        Assert.True(store.Contains("out-of-band"));
        Assert.False(store.Contains("a"));
        Assert.True(store.IsTombstoned("a"));
    }

    [Fact]
    public async Task Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval()
    {
        var (mirror, storage, store, _, invalidator) = await CreateAsync();
        var operatorCiphertext = new byte[] { 200, 201, 202 };
        // Strictly newer than Art()'s fixed UpdatedUtc (2026-09-28T01:00:00Z) — legitimately
        // wins LWW, exactly as the real store would order it.
        store.Seed("a",
            updatedUtc: new DateTimeOffset(2026, 9, 28, 2, 0, 0, TimeSpan.Zero),
            revision: 5,
            ciphertext: operatorCiphertext);

        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        var r1 = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Empty(r1.Applied);
        var message = Assert.Single(r1.Failed);
        Assert.Contains("'a'", message);
        Assert.Contains("newer local value", message);
        Assert.DoesNotContain("did not decrypt", message); // distinct from the master-key message
        Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext); // untouched
        Assert.Empty(invalidator.Invalidated); // never applied -> never invalidated

        // Never ledgered, so a later central tombstone must NOT delete the operator's secret.
        await storage.UpsertDeployedSecretsAsync([], "d2");
        var r2 = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Empty(r2.Removed);
        Assert.True(store.Contains("a"));
        Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext);
    }

    [Fact]
    public async Task Reconcile_RemoveThenReAdd_TombstoneIsRecovered_ReappliesAndVerifies()
    {
        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
        var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        var firstApply = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Equal(["a"], firstApply.Applied);

        // Site stops referencing it: central tombstones the row, this node deletes it. The
        // tombstone is stamped LATER than the originally-shipped UpdatedUtc (01:00).
        store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
        await storage.UpsertDeployedSecretsAsync([], "d2");
        var removePass = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Equal(["a"], removePass.Removed);
        Assert.True(store.IsTombstoned("a"));
        invalidator.Invalidated.Clear();

        // Site references it again — central ships the SAME (unchanged, older-stamped)
        // secret. A naive ApplyReplicatedAsync call is a no-op against the newer tombstone;
        // the mirror must detect and recover this.
        time.Now = new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero); // later than the tombstone
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3");
        var reAddPass = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["a"], reAddPass.Applied);
        Assert.Empty(reAddPass.Failed);
        Assert.True(store.Contains("a"));
        Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); // Art()'s ciphertext, re-materialized
        Assert.Contains("a", invalidator.Invalidated);

        // Ledgering used the SHIPPED artifact's own revision, not the bumped local one, so a
        // later pass with the same shipped row is a true no-op.
        var thirdPass = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Empty(thirdPass.Applied);
        Assert.False(thirdPass.Changed);

        var mirrored = await storage.GetMirroredSecretsAsync();
        Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
    }

    [Fact]
    public async Task Reconcile_ClockSkewedRecoveredRow_StillConvergesOnNextCentralRevision()
    {
        // Fix round 2 (review "clock skew" finding): a tombstone-recovery bump can leave a local
        // row stamped AHEAD of a central UpdatedUtc that arrives afterward (site clock skew, or
        // simply a central edit made before the recovery but shipped after it). The row is still
        // OWNED by this mirror's ledger, so a later genuinely-new central revision must not be
        // misreported as an out-of-band "newer local value" — it must converge.
        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
        var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);

        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        await mirror.ReconcileAsync(CancellationToken.None);

        // Remove then re-add — but the SITE CLOCK IS SKEWED FAR INTO THE FUTURE while the
        // recovery bump happens (a bad RTC/NTP fault), so the recovered row's UpdatedUtc lands
        // far ahead of anything central will ship for a long while.
        store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
        await storage.UpsertDeployedSecretsAsync([], "d2");
        await mirror.ReconcileAsync(CancellationToken.None); // tombstones "a"

        time.Now = new DateTimeOffset(2099, 1, 1, 0, 0, 0, TimeSpan.Zero); // skewed site clock
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3");
        var recoveredPass = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Equal(["a"], recoveredPass.Applied);
        Assert.True(store.Rows["a"].UpdatedUtc.Year >= 2099); // confirms the skewed stamp landed

        // Site clock recovers to normal. Central ships a genuinely NEW revision (different
        // ciphertext, i.e. an actual rotation) with an ordinary timestamp — OLDER than the
        // skewed local stamp, so a naive ApplyReplicatedAsync would no-op against it.
        time.Now = new DateTimeOffset(2026, 9, 29, 0, 0, 0, TimeSpan.Zero);
        var rotatedCiphertext = new byte[] { 7, 7, 7 };
        await storage.UpsertDeployedSecretsAsync([Art("a", 2, ciphertext: rotatedCiphertext)], "d4");

        var convergePass = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["a"], convergePass.Applied);
        Assert.Empty(convergePass.Failed); // NOT "newer local value" — this mirror owns the name
        Assert.Equal(rotatedCiphertext, store.Rows["a"].Ciphertext);
        Assert.Contains("a", invalidator.Invalidated);

        var mirrored = await storage.GetMirroredSecretsAsync();
        Assert.Equal(2, mirrored.Single(m => m.Name == "a").Revision);
    }

    [Fact]
    public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();

        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        var r1 = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Equal(["a"], r1.Applied);

        var r2 = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Empty(r2.Applied);
        Assert.False(r2.Changed);

        await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2");
        var r3 = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Equal(["a"], r3.Applied);
        Assert.Equal(2, store.Rows["a"].Revision);
    }

    // ── Same-revision drift (final review I1) ──

    [Fact]
    public async Task Reconcile_OwnedNameDeletedLocally_IsHealedAndVerified_NotMisreportedAsKekMismatch()
    {
        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
        var (mirror, storage, store, resolver, invalidator) = await CreateAsync(time: time);
        resolver.BackingStore = store; // resolve returns null once the local row is gone
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);

        // An operator deletes the mirror-owned name on this node (library CLI/UI) — the
        // ledger still says r1 is mirrored, and central still ships r1.
        await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
        Assert.True(store.IsTombstoned("a"));
        invalidator.Invalidated.Clear();

        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Empty(r.Failed);
        Assert.Equal(["a"], r.Applied);
        Assert.True(store.Contains("a"));
        Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
        Assert.Contains("a", invalidator.Invalidated);

        // Healed state is stable: the next pass is a no-op that still verifies clean.
        var next = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.False(next.Changed);
        Assert.Empty(next.Failed);
        Assert.Equal(1, (await storage.GetMirroredSecretsAsync()).Single(m => m.Name == "a").Revision);
    }

    [Fact]
    public async Task Reconcile_OwnedNameEditedLocally_IsRevertedToTheDeployedCiphertext()
    {
        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
        var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);

        // An operator overwrites the mirror-owned name locally with a NEWER stamp.
        store.Seed("a",
            updatedUtc: new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero),
            revision: 7,
            ciphertext: [200, 201, 202]);
        invalidator.Invalidated.Clear();

        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Empty(r.Failed);
        Assert.Equal(["a"], r.Applied);
        Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
        Assert.Equal(new byte[] { 30, 31, 32, 33, 34, 35 }, store.Rows["a"].WrappedDek);
        Assert.True(store.Rows["a"].Revision > 7); // re-stamped past the operator's row
        Assert.Contains("a", invalidator.Invalidated);
    }

    [Fact]
    public async Task Reconcile_OwnedNameDrift_LogsOncePerDriftNotEveryPass()
    {
        var logger = new CapturingLogger<DeployedSecretsMirror>();
        var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
        var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, time: time);
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        await mirror.ReconcileAsync(CancellationToken.None);
        await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);

        await mirror.ReconcileAsync(CancellationToken.None);
        await mirror.ReconcileAsync(CancellationToken.None);
        await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("re-applying"));
        Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Error);
    }

    // ── Unreadable row_json (final review M4) ──

    [Fact]
    public async Task Reconcile_UnparseableRowJson_IsAFailedEntryForThatName_OthersStillApply()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("bad", 1), Art("good", 1)], "d1");
        await CorruptRowJsonAsync(storage, "bad");

        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["good"], r.Applied);
        Assert.Equal(["bad"], r.FailedNames);
        var message = Assert.Single(r.Failed);
        Assert.Contains("'bad'", message);
        Assert.Contains("could not be read", message);
        Assert.True(store.Contains("good"));
        Assert.False(store.Contains("bad"));
    }

    [Fact]
    public async Task Reconcile_UnparseableRemovedRow_StillRemovesAMirroredName()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        await mirror.ReconcileAsync(CancellationToken.None);
        await storage.UpsertDeployedSecretsAsync([], "d2"); // soft-removes "a"
        await CorruptRowJsonAsync(storage, "a");

        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["a"], r.Removed);
        Assert.Empty(r.Failed);
        Assert.True(store.IsTombstoned("a"));
    }

    private static async Task CorruptRowJsonAsync(SiteStorageService storage, string name)
    {
        await using var connection = storage.CreateConnection();
        await using var command = connection.CreateCommand();
        command.CommandText = "UPDATE deployed_secrets SET row_json = '{not json' WHERE name = @name";
        command.Parameters.AddWithValue("@name", name);
        Assert.Equal(1, await command.ExecuteNonQueryAsync());
    }

    [Fact]
    public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");

        await mirror.ReconcileAsync(CancellationToken.None);

        var applied = store.Rows["a"];
        Assert.Equal("kek-central", applied.KekId);
        Assert.Equal(1, applied.Revision);
        Assert.Equal(SecretContentType.Text, applied.ContentType);
    }

    [Theory]
    [InlineData(SecretContentType.Text)]
    [InlineData(SecretContentType.ConnectionString)]
    [InlineData(SecretContentType.Json)]
    [InlineData(SecretContentType.BinaryBase64)]
    public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType)
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync(
            [Art("a", 1, contentType: contentType.ToString())], "d1");

        await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(contentType, store.Rows["a"].ContentType);
    }

    // ── Per-row isolation (ruling 2) ──

    [Fact]
    public async Task Reconcile_OneRowThrows_OthersStillApply()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        // "bad" carries an unrecognised ContentType (as a newer central under version skew
        // might ship), so ToStored's Enum.Parse throws for that row only.
        await storage.UpsertDeployedSecretsAsync(
            [Art("bad", 1, contentType: "NotARealContentType"), Art("good", 1)], "d1");

        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["good"], r.Applied);
        var message = Assert.Single(r.Failed);
        Assert.Contains("'bad'", message);
        Assert.True(store.Contains("good"));
        Assert.False(store.Contains("bad"));
    }

    [Fact]
    public async Task Reconcile_OneRowThrows_FailureIsDeSpammedLikeAnyOtherFailure()
    {
        var logger = new CapturingLogger<DeployedSecretsMirror>();
        var (mirror, storage, _, _, _) = await CreateAsync(logger);
        await storage.UpsertDeployedSecretsAsync([Art("bad", 1, contentType: "NotARealContentType")], "d1");

        await mirror.ReconcileAsync(CancellationToken.None);
        await mirror.ReconcileAsync(CancellationToken.None);

        var levels = logger.Entries
            .Where(e => e.Message.Contains("'bad'") && e.Message.Contains("failed to process"))
            .Select(e => e.Level)
            .ToList();
        Assert.Equal([LogLevel.Error, LogLevel.Debug], levels);
    }

    // ── Cache invalidation (ruling 3) ──

    [Fact]
    public async Task Reconcile_SuccessfulApply_InvalidatesCache_BeforeVerify()
    {
        var (mirror, storage, _, _, invalidator) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");

        await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Contains("a", invalidator.Invalidated);
    }

    [Fact]
    public async Task Reconcile_Removal_InvalidatesCache()
    {
        var (mirror, storage, _, _, invalidator) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        await mirror.ReconcileAsync(CancellationToken.None);
        invalidator.Invalidated.Clear();

        await storage.UpsertDeployedSecretsAsync([], "d2");
        await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Contains("a", invalidator.Invalidated);
    }

    [Fact]
    public async Task Reconcile_WithNoInvalidatorRegistered_StillApplies()
    {
        var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        await storage.InitializeAsync();
        var mirror = new DeployedSecretsMirror(
            storage,
            new FakeSecretStore(),
            new FakeSecretResolver(),
            invalidator: null,
            Options.Create(new SiteRuntimeOptions { DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15) }),
            NullLogger<DeployedSecretsMirror>.Instance,
            TimeProvider.System);

        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Equal(["a"], r.Applied);
    }

    // ── Decrypt verification ──

    [Fact]
    public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsReportedWithKekId()
    {
        var (mirror, storage, _, resolver, _) = await CreateAsync();
        resolver.ThrowFor("a");

        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
        var r = await mirror.ReconcileAsync(CancellationToken.None);

        var message = Assert.Single(r.Failed);
        Assert.Contains("kek-central", message);
        Assert.Contains("'a'", message);
        // FailedNames (Task 10): the name behind each Failed entry, in lockstep order, so a
        // caller (DeploymentManagerActor) can tell whether a failure belongs to names it shipped.
        Assert.Equal(["a"], r.FailedNames);
    }

    [Fact]
    public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported()
    {
        var (mirror, storage, _, resolver, _) = await CreateAsync();
        resolver.ReturnNullFor("a");

        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        var r = await mirror.ReconcileAsync(CancellationToken.None);

        Assert.Single(r.Failed);
    }

    [Fact]
    public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied()
    {
        var (mirror, storage, _, resolver, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");

        var r1 = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Empty(r1.Failed);

        // Master key "changes" (or a KEK rotation invalidates the local unwrap) between
        // passes — even though nothing about deployed_secrets changed, the next reconcile
        // must still catch the failure because it re-verifies unconditionally.
        resolver.ThrowFor("a");
        var r2 = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Single(r2.Failed);
        Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified
    }

    // ── Error-log de-spam (ruling 1 of the original task, unaffected by fix round 1) ──

    [Fact]
    public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess()
    {
        var logger = new CapturingLogger<DeployedSecretsMirror>();
        var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
        resolver.ThrowFor("a");
        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");

        await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error
        await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug
        await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug

        var failureEntries = logger.Entries
            .Where(e => e.Message.Contains("did not decrypt"))
            .ToList();
        Assert.Equal(3, failureEntries.Count);
        Assert.Equal(LogLevel.Error, failureEntries[0].Level);
        Assert.Equal(LogLevel.Debug, failureEntries[1].Level);
        Assert.Equal(LogLevel.Debug, failureEntries[2].Level);

        // Now the secret resolves (e.g. the key issue is fixed).
        resolver.ClearFailures();
        var recovered = await mirror.ReconcileAsync(CancellationToken.None);
        Assert.Empty(recovered.Failed);

        // Fails again — must log at Error again, not Debug, since it succeeded in between.
        resolver.ThrowFor("a");
        await mirror.ReconcileAsync(CancellationToken.None);

        var failureEntriesAfterRecovery = logger.Entries
            .Where(e => e.Message.Contains("did not decrypt"))
            .ToList();
        Assert.Equal(4, failureEntriesAfterRecovery.Count);
        Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level);
    }

    [Fact]
    public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween()
    {
        var logger = new CapturingLogger<DeployedSecretsMirror>();
        var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
        resolver.ThrowFor("a");

        await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
        await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1)
        await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat)

        // Central ships a new revision of the same (still-failing) secret.
        await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2");
        await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new)

        var levels = logger.Entries
            .Where(e => e.Message.Contains("did not decrypt"))
            .Select(e => e.Level)
            .ToList();
        Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels);
    }

    // ── Cancellation + concurrency (ruling 6) ──

    [Fact]
    public async Task ReconcileAsync_PreCancelledToken_ThrowsOperationCanceled()
    {
        var (mirror, _, _, _, _) = await CreateAsync();
        using var cts = new CancellationTokenSource();
        cts.Cancel();

        await Assert.ThrowsAnyAsync<OperationCanceledException>(
            () => mirror.ReconcileAsync(cts.Token));
    }

    [Fact]
    public async Task ReconcileAsync_ConcurrentCallers_BothCompleteWithoutDoubleApply()
    {
        var (mirror, storage, store, _, _) = await CreateAsync();
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");

        var t1 = mirror.ReconcileAsync(CancellationToken.None);
        var t2 = mirror.ReconcileAsync(CancellationToken.None);
        var results = await Task.WhenAll(t1, t2);

        // The gate serializes the two passes: whichever runs first applies it, the other
        // sees it already ledgered at the shipped revision and applies nothing.
        Assert.Equal(1, results.Sum(r => r.Applied.Count));
        Assert.True(store.Contains("a"));

        var mirrored = await storage.GetMirroredSecretsAsync();
        Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
    }

    // ── Dispose (ruling 6) ──

    [Fact]
    public void Dispose_DoesNotThrow_AndIsIdempotent()
    {
        var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        var mirror = new DeployedSecretsMirror(
            storage,
            new FakeSecretStore(),
            new FakeSecretResolver(),
            invalidator: null,
            Options.Create(new SiteRuntimeOptions()),
            NullLogger<DeployedSecretsMirror>.Instance,
            TimeProvider.System);

        mirror.Dispose();
        mirror.Dispose();
    }

    // ── Hosted loop ──

    [Fact]
    public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval()
    {
        var (mirror, storage, store, resolver, _) = await CreateAsync(
            interval: TimeSpan.FromMilliseconds(30));
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");

        await mirror.StartAsync(CancellationToken.None);
        try
        {
            await WaitUntilAsync(() => store.Contains("a"));
            Assert.True(store.Contains("a")); // ran once immediately at start

            // Prove the loop keeps ticking (not just the immediate start-of-day pass) by
            // observing further resolver invocations over time.
            var before = resolver.CallCount;
            await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000);
            Assert.True(resolver.CallCount > before + 1);
        }
        finally
        {
            await mirror.StopAsync(CancellationToken.None);
        }
    }

    [Fact]
    public async Task ExecuteAsync_PerRowFailure_IsIsolated_AndLoopRecoversOnceCleared()
    {
        var logger = new CapturingLogger<DeployedSecretsMirror>();
        var (mirror, storage, store, _, _) = await CreateAsync(
            logger, interval: TimeSpan.FromMilliseconds(30));
        await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
        store.ThrowOnApply = true;

        await mirror.StartAsync(CancellationToken.None);
        try
        {
            // A per-row store fault is isolated (ruling 2): it never escapes ReconcileAsync,
            // so it shows up as a row-level Error log, not the pass-level Warning.
            await WaitUntilAsync(() => logger.Entries.Any(
                e => e.Level == LogLevel.Error && e.Message.Contains("failed to process")));

            // The loop must still be alive after the failure — force success and confirm a
            // later pass applies the secret.
            store.ThrowOnApply = false;
            await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000);
            Assert.True(store.Contains("a"));
        }
        finally
        {
            await mirror.StopAsync(CancellationToken.None);
        }
    }

    // ── Harness ──

    private async Task<(
        DeployedSecretsMirror Mirror,
        SiteStorageService Storage,
        FakeSecretStore Store,
        FakeSecretResolver Resolver,
        FakeSecretCacheInvalidator Invalidator)> CreateAsync(
        CapturingLogger<DeployedSecretsMirror>? logger = null,
        TimeSpan? interval = null,
        TimeProvider? time = null)
    {
        var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
        await storage.InitializeAsync();

        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var invalidator = new FakeSecretCacheInvalidator();
        var options = Options.Create(new SiteRuntimeOptions
        {
            DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15),
        });

        var mirror = new DeployedSecretsMirror(
            storage,
            store,
            resolver,
            invalidator,
            options,
            (ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
            time ?? TimeProvider.System);

        return (mirror, storage, store, resolver, invalidator);
    }

    private static SecretArtifact Art(
        string name, long revision, string kek = "kek-1", string contentType = "Text", byte[]? ciphertext = null) => new(
        Name: name,
        ContentType: contentType,
        Ciphertext: ciphertext ?? [1, 2, 3, 4, 5],
        Nonce: [10, 11, 12],
        Tag: [20, 21, 22, 23],
        WrappedDek: [30, 31, 32, 33, 34, 35],
        WrapNonce: [40, 41, 42],
        WrapTag: [50, 51, 52, 53],
        KekId: kek,
        Revision: revision,
        CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
        UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
        Description: "test secret");

    private static async Task WaitUntilAsync(Func<bool> condition, int timeoutMs = 5000)
    {
        var elapsed = 0;
        const int step = 20;
        while (!condition() && elapsed < timeoutMs)
        {
            await Task.Delay(step);
            elapsed += step;
        }
        Assert.True(condition(), "condition not met within timeout");
    }

    /// <summary>Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests.</summary>
    private sealed class CapturingLogger<T> : ILogger<T>
    {
        public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new();

        IDisposable? ILogger.BeginScope<TState>(TState state) => null;
        public bool IsEnabled(LogLevel logLevel) => true;

        public void Log<TState>(
            LogLevel logLevel, EventId eventId, TState state, Exception? exception,
            Func<TState, Exception?, string> formatter)
            => Entries.Enqueue((logLevel, formatter(state, exception)));
    }

    /// <summary>Minimal controllable <see cref="TimeProvider"/> — mirrors the pattern used by <c>ApiMethodCacheTests</c>.</summary>
    private sealed class FakeTimeProvider : TimeProvider
    {
        public DateTimeOffset Now { get; set; }
        public FakeTimeProvider(DateTimeOffset start) => Now = start;
        public override DateTimeOffset GetUtcNow() => Now;
    }

    /// <summary>Records every <see cref="ISecretCacheInvalidator.Invalidate"/> call by secret name.</summary>
    private sealed class FakeSecretCacheInvalidator : ISecretCacheInvalidator
    {
        public List<string> Invalidated { get; } = new();
        public void Invalidate(SecretName name) => Invalidated.Add(name.Value);
    }

    /// <summary>
    /// In-memory <see cref="ISecretStore"/> honouring the REAL <c>SqliteSecretStore</c>
    /// semantics (fix round 1, review Important #1): <see cref="ApplyReplicatedAsync"/> is
    /// strict last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/> and applies
    /// verbatim (no revision bump); <see cref="DeleteAsync"/> tombstones in place (bumps
    /// revision, stamps <see cref="Now"/>) rather than hard-deleting.
    /// </summary>
    private sealed class FakeSecretStore : ISecretStore
    {
        private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);

        /// <summary>
        /// Controllable "now" used to stamp a <see cref="DeleteAsync"/> tombstone —
        /// deterministic, not wall-clock, so tests can control ordering against a shipped
        /// artifact's fixed <c>UpdatedUtc</c>.
        /// </summary>
        public DateTimeOffset Now { get; set; } = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);

        /// <summary>Set to make the next <see cref="ApplyReplicatedAsync"/> call throw, simulating a store fault.</summary>
        public bool ThrowOnApply { get; set; }

        /// <summary>Every row currently held, including tombstones, for direct inspection.</summary>
        public IReadOnlyDictionary<string, StoredSecret> Rows => _rows;

        /// <summary>Seeds a LIVE row directly (bypassing ApplyReplicatedAsync/LWW) — an out-of-band secret.</summary>
        public void Seed(string name, DateTimeOffset updatedUtc, long revision = 1, byte[]? ciphertext = null) =>
            _rows[name] = new StoredSecret
            {
                Name = new SecretName(name),
                ContentType = SecretContentType.Text,
                Ciphertext = ciphertext ?? [9],
                Nonce = [9],
                Tag = [9],
                WrappedDek = [9],
                WrapNonce = [9],
                WrapTag = [9],
                KekId = "kek-out-of-band",
                Revision = revision,
                CreatedUtc = updatedUtc,
                UpdatedUtc = updatedUtc,
            };

        /// <summary>True if the row is present and NOT tombstoned.</summary>
        public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted;

        /// <summary>True if the row is present as a tombstone.</summary>
        public bool IsTombstoned(string name) => _rows.TryGetValue(name, out var row) && row.IsDeleted;

        public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
            Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);

        public Task UpsertAsync(StoredSecret row, CancellationToken ct)
        {
            _rows[row.Name.Value] = row;
            return Task.CompletedTask;
        }

        /// <summary>Tombstones (does not hard-delete) — matches <c>SqliteSecretStore.DeleteAsync</c>.</summary>
        public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
        {
            if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted)
            {
                return Task.FromResult(false);
            }

            _rows[name.Value] = existing with
            {
                IsDeleted = true,
                DeletedUtc = Now,
                UpdatedUtc = Now,
                Revision = existing.Revision + 1,
                UpdatedBy = actor,
            };
            return Task.FromResult(true);
        }

        public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
            Task.FromResult<IReadOnlyList<SecretMetadata>>([]);

        public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
            Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);

        /// <summary>Strict LWW on (UpdatedUtc, Revision) via <see cref="SecretLastWriterWins.IsNewer"/>; applied verbatim — matches <c>SqliteSecretStore</c>.</summary>
        public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
        {
            if (ThrowOnApply)
            {
                throw new InvalidOperationException("simulated store fault");
            }

            if (_rows.TryGetValue(row.Name.Value, out var existing)
                && !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision))
            {
                return Task.CompletedTask; // no-op, matches real semantics
            }

            _rows[row.Name.Value] = row;
            return Task.CompletedTask;
        }

        public Task<bool> ApplyRewrapAsync(
            StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
            Task.FromResult(false);
    }

    /// <summary>In-memory <see cref="ISecretResolver"/> whose per-name resolution outcome tests control.</summary>
    private sealed class FakeSecretResolver : ISecretResolver
    {
        private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);
        private readonly HashSet<string> _nullReturning = new(StringComparer.Ordinal);

        public int CallCount;

        /// <summary>When set, a name that is absent or tombstoned in this store resolves to null, like the real resolver.</summary>
        public FakeSecretStore? BackingStore { get; set; }

        /// <summary>Row absent/tombstoned path — <c>DefaultSecretResolver.GetAsync</c> returns null.</summary>
        public void ReturnNullFor(string name) => _nullReturning.Add(name);

        /// <summary>Decryption/integrity failure path — <c>DefaultSecretResolver.GetAsync</c> rethrows <c>SecretDecryptionException</c>.</summary>
        public void ThrowFor(string name) => _throwing.Add(name);

        public void ClearFailures()
        {
            _nullReturning.Clear();
            _throwing.Clear();
        }

        public Task<string?> GetAsync(SecretName name, CancellationToken ct)
        {
            Interlocked.Increment(ref CallCount);
            if (_throwing.Contains(name.Value))
            {
                throw new InvalidOperationException("simulated decrypt failure");
            }

            if (BackingStore is not null && !BackingStore.Contains(name.Value))
            {
                return Task.FromResult<string?>(null);
            }

            return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
        }
    }
}