using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets; using ZB.MOM.WW.ScadaBridge.TestSupport; using ZB.MOM.WW.Secrets.Abstractions; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets; /// /// Site-secrets artifact deploy: — per-node apply /// + decrypt-verify of deployed_secrets into this node's own secret store. /// /// /// honours the /// real SqliteSecretStore semantics — ApplyReplicatedAsync is strict /// last-writer-wins via , and DeleteAsync /// tombstones rather than hard-deleting — so tests exercise the same conflict shapes the real /// store can produce, not an idealized "always overwrite" double. /// public class DeployedSecretsMirrorTests : IDisposable { private readonly TestLocalDb _localDb; /// Initializes the test class with a temporary LocalDb database. public DeployedSecretsMirrorTests() { _localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror"); } /// Releases the resources used by this instance. public void Dispose() { var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } // ── Apply / remove / out-of-band ── /// Verifies that reconcile applies and verifies deployed secrets and deletes only the names it mirrored. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored() { var (mirror, storage, store, _, _) = await CreateAsync(); // Seeded OLDER than what will be shipped, so the shipped row legitimately wins LWW // and is applied — the strictly-NEWER out-of-band case is covered separately by // Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval. store.Seed("out-of-band", updatedUtc: new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero)); await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1"); var r1 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal)); Assert.Empty(r1.Failed); await storage.UpsertDeployedSecretsAsync([], "d2"); // Simulate: "out-of-band" was never actually mirrored by this node (an operator secret // that happens to share the name) by clearing its ledger row directly — a defensive // check independent of the natural not-ledgered path exercised below. await storage.RemoveMirroredSecretAsync("out-of-band"); var r2 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r2.Removed); Assert.True(store.Contains("out-of-band")); Assert.False(store.Contains("a")); Assert.True(store.IsTombstoned("a")); } /// Verifies that a newer out-of-band secret is neither overwritten nor ledgered and survives a later removal. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval() { var (mirror, storage, store, _, invalidator) = await CreateAsync(); var operatorCiphertext = new byte[] { 200, 201, 202 }; // Strictly newer than Art()'s fixed UpdatedUtc (2026-09-28T01:00:00Z) — legitimately // wins LWW, exactly as the real store would order it. store.Seed("a", updatedUtc: new DateTimeOffset(2026, 9, 28, 2, 0, 0, TimeSpan.Zero), revision: 5, ciphertext: operatorCiphertext); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var r1 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r1.Applied); var message = Assert.Single(r1.Failed); Assert.Contains("'a'", message); Assert.Contains("newer local value", message); Assert.DoesNotContain("did not decrypt", message); // distinct from the master-key message Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext); // untouched Assert.Empty(invalidator.Invalidated); // never applied -> never invalidated // Never ledgered, so a later central tombstone must NOT delete the operator's secret. await storage.UpsertDeployedSecretsAsync([], "d2"); var r2 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r2.Removed); Assert.True(store.Contains("a")); Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext); } /// Verifies that a removed then re-added secret recovers from its tombstone, reapplies and verifies. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_RemoveThenReAdd_TombstoneIsRecovered_ReappliesAndVerifies() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var firstApply = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], firstApply.Applied); // Site stops referencing it: central tombstones the row, this node deletes it. The // tombstone is stamped LATER than the originally-shipped UpdatedUtc (01:00). store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero); await storage.UpsertDeployedSecretsAsync([], "d2"); var removePass = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], removePass.Removed); Assert.True(store.IsTombstoned("a")); invalidator.Invalidated.Clear(); // Site references it again — central ships the SAME (unchanged, older-stamped) // secret. A naive ApplyReplicatedAsync call is a no-op against the newer tombstone; // the mirror must detect and recover this. time.Now = new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero); // later than the tombstone await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3"); var reAddPass = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], reAddPass.Applied); Assert.Empty(reAddPass.Failed); Assert.True(store.Contains("a")); Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); // Art()'s ciphertext, re-materialized Assert.Contains("a", invalidator.Invalidated); // Ledgering used the SHIPPED artifact's own revision, not the bumped local one, so a // later pass with the same shipped row is a true no-op. var thirdPass = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(thirdPass.Applied); Assert.False(thirdPass.Changed); var mirrored = await storage.GetMirroredSecretsAsync(); Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision); } /// Verifies that a clock-skewed recovered row still converges on the next central revision. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_ClockSkewedRecoveredRow_StillConvergesOnNextCentralRevision() { // Fix round 2 (review "clock skew" finding): a tombstone-recovery bump can leave a local // row stamped AHEAD of a central UpdatedUtc that arrives afterward (site clock skew, or // simply a central edit made before the recovery but shipped after it). The row is still // OWNED by this mirror's ledger, so a later genuinely-new central revision must not be // misreported as an out-of-band "newer local value" — it must converge. var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); // Remove then re-add — but the SITE CLOCK IS SKEWED FAR INTO THE FUTURE while the // recovery bump happens (a bad RTC/NTP fault), so the recovered row's UpdatedUtc lands // far ahead of anything central will ship for a long while. store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero); await storage.UpsertDeployedSecretsAsync([], "d2"); await mirror.ReconcileAsync(CancellationToken.None); // tombstones "a" time.Now = new DateTimeOffset(2099, 1, 1, 0, 0, 0, TimeSpan.Zero); // skewed site clock await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3"); var recoveredPass = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], recoveredPass.Applied); Assert.True(store.Rows["a"].UpdatedUtc.Year >= 2099); // confirms the skewed stamp landed // Site clock recovers to normal. Central ships a genuinely NEW revision (different // ciphertext, i.e. an actual rotation) with an ordinary timestamp — OLDER than the // skewed local stamp, so a naive ApplyReplicatedAsync would no-op against it. time.Now = new DateTimeOffset(2026, 9, 29, 0, 0, 0, TimeSpan.Zero); var rotatedCiphertext = new byte[] { 7, 7, 7 }; await storage.UpsertDeployedSecretsAsync([Art("a", 2, ciphertext: rotatedCiphertext)], "d4"); var convergePass = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], convergePass.Applied); Assert.Empty(convergePass.Failed); // NOT "newer local value" — this mirror owns the name Assert.Equal(rotatedCiphertext, store.Rows["a"].Ciphertext); Assert.Contains("a", invalidator.Invalidated); var mirrored = await storage.GetMirroredSecretsAsync(); Assert.Equal(2, mirrored.Single(m => m.Name == "a").Revision); } /// Verifies that reconcile reapplies a secret only when its revision changes. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges() { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var r1 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r1.Applied); var r2 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r2.Applied); Assert.False(r2.Changed); await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2"); var r3 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r3.Applied); Assert.Equal(2, store.Rows["a"].Revision); } // ── Same-revision drift (final review I1) ── /// Verifies that an owned name deleted locally is healed and verified rather than misreported as a KEK mismatch. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OwnedNameDeletedLocally_IsHealedAndVerified_NotMisreportedAsKekMismatch() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero)); var (mirror, storage, store, resolver, invalidator) = await CreateAsync(time: time); resolver.BackingStore = store; // resolve returns null once the local row is gone await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied); // An operator deletes the mirror-owned name on this node (library CLI/UI) — the // ledger still says r1 is mirrored, and central still ships r1. await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None); Assert.True(store.IsTombstoned("a")); invalidator.Invalidated.Clear(); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r.Failed); Assert.Equal(["a"], r.Applied); Assert.True(store.Contains("a")); Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); Assert.Contains("a", invalidator.Invalidated); // Healed state is stable: the next pass is a no-op that still verifies clean. var next = await mirror.ReconcileAsync(CancellationToken.None); Assert.False(next.Changed); Assert.Empty(next.Failed); Assert.Equal(1, (await storage.GetMirroredSecretsAsync()).Single(m => m.Name == "a").Revision); } /// Verifies that an owned name edited locally is reverted to the deployed ciphertext. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OwnedNameEditedLocally_IsRevertedToTheDeployedCiphertext() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero)); var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied); // An operator overwrites the mirror-owned name locally with a NEWER stamp. store.Seed("a", updatedUtc: new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero), revision: 7, ciphertext: [200, 201, 202]); invalidator.Invalidated.Clear(); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r.Failed); Assert.Equal(["a"], r.Applied); Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); Assert.Equal(new byte[] { 30, 31, 32, 33, 34, 35 }, store.Rows["a"].WrappedDek); Assert.True(store.Rows["a"].Revision > 7); // re-stamped past the operator's row Assert.Contains("a", invalidator.Invalidated); } /// Verifies that owned-name drift is logged once per drift rather than on every pass. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OwnedNameDrift_LogsOncePerDriftNotEveryPass() { var logger = new CapturingLogger(); var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero)); var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, time: time); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None); await mirror.ReconcileAsync(CancellationToken.None); await mirror.ReconcileAsync(CancellationToken.None); await mirror.ReconcileAsync(CancellationToken.None); Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("re-applying")); Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Error); } // ── Unreadable row_json ── /// Verifies that an unparseable row JSON fails only that name while other names still apply. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_UnparseableRowJson_IsAFailedEntryForThatName_OthersStillApply() { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("bad", 1), Art("good", 1)], "d1"); await CorruptRowJsonAsync(storage, "bad"); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["good"], r.Applied); Assert.Equal(["bad"], r.FailedNames); var message = Assert.Single(r.Failed); Assert.Contains("'bad'", message); Assert.Contains("could not be read", message); Assert.True(store.Contains("good")); Assert.False(store.Contains("bad")); } /// Verifies that an unparseable removed row still removes a mirrored name. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_UnparseableRemovedRow_StillRemovesAMirroredName() { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); await storage.UpsertDeployedSecretsAsync([], "d2"); // soft-removes "a" await CorruptRowJsonAsync(storage, "a"); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r.Removed); Assert.Empty(r.Failed); Assert.True(store.IsTombstoned("a")); } private static async Task CorruptRowJsonAsync(SiteStorageService storage, string name) { await using var connection = storage.CreateConnection(); await using var command = connection.CreateCommand(); command.CommandText = "UPDATE deployed_secrets SET row_json = '{not json' WHERE name = @name"; command.Parameters.AddWithValue("@name", name); Assert.Equal(1, await command.ExecuteNonQueryAsync()); } /// Verifies that reconcile applies through ApplyReplicatedAsync with a row mapped from the secret artifact. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact() { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1"); await mirror.ReconcileAsync(CancellationToken.None); var applied = store.Rows["a"]; Assert.Equal("kek-central", applied.KekId); Assert.Equal(1, applied.Revision); Assert.Equal(SecretContentType.Text, applied.ContentType); } /// Verifies that reconcile round-trips every secret content type. /// The secret content type under test. /// A task that represents the asynchronous operation. [Theory] [InlineData(SecretContentType.Text)] [InlineData(SecretContentType.ConnectionString)] [InlineData(SecretContentType.Json)] [InlineData(SecretContentType.BinaryBase64)] public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType) { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync( [Art("a", 1, contentType: contentType.ToString())], "d1"); await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(contentType, store.Rows["a"].ContentType); } // ── Per-row isolation (ruling 2) ── /// Verifies that when one row throws the other rows still apply. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OneRowThrows_OthersStillApply() { var (mirror, storage, store, _, _) = await CreateAsync(); // "bad" carries an unrecognised ContentType (as a newer central under version skew // might ship), so ToStored's Enum.Parse throws for that row only. await storage.UpsertDeployedSecretsAsync( [Art("bad", 1, contentType: "NotARealContentType"), Art("good", 1)], "d1"); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["good"], r.Applied); var message = Assert.Single(r.Failed); Assert.Contains("'bad'", message); Assert.True(store.Contains("good")); Assert.False(store.Contains("bad")); } /// Verifies that a throwing row's failure is de-spammed like any other failure. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_OneRowThrows_FailureIsDeSpammedLikeAnyOtherFailure() { var logger = new CapturingLogger(); var (mirror, storage, _, _, _) = await CreateAsync(logger); await storage.UpsertDeployedSecretsAsync([Art("bad", 1, contentType: "NotARealContentType")], "d1"); await mirror.ReconcileAsync(CancellationToken.None); await mirror.ReconcileAsync(CancellationToken.None); var levels = logger.Entries .Where(e => e.Message.Contains("'bad'") && e.Message.Contains("failed to process")) .Select(e => e.Level) .ToList(); Assert.Equal([LogLevel.Error, LogLevel.Debug], levels); } // ── Cache invalidation (ruling 3) ── /// Verifies that a successful apply invalidates the cache before verification. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_SuccessfulApply_InvalidatesCache_BeforeVerify() { var (mirror, storage, _, _, invalidator) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); Assert.Contains("a", invalidator.Invalidated); } /// Verifies that a removal invalidates the cache. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_Removal_InvalidatesCache() { var (mirror, storage, _, _, invalidator) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); invalidator.Invalidated.Clear(); await storage.UpsertDeployedSecretsAsync([], "d2"); await mirror.ReconcileAsync(CancellationToken.None); Assert.Contains("a", invalidator.Invalidated); } /// Verifies that reconcile still applies when no cache invalidator is registered. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_WithNoInvalidatorRegistered_StillApplies() { var storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); await storage.InitializeAsync(); var mirror = new DeployedSecretsMirror( storage, new FakeSecretStore(), new FakeSecretResolver(), invalidator: null, Options.Create(new SiteRuntimeOptions { DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15) }), NullLogger.Instance, TimeProvider.System); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r.Applied); } // ── Decrypt verification ── /// Verifies that a resolver decrypt exception is reported together with the KEK id. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsReportedWithKekId() { var (mirror, storage, _, resolver, _) = await CreateAsync(); resolver.ThrowFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1"); var r = await mirror.ReconcileAsync(CancellationToken.None); var message = Assert.Single(r.Failed); Assert.Contains("kek-central", message); Assert.Contains("'a'", message); // FailedNames: the name behind each Failed entry, in lockstep order, so a // caller (DeploymentManagerActor) can tell whether a failure belongs to names it shipped. Assert.Equal(["a"], r.FailedNames); } /// Verifies that a null resolve is also reported as a decrypt failure. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported() { var (mirror, storage, _, resolver, _) = await CreateAsync(); resolver.ReturnNullFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Single(r.Failed); } /// Verifies that reconcile verifies on every pass even when the secret is already applied. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied() { var (mirror, storage, _, resolver, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var r1 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(r1.Failed); // Master key "changes" (or a KEK rotation invalidates the local unwrap) between // passes — even though nothing about deployed_secrets changed, the next reconcile // must still catch the failure because it re-verifies unconditionally. resolver.ThrowFor("a"); var r2 = await mirror.ReconcileAsync(CancellationToken.None); Assert.Single(r2.Failed); Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified } // ── Error-log de-spam (ruling 1 of the original task, unaffected by fix round 1) ── /// Verifies that a decrypt failure logs an error the first time, then debug, then an error again after a success. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess() { var logger = new CapturingLogger(); var (mirror, storage, _, resolver, _) = await CreateAsync(logger); resolver.ThrowFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1"); await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug var failureEntries = logger.Entries .Where(e => e.Message.Contains("did not decrypt")) .ToList(); Assert.Equal(3, failureEntries.Count); Assert.Equal(LogLevel.Error, failureEntries[0].Level); Assert.Equal(LogLevel.Debug, failureEntries[1].Level); Assert.Equal(LogLevel.Debug, failureEntries[2].Level); // Now the secret resolves (e.g. the key issue is fixed). resolver.ClearFailures(); var recovered = await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(recovered.Failed); // Fails again — must log at Error again, not Debug, since it succeeded in between. resolver.ThrowFor("a"); await mirror.ReconcileAsync(CancellationToken.None); var failureEntriesAfterRecovery = logger.Entries .Where(e => e.Message.Contains("did not decrypt")) .ToList(); Assert.Equal(4, failureEntriesAfterRecovery.Count); Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level); } /// Verifies that a decrypt failure on a new revision logs an error again even without an intervening success. /// A task that represents the asynchronous operation. [Fact] public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween() { var logger = new CapturingLogger(); var (mirror, storage, _, resolver, _) = await CreateAsync(logger); resolver.ThrowFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1"); await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1) await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat) // Central ships a new revision of the same (still-failing) secret. await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2"); await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new) var levels = logger.Entries .Where(e => e.Message.Contains("did not decrypt")) .Select(e => e.Level) .ToList(); Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels); } // ── Cancellation + concurrency (ruling 6) ── /// Verifies that a pre-cancelled token makes reconcile throw an . /// A task that represents the asynchronous operation. [Fact] public async Task ReconcileAsync_PreCancelledToken_ThrowsOperationCanceled() { var (mirror, _, _, _, _) = await CreateAsync(); using var cts = new CancellationTokenSource(); cts.Cancel(); await Assert.ThrowsAnyAsync( () => mirror.ReconcileAsync(cts.Token)); } /// Verifies that concurrent reconcile callers both complete without applying twice. /// A task that represents the asynchronous operation. [Fact] public async Task ReconcileAsync_ConcurrentCallers_BothCompleteWithoutDoubleApply() { var (mirror, storage, store, _, _) = await CreateAsync(); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); var t1 = mirror.ReconcileAsync(CancellationToken.None); var t2 = mirror.ReconcileAsync(CancellationToken.None); var results = await Task.WhenAll(t1, t2); // The gate serializes the two passes: whichever runs first applies it, the other // sees it already ledgered at the shipped revision and applies nothing. Assert.Equal(1, results.Sum(r => r.Applied.Count)); Assert.True(store.Contains("a")); var mirrored = await storage.GetMirroredSecretsAsync(); Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision); } // ── Dispose (ruling 6) ── /// Verifies that disposing the mirror does not throw and is idempotent. [Fact] public void Dispose_DoesNotThrow_AndIsIdempotent() { var storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); var mirror = new DeployedSecretsMirror( storage, new FakeSecretStore(), new FakeSecretResolver(), invalidator: null, Options.Create(new SiteRuntimeOptions()), NullLogger.Instance, TimeProvider.System); mirror.Dispose(); mirror.Dispose(); } // ── Hosted loop ── /// Verifies that the background loop runs once at start and again every interval. /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval() { var (mirror, storage, store, resolver, _) = await CreateAsync( interval: TimeSpan.FromMilliseconds(30)); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.StartAsync(CancellationToken.None); try { await WaitUntilAsync(() => store.Contains("a")); Assert.True(store.Contains("a")); // ran once immediately at start // Prove the loop keeps ticking (not just the immediate start-of-day pass) by // observing further resolver invocations over time. var before = resolver.CallCount; await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000); Assert.True(resolver.CallCount > before + 1); } finally { await mirror.StopAsync(CancellationToken.None); } } /// Verifies that a per-row failure is isolated and the loop recovers once the failure clears. /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_PerRowFailure_IsIsolated_AndLoopRecoversOnceCleared() { var logger = new CapturingLogger(); var (mirror, storage, store, _, _) = await CreateAsync( logger, interval: TimeSpan.FromMilliseconds(30)); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); store.ThrowOnApply = true; await mirror.StartAsync(CancellationToken.None); try { // A per-row store fault is isolated (ruling 2): it never escapes ReconcileAsync, // so it shows up as a row-level Error log, not the pass-level Warning. await WaitUntilAsync(() => logger.Entries.Any( e => e.Level == LogLevel.Error && e.Message.Contains("failed to process"))); // The loop must still be alive after the failure — force success and confirm a // later pass applies the secret. store.ThrowOnApply = false; await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000); Assert.True(store.Contains("a")); } finally { await mirror.StopAsync(CancellationToken.None); } } // ── Per-node status row for central ── /// Verifies that the first pass after startup writes this node's status row even when healthy. /// A task that represents the asynchronous operation. [Fact] public async Task Status_FirstPassAfterStartup_WritesThisNodesRow_EvenWhenHealthy() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b")); await mirror.ReconcileAsync(CancellationToken.None); var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); Assert.Equal("node-b", status.NodeName); Assert.Empty(status.Failures); Assert.Equal(time.Now, status.CheckedAtUtc); } /// Verifies that an unchanged failure set is not rewritten on later passes. /// A task that represents the asynchronous operation. [Fact] public async Task Status_UnchangedFailureSet_IsNotRewrittenOnLaterPasses() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b")); resolver.ThrowFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1"); await mirror.ReconcileAsync(CancellationToken.None); var firstWrite = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); // Same failure on every later pass: the row is replicated, so it must not be rewritten. time.Now = time.Now.AddMinutes(5); await mirror.ReconcileAsync(CancellationToken.None); await mirror.ReconcileAsync(CancellationToken.None); var after = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); Assert.Equal(firstWrite.CheckedAtUtc, after.CheckedAtUtc); var failure = Assert.Single(after.Failures); Assert.Equal("a", failure.SecretName); Assert.Contains("kek-central", failure.Message); } /// Verifies that an unchanged failure set is refreshed once the refresh interval has passed. /// A task that represents the asynchronous operation. [Fact] public async Task Status_UnchangedFailureSet_IsRefreshedOnceTheRefreshIntervalHasPassed() { // The row's checked_at is a heartbeat: SecretMirrorStatusReporter prunes rows that stop // being refreshed, so a live node must re-write an unchanged row every refresh interval. var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b")); await mirror.ReconcileAsync(CancellationToken.None); var first = Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc; time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval - TimeSpan.FromSeconds(1); await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(first, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc); time.Now = first + new SiteRuntimeOptions().SecretMirrorStatusRefreshInterval; await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(time.Now, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc); } /// Verifies that a status row deleted by a peer prune is rewritten on the next pass. /// A task that represents the asynchronous operation. [Fact] public async Task Status_RowDeletedByAPeerPrune_IsRewrittenOnTheNextPass() { // Rig finding: when both nodes restart together, the peer's reporter can prune this // node's old row concurrently with this node's fresh write, and the replicated delete // wins. The mirror notices its row is gone and writes it again. var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b")); await mirror.ReconcileAsync(CancellationToken.None); var written = Assert.Single(await storage.GetSecretMirrorStatusesAsync()).CheckedAtUtc; Assert.True(await storage.DeleteSecretMirrorStatusIfUnchangedAsync("node-b", written)); time.Now = time.Now.AddSeconds(15); await mirror.ReconcileAsync(CancellationToken.None); var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); Assert.Equal("node-b", status.NodeName); Assert.Equal(time.Now, status.CheckedAtUtc); } /// Verifies that a failure set change rewrites the status row and that it clears once healed. /// A task that represents the asynchronous operation. [Fact] public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed() { var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero)); var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-a")); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); Assert.Empty(Assert.Single(await storage.GetSecretMirrorStatusesAsync()).Failures); // A decrypt failure appears -> row rewritten with the failure. resolver.ThrowFor("a"); time.Now = time.Now.AddMinutes(1); await mirror.ReconcileAsync(CancellationToken.None); var failing = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); Assert.Equal("a", Assert.Single(failing.Failures).SecretName); Assert.Equal(time.Now, failing.CheckedAtUtc); // Healed -> row rewritten empty. resolver.ClearFailures(); time.Now = time.Now.AddMinutes(1); await mirror.ReconcileAsync(CancellationToken.None); var healed = Assert.Single(await storage.GetSecretMirrorStatusesAsync()); Assert.Empty(healed.Failures); Assert.Equal(time.Now, healed.CheckedAtUtc); } /// Verifies that the status row carries only names and messages, never ciphertext. /// A task that represents the asynchronous operation. [Fact] public async Task Status_RowCarriesNamesAndMessagesOnly_NeverCiphertext() { var (mirror, storage, _, resolver, _) = await CreateAsync(nodeIdentity: new FixedNodeIdentity("node-a")); resolver.ThrowFor("a"); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await mirror.ReconcileAsync(CancellationToken.None); await using var connection = storage.CreateConnection(); await using var command = connection.CreateCommand(); command.CommandText = "SELECT failures_json FROM secret_mirror_status WHERE node_name = 'node-a'"; var json = (string)(await command.ExecuteScalarAsync())!; Assert.Contains("\"SecretName\":\"a\"", json); Assert.DoesNotContain("Ciphertext", json); Assert.DoesNotContain(Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5 }), json); } /// Verifies that a status write failure does not break the pass and is retried on the next pass. /// A task that represents the asynchronous operation. [Fact] public async Task Status_WriteFailure_DoesNotBreakThePass_AndIsRetriedNextPass() { var logger = new CapturingLogger(); var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, nodeIdentity: new FixedNodeIdentity("node-a")); await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1"); await using (var connection = storage.CreateConnection()) await using (var drop = connection.CreateCommand()) { drop.CommandText = "DROP TABLE secret_mirror_status"; await drop.ExecuteNonQueryAsync(); } var r = await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(["a"], r.Applied); Assert.True(store.Contains("a")); Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("mirror status")); // Table back (e.g. transient fault cleared): the unchanged failure set is still // written, because the earlier write never landed. await storage.InitializeAsync(); await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal("node-a", Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName); } /// Verifies that without a node identity the status row falls back to the machine name. /// A task that represents the asynchronous operation. [Fact] public async Task Status_WithoutNodeIdentity_FallsBackToMachineName() { var (mirror, storage, _, _, _) = await CreateAsync(); await mirror.ReconcileAsync(CancellationToken.None); Assert.Equal(Environment.MachineName, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName); } private sealed class FixedNodeIdentity(string? name) : INodeIdentityProvider { /// public string? NodeName { get; } = name; } // ── Harness ── private async Task<( DeployedSecretsMirror Mirror, SiteStorageService Storage, FakeSecretStore Store, FakeSecretResolver Resolver, FakeSecretCacheInvalidator Invalidator)> CreateAsync( CapturingLogger? logger = null, TimeSpan? interval = null, TimeProvider? time = null, INodeIdentityProvider? nodeIdentity = null) { var storage = new SiteStorageService(_localDb.Db, NullLogger.Instance); await storage.InitializeAsync(); var store = new FakeSecretStore(); var resolver = new FakeSecretResolver(); var invalidator = new FakeSecretCacheInvalidator(); var options = Options.Create(new SiteRuntimeOptions { DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15), }); var mirror = new DeployedSecretsMirror( storage, store, resolver, invalidator, options, (ILogger?)logger ?? NullLogger.Instance, time ?? TimeProvider.System, nodeIdentity); return (mirror, storage, store, resolver, invalidator); } private static SecretArtifact Art( string name, long revision, string kek = "kek-1", string contentType = "Text", byte[]? ciphertext = null) => new( Name: name, ContentType: contentType, Ciphertext: ciphertext ?? [1, 2, 3, 4, 5], Nonce: [10, 11, 12], Tag: [20, 21, 22, 23], WrappedDek: [30, 31, 32, 33, 34, 35], WrapNonce: [40, 41, 42], WrapTag: [50, 51, 52, 53], KekId: kek, Revision: revision, CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero), UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero), Description: "test secret"); private static async Task WaitUntilAsync(Func condition, int timeoutMs = 5000) { var elapsed = 0; const int step = 20; while (!condition() && elapsed < timeoutMs) { await Task.Delay(step); elapsed += step; } Assert.True(condition(), "condition not met within timeout"); } /// Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests. private sealed class CapturingLogger : ILogger { /// Gets the captured log entries in emission order. public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new(); /// IDisposable? ILogger.BeginScope(TState state) => null; /// Reports whether logging is enabled for the given level. /// The log level. /// True if the level is enabled; otherwise false. public bool IsEnabled(LogLevel logLevel) => true; /// Records the log entry for later assertions. /// The type of the state object. /// The log level. /// The event id. /// The scope or log state. /// The exception associated with the entry, if any. /// Formats the state and exception into the message text. public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => Entries.Enqueue((logLevel, formatter(state, exception))); } /// Minimal controllable — mirrors the pattern used by ApiMethodCacheTests. private sealed class FakeTimeProvider : TimeProvider { /// Gets or sets the current time reported by this provider. public DateTimeOffset Now { get; set; } /// Initializes the provider at the given start time. /// The initial time. public FakeTimeProvider(DateTimeOffset start) => Now = start; /// public override DateTimeOffset GetUtcNow() => Now; } /// Records every call by secret name. private sealed class FakeSecretCacheInvalidator : ISecretCacheInvalidator { /// Gets the secret names invalidated so far, in call order. public List Invalidated { get; } = new(); /// Records the invalidated secret name. /// The secret name. public void Invalidate(SecretName name) => Invalidated.Add(name.Value); } /// /// In-memory honouring the REAL SqliteSecretStore /// semantics: is /// strict last-writer-wins via and applies /// verbatim (no revision bump); tombstones in place (bumps /// revision, stamps ) rather than hard-deleting. /// private sealed class FakeSecretStore : ISecretStore { private readonly Dictionary _rows = new(StringComparer.Ordinal); /// /// Controllable "now" used to stamp a tombstone — /// deterministic, not wall-clock, so tests can control ordering against a shipped /// artifact's fixed UpdatedUtc. /// public DateTimeOffset Now { get; set; } = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero); /// Set to make the next call throw, simulating a store fault. public bool ThrowOnApply { get; set; } /// Every row currently held, including tombstones, for direct inspection. public IReadOnlyDictionary Rows => _rows; /// Seeds a LIVE row directly (bypassing ApplyReplicatedAsync/LWW) — an out-of-band secret. /// The secret name. /// The last-updated timestamp stamped on the row. /// The row revision. /// The ciphertext bytes, or null for a default placeholder. public void Seed(string name, DateTimeOffset updatedUtc, long revision = 1, byte[]? ciphertext = null) => _rows[name] = new StoredSecret { Name = new SecretName(name), ContentType = SecretContentType.Text, Ciphertext = ciphertext ?? [9], Nonce = [9], Tag = [9], WrappedDek = [9], WrapNonce = [9], WrapTag = [9], KekId = "kek-out-of-band", Revision = revision, CreatedUtc = updatedUtc, UpdatedUtc = updatedUtc, }; /// True if the row is present and NOT tombstoned. /// The secret name. /// True if a live row exists for the name. public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted; /// True if the row is present as a tombstone. /// The secret name. /// True if the row exists and is tombstoned. public bool IsTombstoned(string name) => _rows.TryGetValue(name, out var row) && row.IsDeleted; /// Looks up the secret by name. /// The secret name. /// A token to cancel the operation. /// The secret, or null when none matches. public Task GetAsync(SecretName name, CancellationToken ct) => Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null); /// Stores or replaces the row in the in-memory fake. /// The secret row. /// A token to cancel the operation. /// A task that represents the asynchronous operation. public Task UpsertAsync(StoredSecret row, CancellationToken ct) { _rows[row.Name.Value] = row; return Task.CompletedTask; } /// Tombstones (does not hard-delete) — matches SqliteSecretStore.DeleteAsync. /// The secret name. /// The actor recorded on the tombstone. /// The cancellation token. /// True if a live row was tombstoned; false if it was absent or already deleted. public Task DeleteAsync(SecretName name, string? actor, CancellationToken ct) { if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted) { return Task.FromResult(false); } _rows[name.Value] = existing with { IsDeleted = true, DeletedUtc = Now, UpdatedUtc = Now, Revision = existing.Revision + 1, UpdatedBy = actor, }; return Task.FromResult(true); } /// Returns an empty secret listing. /// Whether deleted secrets are included, which the fake ignores. /// A token to cancel the operation. /// An empty list. public Task> ListAsync(bool includeDeleted, CancellationToken ct) => Task.FromResult>([]); /// Returns an empty manifest. /// A token to cancel the operation. /// An empty manifest. public Task> GetManifestAsync(CancellationToken ct) => Task.FromResult>([]); /// Strict LWW on (UpdatedUtc, Revision) via ; applied verbatim — matches SqliteSecretStore. /// The replicated row to apply. /// The cancellation token. /// A task that represents the asynchronous operation. public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) { if (ThrowOnApply) { throw new InvalidOperationException("simulated store fault"); } if (_rows.TryGetValue(row.Name.Value, out var existing) && !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision)) { return Task.CompletedTask; // no-op, matches real semantics } _rows[row.Name.Value] = row; return Task.CompletedTask; } /// Reports that no rewrap was applied. /// The rewrapped secret row. /// The wrapped data key the row is expected to hold currently. /// A token to cancel the operation. /// Always false. public Task ApplyRewrapAsync( StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) => Task.FromResult(false); } /// In-memory whose per-name resolution outcome tests control. private sealed class FakeSecretResolver : ISecretResolver { private readonly HashSet _throwing = new(StringComparer.Ordinal); private readonly HashSet _nullReturning = new(StringComparer.Ordinal); public int CallCount; /// When set, a name that is absent or tombstoned in this store resolves to null, like the real resolver. public FakeSecretStore? BackingStore { get; set; } /// Row absent/tombstoned path — DefaultSecretResolver.GetAsync returns null. /// The secret name that should resolve to null. public void ReturnNullFor(string name) => _nullReturning.Add(name); /// Decryption/integrity failure path — DefaultSecretResolver.GetAsync rethrows SecretDecryptionException. /// The secret name whose resolution should throw. public void ThrowFor(string name) => _throwing.Add(name); /// Clears every configured null-returning and throwing name. public void ClearFailures() { _nullReturning.Clear(); _throwing.Clear(); } /// Looks up the secret by name. /// The secret name. /// A token to cancel the operation. /// The secret, or null when none matches. public Task GetAsync(SecretName name, CancellationToken ct) { Interlocked.Increment(ref CallCount); if (_throwing.Contains(name.Value)) { throw new InvalidOperationException("simulated decrypt failure"); } if (BackingStore is not null && !BackingStore.Contains(name.Value)) { return Task.FromResult(null); } return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value"); } } }