Joseph Doherty
4d7d6456e3
Fix round 1 (review task-9-review.md), all rulings: 1. (Important 1) ApplyReplicatedAsync is no longer trusted blindly. Every apply is confirmed by a store.GetAsync read-back before ledgering: a tombstoned local row (routine remove-then-re-add) is safely re-stamped past the tombstone and re-applied (confirmed neither UpdatedUtc nor Revision is AEAD-bound in AesGcmEnvelopeCipher, so the bump cannot break decryption); a live local row with different ciphertext (a strictly-newer out-of-band secret) is left untouched, never ledgered, never verified against, and reported with a distinct Failed message so a later central tombstone can never delete it via this mirror. 2. (Important 2) Per-row try/catch (rethrow only on ct.IsCancellationRequested) so one poisoned row (e.g. an unrecognised ContentType from a newer central) no longer blocks every row after it. 3. (Important 3) Optional ISecretCacheInvalidator, invalidated after every successful apply/delete and before verify, so a rotated/removed secret's plaintext cannot be served stale for the remainder of the resolver's TTL. 4. (Minor 4) Loop and VerifyAsync OperationCanceledException handling now filters on ct/stoppingToken.IsCancellationRequested specifically, rather than swallowing every OCE. 5. (Minor 5) Class remarks note the mirror assumes a node-local ISecretStore; Task 11 will reject SqlServer secrets replication on the Site role. 6. (Minor 6/7) Added cancellation + concurrent-callers tests, dropped the redundant FailFor (tests now use ThrowFor/ReturnNullFor to exercise distinct decrypt-failure paths), and Dispose() now disposes the gate SemaphoreSlim. Verify: dotnet test tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests --filter FullyQualifiedName~DeployedSecretsMirrorTests (24/24); full SiteRuntime.Tests suite 876/876.
719 lines
30 KiB
C#
719 lines
30 KiB
C#
using System.Collections.Concurrent;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
|
|
|
|
/// <summary>
|
|
/// Task 9, site-secrets artifact deploy: <see cref="DeployedSecretsMirror"/> — per-node apply
|
|
/// + decrypt-verify of <c>deployed_secrets</c> into this node's own secret store.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Fix round 1 (review <c>task-9-review.md</c>): <see cref="FakeSecretStore"/> now honours the
|
|
/// real <c>SqliteSecretStore</c> semantics — <c>ApplyReplicatedAsync</c> is strict
|
|
/// last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/>, and <c>DeleteAsync</c>
|
|
/// tombstones rather than hard-deleting — so tests exercise the same conflict shapes the real
|
|
/// store can produce, not an idealized "always overwrite" double.
|
|
/// </remarks>
|
|
public class DeployedSecretsMirrorTests : IDisposable
|
|
{
|
|
private readonly TestLocalDb _localDb;
|
|
|
|
public DeployedSecretsMirrorTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror");
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
// ── Apply / remove / out-of-band ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
// Seeded OLDER than what will be shipped, so the shipped row legitimately wins LWW
|
|
// and is applied — the strictly-NEWER out-of-band case is covered separately by
|
|
// Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval.
|
|
store.Seed("out-of-band", updatedUtc: new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero));
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal));
|
|
Assert.Empty(r1.Failed);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
// Simulate: "out-of-band" was never actually mirrored by this node (an operator secret
|
|
// that happens to share the name) by clearing its ledger row directly — a defensive
|
|
// check independent of the natural not-ledgered path exercised below.
|
|
await storage.RemoveMirroredSecretAsync("out-of-band");
|
|
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r2.Removed);
|
|
Assert.True(store.Contains("out-of-band"));
|
|
Assert.False(store.Contains("a"));
|
|
Assert.True(store.IsTombstoned("a"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval()
|
|
{
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync();
|
|
var operatorCiphertext = new byte[] { 200, 201, 202 };
|
|
// Strictly newer than Art()'s fixed UpdatedUtc (2026-09-28T01:00:00Z) — legitimately
|
|
// wins LWW, exactly as the real store would order it.
|
|
store.Seed("a",
|
|
updatedUtc: new DateTimeOffset(2026, 9, 28, 2, 0, 0, TimeSpan.Zero),
|
|
revision: 5,
|
|
ciphertext: operatorCiphertext);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r1.Applied);
|
|
var message = Assert.Single(r1.Failed);
|
|
Assert.Contains("'a'", message);
|
|
Assert.Contains("newer local value", message);
|
|
Assert.DoesNotContain("did not decrypt", message); // distinct from the master-key message
|
|
Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext); // untouched
|
|
Assert.Empty(invalidator.Invalidated); // never applied -> never invalidated
|
|
|
|
// Never ledgered, so a later central tombstone must NOT delete the operator's secret.
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r2.Removed);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_RemoveThenReAdd_TombstoneIsRecovered_ReappliesAndVerifies()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var firstApply = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], firstApply.Applied);
|
|
|
|
// Site stops referencing it: central tombstones the row, this node deletes it. The
|
|
// tombstone is stamped LATER than the originally-shipped UpdatedUtc (01:00).
|
|
store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
var removePass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], removePass.Removed);
|
|
Assert.True(store.IsTombstoned("a"));
|
|
invalidator.Invalidated.Clear();
|
|
|
|
// Site references it again — central ships the SAME (unchanged, older-stamped)
|
|
// secret. A naive ApplyReplicatedAsync call is a no-op against the newer tombstone;
|
|
// the mirror must detect and recover this.
|
|
time.Now = new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero); // later than the tombstone
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3");
|
|
var reAddPass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], reAddPass.Applied);
|
|
Assert.Empty(reAddPass.Failed);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); // Art()'s ciphertext, re-materialized
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
|
|
// Ledgering used the SHIPPED artifact's own revision, not the bumped local one, so a
|
|
// later pass with the same shipped row is a true no-op.
|
|
var thirdPass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(thirdPass.Applied);
|
|
Assert.False(thirdPass.Changed);
|
|
|
|
var mirrored = await storage.GetMirroredSecretsAsync();
|
|
Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], r1.Applied);
|
|
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(r2.Applied);
|
|
Assert.False(r2.Changed);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2");
|
|
var r3 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], r3.Applied);
|
|
Assert.Equal(2, store.Rows["a"].Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var applied = store.Rows["a"];
|
|
Assert.Equal("kek-central", applied.KekId);
|
|
Assert.Equal(1, applied.Revision);
|
|
Assert.Equal(SecretContentType.Text, applied.ContentType);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(SecretContentType.Text)]
|
|
[InlineData(SecretContentType.ConnectionString)]
|
|
[InlineData(SecretContentType.Json)]
|
|
[InlineData(SecretContentType.BinaryBase64)]
|
|
public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType)
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync(
|
|
[Art("a", 1, contentType: contentType.ToString())], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(contentType, store.Rows["a"].ContentType);
|
|
}
|
|
|
|
// ── Per-row isolation (ruling 2) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OneRowThrows_OthersStillApply()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
// "bad" carries an unrecognised ContentType (as a newer central under version skew
|
|
// might ship), so ToStored's Enum.Parse throws for that row only.
|
|
await storage.UpsertDeployedSecretsAsync(
|
|
[Art("bad", 1, contentType: "NotARealContentType"), Art("good", 1)], "d1");
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["good"], r.Applied);
|
|
var message = Assert.Single(r.Failed);
|
|
Assert.Contains("'bad'", message);
|
|
Assert.True(store.Contains("good"));
|
|
Assert.False(store.Contains("bad"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OneRowThrows_FailureIsDeSpammedLikeAnyOtherFailure()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, _, _) = await CreateAsync(logger);
|
|
await storage.UpsertDeployedSecretsAsync([Art("bad", 1, contentType: "NotARealContentType")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var levels = logger.Entries
|
|
.Where(e => e.Message.Contains("'bad'") && e.Message.Contains("failed to process"))
|
|
.Select(e => e.Level)
|
|
.ToList();
|
|
Assert.Equal([LogLevel.Error, LogLevel.Debug], levels);
|
|
}
|
|
|
|
// ── Cache invalidation (ruling 3) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_SuccessfulApply_InvalidatesCache_BeforeVerify()
|
|
{
|
|
var (mirror, storage, _, _, invalidator) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_Removal_InvalidatesCache()
|
|
{
|
|
var (mirror, storage, _, _, invalidator) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
invalidator.Invalidated.Clear();
|
|
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_WithNoInvalidatorRegistered_StillApplies()
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await storage.InitializeAsync();
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
new FakeSecretStore(),
|
|
new FakeSecretResolver(),
|
|
invalidator: null,
|
|
Options.Create(new SiteRuntimeOptions { DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15) }),
|
|
NullLogger<DeployedSecretsMirror>.Instance,
|
|
TimeProvider.System);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r.Applied);
|
|
}
|
|
|
|
// ── Decrypt verification ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsReportedWithKekId()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
resolver.ThrowFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var message = Assert.Single(r.Failed);
|
|
Assert.Contains("kek-central", message);
|
|
Assert.Contains("'a'", message);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
resolver.ReturnNullFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Single(r.Failed);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(r1.Failed);
|
|
|
|
// Master key "changes" (or a KEK rotation invalidates the local unwrap) between
|
|
// passes — even though nothing about deployed_secrets changed, the next reconcile
|
|
// must still catch the failure because it re-verifies unconditionally.
|
|
resolver.ThrowFor("a");
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Single(r2.Failed);
|
|
Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified
|
|
}
|
|
|
|
// ── Error-log de-spam (ruling 1 of the original task, unaffected by fix round 1) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
|
|
resolver.ThrowFor("a");
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug
|
|
|
|
var failureEntries = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.ToList();
|
|
Assert.Equal(3, failureEntries.Count);
|
|
Assert.Equal(LogLevel.Error, failureEntries[0].Level);
|
|
Assert.Equal(LogLevel.Debug, failureEntries[1].Level);
|
|
Assert.Equal(LogLevel.Debug, failureEntries[2].Level);
|
|
|
|
// Now the secret resolves (e.g. the key issue is fixed).
|
|
resolver.ClearFailures();
|
|
var recovered = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(recovered.Failed);
|
|
|
|
// Fails again — must log at Error again, not Debug, since it succeeded in between.
|
|
resolver.ThrowFor("a");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var failureEntriesAfterRecovery = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.ToList();
|
|
Assert.Equal(4, failureEntriesAfterRecovery.Count);
|
|
Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
|
|
resolver.ThrowFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1)
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat)
|
|
|
|
// Central ships a new revision of the same (still-failing) secret.
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2");
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new)
|
|
|
|
var levels = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.Select(e => e.Level)
|
|
.ToList();
|
|
Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels);
|
|
}
|
|
|
|
// ── Cancellation + concurrency (ruling 6) ──
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_PreCancelledToken_ThrowsOperationCanceled()
|
|
{
|
|
var (mirror, _, _, _, _) = await CreateAsync();
|
|
using var cts = new CancellationTokenSource();
|
|
cts.Cancel();
|
|
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
|
() => mirror.ReconcileAsync(cts.Token));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_ConcurrentCallers_BothCompleteWithoutDoubleApply()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
var t1 = mirror.ReconcileAsync(CancellationToken.None);
|
|
var t2 = mirror.ReconcileAsync(CancellationToken.None);
|
|
var results = await Task.WhenAll(t1, t2);
|
|
|
|
// The gate serializes the two passes: whichever runs first applies it, the other
|
|
// sees it already ledgered at the shipped revision and applies nothing.
|
|
Assert.Equal(1, results.Sum(r => r.Applied.Count));
|
|
Assert.True(store.Contains("a"));
|
|
|
|
var mirrored = await storage.GetMirroredSecretsAsync();
|
|
Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
// ── Dispose (ruling 6) ──
|
|
|
|
[Fact]
|
|
public void Dispose_DoesNotThrow_AndIsIdempotent()
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
new FakeSecretStore(),
|
|
new FakeSecretResolver(),
|
|
invalidator: null,
|
|
Options.Create(new SiteRuntimeOptions()),
|
|
NullLogger<DeployedSecretsMirror>.Instance,
|
|
TimeProvider.System);
|
|
|
|
mirror.Dispose();
|
|
mirror.Dispose();
|
|
}
|
|
|
|
// ── Hosted loop ──
|
|
|
|
[Fact]
|
|
public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval()
|
|
{
|
|
var (mirror, storage, store, resolver, _) = await CreateAsync(
|
|
interval: TimeSpan.FromMilliseconds(30));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.StartAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
await WaitUntilAsync(() => store.Contains("a"));
|
|
Assert.True(store.Contains("a")); // ran once immediately at start
|
|
|
|
// Prove the loop keeps ticking (not just the immediate start-of-day pass) by
|
|
// observing further resolver invocations over time.
|
|
var before = resolver.CallCount;
|
|
await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000);
|
|
Assert.True(resolver.CallCount > before + 1);
|
|
}
|
|
finally
|
|
{
|
|
await mirror.StopAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ExecuteAsync_PerRowFailure_IsIsolated_AndLoopRecoversOnceCleared()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, store, _, _) = await CreateAsync(
|
|
logger, interval: TimeSpan.FromMilliseconds(30));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
store.ThrowOnApply = true;
|
|
|
|
await mirror.StartAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
// A per-row store fault is isolated (ruling 2): it never escapes ReconcileAsync,
|
|
// so it shows up as a row-level Error log, not the pass-level Warning.
|
|
await WaitUntilAsync(() => logger.Entries.Any(
|
|
e => e.Level == LogLevel.Error && e.Message.Contains("failed to process")));
|
|
|
|
// The loop must still be alive after the failure — force success and confirm a
|
|
// later pass applies the secret.
|
|
store.ThrowOnApply = false;
|
|
await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000);
|
|
Assert.True(store.Contains("a"));
|
|
}
|
|
finally
|
|
{
|
|
await mirror.StopAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
// ── Harness ──
|
|
|
|
private async Task<(
|
|
DeployedSecretsMirror Mirror,
|
|
SiteStorageService Storage,
|
|
FakeSecretStore Store,
|
|
FakeSecretResolver Resolver,
|
|
FakeSecretCacheInvalidator Invalidator)> CreateAsync(
|
|
CapturingLogger<DeployedSecretsMirror>? logger = null,
|
|
TimeSpan? interval = null,
|
|
TimeProvider? time = null)
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await storage.InitializeAsync();
|
|
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var invalidator = new FakeSecretCacheInvalidator();
|
|
var options = Options.Create(new SiteRuntimeOptions
|
|
{
|
|
DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15),
|
|
});
|
|
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
store,
|
|
resolver,
|
|
invalidator,
|
|
options,
|
|
(ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
|
|
time ?? TimeProvider.System);
|
|
|
|
return (mirror, storage, store, resolver, invalidator);
|
|
}
|
|
|
|
private static SecretArtifact Art(
|
|
string name, long revision, string kek = "kek-1", string contentType = "Text") => new(
|
|
Name: name,
|
|
ContentType: contentType,
|
|
Ciphertext: [1, 2, 3, 4, 5],
|
|
Nonce: [10, 11, 12],
|
|
Tag: [20, 21, 22, 23],
|
|
WrappedDek: [30, 31, 32, 33, 34, 35],
|
|
WrapNonce: [40, 41, 42],
|
|
WrapTag: [50, 51, 52, 53],
|
|
KekId: kek,
|
|
Revision: revision,
|
|
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
|
|
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
|
|
Description: "test secret");
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, int timeoutMs = 5000)
|
|
{
|
|
var elapsed = 0;
|
|
const int step = 20;
|
|
while (!condition() && elapsed < timeoutMs)
|
|
{
|
|
await Task.Delay(step);
|
|
elapsed += step;
|
|
}
|
|
Assert.True(condition(), "condition not met within timeout");
|
|
}
|
|
|
|
/// <summary>Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests.</summary>
|
|
private sealed class CapturingLogger<T> : ILogger<T>
|
|
{
|
|
public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new();
|
|
|
|
IDisposable? ILogger.BeginScope<TState>(TState state) => null;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(
|
|
LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
|
Func<TState, Exception?, string> formatter)
|
|
=> Entries.Enqueue((logLevel, formatter(state, exception)));
|
|
}
|
|
|
|
/// <summary>Minimal controllable <see cref="TimeProvider"/> — mirrors the pattern used by <c>ApiMethodCacheTests</c>.</summary>
|
|
private sealed class FakeTimeProvider : TimeProvider
|
|
{
|
|
public DateTimeOffset Now { get; set; }
|
|
public FakeTimeProvider(DateTimeOffset start) => Now = start;
|
|
public override DateTimeOffset GetUtcNow() => Now;
|
|
}
|
|
|
|
/// <summary>Records every <see cref="ISecretCacheInvalidator.Invalidate"/> call by secret name.</summary>
|
|
private sealed class FakeSecretCacheInvalidator : ISecretCacheInvalidator
|
|
{
|
|
public List<string> Invalidated { get; } = new();
|
|
public void Invalidate(SecretName name) => Invalidated.Add(name.Value);
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-memory <see cref="ISecretStore"/> honouring the REAL <c>SqliteSecretStore</c>
|
|
/// semantics (fix round 1, review Important #1): <see cref="ApplyReplicatedAsync"/> is
|
|
/// strict last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/> and applies
|
|
/// verbatim (no revision bump); <see cref="DeleteAsync"/> tombstones in place (bumps
|
|
/// revision, stamps <see cref="Now"/>) rather than hard-deleting.
|
|
/// </summary>
|
|
private sealed class FakeSecretStore : ISecretStore
|
|
{
|
|
private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);
|
|
|
|
/// <summary>
|
|
/// Controllable "now" used to stamp a <see cref="DeleteAsync"/> tombstone —
|
|
/// deterministic, not wall-clock, so tests can control ordering against a shipped
|
|
/// artifact's fixed <c>UpdatedUtc</c>.
|
|
/// </summary>
|
|
public DateTimeOffset Now { get; set; } = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
/// <summary>Set to make the next <see cref="ApplyReplicatedAsync"/> call throw, simulating a store fault.</summary>
|
|
public bool ThrowOnApply { get; set; }
|
|
|
|
/// <summary>Every row currently held, including tombstones, for direct inspection.</summary>
|
|
public IReadOnlyDictionary<string, StoredSecret> Rows => _rows;
|
|
|
|
/// <summary>Seeds a LIVE row directly (bypassing ApplyReplicatedAsync/LWW) — an out-of-band secret.</summary>
|
|
public void Seed(string name, DateTimeOffset updatedUtc, long revision = 1, byte[]? ciphertext = null) =>
|
|
_rows[name] = new StoredSecret
|
|
{
|
|
Name = new SecretName(name),
|
|
ContentType = SecretContentType.Text,
|
|
Ciphertext = ciphertext ?? [9],
|
|
Nonce = [9],
|
|
Tag = [9],
|
|
WrappedDek = [9],
|
|
WrapNonce = [9],
|
|
WrapTag = [9],
|
|
KekId = "kek-out-of-band",
|
|
Revision = revision,
|
|
CreatedUtc = updatedUtc,
|
|
UpdatedUtc = updatedUtc,
|
|
};
|
|
|
|
/// <summary>True if the row is present and NOT tombstoned.</summary>
|
|
public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted;
|
|
|
|
/// <summary>True if the row is present as a tombstone.</summary>
|
|
public bool IsTombstoned(string name) => _rows.TryGetValue(name, out var row) && row.IsDeleted;
|
|
|
|
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
|
|
Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);
|
|
|
|
public Task UpsertAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Tombstones (does not hard-delete) — matches <c>SqliteSecretStore.DeleteAsync</c>.</summary>
|
|
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
|
|
{
|
|
if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted)
|
|
{
|
|
return Task.FromResult(false);
|
|
}
|
|
|
|
_rows[name.Value] = existing with
|
|
{
|
|
IsDeleted = true,
|
|
DeletedUtc = Now,
|
|
UpdatedUtc = Now,
|
|
Revision = existing.Revision + 1,
|
|
UpdatedBy = actor,
|
|
};
|
|
return Task.FromResult(true);
|
|
}
|
|
|
|
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretMetadata>>([]);
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
|
|
/// <summary>Strict LWW on (UpdatedUtc, Revision) via <see cref="SecretLastWriterWins.IsNewer"/>; applied verbatim — matches <c>SqliteSecretStore</c>.</summary>
|
|
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
if (ThrowOnApply)
|
|
{
|
|
throw new InvalidOperationException("simulated store fault");
|
|
}
|
|
|
|
if (_rows.TryGetValue(row.Name.Value, out var existing)
|
|
&& !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision))
|
|
{
|
|
return Task.CompletedTask; // no-op, matches real semantics
|
|
}
|
|
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task<bool> ApplyRewrapAsync(
|
|
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
|
Task.FromResult(false);
|
|
}
|
|
|
|
/// <summary>In-memory <see cref="ISecretResolver"/> whose per-name resolution outcome tests control.</summary>
|
|
private sealed class FakeSecretResolver : ISecretResolver
|
|
{
|
|
private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);
|
|
private readonly HashSet<string> _nullReturning = new(StringComparer.Ordinal);
|
|
|
|
public int CallCount;
|
|
|
|
/// <summary>Row absent/tombstoned path — <c>DefaultSecretResolver.GetAsync</c> returns null.</summary>
|
|
public void ReturnNullFor(string name) => _nullReturning.Add(name);
|
|
|
|
/// <summary>Decryption/integrity failure path — <c>DefaultSecretResolver.GetAsync</c> rethrows <c>SecretDecryptionException</c>.</summary>
|
|
public void ThrowFor(string name) => _throwing.Add(name);
|
|
|
|
public void ClearFailures()
|
|
{
|
|
_nullReturning.Clear();
|
|
_throwing.Clear();
|
|
}
|
|
|
|
public Task<string?> GetAsync(SecretName name, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref CallCount);
|
|
if (_throwing.Contains(name.Value))
|
|
{
|
|
throw new InvalidOperationException("simulated decrypt failure");
|
|
}
|
|
|
|
return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
|
|
}
|
|
}
|
|
}
|