Joseph Doherty
0289d51137
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
1024 lines
44 KiB
C#
1024 lines
44 KiB
C#
using System.Collections.Concurrent;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Secrets;
|
|
|
|
/// <summary>
|
|
/// Task 9, site-secrets artifact deploy: <see cref="DeployedSecretsMirror"/> — per-node apply
|
|
/// + decrypt-verify of <c>deployed_secrets</c> into this node's own secret store.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Fix round 1 (review <c>task-9-review.md</c>): <see cref="FakeSecretStore"/> now honours the
|
|
/// real <c>SqliteSecretStore</c> semantics — <c>ApplyReplicatedAsync</c> is strict
|
|
/// last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/>, and <c>DeleteAsync</c>
|
|
/// tombstones rather than hard-deleting — so tests exercise the same conflict shapes the real
|
|
/// store can produce, not an idealized "always overwrite" double.
|
|
/// </remarks>
|
|
public class DeployedSecretsMirrorTests : IDisposable
|
|
{
|
|
private readonly TestLocalDb _localDb;
|
|
|
|
public DeployedSecretsMirrorTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("deployed-secrets-mirror");
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
// ── Apply / remove / out-of-band ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_Applies_Verifies_AndOnlyDeletesWhatItMirrored()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
// Seeded OLDER than what will be shipped, so the shipped row legitimately wins LWW
|
|
// and is applied — the strictly-NEWER out-of-band case is covered separately by
|
|
// Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval.
|
|
store.Seed("out-of-band", updatedUtc: new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero));
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1), Art("out-of-band", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(new[] { "a", "out-of-band" }, r1.Applied.OrderBy(x => x, StringComparer.Ordinal));
|
|
Assert.Empty(r1.Failed);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
// Simulate: "out-of-band" was never actually mirrored by this node (an operator secret
|
|
// that happens to share the name) by clearing its ledger row directly — a defensive
|
|
// check independent of the natural not-ledgered path exercised below.
|
|
await storage.RemoveMirroredSecretAsync("out-of-band");
|
|
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r2.Removed);
|
|
Assert.True(store.Contains("out-of-band"));
|
|
Assert.False(store.Contains("a"));
|
|
Assert.True(store.IsTombstoned("a"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OutOfBandNewerSecret_NotOverwritten_NotLedgered_AndSurvivesLaterRemoval()
|
|
{
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync();
|
|
var operatorCiphertext = new byte[] { 200, 201, 202 };
|
|
// Strictly newer than Art()'s fixed UpdatedUtc (2026-09-28T01:00:00Z) — legitimately
|
|
// wins LWW, exactly as the real store would order it.
|
|
store.Seed("a",
|
|
updatedUtc: new DateTimeOffset(2026, 9, 28, 2, 0, 0, TimeSpan.Zero),
|
|
revision: 5,
|
|
ciphertext: operatorCiphertext);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r1.Applied);
|
|
var message = Assert.Single(r1.Failed);
|
|
Assert.Contains("'a'", message);
|
|
Assert.Contains("newer local value", message);
|
|
Assert.DoesNotContain("did not decrypt", message); // distinct from the master-key message
|
|
Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext); // untouched
|
|
Assert.Empty(invalidator.Invalidated); // never applied -> never invalidated
|
|
|
|
// Never ledgered, so a later central tombstone must NOT delete the operator's secret.
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r2.Removed);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Equal(operatorCiphertext, store.Rows["a"].Ciphertext);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_RemoveThenReAdd_TombstoneIsRecovered_ReappliesAndVerifies()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var firstApply = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], firstApply.Applied);
|
|
|
|
// Site stops referencing it: central tombstones the row, this node deletes it. The
|
|
// tombstone is stamped LATER than the originally-shipped UpdatedUtc (01:00).
|
|
store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
var removePass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], removePass.Removed);
|
|
Assert.True(store.IsTombstoned("a"));
|
|
invalidator.Invalidated.Clear();
|
|
|
|
// Site references it again — central ships the SAME (unchanged, older-stamped)
|
|
// secret. A naive ApplyReplicatedAsync call is a no-op against the newer tombstone;
|
|
// the mirror must detect and recover this.
|
|
time.Now = new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero); // later than the tombstone
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3");
|
|
var reAddPass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], reAddPass.Applied);
|
|
Assert.Empty(reAddPass.Failed);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext); // Art()'s ciphertext, re-materialized
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
|
|
// Ledgering used the SHIPPED artifact's own revision, not the bumped local one, so a
|
|
// later pass with the same shipped row is a true no-op.
|
|
var thirdPass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(thirdPass.Applied);
|
|
Assert.False(thirdPass.Changed);
|
|
|
|
var mirrored = await storage.GetMirroredSecretsAsync();
|
|
Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_ClockSkewedRecoveredRow_StillConvergesOnNextCentralRevision()
|
|
{
|
|
// Fix round 2 (review "clock skew" finding): a tombstone-recovery bump can leave a local
|
|
// row stamped AHEAD of a central UpdatedUtc that arrives afterward (site clock skew, or
|
|
// simply a central edit made before the recovery but shipped after it). The row is still
|
|
// OWNED by this mirror's ledger, so a later genuinely-new central revision must not be
|
|
// misreported as an out-of-band "newer local value" — it must converge.
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Remove then re-add — but the SITE CLOCK IS SKEWED FAR INTO THE FUTURE while the
|
|
// recovery bump happens (a bad RTC/NTP fault), so the recovered row's UpdatedUtc lands
|
|
// far ahead of anything central will ship for a long while.
|
|
store.Now = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
await mirror.ReconcileAsync(CancellationToken.None); // tombstones "a"
|
|
|
|
time.Now = new DateTimeOffset(2099, 1, 1, 0, 0, 0, TimeSpan.Zero); // skewed site clock
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d3");
|
|
var recoveredPass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], recoveredPass.Applied);
|
|
Assert.True(store.Rows["a"].UpdatedUtc.Year >= 2099); // confirms the skewed stamp landed
|
|
|
|
// Site clock recovers to normal. Central ships a genuinely NEW revision (different
|
|
// ciphertext, i.e. an actual rotation) with an ordinary timestamp — OLDER than the
|
|
// skewed local stamp, so a naive ApplyReplicatedAsync would no-op against it.
|
|
time.Now = new DateTimeOffset(2026, 9, 29, 0, 0, 0, TimeSpan.Zero);
|
|
var rotatedCiphertext = new byte[] { 7, 7, 7 };
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 2, ciphertext: rotatedCiphertext)], "d4");
|
|
|
|
var convergePass = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], convergePass.Applied);
|
|
Assert.Empty(convergePass.Failed); // NOT "newer local value" — this mirror owns the name
|
|
Assert.Equal(rotatedCiphertext, store.Rows["a"].Ciphertext);
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
|
|
var mirrored = await storage.GetMirroredSecretsAsync();
|
|
Assert.Equal(2, mirrored.Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_ReAppliesOnlyWhenRevisionChanges()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], r1.Applied);
|
|
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(r2.Applied);
|
|
Assert.False(r2.Changed);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 2)], "d2");
|
|
var r3 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal(["a"], r3.Applied);
|
|
Assert.Equal(2, store.Rows["a"].Revision);
|
|
}
|
|
|
|
// ── Same-revision drift (final review I1) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OwnedNameDeletedLocally_IsHealedAndVerified_NotMisreportedAsKekMismatch()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, resolver, invalidator) = await CreateAsync(time: time);
|
|
resolver.BackingStore = store; // resolve returns null once the local row is gone
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
|
|
|
|
// An operator deletes the mirror-owned name on this node (library CLI/UI) — the
|
|
// ledger still says r1 is mirrored, and central still ships r1.
|
|
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
|
|
Assert.True(store.IsTombstoned("a"));
|
|
invalidator.Invalidated.Clear();
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r.Failed);
|
|
Assert.Equal(["a"], r.Applied);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
|
|
// Healed state is stable: the next pass is a no-op that still verifies clean.
|
|
var next = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.False(next.Changed);
|
|
Assert.Empty(next.Failed);
|
|
Assert.Equal(1, (await storage.GetMirroredSecretsAsync()).Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OwnedNameEditedLocally_IsRevertedToTheDeployedCiphertext()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
|
|
|
|
// An operator overwrites the mirror-owned name locally with a NEWER stamp.
|
|
store.Seed("a",
|
|
updatedUtc: new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero),
|
|
revision: 7,
|
|
ciphertext: [200, 201, 202]);
|
|
invalidator.Invalidated.Clear();
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Empty(r.Failed);
|
|
Assert.Equal(["a"], r.Applied);
|
|
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
|
|
Assert.Equal(new byte[] { 30, 31, 32, 33, 34, 35 }, store.Rows["a"].WrappedDek);
|
|
Assert.True(store.Rows["a"].Revision > 7); // re-stamped past the operator's row
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OwnedNameDrift_LogsOncePerDriftNotEveryPass()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, time: time);
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("re-applying"));
|
|
Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Error);
|
|
}
|
|
|
|
// ── Unreadable row_json (final review M4) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_UnparseableRowJson_IsAFailedEntryForThatName_OthersStillApply()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("bad", 1), Art("good", 1)], "d1");
|
|
await CorruptRowJsonAsync(storage, "bad");
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["good"], r.Applied);
|
|
Assert.Equal(["bad"], r.FailedNames);
|
|
var message = Assert.Single(r.Failed);
|
|
Assert.Contains("'bad'", message);
|
|
Assert.Contains("could not be read", message);
|
|
Assert.True(store.Contains("good"));
|
|
Assert.False(store.Contains("bad"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_UnparseableRemovedRow_StillRemovesAMirroredName()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await storage.UpsertDeployedSecretsAsync([], "d2"); // soft-removes "a"
|
|
await CorruptRowJsonAsync(storage, "a");
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r.Removed);
|
|
Assert.Empty(r.Failed);
|
|
Assert.True(store.IsTombstoned("a"));
|
|
}
|
|
|
|
private static async Task CorruptRowJsonAsync(SiteStorageService storage, string name)
|
|
{
|
|
await using var connection = storage.CreateConnection();
|
|
await using var command = connection.CreateCommand();
|
|
command.CommandText = "UPDATE deployed_secrets SET row_json = '{not json' WHERE name = @name";
|
|
command.Parameters.AddWithValue("@name", name);
|
|
Assert.Equal(1, await command.ExecuteNonQueryAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var applied = store.Rows["a"];
|
|
Assert.Equal("kek-central", applied.KekId);
|
|
Assert.Equal(1, applied.Revision);
|
|
Assert.Equal(SecretContentType.Text, applied.ContentType);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(SecretContentType.Text)]
|
|
[InlineData(SecretContentType.ConnectionString)]
|
|
[InlineData(SecretContentType.Json)]
|
|
[InlineData(SecretContentType.BinaryBase64)]
|
|
public async Task Reconcile_RoundTripsEveryContentType(SecretContentType contentType)
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync(
|
|
[Art("a", 1, contentType: contentType.ToString())], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(contentType, store.Rows["a"].ContentType);
|
|
}
|
|
|
|
// ── Per-row isolation (ruling 2) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OneRowThrows_OthersStillApply()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
// "bad" carries an unrecognised ContentType (as a newer central under version skew
|
|
// might ship), so ToStored's Enum.Parse throws for that row only.
|
|
await storage.UpsertDeployedSecretsAsync(
|
|
[Art("bad", 1, contentType: "NotARealContentType"), Art("good", 1)], "d1");
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["good"], r.Applied);
|
|
var message = Assert.Single(r.Failed);
|
|
Assert.Contains("'bad'", message);
|
|
Assert.True(store.Contains("good"));
|
|
Assert.False(store.Contains("bad"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_OneRowThrows_FailureIsDeSpammedLikeAnyOtherFailure()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, _, _) = await CreateAsync(logger);
|
|
await storage.UpsertDeployedSecretsAsync([Art("bad", 1, contentType: "NotARealContentType")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var levels = logger.Entries
|
|
.Where(e => e.Message.Contains("'bad'") && e.Message.Contains("failed to process"))
|
|
.Select(e => e.Level)
|
|
.ToList();
|
|
Assert.Equal([LogLevel.Error, LogLevel.Debug], levels);
|
|
}
|
|
|
|
// ── Cache invalidation (ruling 3) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_SuccessfulApply_InvalidatesCache_BeforeVerify()
|
|
{
|
|
var (mirror, storage, _, _, invalidator) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_Removal_InvalidatesCache()
|
|
{
|
|
var (mirror, storage, _, _, invalidator) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
invalidator.Invalidated.Clear();
|
|
|
|
await storage.UpsertDeployedSecretsAsync([], "d2");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Contains("a", invalidator.Invalidated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_WithNoInvalidatorRegistered_StillApplies()
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await storage.InitializeAsync();
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
new FakeSecretStore(),
|
|
new FakeSecretResolver(),
|
|
invalidator: null,
|
|
Options.Create(new SiteRuntimeOptions { DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15) }),
|
|
NullLogger<DeployedSecretsMirror>.Instance,
|
|
TimeProvider.System);
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r.Applied);
|
|
}
|
|
|
|
// ── Decrypt verification ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_ExceptionFromResolver_IsReportedWithKekId()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
resolver.ThrowFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var message = Assert.Single(r.Failed);
|
|
Assert.Contains("kek-central", message);
|
|
Assert.Contains("'a'", message);
|
|
// FailedNames (Task 10): the name behind each Failed entry, in lockstep order, so a
|
|
// caller (DeploymentManagerActor) can tell whether a failure belongs to names it shipped.
|
|
Assert.Equal(["a"], r.FailedNames);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_NullResolve_IsAlsoReported()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
resolver.ReturnNullFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Single(r.Failed);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_VerifiesEveryPass_EvenWhenAlreadyApplied()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
var r1 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(r1.Failed);
|
|
|
|
// Master key "changes" (or a KEK rotation invalidates the local unwrap) between
|
|
// passes — even though nothing about deployed_secrets changed, the next reconcile
|
|
// must still catch the failure because it re-verifies unconditionally.
|
|
resolver.ThrowFor("a");
|
|
var r2 = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Single(r2.Failed);
|
|
Assert.Empty(r2.Applied); // unchanged revision — not re-applied, only re-verified
|
|
}
|
|
|
|
// ── Error-log de-spam (ruling 1 of the original task, unaffected by fix round 1) ──
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_LogsErrorOnlyFirstTime_ThenDebug_ThenErrorAgainAfterASuccess()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
|
|
resolver.ThrowFor("a");
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 1st failure: Error
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 2nd identical failure: Debug
|
|
await mirror.ReconcileAsync(CancellationToken.None); // 3rd identical failure: Debug
|
|
|
|
var failureEntries = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.ToList();
|
|
Assert.Equal(3, failureEntries.Count);
|
|
Assert.Equal(LogLevel.Error, failureEntries[0].Level);
|
|
Assert.Equal(LogLevel.Debug, failureEntries[1].Level);
|
|
Assert.Equal(LogLevel.Debug, failureEntries[2].Level);
|
|
|
|
// Now the secret resolves (e.g. the key issue is fixed).
|
|
resolver.ClearFailures();
|
|
var recovered = await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(recovered.Failed);
|
|
|
|
// Fails again — must log at Error again, not Debug, since it succeeded in between.
|
|
resolver.ThrowFor("a");
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var failureEntriesAfterRecovery = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.ToList();
|
|
Assert.Equal(4, failureEntriesAfterRecovery.Count);
|
|
Assert.Equal(LogLevel.Error, failureEntriesAfterRecovery[3].Level);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reconcile_DecryptFailure_NewRevision_LogsErrorAgain_EvenWithoutASuccessInBetween()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(logger);
|
|
resolver.ThrowFor("a");
|
|
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-1")], "d1");
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Error (rev 1)
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Debug (rev 1, repeat)
|
|
|
|
// Central ships a new revision of the same (still-failing) secret.
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 2, kek: "kek-1")], "d2");
|
|
await mirror.ReconcileAsync(CancellationToken.None); // Error again (rev 2 is new)
|
|
|
|
var levels = logger.Entries
|
|
.Where(e => e.Message.Contains("did not decrypt"))
|
|
.Select(e => e.Level)
|
|
.ToList();
|
|
Assert.Equal([LogLevel.Error, LogLevel.Debug, LogLevel.Error], levels);
|
|
}
|
|
|
|
// ── Cancellation + concurrency (ruling 6) ──
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_PreCancelledToken_ThrowsOperationCanceled()
|
|
{
|
|
var (mirror, _, _, _, _) = await CreateAsync();
|
|
using var cts = new CancellationTokenSource();
|
|
cts.Cancel();
|
|
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
|
() => mirror.ReconcileAsync(cts.Token));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_ConcurrentCallers_BothCompleteWithoutDoubleApply()
|
|
{
|
|
var (mirror, storage, store, _, _) = await CreateAsync();
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
var t1 = mirror.ReconcileAsync(CancellationToken.None);
|
|
var t2 = mirror.ReconcileAsync(CancellationToken.None);
|
|
var results = await Task.WhenAll(t1, t2);
|
|
|
|
// The gate serializes the two passes: whichever runs first applies it, the other
|
|
// sees it already ledgered at the shipped revision and applies nothing.
|
|
Assert.Equal(1, results.Sum(r => r.Applied.Count));
|
|
Assert.True(store.Contains("a"));
|
|
|
|
var mirrored = await storage.GetMirroredSecretsAsync();
|
|
Assert.Equal(1, mirrored.Single(m => m.Name == "a").Revision);
|
|
}
|
|
|
|
// ── Dispose (ruling 6) ──
|
|
|
|
[Fact]
|
|
public void Dispose_DoesNotThrow_AndIsIdempotent()
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
new FakeSecretStore(),
|
|
new FakeSecretResolver(),
|
|
invalidator: null,
|
|
Options.Create(new SiteRuntimeOptions()),
|
|
NullLogger<DeployedSecretsMirror>.Instance,
|
|
TimeProvider.System);
|
|
|
|
mirror.Dispose();
|
|
mirror.Dispose();
|
|
}
|
|
|
|
// ── Hosted loop ──
|
|
|
|
[Fact]
|
|
public async Task ExecuteAsync_RunsOnceAtStart_AndAgainEveryInterval()
|
|
{
|
|
var (mirror, storage, store, resolver, _) = await CreateAsync(
|
|
interval: TimeSpan.FromMilliseconds(30));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.StartAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
await WaitUntilAsync(() => store.Contains("a"));
|
|
Assert.True(store.Contains("a")); // ran once immediately at start
|
|
|
|
// Prove the loop keeps ticking (not just the immediate start-of-day pass) by
|
|
// observing further resolver invocations over time.
|
|
var before = resolver.CallCount;
|
|
await WaitUntilAsync(() => resolver.CallCount > before + 1, timeoutMs: 2000);
|
|
Assert.True(resolver.CallCount > before + 1);
|
|
}
|
|
finally
|
|
{
|
|
await mirror.StopAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ExecuteAsync_PerRowFailure_IsIsolated_AndLoopRecoversOnceCleared()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, store, _, _) = await CreateAsync(
|
|
logger, interval: TimeSpan.FromMilliseconds(30));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
store.ThrowOnApply = true;
|
|
|
|
await mirror.StartAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
// A per-row store fault is isolated (ruling 2): it never escapes ReconcileAsync,
|
|
// so it shows up as a row-level Error log, not the pass-level Warning.
|
|
await WaitUntilAsync(() => logger.Entries.Any(
|
|
e => e.Level == LogLevel.Error && e.Message.Contains("failed to process")));
|
|
|
|
// The loop must still be alive after the failure — force success and confirm a
|
|
// later pass applies the secret.
|
|
store.ThrowOnApply = false;
|
|
await WaitUntilAsync(() => store.Contains("a"), timeoutMs: 3000);
|
|
Assert.True(store.Contains("a"));
|
|
}
|
|
finally
|
|
{
|
|
await mirror.StopAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
// ── Per-node status row for central (review finding M1) ──
|
|
|
|
[Fact]
|
|
public async Task Status_FirstPassAfterStartup_WritesThisNodesRow_EvenWhenHealthy()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, _, _, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var status = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Equal("node-b", status.NodeName);
|
|
Assert.Empty(status.Failures);
|
|
Assert.Equal(time.Now, status.CheckedAtUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Status_UnchangedFailureSet_IsNotRewrittenOnLaterPasses()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-b"));
|
|
resolver.ThrowFor("a");
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1, kek: "kek-central")], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
var firstWrite = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
|
|
|
// Same failure on every later pass: the row is replicated, so it must not be rewritten.
|
|
time.Now = time.Now.AddMinutes(5);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
var after = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Equal(firstWrite.CheckedAtUtc, after.CheckedAtUtc);
|
|
var failure = Assert.Single(after.Failures);
|
|
Assert.Equal("a", failure.SecretName);
|
|
Assert.Contains("kek-central", failure.Message);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Status_FailureSetChange_RewritesTheRow_AndClearsWhenHealed()
|
|
{
|
|
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 6, 0, 0, TimeSpan.Zero));
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(time: time, nodeIdentity: new FixedNodeIdentity("node-a"));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Empty(Assert.Single(await storage.GetSecretMirrorStatusesAsync()).Failures);
|
|
|
|
// A decrypt failure appears -> row rewritten with the failure.
|
|
resolver.ThrowFor("a");
|
|
time.Now = time.Now.AddMinutes(1);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
var failing = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Equal("a", Assert.Single(failing.Failures).SecretName);
|
|
Assert.Equal(time.Now, failing.CheckedAtUtc);
|
|
|
|
// Healed -> row rewritten empty.
|
|
resolver.ClearFailures();
|
|
time.Now = time.Now.AddMinutes(1);
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
var healed = Assert.Single(await storage.GetSecretMirrorStatusesAsync());
|
|
Assert.Empty(healed.Failures);
|
|
Assert.Equal(time.Now, healed.CheckedAtUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Status_RowCarriesNamesAndMessagesOnly_NeverCiphertext()
|
|
{
|
|
var (mirror, storage, _, resolver, _) = await CreateAsync(nodeIdentity: new FixedNodeIdentity("node-a"));
|
|
resolver.ThrowFor("a");
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
await using var connection = storage.CreateConnection();
|
|
await using var command = connection.CreateCommand();
|
|
command.CommandText = "SELECT failures_json FROM secret_mirror_status WHERE node_name = 'node-a'";
|
|
var json = (string)(await command.ExecuteScalarAsync())!;
|
|
Assert.Contains("\"SecretName\":\"a\"", json);
|
|
Assert.DoesNotContain("Ciphertext", json);
|
|
Assert.DoesNotContain(Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5 }), json);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Status_WriteFailure_DoesNotBreakThePass_AndIsRetriedNextPass()
|
|
{
|
|
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
|
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, nodeIdentity: new FixedNodeIdentity("node-a"));
|
|
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
|
|
|
await using (var connection = storage.CreateConnection())
|
|
await using (var drop = connection.CreateCommand())
|
|
{
|
|
drop.CommandText = "DROP TABLE secret_mirror_status";
|
|
await drop.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(["a"], r.Applied);
|
|
Assert.True(store.Contains("a"));
|
|
Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("mirror status"));
|
|
|
|
// Table back (e.g. transient fault cleared): the unchanged failure set is still
|
|
// written, because the earlier write never landed.
|
|
await storage.InitializeAsync();
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
Assert.Equal("node-a", Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Status_WithoutNodeIdentity_FallsBackToMachineName()
|
|
{
|
|
var (mirror, storage, _, _, _) = await CreateAsync();
|
|
|
|
await mirror.ReconcileAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(Environment.MachineName, Assert.Single(await storage.GetSecretMirrorStatusesAsync()).NodeName);
|
|
}
|
|
|
|
private sealed class FixedNodeIdentity(string? name) : INodeIdentityProvider
|
|
{
|
|
public string? NodeName { get; } = name;
|
|
}
|
|
|
|
// ── Harness ──
|
|
|
|
private async Task<(
|
|
DeployedSecretsMirror Mirror,
|
|
SiteStorageService Storage,
|
|
FakeSecretStore Store,
|
|
FakeSecretResolver Resolver,
|
|
FakeSecretCacheInvalidator Invalidator)> CreateAsync(
|
|
CapturingLogger<DeployedSecretsMirror>? logger = null,
|
|
TimeSpan? interval = null,
|
|
TimeProvider? time = null,
|
|
INodeIdentityProvider? nodeIdentity = null)
|
|
{
|
|
var storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
await storage.InitializeAsync();
|
|
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var invalidator = new FakeSecretCacheInvalidator();
|
|
var options = Options.Create(new SiteRuntimeOptions
|
|
{
|
|
DeployedSecretsMirrorInterval = interval ?? TimeSpan.FromSeconds(15),
|
|
});
|
|
|
|
var mirror = new DeployedSecretsMirror(
|
|
storage,
|
|
store,
|
|
resolver,
|
|
invalidator,
|
|
options,
|
|
(ILogger<DeployedSecretsMirror>?)logger ?? NullLogger<DeployedSecretsMirror>.Instance,
|
|
time ?? TimeProvider.System,
|
|
nodeIdentity);
|
|
|
|
return (mirror, storage, store, resolver, invalidator);
|
|
}
|
|
|
|
private static SecretArtifact Art(
|
|
string name, long revision, string kek = "kek-1", string contentType = "Text", byte[]? ciphertext = null) => new(
|
|
Name: name,
|
|
ContentType: contentType,
|
|
Ciphertext: ciphertext ?? [1, 2, 3, 4, 5],
|
|
Nonce: [10, 11, 12],
|
|
Tag: [20, 21, 22, 23],
|
|
WrappedDek: [30, 31, 32, 33, 34, 35],
|
|
WrapNonce: [40, 41, 42],
|
|
WrapTag: [50, 51, 52, 53],
|
|
KekId: kek,
|
|
Revision: revision,
|
|
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
|
|
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
|
|
Description: "test secret");
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, int timeoutMs = 5000)
|
|
{
|
|
var elapsed = 0;
|
|
const int step = 20;
|
|
while (!condition() && elapsed < timeoutMs)
|
|
{
|
|
await Task.Delay(step);
|
|
elapsed += step;
|
|
}
|
|
Assert.True(condition(), "condition not met within timeout");
|
|
}
|
|
|
|
/// <summary>Thread-safe capturing logger so tests can assert on emitted log levels/messages, mirroring SiteReconciliationActorTests.</summary>
|
|
private sealed class CapturingLogger<T> : ILogger<T>
|
|
{
|
|
public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new();
|
|
|
|
IDisposable? ILogger.BeginScope<TState>(TState state) => null;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(
|
|
LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
|
Func<TState, Exception?, string> formatter)
|
|
=> Entries.Enqueue((logLevel, formatter(state, exception)));
|
|
}
|
|
|
|
/// <summary>Minimal controllable <see cref="TimeProvider"/> — mirrors the pattern used by <c>ApiMethodCacheTests</c>.</summary>
|
|
private sealed class FakeTimeProvider : TimeProvider
|
|
{
|
|
public DateTimeOffset Now { get; set; }
|
|
public FakeTimeProvider(DateTimeOffset start) => Now = start;
|
|
public override DateTimeOffset GetUtcNow() => Now;
|
|
}
|
|
|
|
/// <summary>Records every <see cref="ISecretCacheInvalidator.Invalidate"/> call by secret name.</summary>
|
|
private sealed class FakeSecretCacheInvalidator : ISecretCacheInvalidator
|
|
{
|
|
public List<string> Invalidated { get; } = new();
|
|
public void Invalidate(SecretName name) => Invalidated.Add(name.Value);
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-memory <see cref="ISecretStore"/> honouring the REAL <c>SqliteSecretStore</c>
|
|
/// semantics (fix round 1, review Important #1): <see cref="ApplyReplicatedAsync"/> is
|
|
/// strict last-writer-wins via <see cref="SecretLastWriterWins.IsNewer"/> and applies
|
|
/// verbatim (no revision bump); <see cref="DeleteAsync"/> tombstones in place (bumps
|
|
/// revision, stamps <see cref="Now"/>) rather than hard-deleting.
|
|
/// </summary>
|
|
private sealed class FakeSecretStore : ISecretStore
|
|
{
|
|
private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);
|
|
|
|
/// <summary>
|
|
/// Controllable "now" used to stamp a <see cref="DeleteAsync"/> tombstone —
|
|
/// deterministic, not wall-clock, so tests can control ordering against a shipped
|
|
/// artifact's fixed <c>UpdatedUtc</c>.
|
|
/// </summary>
|
|
public DateTimeOffset Now { get; set; } = new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
/// <summary>Set to make the next <see cref="ApplyReplicatedAsync"/> call throw, simulating a store fault.</summary>
|
|
public bool ThrowOnApply { get; set; }
|
|
|
|
/// <summary>Every row currently held, including tombstones, for direct inspection.</summary>
|
|
public IReadOnlyDictionary<string, StoredSecret> Rows => _rows;
|
|
|
|
/// <summary>Seeds a LIVE row directly (bypassing ApplyReplicatedAsync/LWW) — an out-of-band secret.</summary>
|
|
public void Seed(string name, DateTimeOffset updatedUtc, long revision = 1, byte[]? ciphertext = null) =>
|
|
_rows[name] = new StoredSecret
|
|
{
|
|
Name = new SecretName(name),
|
|
ContentType = SecretContentType.Text,
|
|
Ciphertext = ciphertext ?? [9],
|
|
Nonce = [9],
|
|
Tag = [9],
|
|
WrappedDek = [9],
|
|
WrapNonce = [9],
|
|
WrapTag = [9],
|
|
KekId = "kek-out-of-band",
|
|
Revision = revision,
|
|
CreatedUtc = updatedUtc,
|
|
UpdatedUtc = updatedUtc,
|
|
};
|
|
|
|
/// <summary>True if the row is present and NOT tombstoned.</summary>
|
|
public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted;
|
|
|
|
/// <summary>True if the row is present as a tombstone.</summary>
|
|
public bool IsTombstoned(string name) => _rows.TryGetValue(name, out var row) && row.IsDeleted;
|
|
|
|
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
|
|
Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);
|
|
|
|
public Task UpsertAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Tombstones (does not hard-delete) — matches <c>SqliteSecretStore.DeleteAsync</c>.</summary>
|
|
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
|
|
{
|
|
if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted)
|
|
{
|
|
return Task.FromResult(false);
|
|
}
|
|
|
|
_rows[name.Value] = existing with
|
|
{
|
|
IsDeleted = true,
|
|
DeletedUtc = Now,
|
|
UpdatedUtc = Now,
|
|
Revision = existing.Revision + 1,
|
|
UpdatedBy = actor,
|
|
};
|
|
return Task.FromResult(true);
|
|
}
|
|
|
|
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretMetadata>>([]);
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
|
|
/// <summary>Strict LWW on (UpdatedUtc, Revision) via <see cref="SecretLastWriterWins.IsNewer"/>; applied verbatim — matches <c>SqliteSecretStore</c>.</summary>
|
|
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
if (ThrowOnApply)
|
|
{
|
|
throw new InvalidOperationException("simulated store fault");
|
|
}
|
|
|
|
if (_rows.TryGetValue(row.Name.Value, out var existing)
|
|
&& !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision))
|
|
{
|
|
return Task.CompletedTask; // no-op, matches real semantics
|
|
}
|
|
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task<bool> ApplyRewrapAsync(
|
|
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
|
Task.FromResult(false);
|
|
}
|
|
|
|
/// <summary>In-memory <see cref="ISecretResolver"/> whose per-name resolution outcome tests control.</summary>
|
|
private sealed class FakeSecretResolver : ISecretResolver
|
|
{
|
|
private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);
|
|
private readonly HashSet<string> _nullReturning = new(StringComparer.Ordinal);
|
|
|
|
public int CallCount;
|
|
|
|
/// <summary>When set, a name that is absent or tombstoned in this store resolves to null, like the real resolver.</summary>
|
|
public FakeSecretStore? BackingStore { get; set; }
|
|
|
|
/// <summary>Row absent/tombstoned path — <c>DefaultSecretResolver.GetAsync</c> returns null.</summary>
|
|
public void ReturnNullFor(string name) => _nullReturning.Add(name);
|
|
|
|
/// <summary>Decryption/integrity failure path — <c>DefaultSecretResolver.GetAsync</c> rethrows <c>SecretDecryptionException</c>.</summary>
|
|
public void ThrowFor(string name) => _throwing.Add(name);
|
|
|
|
public void ClearFailures()
|
|
{
|
|
_nullReturning.Clear();
|
|
_throwing.Clear();
|
|
}
|
|
|
|
public Task<string?> GetAsync(SecretName name, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref CallCount);
|
|
if (_throwing.Contains(name.Value))
|
|
{
|
|
throw new InvalidOperationException("simulated decrypt failure");
|
|
}
|
|
|
|
if (BackingStore is not null && !BackingStore.Contains(name.Value))
|
|
{
|
|
return Task.FromResult<string?>(null);
|
|
}
|
|
|
|
return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
|
|
}
|
|
}
|
|
}
|