fix(site-secrets): mirror heals same-revision local drift and isolates unreadable rows

Final review I1: a name already ledgered at the shipped revision is now read back every
pass; a missing, tombstoned or edited local row is healed through the mirror-owned re-stamp
path (bump, apply, read back, invalidate) before verify, instead of a local delete being
misreported as a KEK mismatch that blocked every later deploy. Drift logs Warning once per
occurrence. M4: row_json that cannot be parsed becomes a Failed entry for that name via the
new SiteStorageService.GetDeployedSecretsTolerantAsync; an unreadable removed row is still
honoured by name. M10: invalidator ctor doc corrected.
This commit is contained in:
Joseph Doherty
2026-09-28 21:08:34 -04:00
parent 9835ef8be3
commit 6f1787abe1
3 changed files with 277 additions and 27 deletions
@@ -205,6 +205,126 @@ public class DeployedSecretsMirrorTests : IDisposable
Assert.Equal(2, store.Rows["a"].Revision);
}
// ── Same-revision drift (final review I1) ──
[Fact]
public async Task Reconcile_OwnedNameDeletedLocally_IsHealedAndVerified_NotMisreportedAsKekMismatch()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
var (mirror, storage, store, resolver, invalidator) = await CreateAsync(time: time);
resolver.BackingStore = store; // resolve returns null once the local row is gone
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
// An operator deletes the mirror-owned name on this node (library CLI/UI) — the
// ledger still says r1 is mirrored, and central still ships r1.
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
Assert.True(store.IsTombstoned("a"));
invalidator.Invalidated.Clear();
var r = await mirror.ReconcileAsync(CancellationToken.None);
Assert.Empty(r.Failed);
Assert.Equal(["a"], r.Applied);
Assert.True(store.Contains("a"));
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
Assert.Contains("a", invalidator.Invalidated);
// Healed state is stable: the next pass is a no-op that still verifies clean.
var next = await mirror.ReconcileAsync(CancellationToken.None);
Assert.False(next.Changed);
Assert.Empty(next.Failed);
Assert.Equal(1, (await storage.GetMirroredSecretsAsync()).Single(m => m.Name == "a").Revision);
}
[Fact]
public async Task Reconcile_OwnedNameEditedLocally_IsRevertedToTheDeployedCiphertext()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
// An operator overwrites the mirror-owned name locally with a NEWER stamp.
store.Seed("a",
updatedUtc: new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero),
revision: 7,
ciphertext: [200, 201, 202]);
invalidator.Invalidated.Clear();
var r = await mirror.ReconcileAsync(CancellationToken.None);
Assert.Empty(r.Failed);
Assert.Equal(["a"], r.Applied);
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
Assert.Equal(new byte[] { 30, 31, 32, 33, 34, 35 }, store.Rows["a"].WrappedDek);
Assert.True(store.Rows["a"].Revision > 7); // re-stamped past the operator's row
Assert.Contains("a", invalidator.Invalidated);
}
[Fact]
public async Task Reconcile_OwnedNameDrift_LogsOncePerDriftNotEveryPass()
{
var logger = new CapturingLogger<DeployedSecretsMirror>();
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, time: time);
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
await mirror.ReconcileAsync(CancellationToken.None);
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
await mirror.ReconcileAsync(CancellationToken.None);
await mirror.ReconcileAsync(CancellationToken.None);
await mirror.ReconcileAsync(CancellationToken.None);
Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("re-applying"));
Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Error);
}
// ── Unreadable row_json (final review M4) ──
[Fact]
public async Task Reconcile_UnparseableRowJson_IsAFailedEntryForThatName_OthersStillApply()
{
var (mirror, storage, store, _, _) = await CreateAsync();
await storage.UpsertDeployedSecretsAsync([Art("bad", 1), Art("good", 1)], "d1");
await CorruptRowJsonAsync(storage, "bad");
var r = await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(["good"], r.Applied);
Assert.Equal(["bad"], r.FailedNames);
var message = Assert.Single(r.Failed);
Assert.Contains("'bad'", message);
Assert.Contains("could not be read", message);
Assert.True(store.Contains("good"));
Assert.False(store.Contains("bad"));
}
[Fact]
public async Task Reconcile_UnparseableRemovedRow_StillRemovesAMirroredName()
{
var (mirror, storage, store, _, _) = await CreateAsync();
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
await mirror.ReconcileAsync(CancellationToken.None);
await storage.UpsertDeployedSecretsAsync([], "d2"); // soft-removes "a"
await CorruptRowJsonAsync(storage, "a");
var r = await mirror.ReconcileAsync(CancellationToken.None);
Assert.Equal(["a"], r.Removed);
Assert.Empty(r.Failed);
Assert.True(store.IsTombstoned("a"));
}
private static async Task CorruptRowJsonAsync(SiteStorageService storage, string name)
{
await using var connection = storage.CreateConnection();
await using var command = connection.CreateCommand();
command.CommandText = "UPDATE deployed_secrets SET row_json = '{not json' WHERE name = @name";
command.Parameters.AddWithValue("@name", name);
Assert.Equal(1, await command.ExecuteNonQueryAsync());
}
[Fact]
public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
{
@@ -740,6 +860,9 @@ public class DeployedSecretsMirrorTests : IDisposable
public int CallCount;
/// <summary>When set, a name that is absent or tombstoned in this store resolves to null, like the real resolver.</summary>
public FakeSecretStore? BackingStore { get; set; }
/// <summary>Row absent/tombstoned path — <c>DefaultSecretResolver.GetAsync</c> returns null.</summary>
public void ReturnNullFor(string name) => _nullReturning.Add(name);
@@ -760,6 +883,11 @@ public class DeployedSecretsMirrorTests : IDisposable
throw new InvalidOperationException("simulated decrypt failure");
}
if (BackingStore is not null && !BackingStore.Contains(name.Value))
{
return Task.FromResult<string?>(null);
}
return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
}
}