fix(site-secrets): mirror heals same-revision local drift and isolates unreadable rows
Final review I1: a name already ledgered at the shipped revision is now read back every pass; a missing, tombstoned or edited local row is healed through the mirror-owned re-stamp path (bump, apply, read back, invalidate) before verify, instead of a local delete being misreported as a KEK mismatch that blocked every later deploy. Drift logs Warning once per occurrence. M4: row_json that cannot be parsed becomes a Failed entry for that name via the new SiteStorageService.GetDeployedSecretsTolerantAsync; an unreadable removed row is still honoured by name. M10: invalidator ctor doc corrected.
This commit is contained in:
@@ -205,6 +205,126 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
Assert.Equal(2, store.Rows["a"].Revision);
|
||||
}
|
||||
|
||||
// ── Same-revision drift (final review I1) ──
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_OwnedNameDeletedLocally_IsHealedAndVerified_NotMisreportedAsKekMismatch()
|
||||
{
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, store, resolver, invalidator) = await CreateAsync(time: time);
|
||||
resolver.BackingStore = store; // resolve returns null once the local row is gone
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
|
||||
|
||||
// An operator deletes the mirror-owned name on this node (library CLI/UI) — the
|
||||
// ledger still says r1 is mirrored, and central still ships r1.
|
||||
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
|
||||
Assert.True(store.IsTombstoned("a"));
|
||||
invalidator.Invalidated.Clear();
|
||||
|
||||
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(r.Failed);
|
||||
Assert.Equal(["a"], r.Applied);
|
||||
Assert.True(store.Contains("a"));
|
||||
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
|
||||
Assert.Contains("a", invalidator.Invalidated);
|
||||
|
||||
// Healed state is stable: the next pass is a no-op that still verifies clean.
|
||||
var next = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
Assert.False(next.Changed);
|
||||
Assert.Empty(next.Failed);
|
||||
Assert.Equal(1, (await storage.GetMirroredSecretsAsync()).Single(m => m.Name == "a").Revision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_OwnedNameEditedLocally_IsRevertedToTheDeployedCiphertext()
|
||||
{
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, store, _, invalidator) = await CreateAsync(time: time);
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
Assert.Equal(["a"], (await mirror.ReconcileAsync(CancellationToken.None)).Applied);
|
||||
|
||||
// An operator overwrites the mirror-owned name locally with a NEWER stamp.
|
||||
store.Seed("a",
|
||||
updatedUtc: new DateTimeOffset(2026, 9, 28, 12, 0, 0, TimeSpan.Zero),
|
||||
revision: 7,
|
||||
ciphertext: [200, 201, 202]);
|
||||
invalidator.Invalidated.Clear();
|
||||
|
||||
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(r.Failed);
|
||||
Assert.Equal(["a"], r.Applied);
|
||||
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, store.Rows["a"].Ciphertext);
|
||||
Assert.Equal(new byte[] { 30, 31, 32, 33, 34, 35 }, store.Rows["a"].WrappedDek);
|
||||
Assert.True(store.Rows["a"].Revision > 7); // re-stamped past the operator's row
|
||||
Assert.Contains("a", invalidator.Invalidated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_OwnedNameDrift_LogsOncePerDriftNotEveryPass()
|
||||
{
|
||||
var logger = new CapturingLogger<DeployedSecretsMirror>();
|
||||
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.Zero));
|
||||
var (mirror, storage, store, _, _) = await CreateAsync(logger: logger, time: time);
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
await store.DeleteAsync(new SecretName("a"), "operator", CancellationToken.None);
|
||||
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning && e.Message.Contains("re-applying"));
|
||||
Assert.DoesNotContain(logger.Entries, e => e.Level == LogLevel.Error);
|
||||
}
|
||||
|
||||
// ── Unreadable row_json (final review M4) ──
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_UnparseableRowJson_IsAFailedEntryForThatName_OthersStillApply()
|
||||
{
|
||||
var (mirror, storage, store, _, _) = await CreateAsync();
|
||||
await storage.UpsertDeployedSecretsAsync([Art("bad", 1), Art("good", 1)], "d1");
|
||||
await CorruptRowJsonAsync(storage, "bad");
|
||||
|
||||
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(["good"], r.Applied);
|
||||
Assert.Equal(["bad"], r.FailedNames);
|
||||
var message = Assert.Single(r.Failed);
|
||||
Assert.Contains("'bad'", message);
|
||||
Assert.Contains("could not be read", message);
|
||||
Assert.True(store.Contains("good"));
|
||||
Assert.False(store.Contains("bad"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_UnparseableRemovedRow_StillRemovesAMirroredName()
|
||||
{
|
||||
var (mirror, storage, store, _, _) = await CreateAsync();
|
||||
await storage.UpsertDeployedSecretsAsync([Art("a", 1)], "d1");
|
||||
await mirror.ReconcileAsync(CancellationToken.None);
|
||||
await storage.UpsertDeployedSecretsAsync([], "d2"); // soft-removes "a"
|
||||
await CorruptRowJsonAsync(storage, "a");
|
||||
|
||||
var r = await mirror.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(["a"], r.Removed);
|
||||
Assert.Empty(r.Failed);
|
||||
Assert.True(store.IsTombstoned("a"));
|
||||
}
|
||||
|
||||
private static async Task CorruptRowJsonAsync(SiteStorageService storage, string name)
|
||||
{
|
||||
await using var connection = storage.CreateConnection();
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "UPDATE deployed_secrets SET row_json = '{not json' WHERE name = @name";
|
||||
command.Parameters.AddWithValue("@name", name);
|
||||
Assert.Equal(1, await command.ExecuteNonQueryAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_AppliesThroughApplyReplicatedAsync_MappedFromSecretArtifact()
|
||||
{
|
||||
@@ -740,6 +860,9 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
|
||||
public int CallCount;
|
||||
|
||||
/// <summary>When set, a name that is absent or tombstoned in this store resolves to null, like the real resolver.</summary>
|
||||
public FakeSecretStore? BackingStore { get; set; }
|
||||
|
||||
/// <summary>Row absent/tombstoned path — <c>DefaultSecretResolver.GetAsync</c> returns null.</summary>
|
||||
public void ReturnNullFor(string name) => _nullReturning.Add(name);
|
||||
|
||||
@@ -760,6 +883,11 @@ public class DeployedSecretsMirrorTests : IDisposable
|
||||
throw new InvalidOperationException("simulated decrypt failure");
|
||||
}
|
||||
|
||||
if (BackingStore is not null && !BackingStore.Contains(name.Value))
|
||||
{
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
|
||||
return Task.FromResult(_nullReturning.Contains(name.Value) ? null : "decrypted-value");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user