feat(secrets): build G-8 KEK rotation (RewrapAll); design+plan G-7 clustered replication

G-8 (KEK rotation) — built in ZB.MOM.WW.Secrets, lib 0.1.2->0.1.3:
- ISecretCipher.Rewrap(row, oldKek, newKek): re-wraps the per-secret DEK only
  (bodies never re-encrypted; revision/timestamps preserved -> invisible to
  cluster LWW). Fail-closed on wrong old-KEK id, wrong key bytes, and malformed
  wraps; DEK zeroed on all paths.
- ISecretStore.ApplyRewrapAsync(rewrappedRow, expectedCurrentWrappedDek):
  updates only the 4 wrap columns + kek_id, compare-and-swap on the current
  wrapped DEK so a concurrent set/rotate cannot corrupt a row (closes a
  review-caught TOCTOU).
- KekRotationService.RewrapAllAsync + RewrapReport: enumerate all rows incl.
  tombstones, idempotent/resumable skip-already-current, bounded CAS-retry,
  fail-closed on unknown/identical KEK.
- `secret rewrap-all` CLI verb: key material only via env-var name / file path,
  JSON counts report; README section + operator runbook.

Verified: full offline suite green (82 core + 15 UI, 0 regressions) + end-to-end
CLI smoke + adversarial crypto review (all 7 categories PASS; TOCTOU fixed).

G-7 (clustered replication) — designed + planned, no code:
- Fork resolved to build Option A (shared SQL-Server ISecretStore); Akka
  replicator ZB.MOM.WW.Secrets.Akka is a deferred phase-2. Design doc +
  executable plan + .tasks.json under docs/plans/2026-07-17-secrets-g7-*.

Tracking: components/secrets/GAPS.md + CLAUDE.md secrets row updated.
This commit is contained in:
Joseph Doherty
2026-07-17 02:55:43 -04:00
parent b009507e10
commit d82d3451e7
24 changed files with 1280 additions and 12 deletions
@@ -19,6 +19,7 @@ public sealed class SecretCommandsTests : IAsyncLifetime, IDisposable
private readonly SecretsSqliteConnectionFactory _factory;
private readonly SqliteSecretStore _store;
private readonly FakeMasterKeyProvider _oldKek = new("kek-cli");
private readonly AesGcmEnvelopeCipher _cipher;
private readonly DefaultSecretResolver _resolver;
@@ -26,7 +27,7 @@ public sealed class SecretCommandsTests : IAsyncLifetime, IDisposable
{
_factory = new SecretsSqliteConnectionFactory(_dbPath);
_store = new SqliteSecretStore(_factory);
_cipher = new AesGcmEnvelopeCipher(new FakeMasterKeyProvider("kek-cli"));
_cipher = new AesGcmEnvelopeCipher(_oldKek);
// A near-zero TTL keeps the resolver from serving a stale plaintext across a rotate/remove.
_resolver = new DefaultSecretResolver(
_store, _cipher, NoOpAuditWriter.Instance, TimeSpan.Zero);
@@ -53,6 +54,45 @@ public sealed class SecretCommandsTests : IAsyncLifetime, IDisposable
private static CancellationToken Ct => CancellationToken.None;
[Fact]
public async Task RewrapAll_MigratesRowsToNewKek_ReportsCountsWithoutLeakingValues()
{
var (seed, _) = NewSut();
await seed.SetAsync("sql/a", "value-a", SecretContentType.Text, null, "alice", Ct);
await seed.SetAsync("sql/b", "value-b", SecretContentType.Text, null, "alice", Ct);
var newKek = new FakeMasterKeyProvider("kek-new");
var (cmd, output) = NewSut();
int rc = await cmd.RewrapAllAsync(_oldKek, newKek, Ct);
Assert.Equal(0, rc);
string outStr = output.ToString();
Assert.Contains("\"action\":\"rewrap-all\"", outStr);
Assert.Contains("\"total\":2", outStr);
Assert.Contains("\"rewrapped\":2", outStr);
Assert.Contains("\"alreadyCurrent\":0", outStr);
// The report carries counts only — never a secret value.
Assert.DoesNotContain("value-a", outStr);
Assert.DoesNotContain("value-b", outStr);
// The rows now decrypt under the NEW KEK (proof the rewrap actually re-keyed them).
var newCipher = new AesGcmEnvelopeCipher(newKek);
Assert.Equal("value-a", newCipher.Decrypt((await _store.GetAsync(new SecretName("sql/a"), Ct))!));
}
[Fact]
public async Task RewrapAll_IdenticalKek_ReturnsErrorExitCode()
{
var (seed, _) = NewSut();
await seed.SetAsync("sql/a", "value-a", SecretContentType.Text, null, "alice", Ct);
var (cmd, output) = NewSut();
int rc = await cmd.RewrapAllAsync(_oldKek, new FakeMasterKeyProvider("kek-cli"), Ct);
Assert.Equal(1, rc);
Assert.Contains("\"error\"", output.ToString());
}
[Fact]
public async Task Set_Then_Get_RoundTrips()
{
@@ -95,6 +95,97 @@ public class AesGcmEnvelopeCipherTests
Assert.Contains("unknown KEK", ex.Message, StringComparison.Ordinal);
}
[Fact]
public void Rewrap_RewrapsDek_BodyStillDecryptsUnderNewKek()
{
var oldKek = new FakeMasterKeyProvider("kek-old");
var newKek = new FakeMasterKeyProvider("kek-new");
var oldCipher = new AesGcmEnvelopeCipher(oldKek);
var name = new SecretName("sql/foo");
StoredSecret row = oldCipher.Encrypt(name, "hunter2", SecretContentType.Text);
Assert.Equal("kek-old", row.KekId);
// Rewrap is provider-explicit — it can be driven through any cipher instance.
StoredSecret rewrapped = oldCipher.Rewrap(row, oldKek, newKek);
Assert.Equal("kek-new", rewrapped.KekId);
// The DEK-sealed body is untouched; only the KEK-wrap envelope changed.
Assert.True(rewrapped.Ciphertext.AsSpan().SequenceEqual(row.Ciphertext));
Assert.True(rewrapped.Nonce.AsSpan().SequenceEqual(row.Nonce));
Assert.True(rewrapped.Tag.AsSpan().SequenceEqual(row.Tag));
Assert.False(rewrapped.WrappedDek.AsSpan().SequenceEqual(row.WrappedDek));
// The rewrapped row decrypts under the NEW KEK...
var newCipher = new AesGcmEnvelopeCipher(newKek);
Assert.Equal("hunter2", newCipher.Decrypt(rewrapped));
// ...and no longer under the OLD KEK (kek_id now names the new KEK; old cipher fails closed).
Assert.Throws<SecretDecryptionException>(() => oldCipher.Decrypt(rewrapped));
}
[Fact]
public void Rewrap_PreservesRevisionTimestampsTombstoneAndMetadata()
{
var oldKek = new FakeMasterKeyProvider("kek-old");
var newKek = new FakeMasterKeyProvider("kek-new");
var cipher = new AesGcmEnvelopeCipher(oldKek);
StoredSecret row = cipher.Encrypt(new SecretName("sql/foo"), "hunter2", SecretContentType.Text) with
{
Revision = 7,
IsDeleted = true,
DeletedUtc = DateTimeOffset.UnixEpoch,
CreatedUtc = DateTimeOffset.UnixEpoch,
UpdatedUtc = DateTimeOffset.UnixEpoch.AddMinutes(5),
CreatedBy = "alice",
UpdatedBy = "bob",
Description = "the foo password",
};
StoredSecret rewrapped = cipher.Rewrap(row, oldKek, newKek);
// Only the wrap envelope + kek_id change — a rewrap must be invisible to (updated_utc, revision) LWW.
Assert.Equal(7, rewrapped.Revision);
Assert.True(rewrapped.IsDeleted);
Assert.Equal(row.DeletedUtc, rewrapped.DeletedUtc);
Assert.Equal(row.CreatedUtc, rewrapped.CreatedUtc);
Assert.Equal(row.UpdatedUtc, rewrapped.UpdatedUtc);
Assert.Equal("alice", rewrapped.CreatedBy);
Assert.Equal("bob", rewrapped.UpdatedBy);
Assert.Equal("the foo password", rewrapped.Description);
Assert.Equal(SecretContentType.Text, rewrapped.ContentType);
}
[Fact]
public void Rewrap_WrongOldKekId_FailsClosed()
{
var actualKek = new FakeMasterKeyProvider("kek-old");
var wrongOld = new FakeMasterKeyProvider("kek-somethingelse");
var newKek = new FakeMasterKeyProvider("kek-new");
var cipher = new AesGcmEnvelopeCipher(actualKek);
StoredSecret row = cipher.Encrypt(new SecretName("sql/foo"), "hunter2", SecretContentType.Text);
// Row is wrapped by "kek-old" but the caller claims the old KEK is "kek-somethingelse".
var ex = Assert.Throws<SecretDecryptionException>(() => cipher.Rewrap(row, wrongOld, newKek));
Assert.Contains("kek-old", ex.Message, StringComparison.Ordinal);
}
[Fact]
public void Rewrap_CorrectKekIdButWrongKeyMaterial_FailsClosed()
{
// Same KEK-id string, different key bytes → the unwrap must fail closed (tag mismatch),
// never emitting a garbage DEK.
var realOld = new FakeMasterKeyProvider("kek-old");
var imposterOld = new FakeMasterKeyProvider("kek-old"); // same id, fresh random bytes
var newKek = new FakeMasterKeyProvider("kek-new");
var cipher = new AesGcmEnvelopeCipher(realOld);
StoredSecret row = cipher.Encrypt(new SecretName("sql/foo"), "hunter2", SecretContentType.Text);
Assert.Throws<SecretDecryptionException>(() => cipher.Rewrap(row, imposterOld, newKek));
}
private static bool Contains(byte[] haystack, byte[] needle)
{
if (needle.Length == 0 || haystack.Length < needle.Length)
@@ -48,4 +48,9 @@ public sealed class CountingSecretStore : ISecretStore
/// <inheritdoc />
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) =>
throw new NotSupportedException();
/// <inheritdoc />
public Task<bool> ApplyRewrapAsync(
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
throw new NotSupportedException();
}
@@ -0,0 +1,163 @@
using Microsoft.Data.Sqlite;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Crypto;
using ZB.MOM.WW.Secrets.Rotation;
using ZB.MOM.WW.Secrets.Sqlite;
using ZB.MOM.WW.Secrets.Tests.Fakes;
namespace ZB.MOM.WW.Secrets.Tests.Rotation;
/// <summary>
/// End-to-end KEK-rotation tests against the REAL SQLite store and REAL AES-256-GCM cipher — the
/// combination that actually proves a re-wrapped row decrypts under the new KEK (and no longer
/// under the old one).
/// </summary>
public sealed class KekRotationServiceTests : IAsyncLifetime, IDisposable
{
private readonly string _dbPath =
Path.Combine(Path.GetTempPath(), $"zb-secrets-rotate-{Guid.NewGuid():N}.db");
private readonly SecretsSqliteConnectionFactory _factory;
private readonly SqliteSecretStore _store;
private readonly FakeMasterKeyProvider _oldKek = new("kek-old");
private readonly FakeMasterKeyProvider _newKek = new("kek-new");
public KekRotationServiceTests()
{
_factory = new SecretsSqliteConnectionFactory(_dbPath);
_store = new SqliteSecretStore(_factory);
}
public async Task InitializeAsync() =>
await new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None);
public Task DisposeAsync() => Task.CompletedTask;
// Seeds a secret sealed under a given KEK provider (defaults to the old KEK).
private async Task SeedAsync(string name, string value, IMasterKeyProvider? kek = null)
{
var cipher = new AesGcmEnvelopeCipher(kek ?? _oldKek);
StoredSecret row = cipher.Encrypt(new SecretName(name), value, SecretContentType.Text);
await _store.UpsertAsync(row, CancellationToken.None);
}
private KekRotationService Service() => new(_store, new AesGcmEnvelopeCipher(_oldKek));
[Fact]
public async Task RewrapAll_MigratesEveryRow_DecryptsUnderNewKekOnly()
{
await SeedAsync("sql/a", "value-a");
await SeedAsync("sql/b", "value-b");
await SeedAsync("ldap/c", "value-c");
RewrapReport report = await Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None);
Assert.Equal(3, report.Total);
Assert.Equal(3, report.Rewrapped);
Assert.Equal(0, report.AlreadyCurrent);
var newCipher = new AesGcmEnvelopeCipher(_newKek);
var oldCipher = new AesGcmEnvelopeCipher(_oldKek);
foreach ((string name, string value) in new[] { ("sql/a", "value-a"), ("sql/b", "value-b"), ("ldap/c", "value-c") })
{
StoredSecret row = (await _store.GetAsync(new SecretName(name), CancellationToken.None))!;
Assert.Equal("kek-new", row.KekId);
// Decrypts cleanly under the NEW KEK...
Assert.Equal(value, newCipher.Decrypt(row));
// ...and fails closed under the OLD KEK (kek_id no longer matches).
Assert.Throws<SecretDecryptionException>(() => oldCipher.Decrypt(row));
}
}
[Fact]
public async Task RewrapAll_IsIdempotent_SecondRunSkipsEverything()
{
await SeedAsync("sql/a", "value-a");
await SeedAsync("sql/b", "value-b");
RewrapReport first = await Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None);
Assert.Equal(2, first.Rewrapped);
RewrapReport second = await Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None);
Assert.Equal(2, second.Total);
Assert.Equal(0, second.Rewrapped);
Assert.Equal(2, second.AlreadyCurrent);
}
[Fact]
public async Task RewrapAll_RewrapsTombstonedRows()
{
await SeedAsync("sql/live", "live");
await SeedAsync("sql/dead", "dead");
await _store.DeleteAsync(new SecretName("sql/dead"), "carol", CancellationToken.None);
RewrapReport report = await Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None);
Assert.Equal(2, report.Rewrapped);
StoredSecret dead = (await _store.GetAsync(new SecretName("sql/dead"), CancellationToken.None))!;
Assert.Equal("kek-new", dead.KekId);
Assert.True(dead.IsDeleted);
}
[Fact]
public async Task RewrapAll_DoesNotBumpRevisionOrUpdatedUtc()
{
await SeedAsync("sql/a", "value-a");
StoredSecret before = (await _store.GetAsync(new SecretName("sql/a"), CancellationToken.None))!;
await Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None);
StoredSecret after = (await _store.GetAsync(new SecretName("sql/a"), CancellationToken.None))!;
Assert.Equal(before.Revision, after.Revision);
Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
}
[Fact]
public async Task RewrapAll_IdenticalKekIds_Throws()
{
await SeedAsync("sql/a", "value-a");
var same = new FakeMasterKeyProvider("kek-old");
await Assert.ThrowsAsync<ArgumentException>(
() => Service().RewrapAllAsync(_oldKek, same, CancellationToken.None));
}
[Fact]
public async Task RewrapAll_RowOnUnknownKek_Aborts_ButKeepsPriorProgress()
{
// Names are scanned in ORDER BY name: "a-old" and "b-old" migrate, then "z-third" (a KEK that
// is neither old nor new) aborts the pass.
await SeedAsync("a-old", "va");
await SeedAsync("b-old", "vb");
await SeedAsync("z-third", "vz", new FakeMasterKeyProvider("kek-third"));
await Assert.ThrowsAsync<SecretDecryptionException>(
() => Service().RewrapAllAsync(_oldKek, _newKek, CancellationToken.None));
// The two old-KEK rows scanned before the anomaly are already persisted on the new KEK,
// so a re-run (after removing the anomaly) resumes cleanly.
Assert.Equal("kek-new", (await _store.GetAsync(new SecretName("a-old"), CancellationToken.None))!.KekId);
Assert.Equal("kek-new", (await _store.GetAsync(new SecretName("b-old"), CancellationToken.None))!.KekId);
Assert.Equal("kek-third", (await _store.GetAsync(new SecretName("z-third"), CancellationToken.None))!.KekId);
}
public void Dispose()
{
SqliteConnection.ClearAllPools();
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (IOException)
{
// Best-effort temp cleanup.
}
}
}
}
@@ -204,6 +204,101 @@ public sealed class SqliteSecretStoreTests : IAsyncLifetime, IDisposable
Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
}
[Fact]
public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
{
DateTimeOffset created = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
StoredSecret seed = MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: created);
await _store.UpsertAsync(seed, CancellationToken.None);
// Bump the revision once so we can prove ApplyRewrap does NOT change it.
await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
Assert.Equal(1, before.Revision);
// Re-wrap: only the four wrap-related fields differ; everything else is carried from `before`.
StoredSecret rewrapped = before with
{
WrappedDek = [90, 91, 92],
WrapNonce = [93, 94, 95],
WrapTag = [96, 97, 98],
KekId = "kek-2",
};
bool applied = await _store.ApplyRewrapAsync(rewrapped, before.WrappedDek, CancellationToken.None);
Assert.True(applied);
StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
// Wrap envelope + kek_id changed...
Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
Assert.Equal("kek-2", after.KekId);
// ...but revision, updated timestamp, the sealed body, and audit stamps are untouched.
Assert.Equal(before.Revision, after.Revision);
Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
Assert.Equal(before.CreatedUtc, after.CreatedUtc);
Assert.Equal(before.UpdatedBy, after.UpdatedBy);
Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
Assert.Equal(before.Nonce, after.Nonce);
Assert.Equal(before.Tag, after.Tag);
}
[Fact]
public async Task ApplyRewrap_RewrapsTombstonedRows()
{
await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
Assert.True(tomb.IsDeleted);
bool applied = await _store.ApplyRewrapAsync(
tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
tomb.WrappedDek,
CancellationToken.None);
Assert.True(applied);
StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
Assert.Equal("kek-2", after.KekId);
// The tombstone survives the rewrap.
Assert.True(after.IsDeleted);
Assert.Equal(tomb.Revision, after.Revision);
}
[Fact]
public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
{
StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
bool applied = await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None);
Assert.False(applied);
}
[Fact]
public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
{
StoredSecret original = MakeSecret("app/cas", ciphertext: [1, 2, 3]);
await _store.UpsertAsync(original, CancellationToken.None);
StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
// Simulate a concurrent set/rotate: a fresh write changes the wrapped DEK out from under us.
await _store.UpsertAsync(
MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
CancellationToken.None);
// A rewrap that still expects the ORIGINAL wrap must NOT apply (0 rows) — it would otherwise
// pair a stale re-wrap with the newer body and corrupt the row.
bool applied = await _store.ApplyRewrapAsync(
before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
before.WrappedDek,
CancellationToken.None);
Assert.False(applied);
StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
// The concurrent write's wrap survived untouched; no stale re-wrap was applied.
Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
Assert.Equal("kek-1", after.KekId);
}
public void Dispose()
{
// Drop pooled connections so the WAL/-shm/-wal sidecars release before we delete.
@@ -42,4 +42,8 @@ public sealed class FakeCipher : ISecretCipher
return Encoding.UTF8.GetString(secret.Ciphertext);
}
/// <inheritdoc />
public StoredSecret Rewrap(StoredSecret secret, IMasterKeyProvider oldKek, IMasterKeyProvider newKek)
=> secret with { KekId = newKek.KekId };
}
@@ -88,4 +88,9 @@ public sealed class RecordingSecretStore : ISecretStore
/// <inheritdoc />
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
=> throw new NotSupportedException();
/// <inheritdoc />
public Task<bool> ApplyRewrapAsync(
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct)
=> throw new NotSupportedException();
}
@@ -124,5 +124,9 @@ public class SecretsListTests : TestContext
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
=> throw new NotSupportedException();
public Task<bool> ApplyRewrapAsync(
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct)
=> throw new NotSupportedException();
}
}