feat(secrets-cli): KEK doctor — per-row diagnosis + guided rewrap remedy
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,157 @@
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Rotation;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Cli.Interactive;
|
||||
|
||||
/// <summary>The per-row verdict of a KEK diagnosis.</summary>
|
||||
public enum RowKekStatus
|
||||
{
|
||||
/// <summary>The session KEK matches the row and successfully unwraps it — the row is healthy.</summary>
|
||||
Ok,
|
||||
|
||||
/// <summary>The row is wrapped under a different KEK than the session's — the session cannot open it.</summary>
|
||||
WrongKek,
|
||||
|
||||
/// <summary>
|
||||
/// The row's <c>kek_id</c> matches the session KEK, yet the DEK unwrap / decrypt fails closed —
|
||||
/// the wrap envelope or sealed body is damaged.
|
||||
/// </summary>
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
/// <summary>One row's KEK verdict: its name, status, and the <c>kek_id</c> it is actually wrapped under.</summary>
|
||||
/// <param name="SecretName">The secret's normalized name.</param>
|
||||
/// <param name="Status">Whether the session KEK opens the row (<see cref="RowKekStatus"/>).</param>
|
||||
/// <param name="RowKekId">
|
||||
/// The <c>kek_id</c> stamped on the row — the key an operator must hunt down when the status is
|
||||
/// <see cref="RowKekStatus.WrongKek"/>.
|
||||
/// </param>
|
||||
public sealed record KekDiagnosis(string SecretName, RowKekStatus Status, string RowKekId);
|
||||
|
||||
/// <summary>
|
||||
/// The full diagnosis: the session's KEK id and the per-row verdicts (ordered by name), including
|
||||
/// tombstoned rows since they still carry a KEK-wrapped DEK and block a rewrap-all.
|
||||
/// </summary>
|
||||
/// <param name="SessionKekId">The <c>kek_id</c> of the KEK the session is currently using.</param>
|
||||
/// <param name="Rows">The per-row verdicts, ordered by secret name.</param>
|
||||
public sealed record KekDoctorReport(string SessionKekId, IReadOnlyList<KekDiagnosis> Rows)
|
||||
{
|
||||
/// <summary>Whether every diagnosed row opens cleanly under the session KEK.</summary>
|
||||
public bool Healthy => Rows.All(r => r.Status == RowKekStatus.Ok);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lockout triage: verifies the session KEK actually opens every stored row and drives the rewrap
|
||||
/// remedy. An operator whose app can no longer decrypt its secrets runs the doctor to learn, per row,
|
||||
/// which KEK each row is wrapped under and whether the session KEK opens it, then remedies the
|
||||
/// wrong-KEK rows with a guided rewrap-all onto the session KEK.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The store surface makes a full decrypt probe possible even for tombstoned rows:
|
||||
/// <see cref="ISecretStore.ListAsync"/> enumerates all rows' metadata (including tombstones with
|
||||
/// <c>includeDeleted: true</c>) and <see cref="ISecretStore.GetAsync"/> returns the full ciphertext
|
||||
/// row for any name <b>including a tombstoned one</b>, so every row — live or dead — is probed the
|
||||
/// same way. Plaintext produced by the probe is discarded immediately and never surfaced.
|
||||
/// </remarks>
|
||||
public sealed class KekDoctor
|
||||
{
|
||||
/// <summary>
|
||||
/// Diagnoses every stored row (including tombstones) against the session KEK: a row whose
|
||||
/// <c>kek_id</c> differs from the session KEK is reported <see cref="RowKekStatus.WrongKek"/>
|
||||
/// with no decrypt attempted; a row whose <c>kek_id</c> matches is decrypt-probed and reported
|
||||
/// <see cref="RowKekStatus.Ok"/> or, if the unwrap fails closed, <see cref="RowKekStatus.Corrupt"/>.
|
||||
/// </summary>
|
||||
/// <param name="session">The open session; must not be degraded (a KEK is required).</param>
|
||||
/// <param name="ct">A token to cancel the diagnosis.</param>
|
||||
/// <returns>A <see cref="KekDoctorReport"/> with the per-row verdicts, ordered by name.</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="session"/> is <see langword="null"/>.</exception>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// The session is degraded (no KEK / cipher). The operator must re-open the session supplying a
|
||||
/// master key before the doctor can probe rows.
|
||||
/// </exception>
|
||||
public async Task<KekDoctorReport> DiagnoseAsync(SecretsSession session, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(session);
|
||||
|
||||
if (session.MasterKey is null || session.Cipher is null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"The session is degraded: no master key (KEK) is available, so rows cannot be probed. " +
|
||||
"Re-open the session and supply the key material (for example, paste it at the prompt).");
|
||||
}
|
||||
|
||||
string sessionKekId = session.MasterKey.KekId;
|
||||
|
||||
// ListAsync gives metadata only; GetAsync fetches the full ciphertext row (tombstones included)
|
||||
// for the decrypt probe. Enumerate ALL rows so tombstones — which still block a rewrap-all — show.
|
||||
IReadOnlyList<SecretMetadata> all =
|
||||
await session.Store.ListAsync(includeDeleted: true, ct).ConfigureAwait(false);
|
||||
|
||||
List<KekDiagnosis> rows = [];
|
||||
foreach (SecretMetadata meta in all.OrderBy(m => m.Name.Value, StringComparer.Ordinal))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
// A row under a different KEK cannot be opened by this session — report it (with its own
|
||||
// kek_id so the operator knows which key to hunt) WITHOUT attempting a doomed decrypt.
|
||||
if (!string.Equals(meta.KekId, sessionKekId, StringComparison.Ordinal))
|
||||
{
|
||||
rows.Add(new KekDiagnosis(meta.Name.Value, RowKekStatus.WrongKek, meta.KekId));
|
||||
continue;
|
||||
}
|
||||
|
||||
StoredSecret? row = await session.Store.GetAsync(meta.Name, ct).ConfigureAwait(false);
|
||||
if (row is null)
|
||||
{
|
||||
// Vanished between the list and the fetch (a concurrent hard-absence) — nothing to probe.
|
||||
continue;
|
||||
}
|
||||
|
||||
RowKekStatus status;
|
||||
try
|
||||
{
|
||||
// The kek_id matches: probe the actual unwrap/decrypt. Plaintext is discarded at once.
|
||||
_ = session.Cipher.Decrypt(row);
|
||||
status = RowKekStatus.Ok;
|
||||
}
|
||||
catch (SecretDecryptionException)
|
||||
{
|
||||
status = RowKekStatus.Corrupt;
|
||||
}
|
||||
|
||||
rows.Add(new KekDiagnosis(meta.Name.Value, status, row.KekId));
|
||||
}
|
||||
|
||||
return new KekDoctorReport(sessionKekId, rows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remedies wrong-KEK rows by re-wrapping every row from <paramref name="oldKek"/> onto the
|
||||
/// session's KEK, delegating to <see cref="KekRotationService.RewrapAllAsync"/> (idempotent, and
|
||||
/// fail-closed on rows wrapped by neither the old nor the session KEK).
|
||||
/// </summary>
|
||||
/// <param name="session">The open session; must not be degraded (its KEK is the rewrap target).</param>
|
||||
/// <param name="oldKek">The provider for the KEK the wrong-KEK rows are currently wrapped under.</param>
|
||||
/// <param name="ct">A token to cancel the pass (already-persisted re-wraps are retained).</param>
|
||||
/// <returns>A <see cref="RewrapReport"/> summarizing the pass (no secret material).</returns>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// <paramref name="session"/> or <paramref name="oldKek"/> is <see langword="null"/>.
|
||||
/// </exception>
|
||||
/// <exception cref="InvalidOperationException">The session is degraded (no KEK / cipher).</exception>
|
||||
public Task<RewrapReport> RewrapAllAsync(
|
||||
SecretsSession session, IMasterKeyProvider oldKek, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(session);
|
||||
ArgumentNullException.ThrowIfNull(oldKek);
|
||||
|
||||
if (session.MasterKey is null || session.Cipher is null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"The session is degraded: no master key (KEK) is available to re-wrap onto. " +
|
||||
"Re-open the session and supply the key material before running the rewrap remedy.");
|
||||
}
|
||||
|
||||
return new KekRotationService(session.Store, session.Cipher)
|
||||
.RewrapAllAsync(oldKek, session.MasterKey, ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Cli.Interactive;
|
||||
using ZB.MOM.WW.Secrets.Crypto;
|
||||
using ZB.MOM.WW.Secrets.MasterKey;
|
||||
using ZB.MOM.WW.Secrets.Rotation;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Tests.Cli.Interactive;
|
||||
|
||||
/// <summary>
|
||||
/// Exercises <see cref="KekDoctor"/> against the REAL SQLite store and REAL AES-256-GCM cipher — the
|
||||
/// lockout-triage path an operator runs to learn, per row, which KEK it is wrapped under and whether
|
||||
/// the session KEK actually opens it, then remedies via a guided rewrap-all. KEK-A and KEK-B are two
|
||||
/// distinct 32-byte <see cref="LiteralMasterKeyProvider"/> keys (distinct derived <c>kek_id</c>s).
|
||||
/// </summary>
|
||||
public sealed class KekDoctorTests : IAsyncLifetime, IDisposable
|
||||
{
|
||||
private readonly string _dbPath =
|
||||
Path.Combine(Path.GetTempPath(), $"zb-secrets-doctor-{Guid.NewGuid():N}.db");
|
||||
|
||||
private readonly SecretsSqliteConnectionFactory _factory;
|
||||
private readonly SqliteSecretStore _store;
|
||||
|
||||
// Two distinct 32-byte keys → two distinct derived kek_ids (KEK-A / KEK-B).
|
||||
private readonly LiteralMasterKeyProvider _kekA =
|
||||
new(Convert.ToBase64String(FillKey(0xA1)));
|
||||
private readonly LiteralMasterKeyProvider _kekB =
|
||||
new(Convert.ToBase64String(FillKey(0xB2)));
|
||||
|
||||
public KekDoctorTests()
|
||||
{
|
||||
_factory = new SecretsSqliteConnectionFactory(_dbPath);
|
||||
_store = new SqliteSecretStore(_factory);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync() =>
|
||||
await new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None);
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
private static byte[] FillKey(byte b)
|
||||
{
|
||||
byte[] key = new byte[32];
|
||||
Array.Fill(key, b);
|
||||
return key;
|
||||
}
|
||||
|
||||
// A live session over the shared store, keyed by the supplied provider.
|
||||
private SecretsSession Session(IMasterKeyProvider kek) => new()
|
||||
{
|
||||
Target = new TargetConfigReader().Manual(_dbPath, new MasterKeyOptions()),
|
||||
Store = _store,
|
||||
Migrator = new SqliteSecretsStoreMigrator(_factory),
|
||||
MasterKey = kek,
|
||||
Cipher = new AesGcmEnvelopeCipher(kek),
|
||||
StoreKind = "sqlite",
|
||||
};
|
||||
|
||||
// A degraded session: no KEK, no cipher (metadata-only).
|
||||
private SecretsSession DegradedSession() => new()
|
||||
{
|
||||
Target = new TargetConfigReader().Manual(_dbPath, new MasterKeyOptions()),
|
||||
Store = _store,
|
||||
Migrator = new SqliteSecretsStoreMigrator(_factory),
|
||||
MasterKey = null,
|
||||
Cipher = null,
|
||||
StoreKind = "sqlite",
|
||||
};
|
||||
|
||||
private async Task SeedAsync(string name, string value, IMasterKeyProvider kek)
|
||||
{
|
||||
var cipher = new AesGcmEnvelopeCipher(kek);
|
||||
StoredSecret row = cipher.Encrypt(new SecretName(name), value, SecretContentType.Text);
|
||||
await _store.UpsertAsync(row, CancellationToken.None);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Diagnose_reports_ok_rows_under_session_kek()
|
||||
{
|
||||
await SeedAsync("sql/a", "value-a", _kekA);
|
||||
await SeedAsync("ldap/b", "value-b", _kekA);
|
||||
|
||||
KekDoctorReport report = await new KekDoctor().DiagnoseAsync(Session(_kekA), CancellationToken.None);
|
||||
|
||||
Assert.True(report.Healthy);
|
||||
Assert.Equal(_kekA.KekId, report.SessionKekId);
|
||||
Assert.Equal(2, report.Rows.Count);
|
||||
Assert.All(report.Rows, r => Assert.Equal(RowKekStatus.Ok, r.Status));
|
||||
// Rows ordered by name: "ldap/b" precedes "sql/a".
|
||||
Assert.Equal(["ldap/b", "sql/a"], report.Rows.Select(r => r.SecretName));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Diagnose_reports_wrong_kek_rows_with_their_kekid()
|
||||
{
|
||||
// Rows sealed under A; the session runs on B → the operator must learn A is the KEK to hunt.
|
||||
await SeedAsync("sql/a", "value-a", _kekA);
|
||||
|
||||
KekDoctorReport report = await new KekDoctor().DiagnoseAsync(Session(_kekB), CancellationToken.None);
|
||||
|
||||
Assert.False(report.Healthy);
|
||||
Assert.Equal(_kekB.KekId, report.SessionKekId);
|
||||
KekDiagnosis row = Assert.Single(report.Rows);
|
||||
Assert.Equal(RowKekStatus.WrongKek, row.Status);
|
||||
Assert.Equal(_kekA.KekId, row.RowKekId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Diagnose_reports_corrupt_row_when_kekid_matches_but_unwrap_fails()
|
||||
{
|
||||
await SeedAsync("sql/a", "value-a", _kekA);
|
||||
|
||||
// Tamper one byte of the DEK wrap tag: kek_id still matches the session KEK, but the unwrap
|
||||
// (and therefore the decrypt probe) now fails closed. UpsertAsync bumps revision — irrelevant.
|
||||
StoredSecret row = (await _store.GetAsync(new SecretName("sql/a"), CancellationToken.None))!;
|
||||
byte[] tag = (byte[])row.WrapTag.Clone();
|
||||
tag[0] ^= 0xFF;
|
||||
await _store.UpsertAsync(row with { WrapTag = tag }, CancellationToken.None);
|
||||
|
||||
KekDoctorReport report = await new KekDoctor().DiagnoseAsync(Session(_kekA), CancellationToken.None);
|
||||
|
||||
Assert.False(report.Healthy);
|
||||
KekDiagnosis diag = Assert.Single(report.Rows);
|
||||
Assert.Equal(RowKekStatus.Corrupt, diag.Status);
|
||||
Assert.Equal(_kekA.KekId, diag.RowKekId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Diagnose_includes_tombstoned_rows()
|
||||
{
|
||||
await SeedAsync("sql/live", "live", _kekA);
|
||||
await SeedAsync("sql/dead", "dead", _kekA);
|
||||
await _store.DeleteAsync(new SecretName("sql/dead"), "carol", CancellationToken.None);
|
||||
|
||||
KekDoctorReport report = await new KekDoctor().DiagnoseAsync(Session(_kekA), CancellationToken.None);
|
||||
|
||||
// A tombstoned row still appears — it blocks rewrap-all (it carries a KEK-wrapped DEK).
|
||||
Assert.Equal(2, report.Rows.Count);
|
||||
Assert.Contains(report.Rows, r => r.SecretName == "sql/dead");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Diagnose_throws_on_degraded_session()
|
||||
{
|
||||
InvalidOperationException ex = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => new KekDoctor().DiagnoseAsync(DegradedSession(), CancellationToken.None));
|
||||
|
||||
Assert.Contains("key", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RewrapAll_moves_wrong_kek_rows_onto_session_kek()
|
||||
{
|
||||
await SeedAsync("sql/a", "value-a", _kekA);
|
||||
await SeedAsync("sql/b", "value-b", _kekA);
|
||||
|
||||
// Session runs on B; rows are on A → rewrap A onto the session KEK (B).
|
||||
SecretsSession session = Session(_kekB);
|
||||
RewrapReport report = await new KekDoctor().RewrapAllAsync(session, _kekA, CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, report.Total);
|
||||
Assert.Equal(2, report.Rewrapped);
|
||||
Assert.Equal(0, report.AlreadyCurrent);
|
||||
|
||||
// Rows now decrypt under the session KEK (B), and the doctor re-diagnoses healthy.
|
||||
var cipherB = new AesGcmEnvelopeCipher(_kekB);
|
||||
StoredSecret a = (await _store.GetAsync(new SecretName("sql/a"), CancellationToken.None))!;
|
||||
Assert.Equal(_kekB.KekId, a.KekId);
|
||||
Assert.Equal("value-a", cipherB.Decrypt(a));
|
||||
|
||||
KekDoctorReport after = await new KekDoctor().DiagnoseAsync(session, CancellationToken.None);
|
||||
Assert.True(after.Healthy);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort temp cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user