diff --git a/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli/Interactive/Flows/KekDoctorFlow.cs b/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli/Interactive/Flows/KekDoctorFlow.cs
new file mode 100644
index 0000000..ea4d5b8
--- /dev/null
+++ b/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli/Interactive/Flows/KekDoctorFlow.cs
@@ -0,0 +1,245 @@
+using System.Globalization;
+using Spectre.Console;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.MasterKey;
+using ZB.MOM.WW.Secrets.Rotation;
+
+namespace ZB.MOM.WW.Secrets.Cli.Interactive.Flows;
+
+///
+/// The lockout-recovery flow. An operator whose app can no longer decrypt its secrets runs the doctor to
+/// learn, per row, which KEK each row is wrapped under and whether the session KEK opens it (via
+/// ), then chooses a remedy for the wrong-KEK (and corrupt) rows:
+///
+/// - Rewrap from old KEK — the operator supplies the KEK the rows are currently wrapped under
+/// (from an environment variable, a key file, or a pasted base64 key) and the flow re-wraps every row onto
+/// the session KEK after an explicit, exactly-scoped confirmation. This is the non-destructive remedy: the
+/// secret bodies are preserved, only the DEK wrap changes.
+/// - Old KEK is lost — re-set affected secrets — when the old KEK is unrecoverable, each affected
+/// row is offered a masked re-set (a fresh value sealed under the session KEK). This is the destructive
+/// last resort and the only remedy for a row (whose body cannot be
+/// re-wrapped or preserved).
+///
+/// Requires a KEK-capable session (the shell upgrades a degraded session before dispatch). No key material
+/// or typed value is ever echoed: the pasted old KEK and re-set values use masked prompts, and every
+/// operator-controlled string (secret names, KEK ids, error messages) is markup-escaped.
+///
+public sealed class KekDoctorFlow : IInteractiveFlow
+{
+ private const string RewrapChoice = "Rewrap from old KEK";
+ private const string LostKekChoice = "Old KEK is lost — re-set affected secrets";
+ private const string BackChoice = "Back";
+
+ private const string PasteSource = "Paste base64 key";
+ private const string EnvSource = "Environment variable";
+ private const string FileSource = "Key file";
+
+ ///
+ public string Title => "KEK doctor (lockout recovery)";
+
+ ///
+ public bool RequiresKek => true;
+
+ ///
+ public async Task RunAsync(IAnsiConsole console, SecretsSession session, CancellationToken ct)
+ {
+ ArgumentNullException.ThrowIfNull(console);
+ ArgumentNullException.ThrowIfNull(session);
+
+ var doctor = new KekDoctor();
+
+ // Diagnose is deliberately not wrapped: a degraded-session or store fault is the shell's to contain.
+ KekDoctorReport report = await doctor.DiagnoseAsync(session, ct).ConfigureAwait(false);
+ RenderSummary(console, report);
+
+ if (report.Healthy)
+ {
+ console.MarkupLine("[green]All rows open under the session KEK — no lockout. Nothing to remedy.[/]");
+ return;
+ }
+
+ string choice = console.Prompt(new SelectionPrompt()
+ .Title("Choose a [yellow]remedy[/]")
+ .AddChoices(RewrapChoice, LostKekChoice, BackChoice));
+
+ switch (choice)
+ {
+ case RewrapChoice:
+ await RunRewrapAsync(console, session, doctor, report, ct).ConfigureAwait(false);
+ break;
+ case LostKekChoice:
+ await RunLostKekResetAsync(console, session, report, ct).ConfigureAwait(false);
+ break;
+ default:
+ console.MarkupLine("[grey]No changes made.[/]");
+ return;
+ }
+
+ // Re-diagnose so the operator sees the store's post-remedy state in one closing line.
+ KekDoctorReport after = await doctor.DiagnoseAsync(session, ct).ConfigureAwait(false);
+ console.MarkupLineInterpolated($"Re-diagnosis: {Summarize(after)}.");
+ }
+
+ // Renders the session KEK, the row total, the per-status counts, and the distinct foreign kek_ids the
+ // operator must hunt down. Every operator-controlled value rides an interpolated (auto-escaped) hole.
+ private static void RenderSummary(IAnsiConsole console, KekDoctorReport report)
+ {
+ console.MarkupLineInterpolated(
+ $"Session KEK [cyan]{report.SessionKekId}[/] — {report.Total} row(s) scanned.");
+
+ int ok = Count(report, RowKekStatus.Ok);
+ int wrong = Count(report, RowKekStatus.WrongKek);
+ int corrupt = Count(report, RowKekStatus.Corrupt);
+ console.MarkupLineInterpolated(
+ $"[green]{ok} Ok[/], [yellow]{wrong} wrong-KEK[/], [red]{corrupt} corrupt[/].");
+
+ IReadOnlyList foreign = ForeignKekIds(report);
+ if (foreign.Count > 0)
+ {
+ console.MarkupLineInterpolated($"Foreign KEK id(s) seen: {string.Join(", ", foreign)}.");
+ }
+ }
+
+ // The guided rewrap remedy: acquire the old KEK provider, show EXACTLY what will change, and re-wrap
+ // only after an explicit confirmation (defaulting false). The rewrap itself is the only guarded region —
+ // a wrong pasted key surfaces as a SecretDecryptionException, an old==session KEK as an ArgumentException.
+ private static async Task RunRewrapAsync(
+ IAnsiConsole console, SecretsSession session, KekDoctor doctor, KekDoctorReport report, CancellationToken ct)
+ {
+ IMasterKeyProvider oldKek = PromptForOldKek(console);
+
+ try
+ {
+ string oldKekId = oldKek.KekId;
+ int moving = report.Rows.Count(r => string.Equals(r.RowKekId, oldKekId, StringComparison.Ordinal));
+
+ string confirmText =
+ $"Re-wrap {moving} row(s) from KEK '{Markup.Escape(oldKekId)}' → session KEK " +
+ $"'{Markup.Escape(report.SessionKekId)}'? (rewraps every row on the old KEK)";
+ if (!console.Prompt(new ConfirmationPrompt(confirmText) { DefaultValue = false }))
+ {
+ console.MarkupLine("[grey]Rewrap cancelled — no rows changed.[/]");
+ return;
+ }
+
+ RewrapReport result = await doctor.RewrapAllAsync(session, oldKek, ct).ConfigureAwait(false);
+ console.MarkupLineInterpolated(
+ $"[green]Re-wrapped {result.Rewrapped} row(s)[/] onto the session KEK ({result.AlreadyCurrent} already current) of {result.Total} total.");
+ }
+ catch (ArgumentException ex)
+ {
+ // Old and session KEK are identical — nothing to rotate.
+ console.MarkupLineInterpolated($"[red]Rewrap failed:[/] {ex.Message}");
+ }
+ catch (SecretDecryptionException ex)
+ {
+ // Wrong old key, or a row wrapped by neither the old nor the session KEK (the pass aborted;
+ // any rows re-wrapped before the anomaly stay persisted).
+ console.MarkupLineInterpolated($"[red]Rewrap failed:[/] {ex.Message}");
+ }
+ }
+
+ // Prompts for the old-KEK source and builds the matching provider. Paste feeds a masked base64 entry to
+ // the LiteralMasterKeyProvider; env/file defer to the shared MasterKeyProviderFactory.
+ private static IMasterKeyProvider PromptForOldKek(IAnsiConsole console)
+ {
+ string source = console.Prompt(new SelectionPrompt()
+ .Title("Where is the [yellow]old KEK[/]?")
+ .AddChoices(EnvSource, FileSource, PasteSource));
+
+ switch (source)
+ {
+ case EnvSource:
+ string envVar = console.Prompt(new TextPrompt(" Environment variable name:"));
+ return MasterKeyProviderFactory.Create(new MasterKeyOptions
+ {
+ Source = MasterKeySource.Environment,
+ EnvVarName = envVar,
+ });
+ case FileSource:
+ string path = console.Prompt(new TextPrompt(" Key file path:"));
+ return MasterKeyProviderFactory.Create(new MasterKeyOptions
+ {
+ Source = MasterKeySource.File,
+ FilePath = path,
+ });
+ default:
+ string base64 = console.Prompt(
+ new TextPrompt(" Paste the base64 old KEK:").Secret());
+ return new LiteralMasterKeyProvider(base64);
+ }
+ }
+
+ // The destructive last resort: for every wrong-KEK or corrupt row, offer a masked re-set of a fresh
+ // value sealed under the session KEK. A corrupt row is flagged first (re-set is its only remedy).
+ private static async Task RunLostKekResetAsync(
+ IAnsiConsole console, SecretsSession session, KekDoctorReport report, CancellationToken ct)
+ {
+ IReadOnlyList affected = report.Rows
+ .Where(r => r.Status is RowKekStatus.WrongKek or RowKekStatus.Corrupt)
+ .ToList();
+
+ if (affected.Count == 0)
+ {
+ console.MarkupLine("[grey]No wrong-KEK or corrupt rows to re-set.[/]");
+ return;
+ }
+
+ foreach (KekDiagnosis row in affected)
+ {
+ string escapedName = Markup.Escape(row.SecretName);
+
+ if (row.Status == RowKekStatus.Corrupt)
+ {
+ console.MarkupLineInterpolated(
+ $"[red]'{row.SecretName}' is corrupt under the session KEK — it cannot be re-wrapped or preserved; re-setting a fresh value is the only remedy.[/]");
+ }
+
+ if (!console.Prompt(new ConfirmationPrompt(
+ $"Re-set '{escapedName}' with a new value? (overwrites)") { DefaultValue = false }))
+ {
+ continue;
+ }
+
+ await ResetAsync(console, session, new SecretName(row.SecretName), ct).ConfigureAwait(false);
+ console.MarkupLineInterpolated($"[green]Re-set '{row.SecretName}' under the session KEK.[/]");
+ }
+ }
+
+ // Prompts for a masked value + content type and seals a fresh row under the session cipher, overwriting
+ // the wrong-KEK/corrupt row in place. Mirrors the seed-and-stamp shape used by the other flows.
+ private static async Task ResetAsync(
+ IAnsiConsole console, SecretsSession session, SecretName name, CancellationToken ct)
+ {
+ string value = console.Prompt(
+ new TextPrompt($" New value for [green]{Markup.Escape(name.Value)}[/]:").Secret());
+ SecretContentType contentType = console.Prompt(
+ new SelectionPrompt().Title(" Content type").AddChoices(Enum.GetValues()));
+
+ StoredSecret row = session.Cipher!.Encrypt(name, value, contentType) with
+ {
+ CreatedBy = FlowPrompts.Actor,
+ UpdatedBy = FlowPrompts.Actor,
+ };
+ await session.Store.UpsertAsync(row, ct).ConfigureAwait(false);
+ }
+
+ private static int Count(KekDoctorReport report, RowKekStatus status) =>
+ report.Rows.Count(r => r.Status == status);
+
+ // The distinct foreign kek_ids (wrong-KEK rows only), ordered for a deterministic render.
+ private static IReadOnlyList ForeignKekIds(KekDoctorReport report) =>
+ report.Rows
+ .Where(r => r.Status == RowKekStatus.WrongKek)
+ .Select(r => r.RowKekId)
+ .Distinct(StringComparer.Ordinal)
+ .OrderBy(id => id, StringComparer.Ordinal)
+ .ToList();
+
+ // A deterministic "N Status" count-by-status roll-up, ordered by the status enum.
+ private static string Summarize(KekDoctorReport report) =>
+ string.Join(", ", report.Rows
+ .GroupBy(r => r.Status)
+ .OrderBy(g => g.Key)
+ .Select(g => string.Create(CultureInfo.InvariantCulture, $"{g.Count()} {g.Key}")));
+}
diff --git a/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Cli/Interactive/Flows/KekDoctorFlowTests.cs b/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Cli/Interactive/Flows/KekDoctorFlowTests.cs
new file mode 100644
index 0000000..883f24b
--- /dev/null
+++ b/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Cli/Interactive/Flows/KekDoctorFlowTests.cs
@@ -0,0 +1,205 @@
+using Microsoft.Data.Sqlite;
+using Spectre.Console.Testing;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.Cli.Interactive;
+using ZB.MOM.WW.Secrets.Cli.Interactive.Flows;
+using ZB.MOM.WW.Secrets.Crypto;
+using ZB.MOM.WW.Secrets.MasterKey;
+using ZB.MOM.WW.Secrets.Sqlite;
+
+namespace ZB.MOM.WW.Secrets.Tests.Cli.Interactive.Flows;
+
+///
+/// Drives the lockout-recovery against a REAL temp-SQLite store and REAL
+/// AES-256-GCM cipher. KEK-A and KEK-B are two distinct 32-byte
+/// keys (distinct derived kek_ids): rows are sealed under one and the session opened under the
+/// other to reproduce the wrong-KEK lockout. Each flow renders to a scripted
+/// so the tests assert both the rendered and the resulting store state.
+/// The invariant is that the operator recovers a locked-out store from one screen — either by re-wrapping
+/// from the recovered old KEK or by re-setting the affected secrets — without any key or value ever
+/// reaching the terminal.
+///
+public sealed class KekDoctorFlowTests : IAsyncLifetime, IDisposable
+{
+ private readonly string _dbPath =
+ Path.Combine(Path.GetTempPath(), $"zb-secrets-doctorflow-{Guid.NewGuid():N}.db");
+
+ private readonly SecretsSqliteConnectionFactory _factory;
+ private readonly SqliteSecretStore _store;
+
+ // Two distinct 32-byte keys → two distinct derived kek_ids (KEK-A / KEK-B). The base64 forms are
+ // retained so a test can paste KEK-A at the prompt and assert it never echoes back to the terminal.
+ private readonly string _kekABase64 = Convert.ToBase64String(FillKey(0xA1));
+ private readonly string _kekBBase64 = Convert.ToBase64String(FillKey(0xB2));
+ private readonly LiteralMasterKeyProvider _kekA;
+ private readonly LiteralMasterKeyProvider _kekB;
+
+ public KekDoctorFlowTests()
+ {
+ _factory = new SecretsSqliteConnectionFactory(_dbPath);
+ _store = new SqliteSecretStore(_factory);
+ _kekA = new LiteralMasterKeyProvider(_kekABase64);
+ _kekB = new LiteralMasterKeyProvider(_kekBBase64);
+ }
+
+ public async Task InitializeAsync() =>
+ await new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None);
+
+ public Task DisposeAsync() => Task.CompletedTask;
+
+ private static byte[] FillKey(byte b)
+ {
+ byte[] key = new byte[32];
+ Array.Fill(key, b);
+ return key;
+ }
+
+ private static TestConsole NewConsole()
+ {
+ var console = (TestConsole)new TestConsole().Interactive();
+ console.Profile.Width = 240; // wide enough that the summary lines never truncate a kek_id
+ return console;
+ }
+
+ // A live session over the shared store, keyed by the supplied provider.
+ private SecretsSession Session(IMasterKeyProvider kek) => new()
+ {
+ Target = new TargetConfigReader().Manual(_dbPath, new MasterKeyOptions()),
+ Store = _store,
+ Migrator = new SqliteSecretsStoreMigrator(_factory),
+ MasterKey = kek,
+ Cipher = new AesGcmEnvelopeCipher(kek),
+ StoreKind = "sqlite",
+ };
+
+ private async Task SeedAsync(string name, string value, IMasterKeyProvider kek)
+ {
+ var cipher = new AesGcmEnvelopeCipher(kek);
+ StoredSecret row = cipher.Encrypt(new SecretName(name), value, SecretContentType.Text);
+ await _store.UpsertAsync(row, CancellationToken.None);
+ }
+
+ private async Task GetAsync(string name) =>
+ (await _store.GetAsync(new SecretName(name), CancellationToken.None))!;
+
+ [Fact]
+ public async Task Healthy_store_renders_all_ok_summary()
+ {
+ await SeedAsync("svc/a", "value-a", _kekA);
+ await SeedAsync("svc/b", "value-b", _kekA);
+
+ TestConsole console = NewConsole();
+ await new KekDoctorFlow().RunAsync(console, Session(_kekA), CancellationToken.None);
+
+ string output = console.Output;
+ Assert.Contains(_kekA.KekId, output, StringComparison.Ordinal); // the session KEK is surfaced
+ Assert.Contains("All rows open", output, StringComparison.Ordinal); // green all-clear
+ Assert.DoesNotContain("Rewrap from old KEK", output, StringComparison.Ordinal); // no remedy menu
+
+ Assert.True(new KekDoctorFlow().RequiresKek);
+ Assert.Equal("KEK doctor (lockout recovery)", new KekDoctorFlow().Title);
+ }
+
+ [Fact]
+ public async Task Wrong_kek_rows_offer_rewrap_and_rewrap_succeeds()
+ {
+ // Rows sealed under KEK-A; the session runs on KEK-B → both rows are wrong-KEK.
+ await SeedAsync("svc/a", "value-a", _kekA);
+ await SeedAsync("svc/b", "value-b", _kekA);
+ SecretsSession session = Session(_kekB);
+
+ TestConsole console = NewConsole();
+ console.Input.PushKey(ConsoleKey.Enter); // remedy menu: "Rewrap from old KEK" (index 0)
+ console.Input.PushKey(ConsoleKey.DownArrow); // old-key source: skip "Environment variable"
+ console.Input.PushKey(ConsoleKey.DownArrow); // skip "Key file"
+ console.Input.PushKey(ConsoleKey.Enter); // choose "Paste base64 key" (index 2)
+ console.Input.PushTextWithEnter(_kekABase64); // paste KEK-A material (masked)
+ console.Input.PushTextWithEnter("y"); // confirm the rewrap (default is false)
+
+ await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+ // Both rows now decrypt under the session KEK (B).
+ var cipherB = new AesGcmEnvelopeCipher(_kekB);
+ StoredSecret a = await GetAsync("svc/a");
+ StoredSecret b = await GetAsync("svc/b");
+ Assert.Equal(_kekB.KekId, a.KekId);
+ Assert.Equal(_kekB.KekId, b.KekId);
+ Assert.Equal("value-a", cipherB.Decrypt(a));
+ Assert.Equal("value-b", cipherB.Decrypt(b));
+
+ Assert.Contains("Re-wrapped", console.Output, StringComparison.Ordinal);
+ Assert.Contains("2", console.Output, StringComparison.Ordinal); // the rewrap count
+ Assert.DoesNotContain(_kekABase64, console.Output, StringComparison.Ordinal); // pasted key never echoed
+ }
+
+ [Fact]
+ public async Task Rewrap_requires_explicit_confirmation()
+ {
+ await SeedAsync("svc/a", "value-a", _kekA);
+ await SeedAsync("svc/b", "value-b", _kekA);
+ SecretsSession session = Session(_kekB);
+
+ TestConsole console = NewConsole();
+ console.Input.PushKey(ConsoleKey.Enter); // "Rewrap from old KEK"
+ console.Input.PushKey(ConsoleKey.DownArrow);
+ console.Input.PushKey(ConsoleKey.DownArrow);
+ console.Input.PushKey(ConsoleKey.Enter); // "Paste base64 key"
+ console.Input.PushTextWithEnter(_kekABase64);
+ console.Input.PushTextWithEnter("n"); // DECLINE the confirmation
+
+ await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+ // Nothing was re-wrapped: both rows remain under KEK-A.
+ Assert.Equal(_kekA.KekId, (await GetAsync("svc/a")).KekId);
+ Assert.Equal(_kekA.KekId, (await GetAsync("svc/b")).KekId);
+ }
+
+ [Fact]
+ public async Task Lost_kek_path_offers_reset_of_affected_secrets()
+ {
+ // Two wrong-KEK rows (sealed under KEK-A; session on KEK-B). Ordinal order: svc/a precedes svc/b.
+ await SeedAsync("svc/a", "value-a", _kekA);
+ await SeedAsync("svc/b", "value-b", _kekA);
+ SecretsSession session = Session(_kekB);
+
+ TestConsole console = NewConsole();
+ console.Input.PushKey(ConsoleKey.DownArrow); // remedy menu: move to
+ console.Input.PushKey(ConsoleKey.Enter); // "Old KEK is lost — re-set affected secrets" (index 1)
+ console.Input.PushTextWithEnter("y"); // svc/a: accept re-set
+ console.Input.PushTextWithEnter("NEW-A-VALUE"); // masked new value
+ console.Input.PushKey(ConsoleKey.Enter); // content type: Text (first choice)
+ console.Input.PushTextWithEnter("n"); // svc/b: decline re-set
+
+ await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+ // svc/a was re-set under the session KEK (B) with the new value.
+ var cipherB = new AesGcmEnvelopeCipher(_kekB);
+ StoredSecret a = await GetAsync("svc/a");
+ Assert.Equal(_kekB.KekId, a.KekId);
+ Assert.Equal("NEW-A-VALUE", cipherB.Decrypt(a));
+
+ // svc/b was left untouched → still wrong-KEK (under KEK-A).
+ Assert.Equal(_kekA.KekId, (await GetAsync("svc/b")).KekId);
+
+ Assert.DoesNotContain("NEW-A-VALUE", console.Output, StringComparison.Ordinal); // value never echoed
+ }
+
+ public void Dispose()
+ {
+ SqliteConnection.ClearAllPools();
+ foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
+ {
+ try
+ {
+ if (File.Exists(path))
+ {
+ File.Delete(path);
+ }
+ }
+ catch (IOException)
+ {
+ // Best-effort temp cleanup.
+ }
+ }
+ }
+}