feat(secrets-cli): lockout-recovery flow — KEK diagnosis, guided rewrap, lost-KEK reset

ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli/Interactive/Flows/KekDoctorFlow.cs

@@ -0,0 +1,245 @@
+using System.Globalization;
+using Spectre.Console;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.MasterKey;
+using ZB.MOM.WW.Secrets.Rotation;
+
+namespace ZB.MOM.WW.Secrets.Cli.Interactive.Flows;
+
+/// <summary>
+/// The lockout-recovery flow. An operator whose app can no longer decrypt its secrets runs the doctor to
+/// learn, per row, which KEK each row is wrapped under and whether the session KEK opens it (via
+/// <see cref="KekDoctor.DiagnoseAsync"/>), then chooses a remedy for the wrong-KEK (and corrupt) rows:
+/// <list type="bullet">
+/// <item><b>Rewrap from old KEK</b> — the operator supplies the KEK the rows are currently wrapped under
+/// (from an environment variable, a key file, or a pasted base64 key) and the flow re-wraps every row onto
+/// the session KEK after an explicit, exactly-scoped confirmation. This is the non-destructive remedy: the
+/// secret bodies are preserved, only the DEK wrap changes.</item>
+/// <item><b>Old KEK is lost — re-set affected secrets</b> — when the old KEK is unrecoverable, each affected
+/// row is offered a masked re-set (a fresh value sealed under the session KEK). This is the destructive
+/// last resort and the only remedy for a <see cref="RowKekStatus.Corrupt"/> row (whose body cannot be
+/// re-wrapped or preserved).</item>
+/// </list>
+/// Requires a KEK-capable session (the shell upgrades a degraded session before dispatch). No key material
+/// or typed value is ever echoed: the pasted old KEK and re-set values use masked prompts, and every
+/// operator-controlled string (secret names, KEK ids, error messages) is markup-escaped.
+/// </summary>
+public sealed class KekDoctorFlow : IInteractiveFlow
+{
+    private const string RewrapChoice = "Rewrap from old KEK";
+    private const string LostKekChoice = "Old KEK is lost — re-set affected secrets";
+    private const string BackChoice = "Back";
+
+    private const string PasteSource = "Paste base64 key";
+    private const string EnvSource = "Environment variable";
+    private const string FileSource = "Key file";
+
+    /// <inheritdoc />
+    public string Title => "KEK doctor (lockout recovery)";
+
+    /// <inheritdoc />
+    public bool RequiresKek => true;
+
+    /// <inheritdoc />
+    public async Task RunAsync(IAnsiConsole console, SecretsSession session, CancellationToken ct)
+    {
+        ArgumentNullException.ThrowIfNull(console);
+        ArgumentNullException.ThrowIfNull(session);
+
+        var doctor = new KekDoctor();
+
+        // Diagnose is deliberately not wrapped: a degraded-session or store fault is the shell's to contain.
+        KekDoctorReport report = await doctor.DiagnoseAsync(session, ct).ConfigureAwait(false);
+        RenderSummary(console, report);
+
+        if (report.Healthy)
+        {
+            console.MarkupLine("[green]All rows open under the session KEK — no lockout. Nothing to remedy.[/]");
+            return;
+        }
+
+        string choice = console.Prompt(new SelectionPrompt<string>()
+            .Title("Choose a [yellow]remedy[/]")
+            .AddChoices(RewrapChoice, LostKekChoice, BackChoice));
+
+        switch (choice)
+        {
+            case RewrapChoice:
+                await RunRewrapAsync(console, session, doctor, report, ct).ConfigureAwait(false);
+                break;
+            case LostKekChoice:
+                await RunLostKekResetAsync(console, session, report, ct).ConfigureAwait(false);
+                break;
+            default:
+                console.MarkupLine("[grey]No changes made.[/]");
+                return;
+        }
+
+        // Re-diagnose so the operator sees the store's post-remedy state in one closing line.
+        KekDoctorReport after = await doctor.DiagnoseAsync(session, ct).ConfigureAwait(false);
+        console.MarkupLineInterpolated($"Re-diagnosis: {Summarize(after)}.");
+    }
+
+    // Renders the session KEK, the row total, the per-status counts, and the distinct foreign kek_ids the
+    // operator must hunt down. Every operator-controlled value rides an interpolated (auto-escaped) hole.
+    private static void RenderSummary(IAnsiConsole console, KekDoctorReport report)
+    {
+        console.MarkupLineInterpolated(
+            $"Session KEK [cyan]{report.SessionKekId}[/] — {report.Total} row(s) scanned.");
+
+        int ok = Count(report, RowKekStatus.Ok);
+        int wrong = Count(report, RowKekStatus.WrongKek);
+        int corrupt = Count(report, RowKekStatus.Corrupt);
+        console.MarkupLineInterpolated(
+            $"[green]{ok} Ok[/], [yellow]{wrong} wrong-KEK[/], [red]{corrupt} corrupt[/].");
+
+        IReadOnlyList<string> foreign = ForeignKekIds(report);
+        if (foreign.Count > 0)
+        {
+            console.MarkupLineInterpolated($"Foreign KEK id(s) seen: {string.Join(", ", foreign)}.");
+        }
+    }
+
+    // The guided rewrap remedy: acquire the old KEK provider, show EXACTLY what will change, and re-wrap
+    // only after an explicit confirmation (defaulting false). The rewrap itself is the only guarded region —
+    // a wrong pasted key surfaces as a SecretDecryptionException, an old==session KEK as an ArgumentException.
+    private static async Task RunRewrapAsync(
+        IAnsiConsole console, SecretsSession session, KekDoctor doctor, KekDoctorReport report, CancellationToken ct)
+    {
+        IMasterKeyProvider oldKek = PromptForOldKek(console);
+
+        try
+        {
+            string oldKekId = oldKek.KekId;
+            int moving = report.Rows.Count(r => string.Equals(r.RowKekId, oldKekId, StringComparison.Ordinal));
+
+            string confirmText =
+                $"Re-wrap {moving} row(s) from KEK '{Markup.Escape(oldKekId)}' → session KEK " +
+                $"'{Markup.Escape(report.SessionKekId)}'? (rewraps every row on the old KEK)";
+            if (!console.Prompt(new ConfirmationPrompt(confirmText) { DefaultValue = false }))
+            {
+                console.MarkupLine("[grey]Rewrap cancelled — no rows changed.[/]");
+                return;
+            }
+
+            RewrapReport result = await doctor.RewrapAllAsync(session, oldKek, ct).ConfigureAwait(false);
+            console.MarkupLineInterpolated(
+                $"[green]Re-wrapped {result.Rewrapped} row(s)[/] onto the session KEK ({result.AlreadyCurrent} already current) of {result.Total} total.");
+        }
+        catch (ArgumentException ex)
+        {
+            // Old and session KEK are identical — nothing to rotate.
+            console.MarkupLineInterpolated($"[red]Rewrap failed:[/] {ex.Message}");
+        }
+        catch (SecretDecryptionException ex)
+        {
+            // Wrong old key, or a row wrapped by neither the old nor the session KEK (the pass aborted;
+            // any rows re-wrapped before the anomaly stay persisted).
+            console.MarkupLineInterpolated($"[red]Rewrap failed:[/] {ex.Message}");
+        }
+    }
+
+    // Prompts for the old-KEK source and builds the matching provider. Paste feeds a masked base64 entry to
+    // the LiteralMasterKeyProvider; env/file defer to the shared MasterKeyProviderFactory.
+    private static IMasterKeyProvider PromptForOldKek(IAnsiConsole console)
+    {
+        string source = console.Prompt(new SelectionPrompt<string>()
+            .Title("Where is the [yellow]old KEK[/]?")
+            .AddChoices(EnvSource, FileSource, PasteSource));
+
+        switch (source)
+        {
+            case EnvSource:
+                string envVar = console.Prompt(new TextPrompt<string>("  Environment variable name:"));
+                return MasterKeyProviderFactory.Create(new MasterKeyOptions
+                {
+                    Source = MasterKeySource.Environment,
+                    EnvVarName = envVar,
+                });
+            case FileSource:
+                string path = console.Prompt(new TextPrompt<string>("  Key file path:"));
+                return MasterKeyProviderFactory.Create(new MasterKeyOptions
+                {
+                    Source = MasterKeySource.File,
+                    FilePath = path,
+                });
+            default:
+                string base64 = console.Prompt(
+                    new TextPrompt<string>("  Paste the base64 old KEK:").Secret());
+                return new LiteralMasterKeyProvider(base64);
+        }
+    }
+
+    // The destructive last resort: for every wrong-KEK or corrupt row, offer a masked re-set of a fresh
+    // value sealed under the session KEK. A corrupt row is flagged first (re-set is its only remedy).
+    private static async Task RunLostKekResetAsync(
+        IAnsiConsole console, SecretsSession session, KekDoctorReport report, CancellationToken ct)
+    {
+        IReadOnlyList<KekDiagnosis> affected = report.Rows
+            .Where(r => r.Status is RowKekStatus.WrongKek or RowKekStatus.Corrupt)
+            .ToList();
+
+        if (affected.Count == 0)
+        {
+            console.MarkupLine("[grey]No wrong-KEK or corrupt rows to re-set.[/]");
+            return;
+        }
+
+        foreach (KekDiagnosis row in affected)
+        {
+            string escapedName = Markup.Escape(row.SecretName);
+
+            if (row.Status == RowKekStatus.Corrupt)
+            {
+                console.MarkupLineInterpolated(
+                    $"[red]'{row.SecretName}' is corrupt under the session KEK — it cannot be re-wrapped or preserved; re-setting a fresh value is the only remedy.[/]");
+            }
+
+            if (!console.Prompt(new ConfirmationPrompt(
+                    $"Re-set '{escapedName}' with a new value? (overwrites)") { DefaultValue = false }))
+            {
+                continue;
+            }
+
+            await ResetAsync(console, session, new SecretName(row.SecretName), ct).ConfigureAwait(false);
+            console.MarkupLineInterpolated($"[green]Re-set '{row.SecretName}' under the session KEK.[/]");
+        }
+    }
+
+    // Prompts for a masked value + content type and seals a fresh row under the session cipher, overwriting
+    // the wrong-KEK/corrupt row in place. Mirrors the seed-and-stamp shape used by the other flows.
+    private static async Task ResetAsync(
+        IAnsiConsole console, SecretsSession session, SecretName name, CancellationToken ct)
+    {
+        string value = console.Prompt(
+            new TextPrompt<string>($"  New value for [green]{Markup.Escape(name.Value)}[/]:").Secret());
+        SecretContentType contentType = console.Prompt(
+            new SelectionPrompt<SecretContentType>().Title("  Content type").AddChoices(Enum.GetValues<SecretContentType>()));
+
+        StoredSecret row = session.Cipher!.Encrypt(name, value, contentType) with
+        {
+            CreatedBy = FlowPrompts.Actor,
+            UpdatedBy = FlowPrompts.Actor,
+        };
+        await session.Store.UpsertAsync(row, ct).ConfigureAwait(false);
+    }
+
+    private static int Count(KekDoctorReport report, RowKekStatus status) =>
+        report.Rows.Count(r => r.Status == status);
+
+    // The distinct foreign kek_ids (wrong-KEK rows only), ordered for a deterministic render.
+    private static IReadOnlyList<string> ForeignKekIds(KekDoctorReport report) =>
+        report.Rows
+            .Where(r => r.Status == RowKekStatus.WrongKek)
+            .Select(r => r.RowKekId)
+            .Distinct(StringComparer.Ordinal)
+            .OrderBy(id => id, StringComparer.Ordinal)
+            .ToList();
+
+    // A deterministic "N Status" count-by-status roll-up, ordered by the status enum.
+    private static string Summarize(KekDoctorReport report) =>
+        string.Join(", ", report.Rows
+            .GroupBy(r => r.Status)
+            .OrderBy(g => g.Key)
+            .Select(g => string.Create(CultureInfo.InvariantCulture, $"{g.Count()} {g.Key}")));
+}

ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Cli/Interactive/Flows/KekDoctorFlowTests.cs

@@ -0,0 +1,205 @@
+using Microsoft.Data.Sqlite;
+using Spectre.Console.Testing;
+using ZB.MOM.WW.Secrets.Abstractions;
+using ZB.MOM.WW.Secrets.Cli.Interactive;
+using ZB.MOM.WW.Secrets.Cli.Interactive.Flows;
+using ZB.MOM.WW.Secrets.Crypto;
+using ZB.MOM.WW.Secrets.MasterKey;
+using ZB.MOM.WW.Secrets.Sqlite;
+
+namespace ZB.MOM.WW.Secrets.Tests.Cli.Interactive.Flows;
+
+/// <summary>
+/// Drives the lockout-recovery <see cref="KekDoctorFlow"/> against a REAL temp-SQLite store and REAL
+/// AES-256-GCM cipher. KEK-A and KEK-B are two distinct 32-byte <see cref="LiteralMasterKeyProvider"/>
+/// keys (distinct derived <c>kek_id</c>s): rows are sealed under one and the session opened under the
+/// other to reproduce the wrong-KEK lockout. Each flow renders to a scripted <see cref="TestConsole"/>
+/// so the tests assert both the rendered <see cref="TestConsole.Output"/> and the resulting store state.
+/// The invariant is that the operator recovers a locked-out store from one screen — either by re-wrapping
+/// from the recovered old KEK or by re-setting the affected secrets — without any key or value ever
+/// reaching the terminal.
+/// </summary>
+public sealed class KekDoctorFlowTests : IAsyncLifetime, IDisposable
+{
+    private readonly string _dbPath =
+        Path.Combine(Path.GetTempPath(), $"zb-secrets-doctorflow-{Guid.NewGuid():N}.db");
+
+    private readonly SecretsSqliteConnectionFactory _factory;
+    private readonly SqliteSecretStore _store;
+
+    // Two distinct 32-byte keys → two distinct derived kek_ids (KEK-A / KEK-B). The base64 forms are
+    // retained so a test can paste KEK-A at the prompt and assert it never echoes back to the terminal.
+    private readonly string _kekABase64 = Convert.ToBase64String(FillKey(0xA1));
+    private readonly string _kekBBase64 = Convert.ToBase64String(FillKey(0xB2));
+    private readonly LiteralMasterKeyProvider _kekA;
+    private readonly LiteralMasterKeyProvider _kekB;
+
+    public KekDoctorFlowTests()
+    {
+        _factory = new SecretsSqliteConnectionFactory(_dbPath);
+        _store = new SqliteSecretStore(_factory);
+        _kekA = new LiteralMasterKeyProvider(_kekABase64);
+        _kekB = new LiteralMasterKeyProvider(_kekBBase64);
+    }
+
+    public async Task InitializeAsync() =>
+        await new SqliteSecretsStoreMigrator(_factory).MigrateAsync(CancellationToken.None);
+
+    public Task DisposeAsync() => Task.CompletedTask;
+
+    private static byte[] FillKey(byte b)
+    {
+        byte[] key = new byte[32];
+        Array.Fill(key, b);
+        return key;
+    }
+
+    private static TestConsole NewConsole()
+    {
+        var console = (TestConsole)new TestConsole().Interactive();
+        console.Profile.Width = 240; // wide enough that the summary lines never truncate a kek_id
+        return console;
+    }
+
+    // A live session over the shared store, keyed by the supplied provider.
+    private SecretsSession Session(IMasterKeyProvider kek) => new()
+    {
+        Target = new TargetConfigReader().Manual(_dbPath, new MasterKeyOptions()),
+        Store = _store,
+        Migrator = new SqliteSecretsStoreMigrator(_factory),
+        MasterKey = kek,
+        Cipher = new AesGcmEnvelopeCipher(kek),
+        StoreKind = "sqlite",
+    };
+
+    private async Task SeedAsync(string name, string value, IMasterKeyProvider kek)
+    {
+        var cipher = new AesGcmEnvelopeCipher(kek);
+        StoredSecret row = cipher.Encrypt(new SecretName(name), value, SecretContentType.Text);
+        await _store.UpsertAsync(row, CancellationToken.None);
+    }
+
+    private async Task<StoredSecret> GetAsync(string name) =>
+        (await _store.GetAsync(new SecretName(name), CancellationToken.None))!;
+
+    [Fact]
+    public async Task Healthy_store_renders_all_ok_summary()
+    {
+        await SeedAsync("svc/a", "value-a", _kekA);
+        await SeedAsync("svc/b", "value-b", _kekA);
+
+        TestConsole console = NewConsole();
+        await new KekDoctorFlow().RunAsync(console, Session(_kekA), CancellationToken.None);
+
+        string output = console.Output;
+        Assert.Contains(_kekA.KekId, output, StringComparison.Ordinal); // the session KEK is surfaced
+        Assert.Contains("All rows open", output, StringComparison.Ordinal); // green all-clear
+        Assert.DoesNotContain("Rewrap from old KEK", output, StringComparison.Ordinal); // no remedy menu
+
+        Assert.True(new KekDoctorFlow().RequiresKek);
+        Assert.Equal("KEK doctor (lockout recovery)", new KekDoctorFlow().Title);
+    }
+
+    [Fact]
+    public async Task Wrong_kek_rows_offer_rewrap_and_rewrap_succeeds()
+    {
+        // Rows sealed under KEK-A; the session runs on KEK-B → both rows are wrong-KEK.
+        await SeedAsync("svc/a", "value-a", _kekA);
+        await SeedAsync("svc/b", "value-b", _kekA);
+        SecretsSession session = Session(_kekB);
+
+        TestConsole console = NewConsole();
+        console.Input.PushKey(ConsoleKey.Enter); // remedy menu: "Rewrap from old KEK" (index 0)
+        console.Input.PushKey(ConsoleKey.DownArrow); // old-key source: skip "Environment variable"
+        console.Input.PushKey(ConsoleKey.DownArrow); // skip "Key file"
+        console.Input.PushKey(ConsoleKey.Enter); // choose "Paste base64 key" (index 2)
+        console.Input.PushTextWithEnter(_kekABase64); // paste KEK-A material (masked)
+        console.Input.PushTextWithEnter("y"); // confirm the rewrap (default is false)
+
+        await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+        // Both rows now decrypt under the session KEK (B).
+        var cipherB = new AesGcmEnvelopeCipher(_kekB);
+        StoredSecret a = await GetAsync("svc/a");
+        StoredSecret b = await GetAsync("svc/b");
+        Assert.Equal(_kekB.KekId, a.KekId);
+        Assert.Equal(_kekB.KekId, b.KekId);
+        Assert.Equal("value-a", cipherB.Decrypt(a));
+        Assert.Equal("value-b", cipherB.Decrypt(b));
+
+        Assert.Contains("Re-wrapped", console.Output, StringComparison.Ordinal);
+        Assert.Contains("2", console.Output, StringComparison.Ordinal); // the rewrap count
+        Assert.DoesNotContain(_kekABase64, console.Output, StringComparison.Ordinal); // pasted key never echoed
+    }
+
+    [Fact]
+    public async Task Rewrap_requires_explicit_confirmation()
+    {
+        await SeedAsync("svc/a", "value-a", _kekA);
+        await SeedAsync("svc/b", "value-b", _kekA);
+        SecretsSession session = Session(_kekB);
+
+        TestConsole console = NewConsole();
+        console.Input.PushKey(ConsoleKey.Enter); // "Rewrap from old KEK"
+        console.Input.PushKey(ConsoleKey.DownArrow);
+        console.Input.PushKey(ConsoleKey.DownArrow);
+        console.Input.PushKey(ConsoleKey.Enter); // "Paste base64 key"
+        console.Input.PushTextWithEnter(_kekABase64);
+        console.Input.PushTextWithEnter("n"); // DECLINE the confirmation
+
+        await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+        // Nothing was re-wrapped: both rows remain under KEK-A.
+        Assert.Equal(_kekA.KekId, (await GetAsync("svc/a")).KekId);
+        Assert.Equal(_kekA.KekId, (await GetAsync("svc/b")).KekId);
+    }
+
+    [Fact]
+    public async Task Lost_kek_path_offers_reset_of_affected_secrets()
+    {
+        // Two wrong-KEK rows (sealed under KEK-A; session on KEK-B). Ordinal order: svc/a precedes svc/b.
+        await SeedAsync("svc/a", "value-a", _kekA);
+        await SeedAsync("svc/b", "value-b", _kekA);
+        SecretsSession session = Session(_kekB);
+
+        TestConsole console = NewConsole();
+        console.Input.PushKey(ConsoleKey.DownArrow); // remedy menu: move to
+        console.Input.PushKey(ConsoleKey.Enter);     // "Old KEK is lost — re-set affected secrets" (index 1)
+        console.Input.PushTextWithEnter("y");        // svc/a: accept re-set
+        console.Input.PushTextWithEnter("NEW-A-VALUE"); // masked new value
+        console.Input.PushKey(ConsoleKey.Enter);     // content type: Text (first choice)
+        console.Input.PushTextWithEnter("n");        // svc/b: decline re-set
+
+        await new KekDoctorFlow().RunAsync(console, session, CancellationToken.None);
+
+        // svc/a was re-set under the session KEK (B) with the new value.
+        var cipherB = new AesGcmEnvelopeCipher(_kekB);
+        StoredSecret a = await GetAsync("svc/a");
+        Assert.Equal(_kekB.KekId, a.KekId);
+        Assert.Equal("NEW-A-VALUE", cipherB.Decrypt(a));
+
+        // svc/b was left untouched → still wrong-KEK (under KEK-A).
+        Assert.Equal(_kekA.KekId, (await GetAsync("svc/b")).KekId);
+
+        Assert.DoesNotContain("NEW-A-VALUE", console.Output, StringComparison.Ordinal); // value never echoed
+    }
+
+    public void Dispose()
+    {
+        SqliteConnection.ClearAllPools();
+        foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
+        {
+            try
+            {
+                if (File.Exists(path))
+                {
+                    File.Delete(path);
+                }
+            }
+            catch (IOException)
+            {
+                // Best-effort temp cleanup.
+            }
+        }
+    }
+}