fix(controlplane): a PSK secret deleted from the store revokes the key; bound refresh concurrency

RefreshAsync now tells "the store answered: no such secret" (revocation - drop the refreshed key AND the lazily cached one) apart from "the read failed" (keep the previous key). Store reads fan out through Parallel.ForEachAsync capped at MaxConcurrentRefreshReads = 8 instead of one read per cluster at once. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 13:49:56 -04:00
parent 2adeacc655
commit 896aaa2b02
2 changed files with 136 additions and 25 deletions
@@ -78,6 +78,77 @@ public sealed class MeshPskProviderRefreshTests
        psk.ShouldBe("k-1");
    }

+    /// <summary>
+    ///     Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure:
+    ///     the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown
+    ///     read keeps the previous key (the test above); a definitive not-found drops it.
+    /// </summary>
+    [Fact]
+    public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh()
+    {
+        var resolver = new GatedResolver { Value = "k-1" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+        provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service");
+
+        resolver.Value = null;   // the secret was deleted
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
+        provider.Matches("SITE-A", "k-1").ShouldBeFalse();
+    }
+
+    /// <summary>The same revocation reaches a key that was first served from the LAZY cache, not a refresh.</summary>
+    [Fact]
+    public async Task A_deleted_secret_also_evicts_a_lazily_cached_key()
+    {
+        var resolver = new GatedResolver { Value = "k-lazy" };
+        var provider = Provider(resolver);
+        provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue();
+        lazy.ShouldBe("k-lazy");
+
+        resolver.Value = null;
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
+    }
+
+    /// <summary>Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once.</summary>
+    [Fact]
+    public async Task Refresh_store_reads_are_concurrency_bounded()
+    {
+        var resolver = new ConcurrencyTrackingResolver();
+        var provider = Provider(resolver);
+        var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList();
+
+        await provider.RefreshAsync(clusters, CancellationToken.None);
+
+        resolver.Calls.ShouldBe(40);
+        resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads);
+        resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded");
+        foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue();
+    }
+
+    private sealed class ConcurrencyTrackingResolver : ISecretResolver
+    {
+        private int _inFlight;
+        private int _max;
+        private int _calls;
+        public int MaxConcurrent => Volatile.Read(ref _max);
+        public int Calls => Volatile.Read(ref _calls);
+
+        public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
+        {
+            Interlocked.Increment(ref _calls);
+            var now = Interlocked.Increment(ref _inFlight);
+            int seen;
+            while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { }
+            await Task.Delay(20, ct);
+            Interlocked.Decrement(ref _inFlight);
+            return "k-" + name.Value;
+        }
+    }
+
    /// <summary>A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour).</summary>
    [Fact]
    public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily()