fix(SEC-31,SEC-32): re-partition the API-key failure limiter on (peer, key id) with probe admission

The gRPC auth failure limiter partitioned on the key id parsed out of the *unauthenticated* token and rejected with ResourceExhausted before VerifyAsync ran. Key ids are not secret — they ride in every token and are listed on the dashboard — so any network peer could send 10 garbage-secret requests per minute and deny that key indefinitely: the legitimate holder's correct secret was refused before it was ever checked, and the success-path Reset that would clear the block sat behind the verification the block prevented (SEC-31). The tracked map was also flushable — any `a_b_c`-shaped junk minted a fresh partition (the `mxgw` literal was never compared), so ~4096 throwaway tokens evicted a blocked entry and reset the window (SEC-32). ApiKeyFailureLimiter moves from IsBlocked/RecordFailure/Reset(string peer) to a partition-pair API: Check/RecordFailure/Reset(ApiKeyThrottlePartition) with an ApiKeyThrottleDecision result. Two layers share one sliding window — a composite (transport peer, key id) partition at ApiKeyFailureLimit, and a per-key-id aggregate across all peers at the new ApiKeyFailureAggregateLimit (default 30) that bounds a source-rotating sprayer. An over-limit state is now a valve rather than a wall: one request per the new ApiKeyFailureProbeIntervalSeconds (default 5) is admitted through to the real verifier, so the correct secret always reaches the constant-time compare and resets both layers. Guarantees preserved: guessing stays bounded per window, and the failure path still spends no store read per attempt. SEC-32 rides the same change set: the interceptor validates token shape (literal `mxgw` prefix, >= 3 non-empty `_` segments, key id <= 64 chars) before minting a key-id partition, each transport peer may mint at most 32 of them before the overflow collapses onto its fallback partition, and eviction prefers fully expired windows and never drops an over-limit partition below a 2x transient overshoot ceiling. Throttled attempts increment mxgateway.auth.throttled, tagged stage=peer|aggregate only — /metrics is unauthenticated (open SEC-14), so no key material may appear there. Docs in the same commit: GatewayConfiguration limiter rows plus the two new keys, the Authentication hot-path paragraph, the Authorization SEC-11 section, and the limiter / SecurityOptions XML remarks (the old NAT rationale described the defective keying). Tracking rows flipped to Done with a change-log entry. Tests: new ApiKeyFailureLimiterTests (11) covering window pruning, composite vs aggregate trip points, probe cadence, absolute-block mode, reset across both layers, junk-spray eviction resistance, the per-peer cap, and expired-window eviction preference; GatewayGrpcAuthorizationInterceptorTests gains the four SEC-31 contract tests plus NonMxgwToken_FallsBackToTransportPeerPartition (20 total); GatewayOptionsValidatorTests covers both new keys including 0 as a supported disable value (66 total).
2026-08-07 05:39:10 -04:00
parent ead921cace
commit df710e18a9
16 changed files with 1148 additions and 137 deletions
@@ -0,0 +1,265 @@
+using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
+
+namespace ZB.MOM.WW.MxGateway.Tests.Security.Authorization;
+
+/// <summary>
+/// Unit tests for the two-layer API-key failure limiter (SEC-31 / SEC-32): composite
+/// <c>(transport peer, key id)</c> partitions, the cross-peer per-key-id aggregate, probe
+/// admission, the per-peer key-id partition cap, and the eviction preference order.
+/// </summary>
+public sealed class ApiKeyFailureLimiterTests
+{
+    private static readonly TimeSpan Window = TimeSpan.FromSeconds(60);
+    private static readonly TimeSpan ProbeInterval = TimeSpan.FromSeconds(5);
+
+    /// <summary>A partition below the failure limit is admitted without consulting a probe slot.</summary>
+    [Fact]
+    public void Check_BelowLimit_Allows()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        limiter.RecordFailure(partition);
+        limiter.RecordFailure(partition);
+
+        Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition));
+    }
+
+    /// <summary>Reaching the limit inside the window throttles the composite partition.</summary>
+    [Fact]
+    public void Check_AtLimit_ThrottlesCompositePartition()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, partition, 3);
+
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+    }
+
+    /// <summary>Failures older than the sliding window are pruned, releasing the throttle.</summary>
+    [Fact]
+    public void Window_PrunesExpiredFailures_ReleasesThrottle()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, partition, 3);
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+
+        clock.Advance(Window + TimeSpan.FromSeconds(1));
+
+        Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition));
+    }
+
+    /// <summary>
+    /// The composite partition binds a throttle to the failing address: the same key id presented
+    /// from a different transport peer is unaffected. This is the structural half of the SEC-31 fix.
+    /// </summary>
+    [Fact]
+    public void CompositePartition_ThrottleDoesNotFollowKeyIdToAnotherPeer()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 1000);
+
+        RecordFailures(limiter, new ApiKeyThrottlePartition("ipv4:10.0.0.1:1", "victim"), 3);
+
+        Assert.Equal(
+            ApiKeyThrottleDecision.ThrottledByPeer,
+            limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.0.1:1", "victim")));
+        Assert.Equal(
+            ApiKeyThrottleDecision.Allowed,
+            limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.0.2:1", "victim")));
+    }
+
+    /// <summary>
+    /// Failures for one key id spread across many peers trip the per-key aggregate layer, so a
+    /// rotating-source sprayer is still bounded even though no single composite partition trips.
+    /// </summary>
+    [Fact]
+    public void AggregateLayer_TripsAcrossDistinctPeers()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 100, aggregateLimit: 5);
+
+        for (int peer = 0; peer < 5; peer++)
+        {
+            limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.0.0.{peer}:1", "victim"));
+        }
+
+        Assert.Equal(
+            ApiKeyThrottleDecision.ThrottledByAggregate,
+            limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "victim")));
+
+        // The aggregate is per key id: an unrelated key from the same fresh peer is untouched.
+        Assert.Equal(
+            ApiKeyThrottleDecision.Allowed,
+            limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "other-key")));
+    }
+
+    /// <summary>
+    /// A throttled partition is a valve, not a wall: exactly one request per probe interval is
+    /// admitted to the real verifier, so the holder of the correct secret can always get through.
+    /// </summary>
+    [Fact]
+    public void ProbeAdmission_AdmitsOneRequestPerInterval()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, partition, 3);
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+
+        clock.Advance(ProbeInterval);
+        Assert.Equal(ApiKeyThrottleDecision.ProbeAdmitted, limiter.Check(partition));
+
+        // The granted slot is consumed: the next request inside the same interval is throttled.
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+
+        clock.Advance(ProbeInterval);
+        Assert.Equal(ApiKeyThrottleDecision.ProbeAdmitted, limiter.Check(partition));
+    }
+
+    /// <summary>A zero probe interval restores absolute blocking (documented as not recommended).</summary>
+    [Fact]
+    public void ProbeIntervalZero_BlocksAbsolutely()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, probeInterval: TimeSpan.Zero);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, partition, 3);
+
+        // Well past several probe intervals but still inside the failure window: no slot opens.
+        clock.Advance(ProbeInterval + ProbeInterval + ProbeInterval);
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+    }
+
+    /// <summary>A successful verification clears both the composite partition and the key aggregate.</summary>
+    [Fact]
+    public void Reset_ClearsCompositeAndAggregateLayers()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 5);
+        ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, partition, 3);
+        for (int peer = 1; peer < 3; peer++)
+        {
+            limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.0.1.{peer}:1", "victim"));
+        }
+
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition));
+
+        limiter.Reset(partition);
+
+        Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition));
+        Assert.Equal(0, limiter.TrackedAggregateCount);
+        Assert.Equal(
+            ApiKeyThrottleDecision.Allowed,
+            limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "victim")));
+    }
+
+    /// <summary>
+    /// SEC-32: a spray of unique junk partitions (junk tokens resolve to the sender's fallback
+    /// partition, one per address) must not evict a partition that is currently throttled — the
+    /// LRU cap bounds memory, it must not be a reset button for the block.
+    /// </summary>
+    [Fact]
+    public void JunkTokenSpray_DoesNotEvictBlockedEntry()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 8);
+        ApiKeyThrottlePartition blocked = new("ipv4:10.0.0.1:1", "victim");
+
+        RecordFailures(limiter, blocked, 3);
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(blocked));
+
+        for (int i = 0; i < 16; i++)
+        {
+            clock.Advance(TimeSpan.FromMilliseconds(1));
+            limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.9.{i / 256}.{i % 256}:1", KeyId: null));
+        }
+
+        Assert.True(limiter.IsTracked(blocked));
+        Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(blocked));
+    }
+
+    /// <summary>
+    /// SEC-32: one address may mint at most <see cref="ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer"/>
+    /// key-id partitions; the overflow collapses into that address's fallback partition, which then
+    /// throttles the address wholesale instead of letting the spray mint unbounded state.
+    /// </summary>
+    [Fact]
+    public void UniqueMxgwKeyIdSpray_FromOnePeer_CollapsesAtPerPeerCap()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 4096);
+        const string peer = "ipv4:10.0.0.1:1";
+
+        for (int i = 0; i < 1000; i++)
+        {
+            limiter.RecordFailure(new ApiKeyThrottlePartition(peer, $"key{i}"));
+        }
+
+        Assert.True(
+            limiter.TrackedPartitionCount <= ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer + 1,
+            $"expected at most {ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer + 1} partitions, saw {limiter.TrackedPartitionCount}");
+        Assert.Equal(
+            ApiKeyThrottleDecision.ThrottledByPeer,
+            limiter.Check(new ApiKeyThrottlePartition(peer, "key999")));
+    }
+
+    /// <summary>
+    /// SEC-32 eviction preference: with the map at capacity a new failure evicts an entry whose
+    /// window has fully expired rather than an entry that is still counting.
+    /// </summary>
+    [Fact]
+    public void Eviction_PrefersExpiredWindows()
+    {
+        ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch);
+        ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 2);
+        ApiKeyThrottlePartition expired = new("ipv4:10.0.0.1:1", KeyId: null);
+        ApiKeyThrottlePartition active = new("ipv4:10.0.0.2:1", KeyId: null);
+        ApiKeyThrottlePartition arriving = new("ipv4:10.0.0.3:1", KeyId: null);
+
+        limiter.RecordFailure(expired);
+        clock.Advance(Window + TimeSpan.FromSeconds(1));
+        limiter.RecordFailure(active);
+        limiter.RecordFailure(arriving);
+
+        Assert.Equal(2, limiter.TrackedPartitionCount);
+        Assert.False(limiter.IsTracked(expired));
+        Assert.True(limiter.IsTracked(active));
+        Assert.True(limiter.IsTracked(arriving));
+    }
+
+    private static void RecordFailures(ApiKeyFailureLimiter limiter, ApiKeyThrottlePartition partition, int count)
+    {
+        for (int i = 0; i < count; i++)
+        {
+            limiter.RecordFailure(partition);
+        }
+    }
+
+    private static ApiKeyFailureLimiter CreateLimiter(
+        TimeProvider clock,
+        int limit,
+        int aggregateLimit = 0,
+        int maxPartitions = 1024,
+        TimeSpan? probeInterval = null)
+    {
+        return new ApiKeyFailureLimiter(
+            limit,
+            Window,
+            maxPartitions,
+            aggregateLimit,
+            probeInterval ?? ProbeInterval,
+            clock);
+    }
+}