using ZB.MOM.WW.MxGateway.Server.Security.Authorization; using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Security.Authorization; /// /// Unit tests for the two-layer API-key failure limiter (SEC-31 / SEC-32): composite /// (transport peer, key id) partitions, the cross-peer per-key-id aggregate, probe /// admission, the per-peer key-id partition cap, and the eviction preference order. /// public sealed class ApiKeyFailureLimiterTests { private static readonly TimeSpan Window = TimeSpan.FromSeconds(60); private static readonly TimeSpan ProbeInterval = TimeSpan.FromSeconds(5); /// A partition below the failure limit is admitted without consulting a probe slot. [Fact] public void Check_BelowLimit_Allows() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); limiter.RecordFailure(partition); limiter.RecordFailure(partition); Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition)); } /// Reaching the limit inside the window throttles the composite partition. [Fact] public void Check_AtLimit_ThrottlesCompositePartition() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, partition, 3); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); } /// Failures older than the sliding window are pruned, releasing the throttle. [Fact] public void Window_PrunesExpiredFailures_ReleasesThrottle() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, partition, 3); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); clock.Advance(Window + TimeSpan.FromSeconds(1)); Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition)); } /// /// The composite partition binds a throttle to the failing address: the same key id presented /// from a different transport peer is unaffected. This is the structural half of the SEC-31 fix. /// [Fact] public void CompositePartition_ThrottleDoesNotFollowKeyIdToAnotherPeer() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 1000); RecordFailures(limiter, new ApiKeyThrottlePartition("ipv4:10.0.0.1:1", "victim"), 3); Assert.Equal( ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.0.1:1", "victim"))); Assert.Equal( ApiKeyThrottleDecision.Allowed, limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.0.2:1", "victim"))); } /// /// Failures for one key id spread across many peers trip the per-key aggregate layer, so a /// rotating-source sprayer is still bounded even though no single composite partition trips. /// [Fact] public void AggregateLayer_TripsAcrossDistinctPeers() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 100, aggregateLimit: 5); for (int peer = 0; peer < 5; peer++) { limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.0.0.{peer}:1", "victim")); } Assert.Equal( ApiKeyThrottleDecision.ThrottledByAggregate, limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "victim"))); // The aggregate is per key id: an unrelated key from the same fresh peer is untouched. Assert.Equal( ApiKeyThrottleDecision.Allowed, limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "other-key"))); } /// /// A throttled partition is a valve, not a wall: exactly one request per probe interval is /// admitted to the real verifier, so the holder of the correct secret can always get through. /// [Fact] public void ProbeAdmission_AdmitsOneRequestPerInterval() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, partition, 3); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); clock.Advance(ProbeInterval); Assert.Equal(ApiKeyThrottleDecision.ProbeAdmitted, limiter.Check(partition)); // The granted slot is consumed: the next request inside the same interval is throttled. Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); clock.Advance(ProbeInterval); Assert.Equal(ApiKeyThrottleDecision.ProbeAdmitted, limiter.Check(partition)); } /// /// The probe slot is claimed atomically: when a crowd of requests arrives at the same interval /// boundary exactly one is admitted and the rest are still refused. A check-then-act grant would /// let every arrival observe "due" and hand the whole burst through to the verifier. /// [Fact] public void ProbeAdmission_UnderConcurrentArrivals_GrantsExactlyOneSlot() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 5); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); // Five failures on one partition trip both layers (limit 3, aggregate 5), so the concurrent // arrivals contend for the composite probe slot AND the aggregate's. RecordFailures(limiter, partition, 5); clock.Advance(ProbeInterval); // The unsafe window between reading "probe due" and advancing the slot is nanoseconds wide, // so one burst can miss it by luck. Repeating the boundary makes the red reliable, while the // atomic implementation must yield exactly one admission in every round. const int arrivals = 8; const int rounds = 200; int totalAdmitted = 0; for (int round = 0; round < rounds; round++) { // Re-arm: the failures keep both layers over their limits, and recording pushes the next // probe one interval out, which the advance below then reaches. RecordFailures(limiter, partition, 5); clock.Advance(ProbeInterval); ApiKeyThrottleDecision[] decisions = new ApiKeyThrottleDecision[arrivals]; using (Barrier startLine = new(arrivals)) { Thread[] threads = new Thread[arrivals]; for (int index = 0; index < arrivals; index++) { int slot = index; threads[slot] = new Thread(() => { startLine.SignalAndWait(); decisions[slot] = limiter.Check(partition); }); threads[slot].Start(); } foreach (Thread thread in threads) { Assert.True(thread.Join(TimeSpan.FromSeconds(30)), "probe-contention thread did not finish"); } } int admitted = decisions.Count(decision => decision == ApiKeyThrottleDecision.ProbeAdmitted); Assert.Equal( arrivals - admitted, decisions.Count(decision => decision is ApiKeyThrottleDecision.ThrottledByPeer or ApiKeyThrottleDecision.ThrottledByAggregate)); totalAdmitted += admitted; } Assert.Equal(rounds, totalAdmitted); } /// /// The layers claim their probe slots one at a time, so a slot claimed on the composite partition /// must be returned when the aggregate then refuses. Otherwise a refused request would silently /// spend the partition's next slot and push the legitimate holder out by a full interval. /// [Fact] public void ProbeAdmission_WhenAggregateRefuses_ReturnsTheClaimedPeerSlot() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 5); ApiKeyThrottlePartition holder = new("ipv4:10.0.0.1:1", "victim"); // Both layers trip together, arming both slots for t0 + interval. RecordFailures(limiter, holder, 5); // A failure from a second address re-arms only the aggregate (its own composite is far under // the limit), so the aggregate's slot now opens one interval later than the partition's. clock.Advance(TimeSpan.FromSeconds(3)); limiter.RecordFailure(new ApiKeyThrottlePartition("ipv4:10.0.0.2:1", "victim")); // t0 + 5s: the partition's slot is due, the aggregate's is not — the request is refused and // the partition's slot must be handed back rather than consumed. clock.Advance(TimeSpan.FromSeconds(2)); Assert.Equal(ApiKeyThrottleDecision.ThrottledByAggregate, limiter.Check(holder)); // t0 + 8s: the aggregate's slot opens. The partition's slot was restored, so this passes; had // it been consumed above it would not reopen until t0 + 10s and this would be ThrottledByPeer. clock.Advance(TimeSpan.FromSeconds(3)); Assert.Equal(ApiKeyThrottleDecision.ProbeAdmitted, limiter.Check(holder)); } /// /// A compensating release must never undo a re-arm written by a concurrent failure on the same /// partition. Both writers store the identical now + interval value when they share a /// clock tick, so identifying the caller's own reservation by timestamp would let the release /// stomp a fresh re-arm back to an already-due value and reopen the probe slot early. The clock /// is deliberately held still here, which forces exactly that collision. /// [Fact] public void ProbeSlotRestore_DoesNotStompConcurrentRearmAtSameTick() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 5); ApiKeyThrottlePartition holder = new("ipv4:10.0.0.1:1", "victim"); ApiKeyThrottlePartition other = new("ipv4:10.0.0.2:1", "victim"); // Trip both layers, then push the aggregate's slot one interval past the partition's, so // every Check below claims the partition's slot and is then refused by the aggregate — the // claim-and-compensate path under test. RecordFailures(limiter, holder, 5); clock.Advance(TimeSpan.FromSeconds(3)); limiter.RecordFailure(other); clock.Advance(TimeSpan.FromSeconds(2)); // Land a failure on the same partition inside the claim-to-release window — the interleaving // a concurrent RecordFailure produces, forced here so the assertion is deterministic. It // shares the frozen clock tick with the claim, so both write the identical slot value. int interleaved = 0; limiter.ProbeReleaseInterleaveHook = () => { if (Interlocked.Exchange(ref interleaved, 1) == 0) { limiter.RecordFailure(holder); } }; Assert.Equal(ApiKeyThrottleDecision.ThrottledByAggregate, limiter.Check(holder)); Assert.Equal(1, interleaved); limiter.ProbeReleaseInterleaveHook = null; // Drop the aggregate so the next decision reflects the composite partition alone. limiter.Reset(other); // The interleaved failure pushed the slot one interval past the (still unadvanced) clock, so // no probe may be due. Restoring over it would leave the already-due earlier value and hand // the next arrival a free probe. Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(holder)); } /// A zero probe interval restores absolute blocking (documented as not recommended). [Fact] public void ProbeIntervalZero_BlocksAbsolutely() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, probeInterval: TimeSpan.Zero); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, partition, 3); // Well past several probe intervals but still inside the failure window: no slot opens. clock.Advance(ProbeInterval + ProbeInterval + ProbeInterval); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); } /// A successful verification clears both the composite partition and the key aggregate. [Fact] public void Reset_ClearsCompositeAndAggregateLayers() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 5); ApiKeyThrottlePartition partition = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, partition, 3); for (int peer = 1; peer < 3; peer++) { limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.0.1.{peer}:1", "victim")); } Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(partition)); limiter.Reset(partition); Assert.Equal(ApiKeyThrottleDecision.Allowed, limiter.Check(partition)); Assert.Equal(0, limiter.TrackedAggregateCount); Assert.Equal( ApiKeyThrottleDecision.Allowed, limiter.Check(new ApiKeyThrottlePartition("ipv4:10.0.9.9:1", "victim"))); } /// /// A success whose key id was squeezed into the address's shared fallback bucket must not clear /// that bucket: it carries failures from other key ids at the same address, so clearing it would /// make one successful authentication a reset button for an in-progress spray. /// [Fact] public void Reset_WithOverCapKeyId_DoesNotClearSharedFallbackPartition() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 100, maxPartitions: 4096); const string peer = "ipv4:10.0.0.1:1"; // Fill the per-peer key-id cap, then spray past it so the overflow lands on — and trips — // the address's shared fallback partition. for (int i = 0; i < ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer; i++) { limiter.RecordFailure(new ApiKeyThrottlePartition(peer, $"key{i}")); } for (int i = 0; i < 5; i++) { limiter.RecordFailure(new ApiKeyThrottlePartition(peer, $"overflow{i}")); } ApiKeyThrottlePartition overCap = new(peer, "overflow0"); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(overCap)); limiter.Reset(overCap); Assert.True(limiter.IsTracked(overCap)); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(overCap)); } /// /// SEC-32: a spray of unique junk partitions (junk tokens resolve to the sender's fallback /// partition, one per address) must not evict a partition that is currently throttled — the /// LRU cap bounds memory, it must not be a reset button for the block. /// [Fact] public void JunkTokenSpray_DoesNotEvictBlockedEntry() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 8); ApiKeyThrottlePartition blocked = new("ipv4:10.0.0.1:1", "victim"); RecordFailures(limiter, blocked, 3); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(blocked)); for (int i = 0; i < 16; i++) { clock.Advance(TimeSpan.FromMilliseconds(1)); limiter.RecordFailure(new ApiKeyThrottlePartition($"ipv4:10.9.{i / 256}.{i % 256}:1", KeyId: null)); } Assert.True(limiter.IsTracked(blocked)); Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(blocked)); } /// /// SEC-32: one address may mint at most /// key-id partitions; the overflow collapses into that address's fallback partition, which then /// throttles the address wholesale instead of letting the spray mint unbounded state. /// [Fact] public void UniqueMxgwKeyIdSpray_FromOnePeer_CollapsesAtPerPeerCap() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 4096); const string peer = "ipv4:10.0.0.1:1"; for (int i = 0; i < 1000; i++) { limiter.RecordFailure(new ApiKeyThrottlePartition(peer, $"key{i}")); } Assert.True( limiter.TrackedPartitionCount <= ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer + 1, $"expected at most {ApiKeyFailureLimiter.MaxKeyIdPartitionsPerPeer + 1} partitions, saw {limiter.TrackedPartitionCount}"); Assert.Equal( ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(new ApiKeyThrottlePartition(peer, "key999"))); } /// /// SEC-32 eviction preference: with the map at capacity a new failure evicts an entry whose /// window has fully expired rather than an entry that is still counting. /// [Fact] public void Eviction_PrefersExpiredWindows() { ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, maxPartitions: 2); ApiKeyThrottlePartition expired = new("ipv4:10.0.0.1:1", KeyId: null); ApiKeyThrottlePartition active = new("ipv4:10.0.0.2:1", KeyId: null); ApiKeyThrottlePartition arriving = new("ipv4:10.0.0.3:1", KeyId: null); limiter.RecordFailure(expired); clock.Advance(Window + TimeSpan.FromSeconds(1)); limiter.RecordFailure(active); limiter.RecordFailure(arriving); Assert.Equal(2, limiter.TrackedPartitionCount); Assert.False(limiter.IsTracked(expired)); Assert.True(limiter.IsTracked(active)); Assert.True(limiter.IsTracked(arriving)); } private static void RecordFailures(ApiKeyFailureLimiter limiter, ApiKeyThrottlePartition partition, int count) { for (int i = 0; i < count; i++) { limiter.RecordFailure(partition); } } private static ApiKeyFailureLimiter CreateLimiter( TimeProvider clock, int limit, int aggregateLimit = 0, int maxPartitions = 1024, TimeSpan? probeInterval = null) { return new ApiKeyFailureLimiter( limit, Window, maxPartitions, aggregateLimit, probeInterval ?? ProbeInterval, clock); } }