fix(acl): shard eviction swaps the shard object so _globalCount cannot over-subtract; concurrency stress test

2026-08-17 03:55:13 -04:00
parent 97902d6879
commit d884934593
2 changed files with 233 additions and 11 deletions
@@ -163,6 +163,106 @@ public sealed class SessionAclDecisionCacheTests
        neighbourCalls.ShouldBe(1);
    }

+    /// <summary>
+    ///     After a shard is evicted the global counter equals what actually survived — the arithmetic of
+    ///     the swap, pinned single-threaded where it is exact and every off-by-a-shard is visible.
+    /// </summary>
+    [Fact]
+    public void An_eviction_subtracts_exactly_what_it_displaced()
+    {
+        var cache = new SessionAclDecisionCache();
+        const int overshoot = 25;
+
+        for (var i = 0; i < SessionAclDecisionCache.MaxEntriesPerSession + overshoot; i++)
+            Ask(cache, generation: 1, session: "crawler-key", node: $"Plant/dev1/Tag{i}", compute: () => true);
+
+        // The cap is reached on the entry that would have been the 20,001st; everything after it is the
+        // refilled shard. Subtracting anything but the displaced shard's own count would show up here.
+        cache.Count.ShouldBe(overshoot);
+        cache.TrackedCount.ShouldBe(cache.Count);
+        cache.LiveShardCount.ShouldBe(cache.Count);
+    }
+
+    /// <summary>
+    ///     The eviction race. Many writers hammer ONE shard past its ceiling while others insert into it,
+    ///     so evictions and inserts interleave repeatedly; at quiescence the global counter must equal the
+    ///     entries that actually survived.
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         The defect this pins is not reachable single-threaded. Eviction used to empty the shard's
+    ///         dictionary and then reset the counters as two steps: a writer blocked inside
+    ///         <c>TryAdd</c> for the duration of the <c>Clear</c> resumes the moment the table's locks are
+    ///         released — squarely in the window before the counter reset — books an increment, and has it
+    ///         subtracted a second time by the reset that follows. The drift is negative-only, so it never
+    ///         changed a verdict; it made the cache trim progressively earlier than it should.
+    ///     </para>
+    ///     <para>
+    ///         ⚠️ <b>The assertion is equality, not a bound.</b> A ceiling check
+    ///         (<c>TrackedCount &lt;= Count</c>, or <c>&gt;= 0</c>) passes against the broken code, because
+    ///         under-counting is exactly what the broken code does. Only <c>TrackedCount == Count</c>
+    ///         falsifies it — verified by reverting the fix, which turns this test red on the first round.
+    ///     </para>
+    /// </remarks>
+    [Fact]
+    public void Concurrent_eviction_and_insertion_leave_the_counter_exact()
+    {
+        var cache = new SessionAclDecisionCache();
+        const string crawler = "crawler-key";
+        const int crawlers = 8;
+        const int neighbours = 2;
+        const int perCrawler = 40_000;
+        const int perNeighbour = 500;
+
+        // Sized for exposure, not for coverage: one round drives the shard past its ceiling ~16 times, and
+        // the interleaving that used to corrupt the counter needs a writer to land inside the few
+        // nanoseconds between the clear and the counter reset. Fewer rounds let the reverted code pass.
+        const int rounds = 8;
+
+        for (var round = 0; round < rounds; round++)
+        {
+            var currentRound = round;
+            var start = new Barrier(crawlers + neighbours);
+            var workers = new Thread[crawlers + neighbours];
+
+            for (var w = 0; w < workers.Length; w++)
+            {
+                var worker = w;
+
+                // The first `crawlers` threads share ONE shard and drive it past its ceiling many times
+                // over; the rest hold their own shards, so the audit also covers the shards an eviction
+                // must not touch.
+                var session = worker < crawlers ? crawler : $"neighbour-{worker}";
+                var writes = worker < crawlers ? perCrawler : perNeighbour;
+
+                workers[w] = new Thread(() =>
+                {
+                    start.SignalAndWait();
+
+                    // Node identifiers are unique per (round, worker, index), so every store is a genuine
+                    // insert and nothing is lost to a TryAdd collision instead of to the race.
+                    for (var i = 0; i < writes; i++)
+                        cache.GetOrAdd(
+                            generation: 1, session, OpcUaOperation.Read, AddressSpaceRealm.Raw,
+                            $"Plant/dev1/R{currentRound}-W{worker}-T{i}", static () => true);
+                }) { IsBackground = true };
+            }
+
+            foreach (var worker in workers) worker.Start();
+            foreach (var worker in workers) worker.Join();
+            start.Dispose();
+
+            var survived = cache.Count;
+
+            cache.TrackedCount.ShouldBeGreaterThanOrEqualTo(
+                0, $"round {currentRound}: the counter must never be driven negative");
+            cache.LiveShardCount.ShouldBe(
+                survived, $"round {currentRound}: a shard's own counter drifted from its entries");
+            cache.TrackedCount.ShouldBe(
+                survived, $"round {currentRound}: the global counter drifted from the live shards");
+        }
+    }
+
    /// <summary>
    ///     A decision taken while a deployment lands mid-walk is returned but NOT stored, so the new
    ///     generation never inherits a verdict its own trie did not make.