fix(acl): stamp the generation on Shard so a rollover racer cannot serve a stale-generation verdict; eviction-swap capacity hint
This commit is contained in:
Joseph Doherty
@@ -263,6 +263,123 @@ public sealed class SessionAclDecisionCacheTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The store half of the same window: a <c>Set</c> that meets the previous generation's shard must
|
||||
/// displace it, not fill it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Filing into an orphaned shard is never <i>wrong</i> — the read path's generation check means
|
||||
/// nothing stored there can ever be served — which is exactly why it needs its own pin: the symptom
|
||||
/// is silent, and it is a dead cache rather than a leaking one. Every store for that key would land
|
||||
/// somewhere unreadable, consuming the key's whole per-shard budget before an eviction swap finally
|
||||
/// minted a shard for the current generation. Twenty thousand wasted inserts per key per deployment,
|
||||
/// with a hit rate of zero throughout and nothing in the counters to say so.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void A_store_meeting_the_previous_generations_shard_displaces_it()
|
||||
{
|
||||
var cache = new SessionAclDecisionCache();
|
||||
|
||||
Ask(cache, generation: 1, compute: () => true).ShouldBeTrue();
|
||||
cache.AdvanceGenerationForTest(2);
|
||||
|
||||
// Inside the window: the stamp says 2, the shard in the table was built for 1.
|
||||
cache.Set(generation: 2, Session, OpcUaOperation.Read, AddressSpaceRealm.Raw, Node, decision: false);
|
||||
|
||||
cache.TryGet(generation: 2, Session, OpcUaOperation.Read, AddressSpaceRealm.Raw, Node, out var decision)
|
||||
.ShouldBeTrue("the store must have landed in a shard the current generation can read");
|
||||
decision.ShouldBeFalse("and it must be the new verdict, not the one it replaced");
|
||||
|
||||
// The displaced shard was sealed and unbooked, so the survivor is the single new entry.
|
||||
cache.Count.ShouldBe(1);
|
||||
cache.TrackedCount.ShouldBe(1);
|
||||
cache.LiveShardCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The same rollover, arrived at the way production does — a generation moving under live traffic
|
||||
/// rather than through the test seam. No hit may ever return a verdict stored under a different
|
||||
/// generation, and the counters must still settle exactly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The generation is <b>encoded in the stored verdict</b> (<c>gen % 2 == 0</c>), which is what
|
||||
/// makes a leak visible at all: a cache that answers with the right shape but the wrong
|
||||
/// generation's data is indistinguishable from a correct one unless the data says which
|
||||
/// generation it came from. A miss recomputes the encoding for the generation it was asked
|
||||
/// about, so only a hit can ever disagree.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The bumps come from inside the worker mix rather than from a spinning thread of their own,
|
||||
/// and are spaced far enough apart for entries to accumulate between them — a generation moving
|
||||
/// faster than a lookup completes makes every call a miss, which is a cache that is never read
|
||||
/// and therefore a test that pins nothing.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void No_hit_ever_returns_a_verdict_from_another_generation()
|
||||
{
|
||||
var cache = new SessionAclDecisionCache();
|
||||
const int workers = 6;
|
||||
const int writesEach = 30_000;
|
||||
const int bumpEvery = 500;
|
||||
const int rounds = 4;
|
||||
|
||||
// A deliberately narrow node and key space: workers must keep landing on entries their peers
|
||||
// stored, and on shards a bump has just orphaned. Unique identifiers would make every call a miss.
|
||||
const int nodes = 64;
|
||||
const int keys = 2;
|
||||
|
||||
for (var r = 0; r < rounds; r++)
|
||||
{
|
||||
var round = r;
|
||||
var generation = (round * 100L) + 1L;
|
||||
var mismatches = 0;
|
||||
var start = new Barrier(workers);
|
||||
var threads = new Thread[workers];
|
||||
|
||||
for (var w = 0; w < workers; w++)
|
||||
{
|
||||
var worker = w;
|
||||
|
||||
threads[w] = new Thread(() =>
|
||||
{
|
||||
start.SignalAndWait();
|
||||
|
||||
for (var i = 0; i < writesEach; i++)
|
||||
{
|
||||
// One worker advances the generation, so the bump lands in the middle of the others'
|
||||
// traffic rather than beside it.
|
||||
if (worker == 0 && i % bumpEvery == 0) Interlocked.Increment(ref generation);
|
||||
|
||||
var gen = Interlocked.Read(ref generation);
|
||||
var expected = gen % 2 == 0;
|
||||
|
||||
var decided = cache.GetOrAdd(
|
||||
gen, $"key-{worker % keys}", OpcUaOperation.Read, AddressSpaceRealm.Raw,
|
||||
$"Plant/dev1/Tag{i % nodes}", () => expected);
|
||||
|
||||
if (decided != expected) Interlocked.Increment(ref mismatches);
|
||||
}
|
||||
}) { IsBackground = true };
|
||||
}
|
||||
|
||||
foreach (var thread in threads) thread.Start();
|
||||
foreach (var thread in threads) thread.Join();
|
||||
start.Dispose();
|
||||
|
||||
mismatches.ShouldBe(
|
||||
0, $"round {round}: a hit served a verdict taken under a different generation");
|
||||
|
||||
var survived = cache.Count;
|
||||
|
||||
cache.LiveShardCount.ShouldBe(
|
||||
survived, $"round {round}: a shard's own counter drifted from its entries");
|
||||
cache.TrackedCount.ShouldBe(
|
||||
survived, $"round {round}: the global counter drifted from the live shards");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A decision taken while a deployment lands mid-walk is returned but NOT stored, so the new
|
||||
/// generation never inherits a verdict its own trie did not make.
|
||||
@@ -289,6 +406,77 @@ public sealed class SessionAclDecisionCacheTests
|
||||
cache.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The generation-rollover window, made deterministic. Publishing the new generation and purging the
|
||||
/// shards are two steps, and a racer arriving between them sees a generation that already matches
|
||||
/// and shards that have not been dropped yet — so it must not be able to read one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <c>AdvanceGenerationForTest</c> is the racer's view exactly: the stamp moves, nothing clears.
|
||||
/// Before the fix this test served the generation-1 verdict to a generation-2 caller. Key
|
||||
/// revocation and expiry are re-checked above the cache, so the leak was a stale <i>grant walk</i>
|
||||
/// rather than a usable revoked key — real, but narrower than it first looks.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ The assertion is a <b>miss</b>, not a corrected value: serving <c>false</c> from a shard
|
||||
/// built under the old generation would be just as wrong, and a value assertion would pass on it.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void A_shard_built_under_the_previous_generation_is_never_served()
|
||||
{
|
||||
var cache = new SessionAclDecisionCache();
|
||||
|
||||
// GetOrAdd, not a bare Set: Set drops a decision whose generation is not already the stamped one,
|
||||
// and a fresh cache is stamped -1. The lookup inside GetOrAdd is what publishes generation 1.
|
||||
Ask(cache, generation: 1, compute: () => true).ShouldBeTrue();
|
||||
cache.TryGet(generation: 1, Session, OpcUaOperation.Read, AddressSpaceRealm.Raw, Node, out var stored)
|
||||
.ShouldBeTrue("the entry must exist before the window opens, or the test proves nothing");
|
||||
stored.ShouldBeTrue();
|
||||
|
||||
// Stand inside the window: the new generation is published, the shards are not yet purged.
|
||||
cache.AdvanceGenerationForTest(2);
|
||||
|
||||
cache.TryGet(generation: 2, Session, OpcUaOperation.Read, AddressSpaceRealm.Raw, Node, out _)
|
||||
.ShouldBeFalse("a shard may only answer for the generation it was built under");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A shard displaced by the per-shard eviction swap keeps taking inserts correctly, with the counters
|
||||
/// still exact — the behaviour pin behind the capacity hint the swap now passes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Pin, not red-first.</b> The hint sizes a dictionary's initial buckets, which is unobservable
|
||||
/// through the public surface: no assertion here can distinguish a hinted shard from a default one,
|
||||
/// and none tries to. What it pins is that passing it changed nothing about the refill — that the
|
||||
/// replacement shard still accepts entries and still books them exactly once.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void A_shard_replaced_by_the_eviction_swap_still_refills_exactly()
|
||||
{
|
||||
var cache = new SessionAclDecisionCache();
|
||||
const int overshoot = 12;
|
||||
|
||||
for (var i = 0; i < SessionAclDecisionCache.MaxEntriesPerSession + overshoot; i++)
|
||||
Ask(cache, generation: 1, session: "crawler-key", node: $"Plant/dev1/Tag{i}", compute: () => true);
|
||||
|
||||
cache.Count.ShouldBe(overshoot);
|
||||
cache.TrackedCount.ShouldBe(overshoot);
|
||||
cache.LiveShardCount.ShouldBe(overshoot);
|
||||
|
||||
// The refilled shard serves what it was given, and a fresh insert into it still books once.
|
||||
Ask(cache, generation: 1, session: "crawler-key", node: "Plant/dev1/Tag20011",
|
||||
compute: () => throw new InvalidOperationException("this decision was already memoized"))
|
||||
.ShouldBeTrue();
|
||||
Ask(cache, generation: 1, session: "crawler-key", node: "Plant/dev1/Fresh", compute: () => false)
|
||||
.ShouldBeFalse();
|
||||
|
||||
cache.Count.ShouldBe(overshoot + 1);
|
||||
cache.TrackedCount.ShouldBe(cache.Count);
|
||||
cache.LiveShardCount.ShouldBe(cache.Count);
|
||||
}
|
||||
|
||||
private static bool Ask(
|
||||
SessionAclDecisionCache cache,
|
||||
long generation,
|
||||
|
||||
Reference in New Issue
Block a user