lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs

using System.Diagnostics;
using System.Diagnostics.Metrics;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;

namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;

/// <summary>
///     Covers <see cref="BoundedApiKeyStore"/> — the deadline and TTL cache the OPC UA impersonation
///     callback's key lookup runs under (arch-review 03/S18).
/// </summary>
/// <remarks>
///     <para>
///         The deadline cases are driven through the <b>real</b> <see cref="ApiKeyOpcUaUserAuthenticator"/>
///         rather than the store directly, because the fail-closed conversion — a throw becoming the
///         standard opaque denial — is part of what is being asserted, and it lives in the authenticator's
///         catch-all rather than in the decorator.
///     </para>
///     <para>
///         ⚠️ <b>Deadline cases use a REAL short timeout; TTL cases use the fake clock.</b> The hand-rolled
///         <see cref="FakeClock"/> overrides <c>GetUtcNow()</c> only, so <c>CreateTimer</c> falls through to
///         the base implementation and produces a real system timer — advancing the fake clock does not
///         fire a <c>CancelAfter</c>. A deadline test "improved" to advance the fake clock would pass for
///         the wrong reason: the inner task simply never completing.
///     </para>
/// </remarks>
public sealed class BoundedApiKeyStoreTests
{
    private const string Pepper = "unit-test-pepper";
    private const string Principal = "scadabridge-dcl";

    /// <summary>
    ///     A store that never answers must not hold the activation open: the deadline fires and the
    ///     activation is denied.
    /// </summary>
    /// <remarks>
    ///     The SDK raises impersonation under a <c>SessionManager</c>-wide lock, so a lookup that does not
    ///     return stalls every session activation in the server — not just this one. That is the whole of
    ///     03/S18, and it is why the bound is a deadline rather than a cache.
    /// </remarks>
    [Fact]
    public async Task A_blocked_store_denies_without_hanging_the_activation()
    {
        var (presented, inner) = SetUp();
        inner.BlockForever = true;
        var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));

        var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None)
            .WaitAsync(TimeSpan.FromSeconds(5));

        result.Success.ShouldBeFalse();
        result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage);
    }

    /// <summary>
    ///     The positive control for <see cref="A_blocked_store_denies_without_hanging_the_activation"/>:
    ///     the same fake, answering normally, still authenticates.
    /// </summary>
    /// <remarks>
    ///     Without it a build whose decorator denied <b>everything</b> would pass every deadline case in
    ///     this class, and the bound would look correct while having locked the fleet out.
    /// </remarks>
    [Fact]
    public async Task A_reachable_store_still_authenticates()
    {
        var (presented, inner) = SetUp();
        var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));

        var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);

        result.Success.ShouldBeTrue();
        result.Principal.ShouldBe(Principal);
    }

    /// <summary>
    ///     A lookup that completes with a perfectly valid key <b>after</b> the deadline still denies —
    ///     nothing races back in past the bound.
    /// </summary>
    [Fact]
    public async Task A_deadline_never_admits()
    {
        var (presented, inner) = SetUp();
        var gate = new TaskCompletionSource();
        inner.Gate = gate.Task;
        var auth = Auth(Bounded(inner, timeoutMs: 100, cacheSeconds: 0));

        var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);

        // Let the deadline pass, THEN hand the inner store its (valid) answer.
        await Task.Delay(400);
        gate.SetResult();

        var result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
        result.Success.ShouldBeFalse();
        result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage);
    }

    // ---------- the deadline is HARD, not cooperative (S18 live-gate follow-up, 2026-08-20) ----------

    /// <summary>
    ///     An inner store that <b>ignores its cancellation token</b> is abandoned at the deadline anyway:
    ///     control returns to the activation inside the bound, not when the store eventually answers.
    /// </summary>
    /// <remarks>
    ///     <para>
    ///         ⚠️ <b>This is the live-gate failure, and every other deadline case in this class misses
    ///         it.</b> They drive <see cref="GatedApiKeyStore"/>, whose waits take the token, so a
    ///         cooperative <c>CancelAfter</c> is enough to unblock them. Real
    ///         <c>Microsoft.Data.SqlClient</c> is not cooperative while a connect or login is in flight on
    ///         a black-holed socket: the 2026-08-20 gate paused the SQL container and watched a keyed
    ///         activation hang <b>42.9 s</b> past a 2000 ms bound (the Warning fired only at unpause), with
    ///         an unrelated open subscription stalled 57 s behind the <c>SessionManager</c> lock.
    ///     </para>
    ///     <para>
    ///         The wall-clock assertion is what makes this test about the bound rather than about the
    ///         exception type: a cooperative implementation does not throw late here, it does not return
    ///         at all, so the test-side <c>WaitAsync</c> is the red.
    ///     </para>
    /// </remarks>
    [Fact]
    public async Task An_uncancellable_store_is_abandoned_at_the_deadline()
    {
        var (_, inner) = SetUpUncooperative();
        var sut = Bounded(inner, timeoutMs: 200, cacheSeconds: 5);

        var started = Stopwatch.StartNew();
        await Should.ThrowAsync<OperationCanceledException>(
            () => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default)
                .WaitAsync(TimeSpan.FromSeconds(2)));
        started.Stop();

        started.Elapsed.ShouldBeLessThan(
            TimeSpan.FromSeconds(2),
            "the deadline must return control on its own, not wait for the store to answer");

        // The abandoned lookup is released only now — the assertions above already hold without it.
        inner.Release();
    }

    /// <summary>
    ///     A lookup abandoned at the deadline must never land in the cache when it completes later — the
    ///     next activation re-reads the store.
    /// </summary>
    /// <remarks>
    ///     Caching it would be worse than caching a timeout: the answer was taken before the deadline and
    ///     is served after it, so a key revoked or expired in the meantime would be admitted from a
    ///     lookup nobody was still waiting for. The call count is the assertion — a build that stored the
    ///     late result would serve the second call from the cache and leave <c>Calls</c> at 1.
    /// </remarks>
    [Fact]
    public async Task An_abandoned_lookup_that_completes_later_is_not_cached()
    {
        var (_, inner) = SetUpUncooperative();
        var sut = Bounded(inner, timeoutMs: 200, cacheSeconds: 60);

        await Should.ThrowAsync<OperationCanceledException>(
            () => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default)
                .WaitAsync(TimeSpan.FromSeconds(2)));

        // The abandoned inner task now completes, with a perfectly valid key, into nobody's hands.
        inner.Release();
        await inner.Completed.WaitAsync(TimeSpan.FromSeconds(2));

        // Well inside the 60 s TTL: a cached late answer would make this a hit.
        (await sut.FindAsync(inner.KeyId, DateTime.UtcNow, default).WaitAsync(TimeSpan.FromSeconds(2)))
            .ShouldNotBeNull("the store answers immediately once released");
        inner.Calls.ShouldBe(2, "the abandoned lookup must not have populated the cache");
    }

    /// <summary>
    ///     An abandoned lookup that later <b>faults</b> has its exception observed and swallowed, so it
    ///     can never surface as an <c>UnobservedTaskException</c> on the finalizer thread.
    /// </summary>
    /// <remarks>
    ///     Nobody awaits the abandoned task, so without an explicit continuation its fault is unobserved —
    ///     which on some host configurations escalates. The Debug line is the observable proof the
    ///     continuation ran; asserting the absence of a finalizer-thread event instead would depend on GC
    ///     timing and prove nothing on a green run.
    /// </remarks>
    [Fact]
    public async Task An_abandoned_lookup_that_faults_later_is_observed_and_swallowed()
    {
        var (_, inner) = SetUpUncooperative();
        var log = new CapturingLogger();
        var sut = Bounded(inner, timeoutMs: 200, cacheSeconds: 0, log: log);

        await Should.ThrowAsync<OperationCanceledException>(
            () => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default)
                .WaitAsync(TimeSpan.FromSeconds(2)));

        inner.ReleaseWithFault(new InvalidOperationException("SqlClient gave up"));
        await inner.Completed.WaitAsync(TimeSpan.FromSeconds(2));

        // The continuation is a separate task from the one just awaited; give it a moment to run.
        var deadline = Stopwatch.StartNew();
        while (log.Lines.Count == 0 && deadline.Elapsed < TimeSpan.FromSeconds(2))
            await Task.Delay(10);

        log.Lines.ShouldContain(line => line.Contains("abandoned", StringComparison.OrdinalIgnoreCase));
    }

    /// <summary>
    ///     A non-positive <c>StoreTimeoutMs</c> disables the deadline, restoring the unbounded behaviour
    ///     byte-for-byte — the documented rollback.
    /// </summary>
    [Fact]
    public async Task A_zero_timeout_disables_the_deadline()
    {
        var (presented, inner) = SetUp();
        var gate = new TaskCompletionSource();
        inner.Gate = gate.Task;
        var auth = Auth(Bounded(inner, timeoutMs: 0, cacheSeconds: 0));

        var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);

        // Far longer than any deadline this class configures; an active bound would have denied by now.
        await Task.Delay(300);
        pending.IsCompleted.ShouldBeFalse("a disabled deadline must not cancel a slow lookup");

        gate.SetResult();
        (await pending.WaitAsync(TimeSpan.FromSeconds(5))).Success.ShouldBeTrue();
    }

    /// <summary>The anonymous activation path gets the same deadline, and the same denial.</summary>
    /// <remarks>
    ///     It runs on every unauthenticated connect, so leaving it unbounded would leave the
    ///     server-wide-stall hazard fully open on the busiest activation path.
    /// </remarks>
    [Fact]
    public async Task Anonymous_grant_resolution_is_bounded_too()
    {
        var (_, inner) = SetUp();
        inner.BlockForever = true;
        var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));

        var result = await auth.AuthenticateAnonymousAsync(CancellationToken.None)
            .WaitAsync(TimeSpan.FromSeconds(5));

        result.Success.ShouldBeFalse();
    }

    // ---------- TTL read-through cache ----------

    /// <summary>A second activation inside the TTL is served from the cache and never reaches the store.</summary>
    [Fact]
    public async Task A_second_activation_within_the_TTL_does_not_reach_the_store()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);

        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
        clock.Advance(TimeSpan.FromSeconds(2));
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();

        inner.Calls.ShouldBe(1);
    }

    /// <summary>
    ///     FALSIFICATION CONTROL. A key revoked in the store is denied once the TTL has passed — the
    ///     cache expires rather than merely existing.
    /// </summary>
    /// <remarks>
    ///     A cache with no expiry at all passes every other case in this class: the hit test, the
    ///     read-through test and the fault test are all satisfied by "store once, serve forever". This is
    ///     the one that makes <c>StoreCacheSeconds</c> a bound rather than a claim.
    /// </remarks>
    [Fact]
    public async Task A_revoked_key_is_denied_once_the_TTL_has_passed()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);

        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default))
            .ShouldNotBeNull("non-vacuity: the key really was usable before the revocation");

        inner.Revoked = true;

        // Still inside the TTL — the documented, accepted staleness window.
        clock.Advance(TimeSpan.FromSeconds(3));
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();

        // Past it — the store is re-read and the revocation lands.
        clock.Advance(TimeSpan.FromSeconds(3));
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
        inner.Calls.ShouldBe(2);
    }

    /// <summary>A non-positive TTL disables the cache: every call reads through, today's behaviour.</summary>
    [Fact]
    public async Task TTL_zero_reads_through_every_time()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 0, clock);

        for (var i = 0; i < 3; i++)
            await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);

        inner.Calls.ShouldBe(3);
    }

    /// <summary>
    ///     A fault is never cached: the next call reaches the store, so an outage cannot outlive itself.
    /// </summary>
    /// <remarks>
    ///     Caching a deny would be fail-closed and is still wrong — it would go on denying valid keys for
    ///     a further TTL after the database came back.
    /// </remarks>
    [Fact]
    public async Task A_faulted_store_is_not_cached()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 60, clock);
        inner.ThrowsRemaining = 1;

        await Should.ThrowAsync<InvalidOperationException>(
            () => sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default));

        // Same instant — a cached fault would still be "fresh" here, so the clock cannot rescue this.
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
        inner.Calls.ShouldBe(2);
    }

    /// <summary>The anonymous grant lookup is cached on the same terms, keyed by principal.</summary>
    [Fact]
    public async Task Anonymous_grants_are_cached_too()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);

        await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
        await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
        inner.Calls.ShouldBe(1);

        clock.Advance(TimeSpan.FromSeconds(6));
        await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
        inner.Calls.ShouldBe(2);
    }

    /// <summary>
    ///     Past the entry ceiling the cache clears wholesale and keeps answering — a key-id guessing flood
    ///     is a bounded memory event, not an unbounded one.
    /// </summary>
    /// <remarks>
    ///     Wholesale rather than per-entry eviction for the reason <c>SessionAclDecisionCache</c> already
    ///     documents: choosing WHICH entry to drop needs knowledge the cache does not have, and a
    ///     repopulate is cheap and cannot be wrong.
    /// </remarks>
    [Fact]
    public async Task Cache_overflow_clears_wholesale_and_keeps_serving()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock);

        // Seed the real key, then flood past the ceiling with ids that do not exist.
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
        for (var i = 0; i < BoundedApiKeyStore.MaxCacheEntries + 1; i++)
            await sut.FindAsync($"ghost-{i}", clock.GetUtcNow().UtcDateTime, default);

        var callsAfterFlood = inner.Calls;

        // Still correct after the clear — and the real key was evicted, so it is re-read rather than
        // served from a cache the flood was supposed to have emptied.
        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
        inner.Calls.ShouldBe(callsAfterFlood + 1);
    }

    /// <summary>
    ///     A cached key whose <c>ExpiresUtc</c> passes <b>inside</b> the TTL is not served: the store is
    ///     re-read instead.
    /// </summary>
    /// <remarks>
    ///     <para>
    ///         Without this the TTL would resurrect an expired key for up to <c>StoreCacheSeconds</c> —
    ///         the exact axis 03/S19 exists to close, re-opened by 03/S18's own cache. The cache key is
    ///         <c>apiKeyId</c> alone (<c>utcNow</c> changes every call and would defeat the cache), so the
    ///         clock check has to happen on the way OUT of the cache.
    ///     </para>
    ///     <para>
    ///         The inner store is flipped to <c>Revoked</c> first, so "served from cache" and "re-read"
    ///         give different ANSWERS rather than only different call counts — a build that re-read and
    ///         then ignored the result could not pass.
    ///     </para>
    /// </remarks>
    [Fact]
    public async Task A_cached_key_that_expires_by_the_clock_is_not_served()
    {
        var start = new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero);
        var (_, inner) = SetUp(expiresUtc: start.UtcDateTime.AddSeconds(2));
        var clock = new FakeClock(start);
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 60, clock);

        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default))
            .ShouldNotBeNull("non-vacuity: the key was usable and IS now cached");

        inner.Revoked = true;

        // Past ExpiresUtc (+2 s) but far inside the 60 s TTL — the entry is still "fresh" to the cache.
        clock.Advance(TimeSpan.FromSeconds(5));

        (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
        inner.Calls.ShouldBe(2, "the expired entry must be re-read, not served");
    }

    // ---------- metering ----------

    /// <summary>A deadline that fires meters <c>outcome=timeout</c> — the alarm signal for a sick ConfigDb.</summary>
    [Fact]
    public async Task A_timeout_meters_outcome_timeout()
    {
        var (_, inner) = SetUp();
        inner.BlockForever = true;
        using var meter = new MeterRecorder();
        var sut = Bounded(inner, timeoutMs: 100, cacheSeconds: 0);

        await Should.ThrowAsync<OperationCanceledException>(
            () => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default).WaitAsync(TimeSpan.FromSeconds(5)));

        meter.WithOutcome("timeout").ShouldBe(1);
        meter.WithOutcome("cache_hit").ShouldBe(0);
    }

    /// <summary>
    ///     A cache hit meters <c>outcome=cache_hit</c> — the ratio that tells an operator the TTL is
    ///     doing something. A missing stream on a busy fused central is how you notice
    ///     <c>StoreCacheSeconds</c> was set to 0 by accident.
    /// </summary>
    [Fact]
    public async Task A_cache_hit_meters_outcome_cache_hit()
    {
        var (_, inner) = SetUp();
        var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
        using var meter = new MeterRecorder();
        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);

        await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);
        meter.WithOutcome("cache_hit").ShouldBe(0, "the first call is a miss");

        await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);
        meter.WithOutcome("cache_hit").ShouldBe(1);
    }

    /// <summary>A plain read-through — no deadline hit, no cache hit — meters nothing at all.</summary>
    /// <remarks>
    ///     The counter names lookups the bound INTERVENED on. Metering every lookup would bury the two
    ///     outcomes that matter under the ordinary traffic they have to be spotted in.
    /// </remarks>
    [Fact]
    public async Task A_plain_read_through_meters_neither()
    {
        var (_, inner) = SetUp();
        using var meter = new MeterRecorder();
        var sut = Bounded(inner, timeoutMs: 2000, cacheSeconds: 0);

        await sut.FindAsync(inner.KeyId, DateTime.UtcNow, default);

        meter.Total.ShouldBe(0);
    }

    // ---------- helpers ----------

    private static BoundedApiKeyStore Bounded(
        IApiKeyStore inner,
        int timeoutMs,
        int cacheSeconds,
        TimeProvider? clock = null,
        ILogger<BoundedApiKeyStore>? log = null) => new(
        inner,
        Options.Create(new ApiKeyOptions
        {
            Username = "apikey",
            Pepper = Pepper,
            StoreTimeoutMs = timeoutMs,
            StoreCacheSeconds = cacheSeconds,
        }),
        log ?? NullLogger<BoundedApiKeyStore>.Instance,
        clock ?? TimeProvider.System);

    private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new(
        store,
        Options.Create(new ApiKeyOptions { Username = "apikey", Pepper = Pepper }),
        NullLogger<ApiKeyOpcUaUserAuthenticator>.Instance);

    private static (string Presented, GatedApiKeyStore Store) SetUp(DateTime? expiresUtc = null)
    {
        var (presented, apiKeyId, hash) = ApiKeySecret.Generate(Pepper);
        var key = new ApiKey
        {
            ApiKeyId = apiKeyId,
            Name = "test key",
            SecretHash = hash,
            Principal = Principal,
            Enabled = true,
            CreatedUtc = DateTime.UtcNow,
            ExpiresUtc = expiresUtc,
        };
        return (presented, new GatedApiKeyStore(key));
    }

    /// <summary>Builds the same key in front of an inner store that does NOT honour cancellation.</summary>
    /// <returns>The presented credential and the uncooperative store.</returns>
    private static (string Presented, UncooperativeApiKeyStore Store) SetUpUncooperative()
    {
        var (presented, apiKeyId, hash) = ApiKeySecret.Generate(Pepper);
        return (presented, new UncooperativeApiKeyStore(new ApiKey
        {
            ApiKeyId = apiKeyId,
            Name = "test key",
            SecretHash = hash,
            Principal = Principal,
            Enabled = true,
            CreatedUtc = DateTime.UtcNow,
        }));
    }

    /// <summary>
    ///     An inner store whose answer can be withheld — a hang the EF in-memory provider cannot
    ///     represent — and whose calls are counted.
    /// </summary>
    private sealed class GatedApiKeyStore(ApiKey key) : IApiKeyStore
    {
        /// <summary>The id of the key this store holds.</summary>
        public string KeyId => key.ApiKeyId;

        /// <summary>When set, every call awaits this before answering.</summary>
        public Task? Gate { get; set; }

        /// <summary>When true, no call ever completes.</summary>
        public bool BlockForever { get; set; }

        /// <summary>How many calls reached this store (i.e. were not served from the cache).</summary>
        public int Calls => _calls;

        /// <summary>When true, the key lookup answers "no usable key" — a revocation.</summary>
        public bool Revoked { get; set; }

        /// <summary>Number of leading calls that throw before the store starts answering.</summary>
        public int ThrowsRemaining { get; set; }

        private int _calls;

        public async Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
        {
            await WaitAsync(ct).ConfigureAwait(false);
            if (Revoked) return null;
            return string.Equals(apiKeyId, key.ApiKeyId, StringComparison.Ordinal)
                ? new ApiKeyLookup(key, [])
                : null;
        }

        public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
        {
            await WaitAsync(ct).ConfigureAwait(false);
            return [];
        }

        private async Task WaitAsync(CancellationToken ct)
        {
            Interlocked.Increment(ref _calls);

            if (ThrowsRemaining > 0)
            {
                ThrowsRemaining--;
                throw new InvalidOperationException("store is down");
            }

            if (BlockForever) await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
            if (Gate is { } gate) await gate.WaitAsync(ct).ConfigureAwait(false);
        }
    }

    /// <summary>
    ///     An inner store that <b>ignores its cancellation token entirely</b> — the shape real
    ///     <c>Microsoft.Data.SqlClient</c> takes on a black-holed connection, and the one every other fake
    ///     in this class cannot represent.
    /// </summary>
    /// <remarks>
    ///     <see cref="GatedApiKeyStore"/> passes the token into every wait, so a cooperative deadline
    ///     unblocks it and a cooperative implementation looks correct. This one only ever completes when
    ///     the test releases it, so the deadline has to be a real race to return at all.
    /// </remarks>
    private sealed class UncooperativeApiKeyStore(ApiKey key) : IApiKeyStore
    {
        private readonly TaskCompletionSource _gate =
            new(TaskCreationOptions.RunContinuationsAsynchronously);
        private readonly TaskCompletionSource _completed =
            new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _calls;

        /// <summary>The id of the key this store holds.</summary>
        public string KeyId => key.ApiKeyId;

        /// <summary>How many calls reached this store (i.e. were not served from the cache).</summary>
        public int Calls => _calls;

        /// <summary>Completes once an in-flight inner call has run to completion or fault.</summary>
        public Task Completed => _completed.Task;

        /// <summary>Lets the in-flight (and every subsequent) call answer normally.</summary>
        public void Release() => _gate.TrySetResult();

        /// <summary>Lets the in-flight call fail, the way SqlClient eventually gives up on its own.</summary>
        /// <param name="error">The fault to surface from the abandoned inner task.</param>
        public void ReleaseWithFault(Exception error) => _gate.TrySetException(error);

        public async Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
        {
            Interlocked.Increment(ref _calls);
            try
            {
                // ⚠️ ct is deliberately NOT passed — that is the whole point of this fake.
                await _gate.Task.ConfigureAwait(false);
            }
            finally
            {
                _completed.TrySetResult();
            }

            return string.Equals(apiKeyId, key.ApiKeyId, StringComparison.Ordinal)
                ? new ApiKeyLookup(key, [])
                : null;
        }

        public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
        {
            Interlocked.Increment(ref _calls);
            try
            {
                await _gate.Task.ConfigureAwait(false);
            }
            finally
            {
                _completed.TrySetResult();
            }

            return [];
        }
    }

    /// <summary>Collects rendered log messages so a "logged and swallowed" claim can be asserted.</summary>
    private sealed class CapturingLogger : ILogger<BoundedApiKeyStore>
    {
        private readonly List<string> _lines = [];
        private readonly Lock _gate = new();

        /// <summary>Every line rendered so far.</summary>
        public IReadOnlyList<string> Lines
        {
            get { lock (_gate) return [.. _lines]; }
        }

        /// <inheritdoc />
        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;

        /// <inheritdoc />
        public bool IsEnabled(LogLevel logLevel) => true;

        /// <inheritdoc />
        public void Log<TState>(
            LogLevel logLevel,
            EventId eventId,
            TState state,
            Exception? exception,
            Func<TState, Exception?, string> formatter)
        {
            lock (_gate) _lines.Add(formatter(state, exception));
        }
    }

    /// <summary>Collects <c>otopcua.opcua.apikey_store.bounded</c> measurements by their outcome tag.</summary>
    /// <remarks>
    ///     ⚠️ Filtered by instrument name only, which is normally the metrics-test leak (a listener sees
    ///     a PARALLEL test's measurements too). It is safe HERE for two checkable reasons:
    ///     <see cref="BoundedApiKeyStore"/> is the only thing in the product that writes this instrument,
    ///     and this assembly declares <c>CollectionBehavior(DisableTestParallelization = true)</c> in
    ///     <c>AssemblyInfo.cs</c>, so there is no parallel test to leak from. Removing that attribute
    ///     means adding a discriminating tag here.
    /// </remarks>
    private sealed class MeterRecorder : IDisposable
    {
        private readonly MeterListener _listener;
        private readonly List<KeyValuePair<string, object?>[]> _tagSets = [];
        private readonly Lock _gate = new();
        private long _total;

        public MeterRecorder()
        {
            _listener = new MeterListener
            {
                InstrumentPublished = (instrument, listener) =>
                {
                    if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName
                        && instrument.Name == "otopcua.opcua.apikey_store.bounded")
                    {
                        listener.EnableMeasurementEvents(instrument);
                    }
                },
            };
            _listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
            {
                lock (_gate)
                {
                    _total += value;
                    _tagSets.Add(tags.ToArray());
                }
            });
            _listener.Start();
        }

        /// <summary>Every measurement seen, regardless of tags.</summary>
        public long Total
        {
            get { lock (_gate) return _total; }
        }

        /// <summary>How many measurements carried <c>outcome=<paramref name="outcome"/></c>.</summary>
        /// <param name="outcome">The outcome tag value to count.</param>
        /// <returns>The measurement count.</returns>
        public int WithOutcome(string outcome)
        {
            lock (_gate)
                return _tagSets.Count(set => set.Any(t => t.Key == "outcome" && Equals(t.Value, outcome)));
        }

        /// <inheritdoc />
        public void Dispose() => _listener.Dispose();
    }

    /// <summary>
    ///     A hand-advanced clock. Overrides <c>GetUtcNow()</c> only — see the class remarks for why that
    ///     confines it to the TTL cases.
    /// </summary>
    private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider
    {
        private DateTimeOffset _now = utcNow;

        /// <inheritdoc />
        public override DateTimeOffset GetUtcNow() => _now;

        /// <summary>Moves the clock forward.</summary>
        /// <param name="by">How far forward to move.</param>
        public void Advance(TimeSpan by) => _now = _now.Add(by);
    }
}