Merge fix/832-docker-dev-batch (#832 #833 #825 #823)

2026-11-11 02:44:01 -05:00
parent 9c0cdc05ef 2314a53842
commit 2a87d96fac
23 changed files with 1450 additions and 27 deletions
@@ -311,12 +311,14 @@ else
        <ScriptedAlarmModal Visible="_alarmModalVisible" IsNew="_alarmModalIsNew" EquipmentId="@EquipmentId"
                            Existing="_alarmModalExisting" Scripts="_alarmScriptOptions"
                            OnSaved="OnAlarmSavedAsync" OnCancel="@(() => { _alarmModalVisible = false; })" />
-
-        @* One confirm for every destructive row action on this page (R04 H1) — the same RawConfirmDialog
-           /raw routes its deletes through. A mis-click used to delete immediately. *@
-        <RawConfirmDialog @bind-Visible="_confirmVisible" Title="@_confirmTitle" Message="@_confirmMessage"
-                          ConfirmLabel="@_confirmLabel" OnConfirm="RunConfirmedAsync" />
    }
+
+    @* One confirm for every destructive row action on this page (R04 H1) — the same RawConfirmDialog
+       /raw routes its deletes through. A mis-click used to delete immediately. It must render OUTSIDE the
+       tab branches (#832): inside the Alarms branch it existed only on that tab, so the Tags tab's Remove
+       and the Virtual Tags tab's Delete opened a dialog that was never on the page. *@
+    <RawConfirmDialog @bind-Visible="_confirmVisible" Title="@_confirmTitle" Message="@_confirmMessage"
+                      ConfirmLabel="@_confirmLabel" OnConfirm="RunConfirmedAsync" />
 }
 </PageErrorBoundary>

@@ -12,7 +12,16 @@

    @if (_result is not null)
    {
-        @if (_result.Ok)
+        @if (_result.Ok && WarningOf(_result.Message) is { } warning)
+        {
+            @* Reachable but misconfigured (e.g. MTConnect SampleCount > the Agent's bufferSize, #825): a probe
+               that answers Ok with a "WARNING:" message is shown amber with the warning, never a plain green OK. *@
+            <span class="chip chip-warn" title="@warning">
+                OK with warning &middot; @_result.LatencyMs?.ToString("F0") ms
+            </span>
+            <div class="small text-warning mt-1">@warning</div>
+        }
+        else if (_result.Ok)
        {
            <span class="chip chip-ok" title="@($"Probe succeeded in {_result.LatencyMs:F0} ms")">
                OK &middot; @_result.LatencyMs?.ToString("F0") ms
@@ -28,6 +37,16 @@
 </div>

@code {
+    /// <summary>The marker a probe puts in an <c>Ok</c> message to flag a warning (see <c>MTConnectDriverProbe</c>).</summary>
+    internal const string WarningMarker = "WARNING:";
+
+    /// <summary>The warning text after <see cref="WarningMarker"/>, or null when the message carries none.</summary>
+    internal static string? WarningOf(string? message)
+    {
+        var at = message?.IndexOf(WarningMarker, StringComparison.Ordinal) ?? -1;
+        return at < 0 ? null : message![(at + WarningMarker.Length)..].Trim();
+    }
+
    /// <summary>Driver type key — must match an installed <c>IDriverProbe.DriverType</c>.</summary>
    [Parameter, EditorRequired] public string DriverType { get; set; } = "";

@@ -29,5 +29,15 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
        // IReadOnlyCollection<T>, so it can't bind to MinCount's IReadOnlyCollection<T> parameter
        // directly. ToList() bridges to the shared primitive while preserving the count (and message).
        builder.MinCount(o.EnabledSecurityProfiles?.ToList(), 1, "OpcUa:EnabledSecurityProfiles");
+
+        // #833 review: an enabled credential lockout needs positive durations — a zero LockoutSeconds would lock
+        // nothing, a zero FailureWindowSeconds would never accumulate. MaxFailedAttempts <= 0 disables it.
+        var lockout = o.CredentialLockout;
+        builder.RequireThat(lockout is null || lockout.MaxFailedAttempts <= 0
+                            || (lockout.LockoutSeconds > 0 && lockout.FailureWindowSeconds > 0 && lockout.SlotWaitSeconds > 0),
+            "OpcUa:CredentialLockout:LockoutSeconds, OpcUa:CredentialLockout:FailureWindowSeconds and "
+            + "OpcUa:CredentialLockout:SlotWaitSeconds must all be "
+            + "greater than 0 while OpcUa:CredentialLockout:MaxFailedAttempts is greater than 0 (set "
+            + "MaxFailedAttempts to 0 to disable the lockout)");
    }
 }
@@ -113,6 +113,12 @@ public sealed class OpcUaApplicationHostOptions
    ///     own transport defaults, so an absent section changes nothing.
    /// </summary>
    public OpcUaQuotaOptions Quotas { get; set; } = new();
+
+    /// <summary>
+    ///     Failed-credential lockout for UserName activations (<c>OpcUa:CredentialLockout:*</c>, lmxopcua#833).
+    ///     Replaces the SDK's own lockout, which also refused Anonymous; see <see cref="FailedCredentialLockout"/>.
+    /// </summary>
+    public OpcUaCredentialLockoutOptions CredentialLockout { get; set; } = new();
 }

 /// <summary>
@@ -175,6 +181,7 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
    private StandardServer? _server;
    private readonly ITelemetryContext _telemetry = DefaultTelemetry.Create(_ => { });
    private readonly List<IUserTokenAuthenticator> _identityAuthenticators = [];
+    private readonly FailedCredentialLockout _credentialLockout;
    private readonly object _lifetimeLock = new();
    private readonly TaskCompletionSource _disposeCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously);
    private int _disposeStarted;
@@ -191,6 +198,7 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
        _options = options;
        _logger = logger;
        _userAuthenticator = userAuthenticator ?? NullOpcUaUserAuthenticator.Instance;
+        _credentialLockout = new FailedCredentialLockout(options.CredentialLockout ?? new OpcUaCredentialLockoutOptions());
    }

    /// <summary>Gets the OPC Foundation application instance, or null if not yet started.</summary>
@@ -260,14 +268,73 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
            ct.ThrowIfCancellationRequested();
            var args = new ImpersonateEventArgs(context.TokenHandler, context.UserTokenPolicy, context.EndpointDescription);
            var peer = (host._server as OtOpcUaSdkServer)?.ActivationPeerAddress;
-            // Port zero is only an adapter for the existing IP-budget seam: the SDK exposes no peer port.
-            await HandleImpersonationAsync(host._userAuthenticator, args, host._logger, host._options.AllowAnonymous,
-                peer is null ? null : new System.Net.IPEndPoint(peer, 0), ct).ConfigureAwait(false);
-            return ServiceResult.IsBad(args.IdentityValidationError)
-                ? AuthenticationResult.Reject(args.IdentityValidationError)
-                : args.Identity is { } identity
-                    ? AuthenticationResult.Accept(identity)
-                    : AuthenticationResult.Reject(new ServiceResult(StatusCodes.BadIdentityTokenRejected));
+
+            // lmxopcua#833 — only credential-bearing (UserName) activations are counted and refused. Anonymous
+            // never consults the lockout, so failed sign-ins cannot lock a client (or its NAT neighbours) out of
+            // anonymous access the way the SDK's own lockout did.
+            var lockoutKey = tokenType == UserTokenType.UserName
+                ? FailedCredentialLockout.KeyFor(context.ChannelCertificate?.Thumbprint, peer)
+                : null;
+            if (lockoutKey is not null)
+            {
+                // Waits (bounded) for one of this client's verification slots — parallel activations queue, they are
+                // never refused for parallelism. Refused only when locked out, or (uncounted) when no slot freed in time.
+                var decision = await host._credentialLockout.BeginAsync(lockoutKey, ct).ConfigureAwait(false);
+                if (decision.Verdict == CredentialLockoutVerdict.LockedOut)
+                {
+                    // Logged once per lockout episode at Information; every further refusal at Debug, so a client
+                    // hammering a lockout cannot flood the log (#833 review).
+                    var seconds = (long)Math.Ceiling(decision.Remaining.TotalSeconds);
+                    host._logger.Log(decision.FirstRefusal ? LogLevel.Information : LogLevel.Debug,
+                        "OpcUaApplicationHost: UserName activation refused — client {Client} is locked out for {Seconds}s more",
+                        lockoutKey, seconds);
+                    return AuthenticationResult.Reject(ServiceResult.Create(StatusCodes.BadUserAccessDenied,
+                        $"Too many failed authentication attempts. Try again in {seconds} seconds."));
+                }
+
+                if (decision.Verdict == CredentialLockoutVerdict.SlotWaitTimedOut)
+                {
+                    host._logger.LogWarning(
+                        "OpcUaApplicationHost: UserName activation for client {Client} timed out waiting for a "
+                        + "verification slot (OpcUa:CredentialLockout:SlotWaitSeconds); not counted as a failure",
+                        lockoutKey);
+                    return AuthenticationResult.Reject(ServiceResult.Create(StatusCodes.BadTimeout,
+                        "Timed out waiting to verify the credential. Try again."));
+                }
+            }
+
+            // The slot BeginAsync granted is released on every path — completed with the outcome, or abandoned
+            // (uncounted) when the activation was cancelled or threw.
+            var completed = false;
+            try
+            {
+                // Port zero is only an adapter for the existing IP-budget seam: the SDK exposes no peer port.
+                await HandleImpersonationAsync(host._userAuthenticator, args, host._logger, host._options.AllowAnonymous,
+                    peer is null ? null : new System.Net.IPEndPoint(peer, 0), ct).ConfigureAwait(false);
+                var result = ServiceResult.IsBad(args.IdentityValidationError)
+                    ? AuthenticationResult.Reject(args.IdentityValidationError)
+                    : args.Identity is { } identity
+                        ? AuthenticationResult.Accept(identity)
+                        : AuthenticationResult.Reject(new ServiceResult(StatusCodes.BadIdentityTokenRejected));
+
+                if (lockoutKey is not null)
+                {
+                    var admitted = args.Identity is not null && !ServiceResult.IsBad(args.IdentityValidationError);
+                    completed = true;
+                    if (host._credentialLockout.Complete(lockoutKey, admitted))
+                    {
+                        host._logger.LogWarning(
+                            "OpcUaApplicationHost: client {Client} locked out of UserName activation for {Seconds}s after "
+                            + "{Attempts} failed attempts (OpcUa:CredentialLockout); anonymous activation is unaffected",
+                            lockoutKey, host._options.CredentialLockout.LockoutSeconds, host._options.CredentialLockout.MaxFailedAttempts);
+                    }
+                }
+                return result;
+            }
+            finally
+            {
+                if (lockoutKey is not null && !completed) host._credentialLockout.Abandon(lockoutKey);
+            }
        }
    }

@@ -488,7 +555,9 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
    {
        if (!string.IsNullOrWhiteSpace(_options.ApplicationConfigPath))
        {
-            return await _application!.LoadApplicationConfigurationAsync(_options.ApplicationConfigPath, true, ct);
+            var loaded = await _application!.LoadApplicationConfigurationAsync(_options.ApplicationConfigPath, true, ct);
+            if (loaded.ServerConfiguration is { } loadedServer) DisableSdkAuthenticationLockout(loadedServer);
+            return loaded;
        }

        var serverConfig = new ServerConfiguration
@@ -496,6 +565,7 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
            BaseAddresses = [$"opc.tcp://{_options.PublicHostname}:{_options.OpcUaPort}/OtOpcUa"],
        };
        ApplyRequestQuotas(serverConfig, _options.Quotas);
+        DisableSdkAuthenticationLockout(serverConfig);

        serverConfig.SecurityPolicies = BuildSecurityPolicies(_options.EnabledSecurityProfiles).ToArray();
        serverConfig.UserTokenPolicies = BuildUserTokenPolicies(_options.AllowAnonymous).ToArray();
@@ -531,6 +601,15 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
        return config;
    }

+    /// <summary>
+    ///     Switches off the SDK's per-client failed-authentication lockout (lmxopcua#833). It refuses EVERY
+    ///     activation from a locked client — Anonymous included — for 5 minutes; <see cref="FailedCredentialLockout"/>
+    ///     enforces the UserName-only replacement.
+    /// </summary>
+    /// <param name="serverConfig">The configuration to set.</param>
+    internal static void DisableSdkAuthenticationLockout(ServerConfiguration serverConfig) =>
+        serverConfig.MaxFailedAuthenticationAttempts = 0;
+
    /// <summary>
    ///     Applies the request-processing quotas (<c>OpcUa:Quotas:*</c>, review 01 P6 — they were hard-coded at
    ///     5 / 100 / 200) to the SDK server configuration.
@@ -0,0 +1,414 @@
+namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
+
+/// <summary>
+///     The <c>OpcUa:CredentialLockout</c> section: how many failed UserName activations one client may make
+///     before its UserName activations are refused for a while (lmxopcua#833).
+/// </summary>
+/// <remarks>
+///     The defaults are the OPC Foundation SDK 2.0.1 <c>SessionManager</c> lockout this replaces: 5 failures,
+///     each inside 60 s of the previous one, lock the client out for 5 minutes. When enabled
+///     (<see cref="MaxFailedAttempts"/> &gt; 0) every duration must be positive — enforced at host start by
+///     <c>OpcUaApplicationHostOptionsValidator</c>.
+/// </remarks>
+public sealed class OpcUaCredentialLockoutOptions
+{
+    /// <summary>Failed UserName activations that lock a client out; <c>0</c> or less disables the lockout.</summary>
+    public int MaxFailedAttempts { get; set; } = 5;
+
+    /// <summary>A failure more than this many seconds after the previous one starts a fresh count.</summary>
+    public int FailureWindowSeconds { get; set; } = 60;
+
+    /// <summary>How long a locked-out client's UserName activations are refused, in seconds.</summary>
+    public int LockoutSeconds { get; set; } = 300;
+
+    /// <summary>
+    ///     How long a UserName activation waits for one of its client's verification slots before that one attempt
+    ///     is failed (<c>BadTimeout</c>, NOT counted as a credential failure). Only reached when a client has more
+    ///     verifications in progress than <see cref="MaxFailedAttempts"/> minus its recorded failures; a verification
+    ///     is milliseconds, so in practice this bounds a pathological store stall. Kept well under the SDK's
+    ///     operation timeout so the client gets this answer rather than a transport timeout.
+    /// </summary>
+    public int SlotWaitSeconds { get; set; } = 10;
+}
+
+/// <summary>The outcome of <see cref="FailedCredentialLockout.BeginAsync"/>.</summary>
+public enum CredentialLockoutVerdict
+{
+    /// <summary>A verification slot is held — authenticate, then call <see cref="FailedCredentialLockout.Complete"/>.</summary>
+    Proceed,
+
+    /// <summary>The client is locked out; refuse without authenticating.</summary>
+    LockedOut,
+
+    /// <summary>No slot freed within the wait; fail this one attempt, uncounted.</summary>
+    SlotWaitTimedOut,
+}
+
+/// <summary>What <see cref="FailedCredentialLockout.BeginAsync"/> decided.</summary>
+/// <param name="Verdict">Proceed, locked out, or timed out waiting.</param>
+/// <param name="Remaining">Time left on the lockout when <see cref="CredentialLockoutVerdict.LockedOut"/>.</param>
+/// <param name="FirstRefusal">True for the first refusal of a lockout episode (log it at Information, the rest at Debug).</param>
+public readonly record struct CredentialLockoutDecision(
+    CredentialLockoutVerdict Verdict, TimeSpan Remaining, bool FirstRefusal);
+
+/// <summary>
+///     Brute-force lockout for UserName (API-key) activations, scoped so it can never refuse an
+///     <b>anonymous</b> activation (lmxopcua#833).
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Why this exists instead of the SDK's own lockout.</b> OPC Foundation SDK 2.0.1's
+///         <c>SessionManager.ActivateSessionAsync</c> counts every failed identity validation per client and,
+///         at <c>ServerConfiguration.MaxFailedAuthenticationAttempts</c> (default 5), refuses <b>every</b>
+///         activation from that client for 5 minutes with <c>BadUserAccessDenied</c> — the lockout check runs
+///         before the token is even looked at, so it refuses Anonymous too, and a successful anonymous
+///         activation does not clear it. Its key is the client certificate thumbprint when one is presented,
+///         else the peer IP: on a <c>None</c> endpoint the presented certificate is not proven, so a client
+///         could lock out another client's sessions by presenting its certificate, and behind a NAT every
+///         certificate-less client shares one key. The host therefore sets
+///         <c>MaxFailedAuthenticationAttempts = 0</c> and enforces this narrower lockout instead.
+///     </para>
+///     <para>
+///         <b>What it does.</b> Only UserName activations are counted and refused; Anonymous never consults it.
+///         The key is the presented client certificate thumbprint <b>and</b> the observed peer address together.
+///         ⚠️ That only separates clients that present a certificate: on a <c>None</c> endpoint clients commonly
+///         present none, so the key is <b>effectively the IP alone</b> and NAT neighbours there share one count
+///         (for UserName activations only — never for anonymous ones); and when the transport reports no peer
+///         address, every certificate-less client shares the single key <c>-@-</c>. Timing: a failure within
+///         <see cref="OpcUaCredentialLockoutOptions.FailureWindowSeconds"/> of the previous one adds to the
+///         count; at <see cref="OpcUaCredentialLockoutOptions.MaxFailedAttempts"/> the client is refused for
+///         <see cref="OpcUaCredentialLockoutOptions.LockoutSeconds"/>; a refusal during the lockout is not
+///         counted (the lockout never extends itself); an expired lockout starts a fresh count (whatever the
+///         window); a successful UserName activation clears the client's count.
+///     </para>
+///     <para>
+///         <b>Concurrency — parallelism waits, it is never refused.</b> The authenticate call is awaited between the
+///         check and the outcome, so a check and a separate record would let a parallel burst of guesses all pass
+///         before the first failure lands. <see cref="BeginAsync"/> therefore hands out at most
+///         <c>MaxFailedAttempts − recordedFailures</c> verification <b>slots</b> per key; an activation beyond that
+///         <b>waits</b> (FIFO) for a slot, and <see cref="Complete"/> / <see cref="Abandon"/> release it and wake the
+///         next waiter. So no more than <c>MaxFailedAttempts</c> unverified guesses are ever in progress-or-failed
+///         per window, while a legitimate client opening many keyed sessions at once (a ScadaBridge site node behind
+///         one NAT IP) just queues for milliseconds. Waiters are refused only when the key actually locks out (they
+///         never reach the authenticator) or after <see cref="OpcUaCredentialLockoutOptions.SlotWaitSeconds"/>
+///         (uncounted); a cancelled waiter is removed from the queue.
+///     </para>
+///     <para>
+///         Memory is bounded at <see cref="MaxTrackedClients"/>: entries with no failures, slots or waiters are
+///         pruned when it fills, and a client that still finds it full is not tracked (it can only evade by
+///         rotating certificates or addresses, which it could do against the SDK lockout too; API-key secrets are
+///         not guessable).
+///     </para>
+/// </remarks>
+public sealed class FailedCredentialLockout
+{
+    /// <summary>The most clients tracked at once.</summary>
+    public const int MaxTrackedClients = 4096;
+
+    private readonly OpcUaCredentialLockoutOptions _options;
+    private readonly TimeProvider _time;
+    private readonly Dictionary<string, Entry> _entries = new(StringComparer.Ordinal);
+    private readonly object _gate = new();
+
+    /// <summary>Initializes a new instance of the <see cref="FailedCredentialLockout"/> class.</summary>
+    /// <param name="options">The configured thresholds.</param>
+    /// <param name="time">The clock; <see cref="TimeProvider.System"/> when null.</param>
+    public FailedCredentialLockout(OpcUaCredentialLockoutOptions options, TimeProvider? time = null)
+    {
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _time = time ?? TimeProvider.System;
+    }
+
+    /// <summary>Whether the lockout is switched on.</summary>
+    public bool Enabled => _options.MaxFailedAttempts > 0;
+
+    /// <summary>Builds the lockout key for a client.</summary>
+    /// <param name="certificateThumbprint">The presented client certificate's thumbprint, if any.</param>
+    /// <param name="peer">The transport-observed peer address, if known.</param>
+    /// <returns>The key; never null.</returns>
+    public static string KeyFor(string? certificateThumbprint, System.Net.IPAddress? peer) =>
+        $"{(string.IsNullOrEmpty(certificateThumbprint) ? "-" : certificateThumbprint)}@{peer?.ToString() ?? "-"}";
+
+    /// <summary>
+    ///     Acquires a verification slot for <paramref name="key"/>, waiting for one when the client already has as
+    ///     many in progress as it may. On <see cref="CredentialLockoutVerdict.Proceed"/> the caller MUST release the
+    ///     slot with <see cref="Complete"/> or <see cref="Abandon"/>.
+    /// </summary>
+    /// <param name="key">The client key from <see cref="KeyFor"/>.</param>
+    /// <param name="ct">Cancels the wait; a cancelled waiter is dequeued and the exception propagates.</param>
+    /// <returns>The decision.</returns>
+    public async ValueTask<CredentialLockoutDecision> BeginAsync(string key, CancellationToken ct = default)
+    {
+        if (!Enabled) return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
+
+        Waiter waiter;
+        Entry entry;
+        lock (_gate)
+        {
+            var now = _time.GetUtcNow();
+            if (!_entries.TryGetValue(key, out var existing))
+            {
+                if (_entries.Count >= MaxTrackedClients) Prune(now);
+                // Untracked when still full: it gets no slot accounting, as before (see remarks).
+                if (_entries.Count >= MaxTrackedClients) return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
+                existing = new Entry();
+                _entries[key] = existing;
+            }
+
+            entry = existing;
+            Refresh(entry, now);
+            if (TryLockedOut(entry, now, out var refused)) return refused;
+            if (entry.Waiters.Count == 0 && HasFreeSlot(entry))
+            {
+                entry.InFlight++;
+                return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
+            }
+
+            waiter = new Waiter();
+            entry.Waiters.AddLast(waiter.Node = new LinkedListNode<Waiter>(waiter));
+        }
+
+        var wait = TimeSpan.FromSeconds(Math.Max(1, _options.SlotWaitSeconds));
+        try
+        {
+            return await waiter.Completion.Task.WaitAsync(wait, _time, ct).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is TimeoutException or OperationCanceledException)
+        {
+            lock (_gate)
+            {
+                if (waiter.Completion.Task.IsCompletedSuccessfully)
+                {
+                    // Decided in the same instant the wait gave up. A timeout takes the decision as made; a
+                    // cancellation hands a granted slot back (waking the next waiter) so nothing leaks.
+                    var decided = waiter.Completion.Task.Result;
+                    if (ex is not OperationCanceledException) return decided;
+                    if (decided.Verdict == CredentialLockoutVerdict.Proceed) ReleaseCore(key, entry, outcome: null);
+                }
+                else
+                {
+                    entry.Waiters.Remove(waiter.Node!);
+                    waiter.Completion.TrySetCanceled();
+                    RemoveIfIdle(key, entry);
+                }
+            }
+
+            if (ex is OperationCanceledException) throw;
+            return new(CredentialLockoutVerdict.SlotWaitTimedOut, TimeSpan.Zero, false);
+        }
+    }
+
+    /// <summary>Releases a slot from <see cref="BeginAsync"/> and records the activation's outcome.</summary>
+    /// <param name="key">The client key.</param>
+    /// <param name="succeeded">Whether the activation was admitted.</param>
+    /// <returns><see langword="true"/> when this failure locked the client out.</returns>
+    public bool Complete(string key, bool succeeded)
+    {
+        if (!Enabled) return false;
+        lock (_gate)
+        {
+            return _entries.TryGetValue(key, out var entry) && ReleaseCore(key, entry, succeeded);
+        }
+    }
+
+    /// <summary>Releases a slot without recording an outcome (the activation was cancelled or threw).</summary>
+    /// <param name="key">The client key.</param>
+    public void Abandon(string key)
+    {
+        if (!Enabled) return;
+        lock (_gate)
+        {
+            if (_entries.TryGetValue(key, out var entry)) ReleaseCore(key, entry, outcome: null);
+        }
+    }
+
+    /// <summary>Whether <paramref name="key"/> is locked out right now (observation; reserves nothing).</summary>
+    /// <param name="key">The client key from <see cref="KeyFor"/>.</param>
+    /// <param name="remaining">The time left on the lockout when this returns true.</param>
+    /// <returns><see langword="true"/> when the client's UserName activations are refused.</returns>
+    public bool IsLockedOut(string key, out TimeSpan remaining)
+    {
+        remaining = TimeSpan.Zero;
+        if (!Enabled) return false;
+        var now = _time.GetUtcNow();
+        lock (_gate)
+        {
+            if (!_entries.TryGetValue(key, out var entry)) return false;
+            Refresh(entry, now);
+            if (entry.LockedUntil <= now) return false;
+            remaining = entry.LockedUntil - now;
+            return true;
+        }
+    }
+
+    /// <summary>Records one failed UserName activation with no slot (test seam / direct callers).</summary>
+    /// <param name="key">The client key.</param>
+    /// <returns><see langword="true"/> when this failure locked the client out.</returns>
+    public bool RecordFailure(string key)
+    {
+        if (!Enabled) return false;
+        var now = _time.GetUtcNow();
+        lock (_gate)
+        {
+            if (!_entries.TryGetValue(key, out var entry))
+            {
+                if (_entries.Count >= MaxTrackedClients) Prune(now);
+                if (_entries.Count >= MaxTrackedClients) return false;
+                entry = new Entry();
+                _entries[key] = entry;
+            }
+            var locked = RecordFailureCore(entry, now);
+            Pump(entry, now);
+            return locked;
+        }
+    }
+
+    /// <summary>Clears a client's count after a successful UserName activation (no slot).</summary>
+    /// <param name="key">The client key.</param>
+    public void RecordSuccess(string key)
+    {
+        lock (_gate)
+        {
+            if (!_entries.TryGetValue(key, out var entry)) return;
+            ClearCount(entry);
+            Pump(entry, _time.GetUtcNow());
+            RemoveIfIdle(key, entry);
+        }
+    }
+
+    /// <summary>How many clients are tracked (test observation).</summary>
+    internal int TrackedCount
+    {
+        get { lock (_gate) return _entries.Count; }
+    }
+
+    /// <summary>How many activations are waiting for a slot on <paramref name="key"/> (test observation).</summary>
+    internal int WaitingCount(string key)
+    {
+        lock (_gate) return _entries.TryGetValue(key, out var e) ? e.Waiters.Count : 0;
+    }
+
+    private bool HasFreeSlot(Entry entry) => entry.Failures + entry.InFlight < _options.MaxFailedAttempts;
+
+    private bool TryLockedOut(Entry entry, DateTimeOffset now, out CredentialLockoutDecision refused)
+    {
+        refused = default;
+        if (entry.LockedUntil <= now) return false;
+        refused = new(CredentialLockoutVerdict.LockedOut, entry.LockedUntil - now, !entry.RefusalLogged);
+        entry.RefusalLogged = true;
+        return true;
+    }
+
+    /// <summary>Releases one slot, records <paramref name="outcome"/> (null = uncounted) and wakes waiters.</summary>
+    private bool ReleaseCore(string key, Entry entry, bool? outcome)
+    {
+        var now = _time.GetUtcNow();
+        if (entry.InFlight > 0) entry.InFlight--;
+        var lockedNow = false;
+        if (outcome == true) ClearCount(entry);
+        else if (outcome == false) lockedNow = RecordFailureCore(entry, now);
+        Pump(entry, now);
+        RemoveIfIdle(key, entry);
+        return lockedNow;
+    }
+
+    /// <summary>
+    ///     Hands freed slots to waiters in order — or, when the key is locked out, refuses every waiter (they never
+    ///     reach the authenticator). Called under the gate after anything that can free a slot or lock the key.
+    /// </summary>
+    private void Pump(Entry entry, DateTimeOffset now)
+    {
+        Refresh(entry, now);
+        while (entry.Waiters.First is { } first)
+        {
+            if (TryLockedOut(entry, now, out var refused))
+            {
+                entry.Waiters.RemoveFirst();
+                first.Value.Completion.TrySetResult(refused);
+                continue;
+            }
+
+            if (!HasFreeSlot(entry)) return;
+            entry.Waiters.RemoveFirst();
+            entry.InFlight++;
+            if (!first.Value.Completion.TrySetResult(new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false)))
+                entry.InFlight--; // a waiter that already gave up never takes the slot
+        }
+    }
+
+    private static void ClearCount(Entry entry)
+    {
+        entry.Failures = 0;
+        entry.LockedUntil = default;
+        entry.RefusalLogged = false;
+    }
+
+    private bool RecordFailureCore(Entry entry, DateTimeOffset now)
+    {
+        Refresh(entry, now);
+        if (entry.LockedUntil > now) return false; // a refusal (or a late in-flight failure) never extends it
+        entry.Failures++;
+        entry.LastFailure = now;
+        if (entry.Failures < _options.MaxFailedAttempts) return false;
+        entry.LockedUntil = now + TimeSpan.FromSeconds(Math.Max(0, _options.LockoutSeconds));
+        entry.RefusalLogged = false;
+        return entry.LockedUntil > now;
+    }
+
+    /// <summary>
+    ///     Forgets a run of failures that has gone stale: an expired lockout always starts a fresh count (else a
+    ///     <c>LockoutSeconds</c> shorter than <c>FailureWindowSeconds</c> relocked on the first failure after it),
+    ///     and so does a failure older than the window.
+    /// </summary>
+    private void Refresh(Entry entry, DateTimeOffset now)
+    {
+        if (entry.LockedUntil != default && entry.LockedUntil <= now)
+        {
+            ClearCount(entry);
+        }
+        else if (entry.LockedUntil == default && entry.Failures > 0
+                 && now - entry.LastFailure > TimeSpan.FromSeconds(Math.Max(0, _options.FailureWindowSeconds)))
+        {
+            entry.Failures = 0;
+            entry.RefusalLogged = false;
+        }
+    }
+
+    private void RemoveIfIdle(string key, Entry entry)
+    {
+        if (IsIdle(entry, _time.GetUtcNow())) _entries.Remove(key);
+    }
+
+    private void Prune(DateTimeOffset now)
+    {
+        foreach (var key in _entries.Where(e => IsIdle(e.Value, now)).Select(e => e.Key).ToList())
+            _entries.Remove(key);
+    }
+
+    /// <summary>Idle = nothing to remember: no failures, no lockout, no slot held and nobody waiting.</summary>
+    private bool IsIdle(Entry entry, DateTimeOffset now)
+    {
+        Refresh(entry, now);
+        return entry.InFlight == 0 && entry.Waiters.Count == 0 && entry.Failures == 0 && entry.LockedUntil == default;
+    }
+
+    /// <summary>One activation waiting for a slot.</summary>
+    private sealed class Waiter
+    {
+        public readonly TaskCompletionSource<CredentialLockoutDecision> Completion =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        public LinkedListNode<Waiter>? Node;
+    }
+
+    /// <summary>One client's failure count, slots, waiters and lockout. Mutated only under the gate.</summary>
+    private sealed class Entry
+    {
+        public int Failures;
+        public int InFlight;
+        public DateTimeOffset LastFailure;
+        public DateTimeOffset LockedUntil;
+        public bool RefusalLogged;
+        public readonly LinkedList<Waiter> Waiters = new();
+    }
+}