dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(opcua): parallel UserName activations wait for a slot instead of being refused (#833)

Browse Source 
A legitimate consumer opening several keyed sessions at once from one key
(e.g. a ScadaBridge site node behind one NAT IP on a None endpoint) must
never see BadUserAccessDenied for parallelism. BeginAsync now hands out at
most MaxFailedAttempts - recordedFailures verification slots per key;
further activations queue FIFO and are woken as slots free. A waiter is
refused only when the key locks out (never reaching the authenticator) or,
uncounted, after OpcUa:CredentialLockout:SlotWaitSeconds (default 10,
BadTimeout). Cancelled waiters are dequeued; pruning accounts for waiters.
The security property is unchanged: no more than MaxFailedAttempts
unverified guesses in progress-or-failed per window.
This commit is contained in:
Joseph Doherty
2026-11-11 02:43:38 -05:00
parent 61b9adeb9c
commit 2314a53842
9 changed files with 397 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/OpcUaApplicationHostOptionsValidator.cs
+3 -2
View File
@@ -34,8 +34,9 @@ public sealed class OpcUaApplicationHostOptionsValidator : OptionsValidatorBase<
// nothing, a zero FailureWindowSeconds would never accumulate. MaxFailedAttempts <= 0 disables it.
var lockout = o.CredentialLockout;
builder.RequireThat(lockout is null || lockout.MaxFailedAttempts <= 0
|| (lockout.LockoutSeconds > 0 && lockout.FailureWindowSeconds > 0),
"OpcUa:CredentialLockout:LockoutSeconds and OpcUa:CredentialLockout:FailureWindowSeconds must both be "
|| (lockout.LockoutSeconds > 0 && lockout.FailureWindowSeconds > 0 && lockout.SlotWaitSeconds > 0),
"OpcUa:CredentialLockout:LockoutSeconds, OpcUa:CredentialLockout:FailureWindowSeconds and "
+ "OpcUa:CredentialLockout:SlotWaitSeconds must all be "
+ "greater than 0 while OpcUa:CredentialLockout:MaxFailedAttempts is greater than 0 (set "
+ "MaxFailedAttempts to 0 to disable the lockout)");
}
src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OpcUaApplicationHost.cs
+19 -15
View File
@@ -275,31 +275,35 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
var lockoutKey = tokenType == UserTokenType.UserName
? FailedCredentialLockout.KeyFor(context.ChannelCertificate?.Thumbprint, peer)
: null;
if (lockoutKey is not null
&& !host._credentialLockout.TryBegin(lockoutKey, out var refusal, out var remaining, out var firstRefusal))
if (lockoutKey is not null)
{
// Logged once per lockout episode at Information; every further refusal at Debug, so a client hammering
// a lockout cannot flood the log (#833 review).
var level = firstRefusal ? LogLevel.Information : LogLevel.Debug;
if (refusal == CredentialLockoutRefusal.LockedOut)
// Waits (bounded) for one of this client's verification slots — parallel activations queue, they are
// never refused for parallelism. Refused only when locked out, or (uncounted) when no slot freed in time.
var decision = await host._credentialLockout.BeginAsync(lockoutKey, ct).ConfigureAwait(false);
if (decision.Verdict == CredentialLockoutVerdict.LockedOut)
{
var seconds = (long)Math.Ceiling(remaining.TotalSeconds);
host._logger.Log(level,
// Logged once per lockout episode at Information; every further refusal at Debug, so a client
// hammering a lockout cannot flood the log (#833 review).
var seconds = (long)Math.Ceiling(decision.Remaining.TotalSeconds);
host._logger.Log(decision.FirstRefusal ? LogLevel.Information : LogLevel.Debug,
"OpcUaApplicationHost: UserName activation refused — client {Client} is locked out for {Seconds}s more",
lockoutKey, seconds);
return AuthenticationResult.Reject(ServiceResult.Create(StatusCodes.BadUserAccessDenied,
$"Too many failed authentication attempts. Try again in {seconds} seconds."));
}
host._logger.Log(level,
"OpcUaApplicationHost: UserName activation refused — client {Client} already has as many failed plus "
+ "in-progress attempts as OpcUa:CredentialLockout allows",
lockoutKey);
return AuthenticationResult.Reject(ServiceResult.Create(StatusCodes.BadUserAccessDenied,
"Too many authentication attempts in progress. Try again shortly."));
if (decision.Verdict == CredentialLockoutVerdict.SlotWaitTimedOut)
{
host._logger.LogWarning(
"OpcUaApplicationHost: UserName activation for client {Client} timed out waiting for a "
+ "verification slot (OpcUa:CredentialLockout:SlotWaitSeconds); not counted as a failure",
lockoutKey);
return AuthenticationResult.Reject(ServiceResult.Create(StatusCodes.BadTimeout,
"Timed out waiting to verify the credential. Try again."));
}
}
// The slot TryBegin reserved is released on every path — completed with the outcome, or abandoned
// The slot BeginAsync granted is released on every path — completed with the outcome, or abandoned
// (uncounted) when the activation was cancelled or threw.
var completed = false;
try
src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/FailedCredentialLockout.cs
+178 -81
View File
@@ -7,7 +7,7 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
/// <remarks>
/// The defaults are the OPC Foundation SDK 2.0.1 <c>SessionManager</c> lockout this replaces: 5 failures,
/// each inside 60 s of the previous one, lock the client out for 5 minutes. When enabled
/// (<see cref="MaxFailedAttempts"/> &gt; 0) both durations must be positive — enforced at host start by
/// (<see cref="MaxFailedAttempts"/> &gt; 0) every duration must be positive — enforced at host start by
/// <c>OpcUaApplicationHostOptionsValidator</c>.
/// </remarks>
public sealed class OpcUaCredentialLockoutOptions
@@ -20,24 +20,37 @@ public sealed class OpcUaCredentialLockoutOptions
/// <summary>How long a locked-out client's UserName activations are refused, in seconds.</summary>
public int LockoutSeconds { get; set; } = 300;
}
/// <summary>Why <see cref="FailedCredentialLockout.TryBegin"/> refused an activation.</summary>
public enum CredentialLockoutRefusal
{
/// <summary>Not refused.</summary>
None,
/// <summary>The client is locked out.</summary>
LockedOut,
/// <summary>
/// The client's recorded failures plus its activations still being checked already reach the threshold, so
/// one more concurrent attempt could exceed it before any of them has been recorded.
/// How long a UserName activation waits for one of its client's verification slots before that one attempt
/// is failed (<c>BadTimeout</c>, NOT counted as a credential failure). Only reached when a client has more
/// verifications in progress than <see cref="MaxFailedAttempts"/> minus its recorded failures; a verification
/// is milliseconds, so in practice this bounds a pathological store stall. Kept well under the SDK's
/// operation timeout so the client gets this answer rather than a transport timeout.
/// </summary>
TooManyInFlight,
public int SlotWaitSeconds { get; set; } = 10;
}
/// <summary>The outcome of <see cref="FailedCredentialLockout.BeginAsync"/>.</summary>
public enum CredentialLockoutVerdict
{
/// <summary>A verification slot is held — authenticate, then call <see cref="FailedCredentialLockout.Complete"/>.</summary>
Proceed,
/// <summary>The client is locked out; refuse without authenticating.</summary>
LockedOut,
/// <summary>No slot freed within the wait; fail this one attempt, uncounted.</summary>
SlotWaitTimedOut,
}
/// <summary>What <see cref="FailedCredentialLockout.BeginAsync"/> decided.</summary>
/// <param name="Verdict">Proceed, locked out, or timed out waiting.</param>
/// <param name="Remaining">Time left on the lockout when <see cref="CredentialLockoutVerdict.LockedOut"/>.</param>
/// <param name="FirstRefusal">True for the first refusal of a lockout episode (log it at Information, the rest at Debug).</param>
public readonly record struct CredentialLockoutDecision(
CredentialLockoutVerdict Verdict, TimeSpan Remaining, bool FirstRefusal);
/// <summary>
/// Brute-force lockout for UserName (API-key) activations, scoped so it can never refuse an
/// <b>anonymous</b> activation (lmxopcua#833).
@@ -69,16 +82,22 @@ public enum CredentialLockoutRefusal
/// window); a successful UserName activation clears the client's count.
/// </para>
/// <para>
/// <b>Concurrency.</b> The authenticate call is awaited between the check and the outcome, so a check and a
/// separate record would let a burst of parallel activations all pass before the first failure lands.
/// <see cref="TryBegin"/> therefore <b>reserves</b> an in-flight slot under the same lock as the check, and
/// refuses while recorded failures + in-flight attempts already reach the threshold;
/// <see cref="Complete"/> releases the slot and records the outcome.
/// <b>Concurrency — parallelism waits, it is never refused.</b> The authenticate call is awaited between the
/// check and the outcome, so a check and a separate record would let a parallel burst of guesses all pass
/// before the first failure lands. <see cref="BeginAsync"/> therefore hands out at most
/// <c>MaxFailedAttempts − recordedFailures</c> verification <b>slots</b> per key; an activation beyond that
/// <b>waits</b> (FIFO) for a slot, and <see cref="Complete"/> / <see cref="Abandon"/> release it and wake the
/// next waiter. So no more than <c>MaxFailedAttempts</c> unverified guesses are ever in progress-or-failed
/// per window, while a legitimate client opening many keyed sessions at once (a ScadaBridge site node behind
/// one NAT IP) just queues for milliseconds. Waiters are refused only when the key actually locks out (they
/// never reach the authenticator) or after <see cref="OpcUaCredentialLockoutOptions.SlotWaitSeconds"/>
/// (uncounted); a cancelled waiter is removed from the queue.
/// </para>
/// <para>
/// Memory is bounded at <see cref="MaxTrackedClients"/>: expired, idle entries are pruned when it fills,
/// and a client that still finds it full is not tracked (it can only evade by rotating certificates or
/// addresses, which it could do against the SDK lockout too; API-key secrets are not guessable).
/// Memory is bounded at <see cref="MaxTrackedClients"/>: entries with no failures, slots or waiters are
/// pruned when it fills, and a client that still finds it full is not tracked (it can only evade by
/// rotating certificates or addresses, which it could do against the SDK lockout too; API-key secrets are
/// not guessable).
/// </para>
/// </remarks>
public sealed class FailedCredentialLockout
@@ -111,85 +130,95 @@ public sealed class FailedCredentialLockout
$"{(string.IsNullOrEmpty(certificateThumbprint) ? "-" : certificateThumbprint)}@{peer?.ToString() ?? "-"}";
/// <summary>
/// Checks <paramref name="key"/> and, when it may proceed, reserves an in-flight slot that the caller MUST
/// release with <see cref="Complete"/> (only when this returns <see langword="true"/>).
/// Acquires a verification slot for <paramref name="key"/>, waiting for one when the client already has as
/// many in progress as it may. On <see cref="CredentialLockoutVerdict.Proceed"/> the caller MUST release the
/// slot with <see cref="Complete"/> or <see cref="Abandon"/>.
/// </summary>
/// <param name="key">The client key from <see cref="KeyFor"/>.</param>
/// <param name="refusal">Why it was refused, when this returns false.</param>
/// <param name="remaining">The time left on the lockout when refused <see cref="CredentialLockoutRefusal.LockedOut"/>.</param>
/// <param name="firstRefusal">
/// True for the first refusal of this lockout episode (so the caller logs it once at Information and the
/// rest at Debug).
/// </param>
/// <returns><see langword="true"/> when the activation may be authenticated.</returns>
public bool TryBegin(string key, out CredentialLockoutRefusal refusal, out TimeSpan remaining, out bool firstRefusal)
/// <param name="ct">Cancels the wait; a cancelled waiter is dequeued and the exception propagates.</param>
/// <returns>The decision.</returns>
public async ValueTask<CredentialLockoutDecision> BeginAsync(string key, CancellationToken ct = default)
{
refusal = CredentialLockoutRefusal.None;
remaining = TimeSpan.Zero;
firstRefusal = false;
if (!Enabled) return true;
var now = _time.GetUtcNow();
if (!Enabled) return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
Waiter waiter;
Entry entry;
lock (_gate)
{
if (!_entries.TryGetValue(key, out var entry))
var now = _time.GetUtcNow();
if (!_entries.TryGetValue(key, out var existing))
{
if (_entries.Count >= MaxTrackedClients) Prune(now);
if (_entries.Count >= MaxTrackedClients) return true; // untracked: nothing to reserve
entry = new Entry();
_entries[key] = entry;
// Untracked when still full: it gets no slot accounting, as before (see remarks).
if (_entries.Count >= MaxTrackedClients) return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
existing = new Entry();
_entries[key] = existing;
}
entry = existing;
Refresh(entry, now);
if (entry.LockedUntil > now)
if (TryLockedOut(entry, now, out var refused)) return refused;
if (entry.Waiters.Count == 0 && HasFreeSlot(entry))
{
refusal = CredentialLockoutRefusal.LockedOut;
remaining = entry.LockedUntil - now;
firstRefusal = !entry.RefusalLogged;
entry.RefusalLogged = true;
return false;
entry.InFlight++;
return new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false);
}
if (entry.Failures + entry.InFlight >= _options.MaxFailedAttempts)
waiter = new Waiter();
entry.Waiters.AddLast(waiter.Node = new LinkedListNode<Waiter>(waiter));
}
var wait = TimeSpan.FromSeconds(Math.Max(1, _options.SlotWaitSeconds));
try
{
return await waiter.Completion.Task.WaitAsync(wait, _time, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is TimeoutException or OperationCanceledException)
{
lock (_gate)
{
refusal = CredentialLockoutRefusal.TooManyInFlight;
firstRefusal = !entry.RefusalLogged;
entry.RefusalLogged = true;
return false;
if (waiter.Completion.Task.IsCompletedSuccessfully)
{
// Decided in the same instant the wait gave up. A timeout takes the decision as made; a
// cancellation hands a granted slot back (waking the next waiter) so nothing leaks.
var decided = waiter.Completion.Task.Result;
if (ex is not OperationCanceledException) return decided;
if (decided.Verdict == CredentialLockoutVerdict.Proceed) ReleaseCore(key, entry, outcome: null);
}
else
{
entry.Waiters.Remove(waiter.Node!);
waiter.Completion.TrySetCanceled();
RemoveIfIdle(key, entry);
}
}
entry.InFlight++;
return true;
if (ex is OperationCanceledException) throw;
return new(CredentialLockoutVerdict.SlotWaitTimedOut, TimeSpan.Zero, false);
}
}
/// <summary>Releases the slot <see cref="TryBegin"/> reserved and records the activation's outcome.</summary>
/// <summary>Releases a slot from <see cref="BeginAsync"/> and records the activation's outcome.</summary>
/// <param name="key">The client key.</param>
/// <param name="succeeded">Whether the activation was admitted.</param>
/// <returns><see langword="true"/> when this failure locked the client out.</returns>
public bool Complete(string key, bool succeeded)
{
if (!Enabled) return false;
var now = _time.GetUtcNow();
lock (_gate)
{
if (!_entries.TryGetValue(key, out var entry)) return false; // was untracked at TryBegin
if (entry.InFlight > 0) entry.InFlight--;
return succeeded ? RecordSuccessCore(key, entry) : RecordFailureCore(entry, now);
return _entries.TryGetValue(key, out var entry) && ReleaseCore(key, entry, succeeded);
}
}
/// <summary>Releases a reserved slot without recording an outcome (the activation was cancelled or threw).</summary>
/// <summary>Releases a slot without recording an outcome (the activation was cancelled or threw).</summary>
/// <param name="key">The client key.</param>
public void Abandon(string key)
{
if (!Enabled) return;
lock (_gate)
{
if (_entries.TryGetValue(key, out var entry) && entry.InFlight > 0)
{
entry.InFlight--;
if (IsIdle(entry, _time.GetUtcNow())) _entries.Remove(key);
}
if (_entries.TryGetValue(key, out var entry)) ReleaseCore(key, entry, outcome: null);
}
}
@@ -212,7 +241,7 @@ public sealed class FailedCredentialLockout
}
}
/// <summary>Records one failed UserName activation with no reservation (test seam / direct callers).</summary>
/// <summary>Records one failed UserName activation with no slot (test seam / direct callers).</summary>
/// <param name="key">The client key.</param>
/// <returns><see langword="true"/> when this failure locked the client out.</returns>
public bool RecordFailure(string key)
@@ -228,17 +257,22 @@ public sealed class FailedCredentialLockout
entry = new Entry();
_entries[key] = entry;
}
return RecordFailureCore(entry, now);
var locked = RecordFailureCore(entry, now);
Pump(entry, now);
return locked;
}
}
/// <summary>Clears a client's count after a successful UserName activation (no reservation).</summary>
/// <summary>Clears a client's count after a successful UserName activation (no slot).</summary>
/// <param name="key">The client key.</param>
public void RecordSuccess(string key)
{
lock (_gate)
{
if (_entries.TryGetValue(key, out var entry)) RecordSuccessCore(key, entry);
if (!_entries.TryGetValue(key, out var entry)) return;
ClearCount(entry);
Pump(entry, _time.GetUtcNow());
RemoveIfIdle(key, entry);
}
}
@@ -248,16 +282,65 @@ public sealed class FailedCredentialLockout
get { lock (_gate) return _entries.Count; }
}
private bool RecordSuccessCore(string key, Entry entry)
/// <summary>How many activations are waiting for a slot on <paramref name="key"/> (test observation).</summary>
internal int WaitingCount(string key)
{
if (entry.InFlight == 0) _entries.Remove(key);
else
lock (_gate) return _entries.TryGetValue(key, out var e) ? e.Waiters.Count : 0;
}
private bool HasFreeSlot(Entry entry) => entry.Failures + entry.InFlight < _options.MaxFailedAttempts;
private bool TryLockedOut(Entry entry, DateTimeOffset now, out CredentialLockoutDecision refused)
{
refused = default;
if (entry.LockedUntil <= now) return false;
refused = new(CredentialLockoutVerdict.LockedOut, entry.LockedUntil - now, !entry.RefusalLogged);
entry.RefusalLogged = true;
return true;
}
/// <summary>Releases one slot, records <paramref name="outcome"/> (null = uncounted) and wakes waiters.</summary>
private bool ReleaseCore(string key, Entry entry, bool? outcome)
{
var now = _time.GetUtcNow();
if (entry.InFlight > 0) entry.InFlight--;
var lockedNow = false;
if (outcome == true) ClearCount(entry);
else if (outcome == false) lockedNow = RecordFailureCore(entry, now);
Pump(entry, now);
RemoveIfIdle(key, entry);
return lockedNow;
}
/// <summary>
/// Hands freed slots to waiters in order — or, when the key is locked out, refuses every waiter (they never
/// reach the authenticator). Called under the gate after anything that can free a slot or lock the key.
/// </summary>
private void Pump(Entry entry, DateTimeOffset now)
{
Refresh(entry, now);
while (entry.Waiters.First is { } first)
{
entry.Failures = 0;
entry.LockedUntil = default;
entry.RefusalLogged = false;
if (TryLockedOut(entry, now, out var refused))
{
entry.Waiters.RemoveFirst();
first.Value.Completion.TrySetResult(refused);
continue;
}
if (!HasFreeSlot(entry)) return;
entry.Waiters.RemoveFirst();
entry.InFlight++;
if (!first.Value.Completion.TrySetResult(new(CredentialLockoutVerdict.Proceed, TimeSpan.Zero, false)))
entry.InFlight--; // a waiter that already gave up never takes the slot
}
return false;
}
private static void ClearCount(Entry entry)
{
entry.Failures = 0;
entry.LockedUntil = default;
entry.RefusalLogged = false;
}
private bool RecordFailureCore(Entry entry, DateTimeOffset now)
@@ -281,9 +364,7 @@ public sealed class FailedCredentialLockout
{
if (entry.LockedUntil != default && entry.LockedUntil <= now)
{
entry.Failures = 0;
entry.LockedUntil = default;
entry.RefusalLogged = false;
ClearCount(entry);
}
else if (entry.LockedUntil == default && entry.Failures > 0
&& now - entry.LastFailure > TimeSpan.FromSeconds(Math.Max(0, _options.FailureWindowSeconds)))
@@ -293,19 +374,34 @@ public sealed class FailedCredentialLockout
}
}
private void RemoveIfIdle(string key, Entry entry)
{
if (IsIdle(entry, _time.GetUtcNow())) _entries.Remove(key);
}
private void Prune(DateTimeOffset now)
{
foreach (var key in _entries.Where(e => IsIdle(e.Value, now)).Select(e => e.Key).ToList())
_entries.Remove(key);
}
/// <summary>Idle = nothing to remember: no failures, no lockout, no slot held and nobody waiting.</summary>
private bool IsIdle(Entry entry, DateTimeOffset now)
{
Refresh(entry, now);
return entry.InFlight == 0 && entry.Failures == 0 && entry.LockedUntil == default;
return entry.InFlight == 0 && entry.Waiters.Count == 0 && entry.Failures == 0 && entry.LockedUntil == default;
}
/// <summary>One client's failure count, in-flight reservations and lockout. Mutated only under the gate.</summary>
/// <summary>One activation waiting for a slot.</summary>
private sealed class Waiter
{
public readonly TaskCompletionSource<CredentialLockoutDecision> Completion =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public LinkedListNode<Waiter>? Node;
}
/// <summary>One client's failure count, slots, waiters and lockout. Mutated only under the gate.</summary>
private sealed class Entry
{
public int Failures;
@@ -313,5 +409,6 @@ public sealed class FailedCredentialLockout
public DateTimeOffset LastFailure;
public DateTimeOffset LockedUntil;
public bool RefusalLogged;
public readonly LinkedList<Waiter> Waiters = new();
}
}