dohertj2/lmxopcua
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(host): LDAP directory-outage circuit — fail-fast denial during sustained outage (03/S2)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 11:56:45 -04:00
parent 21abe42a90
commit 0c785415a2
3 changed files with 240 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs
+99 -7
View File
@@ -47,6 +47,16 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
/// orphan binds.</summary>
private readonly SemaphoreSlim _gate;
/// <summary>Guards the outage-circuit state (<see cref="_consecutiveSystemFailures"/> +
/// <see cref="_circuitOpenUntilTicks"/>).</summary>
private readonly object _circuitLock = new();
/// <summary>Count of CONSECUTIVE system-side failures; reset by any success / user-side deny.</summary>
private int _consecutiveSystemFailures;
/// <summary><see cref="Environment.TickCount64"/> the circuit stays open until; <c>0</c> = closed.</summary>
private long _circuitOpenUntilTicks;
/// <summary>Constructs the authenticator from its dependencies and the bound LDAP options.</summary>
/// <param name="ldap">The app LDAP auth service (shared with the Admin UI login flow).</param>
/// <param name="scopeFactory">Opens a per-call DI scope to resolve the scoped role mapper.</param>
@@ -70,8 +80,19 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
{
var authTimeout = TimeSpan.FromMilliseconds(Math.Max(1, _options.AuthTimeoutMs));
// Directory-outage circuit: after a run of consecutive system-side failures, fail fast WITHOUT
// touching LDAP for the cooldown window — so a sustained outage cannot serialize every session
// activation behind a per-call stall (the SDK raises this callback under a server-wide lock).
if (IsCircuitOpen())
{
_logger.LogDebug(
"LDAP outage circuit open — fast-denying OPC UA user {User} without a directory call", username);
return OpcUaUserAuthResult.Deny("Authentication backend unavailable");
}
// Bounded in-flight concurrency: acquire a slot within our own wall-clock budget. All slots
// busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed).
// busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed) and do NOT
// feed the outage circuit.
if (!await _gate.WaitAsync(authTimeout, ct).ConfigureAwait(false))
{
_logger.LogWarning(
@@ -95,10 +116,14 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
try
{
return await core.WaitAsync(authTimeout, ct).ConfigureAwait(false);
var outcome = await core.WaitAsync(authTimeout, ct).ConfigureAwait(false);
RecordOutcome(outcome.SystemFailure, username);
return outcome.Result;
}
catch (TimeoutException)
{
// A boundary timeout is directory evidence — it feeds the circuit.
RecordOutcome(systemFailure: true, username);
_logger.LogWarning(
"LDAP authentication timed out after {Timeout} for OPC UA user {User} — denying (fail-closed)",
authTimeout, username);
@@ -106,34 +131,101 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
}
}
/// <summary>Whether the outage circuit is currently open (fast-deny window active). Always false when
/// the circuit is disabled (<see cref="LdapOptions.OutageFailureThreshold"/> ≤ 0).</summary>
private bool IsCircuitOpen()
{
if (_options.OutageFailureThreshold <= 0) return false;
lock (_circuitLock)
{
return _circuitOpenUntilTicks > Environment.TickCount64;
}
}
/// <summary>
/// Updates the outage-circuit state from one authentication outcome. A system-side failure advances
/// the consecutive-failure streak and opens the circuit once it reaches
/// <see cref="LdapOptions.OutageFailureThreshold"/>; any non-system outcome (success or user-side
/// deny) resets the streak and closes the circuit. Half-open behaviour is implicit: once the cooldown
/// elapses <see cref="IsCircuitOpen"/> lets the next call through with the streak still at threshold,
/// so a single further system failure re-opens immediately.
/// </summary>
/// <param name="systemFailure">Whether this outcome was a system-side (directory) failure.</param>
/// <param name="username">The login name, for diagnostics.</param>
private void RecordOutcome(bool systemFailure, string username)
{
if (_options.OutageFailureThreshold <= 0) return; // circuit disabled
lock (_circuitLock)
{
if (systemFailure)
{
_consecutiveSystemFailures++;
if (_consecutiveSystemFailures >= _options.OutageFailureThreshold)
{
var wasOpen = _circuitOpenUntilTicks > Environment.TickCount64;
_circuitOpenUntilTicks =
Environment.TickCount64 + (long)Math.Max(1, _options.OutageCooldownSeconds) * 1000L;
if (!wasOpen)
{
_logger.LogWarning(
"LDAP outage circuit OPENED after {Failures} consecutive system-side failures — " +
"fast-denying for {Cooldown}s (last user {User})",
_consecutiveSystemFailures, _options.OutageCooldownSeconds, username);
}
}
}
else
{
if (_consecutiveSystemFailures >= _options.OutageFailureThreshold)
{
_logger.LogInformation(
"LDAP outage circuit CLOSED — directory recovered (probe succeeded for {User})", username);
}
_consecutiveSystemFailures = 0;
_circuitOpenUntilTicks = 0;
}
}
}
/// <summary>
/// The unbounded core authenticate flow (bind + role-map) that the outer
/// <see cref="AuthenticateUserNameAsync"/> wraps in a wall-clock bound + concurrency cap. Fail-closed:
/// any backend throw becomes a denial rather than escaping into the SDK.
/// any backend throw becomes a denial rather than escaping into the SDK. The returned
/// <see cref="CoreOutcome.SystemFailure"/> flags directory-side faults (a system-side
/// <see cref="LdapAuthResult.IsSystemFailure"/> deny, or an unexpected throw) so the caller can feed
/// the outage circuit; a user-credential deny is NOT a system failure.
/// </summary>
/// <param name="username">The username to authenticate.</param>
/// <param name="password">The cleartext password.</param>
/// <param name="ct">Cancellation token.</param>
private async Task<OpcUaUserAuthResult> AuthenticateCoreAsync(string username, string password, CancellationToken ct)
private async Task<CoreOutcome> AuthenticateCoreAsync(string username, string password, CancellationToken ct)
{
try
{
var result = await _ldap.AuthenticateAsync(username, password, ct).ConfigureAwait(false);
if (!result.Success)
{
return OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials");
return new CoreOutcome(
OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials"), result.IsSystemFailure);
}
var roles = await ResolveRolesAsync(result.Groups, result.Roles, username, ct).ConfigureAwait(false);
return OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles);
return new CoreOutcome(OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles), SystemFailure: false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "LDAP authentication threw for OPC UA user {User}", username);
return OpcUaUserAuthResult.Deny("Authentication backend error");
return new CoreOutcome(OpcUaUserAuthResult.Deny("Authentication backend error"), SystemFailure: true);
}
}
/// <summary>The result of the core authenticate flow plus whether it was a system-side (directory)
/// failure — the signal that drives the outage circuit.</summary>
/// <param name="Result">The auth result to return to the SDK.</param>
/// <param name="SystemFailure">Whether the outcome was a directory-side fault.</param>
private readonly record struct CoreOutcome(OpcUaUserAuthResult Result, bool SystemFailure);
/// <summary>
/// Resolves the user's roles from their LDAP groups via the scoped
/// <see cref="IGroupRoleMapper{TRole}"/>, unioned with any pre-resolved roles (the DevStub