diff --git a/archreview/plans/R2-08-ldap-historyread-async-plan.md.tasks.json b/archreview/plans/R2-08-ldap-historyread-async-plan.md.tasks.json index c76c8bfc..824e864e 100644 --- a/archreview/plans/R2-08-ldap-historyread-async-plan.md.tasks.json +++ b/archreview/plans/R2-08-ldap-historyread-async-plan.md.tasks.json @@ -5,7 +5,7 @@ { "id": "R2-08-T2", "subject": "LdapOptions: ConnectionTimeoutMs/AuthTimeoutMs/MaxConcurrentAuthentications/OutageFailureThreshold/OutageCooldownSeconds + ToLibraryOptions projection + validator rules", "status": "completed", "blockedBy": [] }, { "id": "R2-08-T3", "subject": "LdapAuthResult.IsSystemFailure seam + OtOpcUaLdapAuthService.Adapt mapping (ServiceAccountBindFailed => system-side)", "status": "completed", "blockedBy": [] }, { "id": "R2-08-T4", "subject": "LdapOpcUaUserAuthenticator: WaitAsync wall-clock bound + SemaphoreSlim concurrency cap with release-on-core-completion (turns T1 green)", "status": "completed", "blockedBy": ["R2-08-T1", "R2-08-T2"] }, - { "id": "R2-08-T5", "subject": "Directory-outage circuit: fail-fast deny after N consecutive system-side failures, half-open after cooldown", "status": "pending", "blockedBy": ["R2-08-T3", "R2-08-T4"] }, + { "id": "R2-08-T5", "subject": "Directory-outage circuit: fail-fast deny after N consecutive system-side failures, half-open after cooldown", "status": "completed", "blockedBy": ["R2-08-T3", "R2-08-T4"] }, { "id": "R2-08-T6", "subject": "Outage-sim integration test (real library, unroutable host, via internal HandleImpersonation) + SDK m_eventLock contract doc + docs/security.md", "status": "pending", "blockedBy": ["R2-08-T4"] }, { "id": "R2-08-T7", "subject": "S3 sequential-serving repro test (RED): MaxObservedConcurrency >= 2 on an 8-node Raw batch — fails on current code", "status": "pending", "blockedBy": [] }, { "id": "R2-08-T8", "subject": "ServerHistorianOptions: HistoryReadBatchParallelism/MaxConcurrentHistoryReadBatches/HistoryReadDeadline + Validate warnings + node-manager properties + hosted-service wiring", "status": "pending", "blockedBy": [] }, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs index 2d26b832..bd97cd93 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs @@ -47,6 +47,16 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator /// orphan binds. private readonly SemaphoreSlim _gate; + /// Guards the outage-circuit state ( + + /// ). + private readonly object _circuitLock = new(); + + /// Count of CONSECUTIVE system-side failures; reset by any success / user-side deny. + private int _consecutiveSystemFailures; + + /// the circuit stays open until; 0 = closed. + private long _circuitOpenUntilTicks; + /// Constructs the authenticator from its dependencies and the bound LDAP options. /// The app LDAP auth service (shared with the Admin UI login flow). /// Opens a per-call DI scope to resolve the scoped role mapper. @@ -70,8 +80,19 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator { var authTimeout = TimeSpan.FromMilliseconds(Math.Max(1, _options.AuthTimeoutMs)); + // Directory-outage circuit: after a run of consecutive system-side failures, fail fast WITHOUT + // touching LDAP for the cooldown window — so a sustained outage cannot serialize every session + // activation behind a per-call stall (the SDK raises this callback under a server-wide lock). + if (IsCircuitOpen()) + { + _logger.LogDebug( + "LDAP outage circuit open — fast-denying OPC UA user {User} without a directory call", username); + return OpcUaUserAuthResult.Deny("Authentication backend unavailable"); + } + // Bounded in-flight concurrency: acquire a slot within our own wall-clock budget. All slots - // busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed). + // busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed) and do NOT + // feed the outage circuit. if (!await _gate.WaitAsync(authTimeout, ct).ConfigureAwait(false)) { _logger.LogWarning( @@ -95,10 +116,14 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator try { - return await core.WaitAsync(authTimeout, ct).ConfigureAwait(false); + var outcome = await core.WaitAsync(authTimeout, ct).ConfigureAwait(false); + RecordOutcome(outcome.SystemFailure, username); + return outcome.Result; } catch (TimeoutException) { + // A boundary timeout is directory evidence — it feeds the circuit. + RecordOutcome(systemFailure: true, username); _logger.LogWarning( "LDAP authentication timed out after {Timeout} for OPC UA user {User} — denying (fail-closed)", authTimeout, username); @@ -106,34 +131,101 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator } } + /// Whether the outage circuit is currently open (fast-deny window active). Always false when + /// the circuit is disabled ( ≤ 0). + private bool IsCircuitOpen() + { + if (_options.OutageFailureThreshold <= 0) return false; + lock (_circuitLock) + { + return _circuitOpenUntilTicks > Environment.TickCount64; + } + } + + /// + /// Updates the outage-circuit state from one authentication outcome. A system-side failure advances + /// the consecutive-failure streak and opens the circuit once it reaches + /// ; any non-system outcome (success or user-side + /// deny) resets the streak and closes the circuit. Half-open behaviour is implicit: once the cooldown + /// elapses lets the next call through with the streak still at threshold, + /// so a single further system failure re-opens immediately. + /// + /// Whether this outcome was a system-side (directory) failure. + /// The login name, for diagnostics. + private void RecordOutcome(bool systemFailure, string username) + { + if (_options.OutageFailureThreshold <= 0) return; // circuit disabled + + lock (_circuitLock) + { + if (systemFailure) + { + _consecutiveSystemFailures++; + if (_consecutiveSystemFailures >= _options.OutageFailureThreshold) + { + var wasOpen = _circuitOpenUntilTicks > Environment.TickCount64; + _circuitOpenUntilTicks = + Environment.TickCount64 + (long)Math.Max(1, _options.OutageCooldownSeconds) * 1000L; + if (!wasOpen) + { + _logger.LogWarning( + "LDAP outage circuit OPENED after {Failures} consecutive system-side failures — " + + "fast-denying for {Cooldown}s (last user {User})", + _consecutiveSystemFailures, _options.OutageCooldownSeconds, username); + } + } + } + else + { + if (_consecutiveSystemFailures >= _options.OutageFailureThreshold) + { + _logger.LogInformation( + "LDAP outage circuit CLOSED — directory recovered (probe succeeded for {User})", username); + } + _consecutiveSystemFailures = 0; + _circuitOpenUntilTicks = 0; + } + } + } + /// /// The unbounded core authenticate flow (bind + role-map) that the outer /// wraps in a wall-clock bound + concurrency cap. Fail-closed: - /// any backend throw becomes a denial rather than escaping into the SDK. + /// any backend throw becomes a denial rather than escaping into the SDK. The returned + /// flags directory-side faults (a system-side + /// deny, or an unexpected throw) so the caller can feed + /// the outage circuit; a user-credential deny is NOT a system failure. /// /// The username to authenticate. /// The cleartext password. /// Cancellation token. - private async Task AuthenticateCoreAsync(string username, string password, CancellationToken ct) + private async Task AuthenticateCoreAsync(string username, string password, CancellationToken ct) { try { var result = await _ldap.AuthenticateAsync(username, password, ct).ConfigureAwait(false); if (!result.Success) { - return OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials"); + return new CoreOutcome( + OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials"), result.IsSystemFailure); } var roles = await ResolveRolesAsync(result.Groups, result.Roles, username, ct).ConfigureAwait(false); - return OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles); + return new CoreOutcome(OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles), SystemFailure: false); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "LDAP authentication threw for OPC UA user {User}", username); - return OpcUaUserAuthResult.Deny("Authentication backend error"); + return new CoreOutcome(OpcUaUserAuthResult.Deny("Authentication backend error"), SystemFailure: true); } } + /// The result of the core authenticate flow plus whether it was a system-side (directory) + /// failure — the signal that drives the outage circuit. + /// The auth result to return to the SDK. + /// Whether the outcome was a directory-side fault. + private readonly record struct CoreOutcome(OpcUaUserAuthResult Result, bool SystemFailure); + /// /// Resolves the user's roles from their LDAP groups via the scoped /// , unioned with any pre-resolved roles (the DevStub diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapAuthResilienceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapAuthResilienceTests.cs index 1447894b..58555439 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapAuthResilienceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapAuthResilienceTests.cs @@ -105,6 +105,118 @@ public sealed class LdapAuthResilienceTests await first.WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); } + /// After OutageFailureThreshold consecutive system-side failures the circuit opens and + /// the next call is fast-denied "backend unavailable" WITHOUT touching LDAP (03/S2). + [Fact] + public async Task Consecutive_system_failures_open_circuit_and_fast_deny() + { + var ldap = new DelayingFakeLdap(TimeSpan.Zero, SystemFailure()); + var mapper = new FakeMapper(g => g.ToArray()); + var sut = Build(ldap, mapper, + new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 }); + + for (var i = 0; i < 3; i++) + { + var r = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + r.Success.ShouldBeFalse(); + } + + ldap.Calls.ShouldBe(3); + + var sw = Stopwatch.StartNew(); + var blocked = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + sw.Stop(); + + blocked.Success.ShouldBeFalse(); + blocked.Error.ShouldNotBeNull(); + blocked.Error!.ShouldContain("unavailable"); + ldap.Calls.ShouldBe(3, "an open circuit must fast-deny without reaching the backend"); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1)); + } + + /// A user-side (credential) deny between system failures resets the streak, so the circuit + /// never opens — user denies are not directory evidence (03/S2). + [Fact] + public async Task User_side_deny_resets_the_system_failure_streak() + { + var ldap = new ScriptedFakeLdap( + SystemFailure(), SystemFailure(), UserDeny(), SystemFailure(), SystemFailure()); + var mapper = new FakeMapper(g => g.ToArray()); + var sut = Build(ldap, mapper, + new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 }); + + // 5 scripted calls: sys, sys, user(reset), sys, sys → streak ends at 2 (< 3), circuit stays closed. + for (var i = 0; i < 5; i++) + { + await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + } + + // A 6th call must still REACH the backend (circuit never opened). + await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + ldap.Calls.ShouldBe(6); + } + + /// After the cooldown elapses the circuit half-opens: the next call probes the backend and a + /// success closes the circuit (03/S2). + [Fact] + public async Task Half_open_probe_after_cooldown_closes_on_success() + { + var ldap = new ScriptedFakeLdap( + SystemFailure(), SystemFailure(), SystemFailure(), + new LdapAuthResult(true, "Alice", "alice", new[] { "configeditor" }, Array.Empty(), null)); + var mapper = new FakeMapper(g => g.ToArray()); + var sut = Build(ldap, mapper, + new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 1, AuthTimeoutMs = 5000 }); + + for (var i = 0; i < 3; i++) + await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + ldap.Calls.ShouldBe(3); + + // Circuit open: this call is fast-denied without touching the backend. + var blocked = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + blocked.Error!.ShouldContain("unavailable"); + ldap.Calls.ShouldBe(3); + + // Wait past the cooldown, then a probe reaches the backend and succeeds → circuit closes. + await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken); + + var probe = await sut.AuthenticateUserNameAsync("alice", "secret", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + probe.Success.ShouldBeTrue(); + ldap.Calls.ShouldBe(4, "the half-open probe reaches the backend"); + } + + /// OutageFailureThreshold = 0 disables the circuit — it never opens, every call reaches + /// the backend (03/S2). + [Fact] + public async Task Circuit_disabled_never_opens() + { + var ldap = new DelayingFakeLdap(TimeSpan.Zero, SystemFailure()); + var mapper = new FakeMapper(g => g.ToArray()); + var sut = Build(ldap, mapper, + new LdapOptions { OutageFailureThreshold = 0, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 }); + + for (var i = 0; i < 6; i++) + await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + + ldap.Calls.ShouldBe(6, "a disabled circuit never fast-denies"); + } + + private static LdapAuthResult SystemFailure() => + new(false, null, "u", Array.Empty(), Array.Empty(), "Unexpected authentication error") + { IsSystemFailure = true }; + + private static LdapAuthResult UserDeny() => + new(false, null, "u", Array.Empty(), Array.Empty(), "Invalid username or password") + { IsSystemFailure = false }; + private static LdapOpcUaUserAuthenticator Build( ILdapAuthService ldap, IGroupRoleMapper mapper, LdapOptions options) => new(ldap, ScopeFactoryWith(mapper), Options.Create(options), @@ -187,6 +299,34 @@ public sealed class LdapAuthResilienceTests } } + /// + /// A fake that returns a scripted sequence of results (one per call; + /// the last entry repeats once exhausted). Counts calls so the outage-circuit tests can assert the + /// backend was — or was NOT — reached on a given call. + /// + private sealed class ScriptedFakeLdap : ILdapAuthService + { + private readonly IReadOnlyList _script; + private int _calls; + + /// Initializes the fake with the scripted results, returned in order. + /// The results to return, one per call; the last repeats once exhausted. + public ScriptedFakeLdap(params LdapAuthResult[] script) => _script = script; + + /// The number of times the backend was invoked. + public int Calls => Volatile.Read(ref _calls); + + /// Returns the next scripted result (clamped to the last entry). + /// The username (ignored). + /// The password (ignored). + /// The cancellation token (ignored). + public Task AuthenticateAsync(string username, string password, CancellationToken ct = default) + { + var idx = Interlocked.Increment(ref _calls) - 1; + return Task.FromResult(_script[Math.Min(idx, _script.Count - 1)]); + } + } + /// Test group→role mapper driven by a delegate over the supplied groups. private sealed class FakeMapper(Func, IReadOnlyList> map) : IGroupRoleMapper {