feat(host): LDAP directory-outage circuit — fail-fast denial during sustained outage (03/S2)

archreview/plans/R2-08-ldap-historyread-async-plan.md.tasks.json

@@ -5,7 +5,7 @@
     { "id": "R2-08-T2", "subject": "LdapOptions: ConnectionTimeoutMs/AuthTimeoutMs/MaxConcurrentAuthentications/OutageFailureThreshold/OutageCooldownSeconds + ToLibraryOptions projection + validator rules", "status": "completed", "blockedBy": [] },
     { "id": "R2-08-T3", "subject": "LdapAuthResult.IsSystemFailure seam + OtOpcUaLdapAuthService.Adapt mapping (ServiceAccountBindFailed => system-side)", "status": "completed", "blockedBy": [] },
     { "id": "R2-08-T4", "subject": "LdapOpcUaUserAuthenticator: WaitAsync wall-clock bound + SemaphoreSlim concurrency cap with release-on-core-completion (turns T1 green)", "status": "completed", "blockedBy": ["R2-08-T1", "R2-08-T2"] },
-    { "id": "R2-08-T5", "subject": "Directory-outage circuit: fail-fast deny after N consecutive system-side failures, half-open after cooldown", "status": "pending", "blockedBy": ["R2-08-T3", "R2-08-T4"] },
+    { "id": "R2-08-T5", "subject": "Directory-outage circuit: fail-fast deny after N consecutive system-side failures, half-open after cooldown", "status": "completed", "blockedBy": ["R2-08-T3", "R2-08-T4"] },
     { "id": "R2-08-T6", "subject": "Outage-sim integration test (real library, unroutable host, via internal HandleImpersonation) + SDK m_eventLock contract doc + docs/security.md", "status": "pending", "blockedBy": ["R2-08-T4"] },
     { "id": "R2-08-T7", "subject": "S3 sequential-serving repro test (RED): MaxObservedConcurrency >= 2 on an 8-node Raw batch — fails on current code", "status": "pending", "blockedBy": [] },
     { "id": "R2-08-T8", "subject": "ServerHistorianOptions: HistoryReadBatchParallelism/MaxConcurrentHistoryReadBatches/HistoryReadDeadline + Validate warnings + node-manager properties + hosted-service wiring", "status": "pending", "blockedBy": [] },

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/LdapOpcUaUserAuthenticator.cs

@@ -47,6 +47,16 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
     /// orphan binds.</summary>
     private readonly SemaphoreSlim _gate;
 
+    /// <summary>Guards the outage-circuit state (<see cref="_consecutiveSystemFailures"/> +
+    /// <see cref="_circuitOpenUntilTicks"/>).</summary>
+    private readonly object _circuitLock = new();
+
+    /// <summary>Count of CONSECUTIVE system-side failures; reset by any success / user-side deny.</summary>
+    private int _consecutiveSystemFailures;
+
+    /// <summary><see cref="Environment.TickCount64"/> the circuit stays open until; <c>0</c> = closed.</summary>
+    private long _circuitOpenUntilTicks;
+
     /// <summary>Constructs the authenticator from its dependencies and the bound LDAP options.</summary>
     /// <param name="ldap">The app LDAP auth service (shared with the Admin UI login flow).</param>
     /// <param name="scopeFactory">Opens a per-call DI scope to resolve the scoped role mapper.</param>
@@ -70,8 +80,19 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
     {
         var authTimeout = TimeSpan.FromMilliseconds(Math.Max(1, _options.AuthTimeoutMs));
 
+        // Directory-outage circuit: after a run of consecutive system-side failures, fail fast WITHOUT
+        // touching LDAP for the cooldown window — so a sustained outage cannot serialize every session
+        // activation behind a per-call stall (the SDK raises this callback under a server-wide lock).
+        if (IsCircuitOpen())
+        {
+            _logger.LogDebug(
+                "LDAP outage circuit open — fast-denying OPC UA user {User} without a directory call", username);
+            return OpcUaUserAuthResult.Deny("Authentication backend unavailable");
+        }
+
         // Bounded in-flight concurrency: acquire a slot within our own wall-clock budget. All slots
-        // busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed).
+        // busy is LOCAL backpressure, not directory evidence — deny "busy" (fail-closed) and do NOT
+        // feed the outage circuit.
         if (!await _gate.WaitAsync(authTimeout, ct).ConfigureAwait(false))
         {
             _logger.LogWarning(
@@ -95,10 +116,14 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
 
         try
         {
-            return await core.WaitAsync(authTimeout, ct).ConfigureAwait(false);
+            var outcome = await core.WaitAsync(authTimeout, ct).ConfigureAwait(false);
+            RecordOutcome(outcome.SystemFailure, username);
+            return outcome.Result;
         }
         catch (TimeoutException)
         {
+            // A boundary timeout is directory evidence — it feeds the circuit.
+            RecordOutcome(systemFailure: true, username);
             _logger.LogWarning(
                 "LDAP authentication timed out after {Timeout} for OPC UA user {User} — denying (fail-closed)",
                 authTimeout, username);
@@ -106,34 +131,101 @@ public sealed class LdapOpcUaUserAuthenticator : IOpcUaUserAuthenticator
         }
     }
 
+    /// <summary>Whether the outage circuit is currently open (fast-deny window active). Always false when
+    /// the circuit is disabled (<see cref="LdapOptions.OutageFailureThreshold"/> ≤ 0).</summary>
+    private bool IsCircuitOpen()
+    {
+        if (_options.OutageFailureThreshold <= 0) return false;
+        lock (_circuitLock)
+        {
+            return _circuitOpenUntilTicks > Environment.TickCount64;
+        }
+    }
+
+    /// <summary>
+    /// Updates the outage-circuit state from one authentication outcome. A system-side failure advances
+    /// the consecutive-failure streak and opens the circuit once it reaches
+    /// <see cref="LdapOptions.OutageFailureThreshold"/>; any non-system outcome (success or user-side
+    /// deny) resets the streak and closes the circuit. Half-open behaviour is implicit: once the cooldown
+    /// elapses <see cref="IsCircuitOpen"/> lets the next call through with the streak still at threshold,
+    /// so a single further system failure re-opens immediately.
+    /// </summary>
+    /// <param name="systemFailure">Whether this outcome was a system-side (directory) failure.</param>
+    /// <param name="username">The login name, for diagnostics.</param>
+    private void RecordOutcome(bool systemFailure, string username)
+    {
+        if (_options.OutageFailureThreshold <= 0) return; // circuit disabled
+
+        lock (_circuitLock)
+        {
+            if (systemFailure)
+            {
+                _consecutiveSystemFailures++;
+                if (_consecutiveSystemFailures >= _options.OutageFailureThreshold)
+                {
+                    var wasOpen = _circuitOpenUntilTicks > Environment.TickCount64;
+                    _circuitOpenUntilTicks =
+                        Environment.TickCount64 + (long)Math.Max(1, _options.OutageCooldownSeconds) * 1000L;
+                    if (!wasOpen)
+                    {
+                        _logger.LogWarning(
+                            "LDAP outage circuit OPENED after {Failures} consecutive system-side failures — " +
+                            "fast-denying for {Cooldown}s (last user {User})",
+                            _consecutiveSystemFailures, _options.OutageCooldownSeconds, username);
+                    }
+                }
+            }
+            else
+            {
+                if (_consecutiveSystemFailures >= _options.OutageFailureThreshold)
+                {
+                    _logger.LogInformation(
+                        "LDAP outage circuit CLOSED — directory recovered (probe succeeded for {User})", username);
+                }
+                _consecutiveSystemFailures = 0;
+                _circuitOpenUntilTicks = 0;
+            }
+        }
+    }
+
     /// <summary>
     /// The unbounded core authenticate flow (bind + role-map) that the outer
     /// <see cref="AuthenticateUserNameAsync"/> wraps in a wall-clock bound + concurrency cap. Fail-closed:
-    /// any backend throw becomes a denial rather than escaping into the SDK.
+    /// any backend throw becomes a denial rather than escaping into the SDK. The returned
+    /// <see cref="CoreOutcome.SystemFailure"/> flags directory-side faults (a system-side
+    /// <see cref="LdapAuthResult.IsSystemFailure"/> deny, or an unexpected throw) so the caller can feed
+    /// the outage circuit; a user-credential deny is NOT a system failure.
     /// </summary>
     /// <param name="username">The username to authenticate.</param>
     /// <param name="password">The cleartext password.</param>
     /// <param name="ct">Cancellation token.</param>
-    private async Task<OpcUaUserAuthResult> AuthenticateCoreAsync(string username, string password, CancellationToken ct)
+    private async Task<CoreOutcome> AuthenticateCoreAsync(string username, string password, CancellationToken ct)
     {
         try
         {
             var result = await _ldap.AuthenticateAsync(username, password, ct).ConfigureAwait(false);
             if (!result.Success)
             {
-                return OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials");
+                return new CoreOutcome(
+                    OpcUaUserAuthResult.Deny(result.Error ?? "Invalid credentials"), result.IsSystemFailure);
             }
 
             var roles = await ResolveRolesAsync(result.Groups, result.Roles, username, ct).ConfigureAwait(false);
-            return OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles);
+            return new CoreOutcome(OpcUaUserAuthResult.Allow(result.DisplayName ?? username, roles), SystemFailure: false);
         }
         catch (Exception ex) when (ex is not OperationCanceledException)
         {
             _logger.LogWarning(ex, "LDAP authentication threw for OPC UA user {User}", username);
-            return OpcUaUserAuthResult.Deny("Authentication backend error");
+            return new CoreOutcome(OpcUaUserAuthResult.Deny("Authentication backend error"), SystemFailure: true);
         }
     }
 
+    /// <summary>The result of the core authenticate flow plus whether it was a system-side (directory)
+    /// failure — the signal that drives the outage circuit.</summary>
+    /// <param name="Result">The auth result to return to the SDK.</param>
+    /// <param name="SystemFailure">Whether the outcome was a directory-side fault.</param>
+    private readonly record struct CoreOutcome(OpcUaUserAuthResult Result, bool SystemFailure);
+
     /// <summary>
     /// Resolves the user's roles from their LDAP groups via the scoped
     /// <see cref="IGroupRoleMapper{TRole}"/>, unioned with any pre-resolved roles (the DevStub

tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapAuthResilienceTests.cs

@@ -105,6 +105,118 @@ public sealed class LdapAuthResilienceTests
         await first.WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
     }
 
+    /// <summary>After <c>OutageFailureThreshold</c> consecutive system-side failures the circuit opens and
+    /// the next call is fast-denied "backend unavailable" WITHOUT touching LDAP (03/S2).</summary>
+    [Fact]
+    public async Task Consecutive_system_failures_open_circuit_and_fast_deny()
+    {
+        var ldap = new DelayingFakeLdap(TimeSpan.Zero, SystemFailure());
+        var mapper = new FakeMapper(g => g.ToArray());
+        var sut = Build(ldap, mapper,
+            new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 });
+
+        for (var i = 0; i < 3; i++)
+        {
+            var r = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+                .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+            r.Success.ShouldBeFalse();
+        }
+
+        ldap.Calls.ShouldBe(3);
+
+        var sw = Stopwatch.StartNew();
+        var blocked = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+            .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        sw.Stop();
+
+        blocked.Success.ShouldBeFalse();
+        blocked.Error.ShouldNotBeNull();
+        blocked.Error!.ShouldContain("unavailable");
+        ldap.Calls.ShouldBe(3, "an open circuit must fast-deny without reaching the backend");
+        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1));
+    }
+
+    /// <summary>A user-side (credential) deny between system failures resets the streak, so the circuit
+    /// never opens — user denies are not directory evidence (03/S2).</summary>
+    [Fact]
+    public async Task User_side_deny_resets_the_system_failure_streak()
+    {
+        var ldap = new ScriptedFakeLdap(
+            SystemFailure(), SystemFailure(), UserDeny(), SystemFailure(), SystemFailure());
+        var mapper = new FakeMapper(g => g.ToArray());
+        var sut = Build(ldap, mapper,
+            new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 });
+
+        // 5 scripted calls: sys, sys, user(reset), sys, sys → streak ends at 2 (< 3), circuit stays closed.
+        for (var i = 0; i < 5; i++)
+        {
+            await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+                .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        }
+
+        // A 6th call must still REACH the backend (circuit never opened).
+        await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+            .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        ldap.Calls.ShouldBe(6);
+    }
+
+    /// <summary>After the cooldown elapses the circuit half-opens: the next call probes the backend and a
+    /// success closes the circuit (03/S2).</summary>
+    [Fact]
+    public async Task Half_open_probe_after_cooldown_closes_on_success()
+    {
+        var ldap = new ScriptedFakeLdap(
+            SystemFailure(), SystemFailure(), SystemFailure(),
+            new LdapAuthResult(true, "Alice", "alice", new[] { "configeditor" }, Array.Empty<string>(), null));
+        var mapper = new FakeMapper(g => g.ToArray());
+        var sut = Build(ldap, mapper,
+            new LdapOptions { OutageFailureThreshold = 3, OutageCooldownSeconds = 1, AuthTimeoutMs = 5000 });
+
+        for (var i = 0; i < 3; i++)
+            await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+                .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        ldap.Calls.ShouldBe(3);
+
+        // Circuit open: this call is fast-denied without touching the backend.
+        var blocked = await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+            .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        blocked.Error!.ShouldContain("unavailable");
+        ldap.Calls.ShouldBe(3);
+
+        // Wait past the cooldown, then a probe reaches the backend and succeeds → circuit closes.
+        await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
+
+        var probe = await sut.AuthenticateUserNameAsync("alice", "secret", CancellationToken.None)
+            .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+        probe.Success.ShouldBeTrue();
+        ldap.Calls.ShouldBe(4, "the half-open probe reaches the backend");
+    }
+
+    /// <summary><c>OutageFailureThreshold = 0</c> disables the circuit — it never opens, every call reaches
+    /// the backend (03/S2).</summary>
+    [Fact]
+    public async Task Circuit_disabled_never_opens()
+    {
+        var ldap = new DelayingFakeLdap(TimeSpan.Zero, SystemFailure());
+        var mapper = new FakeMapper(g => g.ToArray());
+        var sut = Build(ldap, mapper,
+            new LdapOptions { OutageFailureThreshold = 0, OutageCooldownSeconds = 60, AuthTimeoutMs = 5000 });
+
+        for (var i = 0; i < 6; i++)
+            await sut.AuthenticateUserNameAsync("u", "p", CancellationToken.None)
+                .WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+
+        ldap.Calls.ShouldBe(6, "a disabled circuit never fast-denies");
+    }
+
+    private static LdapAuthResult SystemFailure() =>
+        new(false, null, "u", Array.Empty<string>(), Array.Empty<string>(), "Unexpected authentication error")
+        { IsSystemFailure = true };
+
+    private static LdapAuthResult UserDeny() =>
+        new(false, null, "u", Array.Empty<string>(), Array.Empty<string>(), "Invalid username or password")
+        { IsSystemFailure = false };
+
     private static LdapOpcUaUserAuthenticator Build(
         ILdapAuthService ldap, IGroupRoleMapper<string> mapper, LdapOptions options) =>
         new(ldap, ScopeFactoryWith(mapper), Options.Create(options),
@@ -187,6 +299,34 @@ public sealed class LdapAuthResilienceTests
         }
     }
 
+    /// <summary>
+    /// A fake <see cref="ILdapAuthService"/> that returns a scripted sequence of results (one per call;
+    /// the last entry repeats once exhausted). Counts calls so the outage-circuit tests can assert the
+    /// backend was — or was NOT — reached on a given call.
+    /// </summary>
+    private sealed class ScriptedFakeLdap : ILdapAuthService
+    {
+        private readonly IReadOnlyList<LdapAuthResult> _script;
+        private int _calls;
+
+        /// <summary>Initializes the fake with the scripted results, returned in order.</summary>
+        /// <param name="script">The results to return, one per call; the last repeats once exhausted.</param>
+        public ScriptedFakeLdap(params LdapAuthResult[] script) => _script = script;
+
+        /// <summary>The number of times the backend was invoked.</summary>
+        public int Calls => Volatile.Read(ref _calls);
+
+        /// <summary>Returns the next scripted result (clamped to the last entry).</summary>
+        /// <param name="username">The username (ignored).</param>
+        /// <param name="password">The password (ignored).</param>
+        /// <param name="ct">The cancellation token (ignored).</param>
+        public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct = default)
+        {
+            var idx = Interlocked.Increment(ref _calls) - 1;
+            return Task.FromResult(_script[Math.Min(idx, _script.Count - 1)]);
+        }
+    }
+
     /// <summary>Test group→role mapper driven by a delegate over the supplied groups.</summary>
     private sealed class FakeMapper(Func<IReadOnlyList<string>, IReadOnlyList<string>> map) : IGroupRoleMapper<string>
     {