diff --git a/ZB.MOM.WW.Auth/Directory.Build.props b/ZB.MOM.WW.Auth/Directory.Build.props index cb4931c..f03af92 100644 --- a/ZB.MOM.WW.Auth/Directory.Build.props +++ b/ZB.MOM.WW.Auth/Directory.Build.props @@ -5,7 +5,7 @@ enable enable latest - 0.1.5 + 0.2.0 true diff --git a/ZB.MOM.WW.Auth/Directory.Packages.props b/ZB.MOM.WW.Auth/Directory.Packages.props index a61d3d3..1e7fff5 100644 --- a/ZB.MOM.WW.Auth/Directory.Packages.props +++ b/ZB.MOM.WW.Auth/Directory.Packages.props @@ -24,6 +24,7 @@ + diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Abstractions/Ldap/LdapContracts.cs b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Abstractions/Ldap/LdapContracts.cs index bc89047..f143205 100644 --- a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Abstractions/Ldap/LdapContracts.cs +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Abstractions/Ldap/LdapContracts.cs @@ -19,6 +19,17 @@ public sealed record LdapOptions public string GroupAttribute { get; init; } = "memberOf"; public int ConnectionTimeoutMs { get; init; } = 10_000; + /// + /// Ordered fallback LDAP endpoints tried when the primary (and any earlier fallback) + /// fails with a system-side error (connect/TLS, service-account bind, or search). + /// Each entry is "host" (adopts ) or "host:port". + /// Empty (the default) disables failover entirely — single-endpoint behavior is + /// unchanged. The TLS validation policy ( + /// / ) applies to every endpoint. Endpoint preference is + /// sticky: the service keeps using the last endpoint that answered until it fails. + /// + public IReadOnlyList FallbackServers { get; init; } = Array.Empty(); + /// /// Optional hook to harden (or, in dev, relax) TLS server-certificate validation for the /// / transports. When diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.AspNetCore/ServiceCollectionExtensions.cs b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.AspNetCore/ServiceCollectionExtensions.cs index e899e47..48e6de6 100644 --- a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.AspNetCore/ServiceCollectionExtensions.cs +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.AspNetCore/ServiceCollectionExtensions.cs @@ -49,11 +49,16 @@ public static class ServiceCollectionExtensions // Fail fast at startup on a misconfigured directory rather than on first login. services.AddSingleton, LdapOptionsValidator>(); - // LdapAuthService is stateless: it holds only a snapshot of LdapOptions and a stateless - // connection factory, and opens/disposes a connection per call. It is not IDisposable. - // Singleton is correct; TryAdd mirrors the pattern in AddZbApiKeyAuth (idempotency). + // LdapAuthService opens/disposes a connection per call and is not IDisposable. Its only + // mutable state is the sticky endpoint preference used by FallbackServers failover — + // which is precisely why the instance must be shared, so a primary-DC outage is paid + // for once rather than on every login. Singleton is correct; TryAdd mirrors the pattern + // in AddZbApiKeyAuth (idempotency). The logger is resolved with GetService, not + // GetRequiredService, so a host without logging configured still resolves the service. services.TryAddSingleton(sp => - new LdapAuthService(sp.GetRequiredService>().Value)); + new LdapAuthService( + sp.GetRequiredService>().Value, + sp.GetService>())); return services; } diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/Internal/LdapEndpoint.cs b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/Internal/LdapEndpoint.cs new file mode 100644 index 0000000..b70a524 --- /dev/null +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/Internal/LdapEndpoint.cs @@ -0,0 +1,60 @@ +namespace ZB.MOM.WW.Auth.Ldap.Internal; + +/// A single LDAP server endpoint resolved from configuration. +internal sealed record LdapEndpoint(string Host, int Port) +{ + /// Renders the endpoint as host:port — the form used in failover log messages. + public override string ToString() => $"{Host}:{Port}"; +} + +/// +/// Parses FallbackServers entries: "host" (adopts the caller's default port) +/// or "host:port". IPv6 literals are not supported — use a DNS name. +/// +internal static class LdapEndpointParser +{ + /// + /// Attempts to parse a single FallbackServers entry. + /// + /// The configured entry, "host" or "host:port". Surrounding whitespace is tolerated. + /// Port adopted when the entry carries no explicit one. + /// The parsed endpoint on success; undefined otherwise. + /// A human-readable reason on failure; empty on success. + /// when the entry parsed. + public static bool TryParse(string entry, int defaultPort, out LdapEndpoint endpoint, out string error) + { + endpoint = null!; + error = ""; + + if (string.IsNullOrWhiteSpace(entry)) + { + error = "entry is empty or whitespace"; + return false; + } + + var parts = entry.Trim().Split(':'); + if (parts.Length > 2) + { + error = $"entry \"{entry.Trim()}\" contains more than one ':' " + + "(IPv6 literals are not supported — use a DNS name)"; + return false; + } + + var host = parts[0].Trim(); + if (host.Length == 0) + { + error = $"entry \"{entry.Trim()}\" has an empty host"; + return false; + } + + var port = defaultPort; + if (parts.Length == 2 && (!int.TryParse(parts[1].Trim(), out port) || port is < 1 or > 65535)) + { + error = $"entry \"{entry.Trim()}\" has an invalid port (must be 1–65535)"; + return false; + } + + endpoint = new LdapEndpoint(host, port); + return true; + } +} diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapAuthService.cs b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapAuthService.cs index 892c173..bbce996 100644 --- a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapAuthService.cs +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapAuthService.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Novell.Directory.Ldap; using ZB.MOM.WW.Auth.Abstractions.Ldap; using ZB.MOM.WW.Auth.Ldap.Internal; @@ -11,6 +12,7 @@ namespace ZB.MOM.WW.Auth.Ldap; /// so the logic is unit-testable without a live server. /// /// +/// /// Fully fail-closed: authentication never throws to the caller — every error, expected /// or unexpected, is mapped to a structured . /// A success is only returned for a user that resolved to exactly one entry, whose password @@ -18,11 +20,38 @@ namespace ZB.MOM.WW.Auth.Ldap; /// Service-account bind failures () are /// kept distinct from end-user bind failures () so /// a system misconfiguration is not mistaken for bad user input. +/// +/// +/// When is non-empty the flow above runs against +/// an ordered endpoint list (primary first, then fallbacks in declared order) and moves on +/// to the next endpoint ONLY on a system-side failure — connect/TLS, service-account bind, +/// or search — i.e. when the endpoint gave no authoritative answer. Any answer from a healthy +/// directory (success, or a user-side failure such as +/// / ) +/// is final and is never retried elsewhere: retrying a rejected password against a second DC +/// would turn one bad-password event into several, tripping account lockouts. Endpoint +/// preference is sticky — the last endpoint that answered is tried first next time — so a +/// primary outage costs its connect timeout once, not once per login. +/// +/// +/// The service is safe to share (it is registered as a singleton). The sticky index is the +/// only mutable state; concurrent logins may race on it, which is benign — the worst outcome +/// is one login starting at a stale endpoint and walking on. +/// /// public sealed class LdapAuthService : ILdapAuthService { private readonly LdapOptions _options; private readonly ILdapConnectionFactory _connectionFactory; + private readonly IReadOnlyList _endpoints; + private readonly ILogger? _logger; + + /// + /// Index into of the endpoint tried first. Written on any + /// authoritative answer from a non-preferred endpoint. Accessed via + /// only; benign races are accepted (see the type remarks). + /// + private int _preferredIndex; /// /// Production constructor: binds against a live directory via the real @@ -31,9 +60,15 @@ public sealed class LdapAuthService : ILdapAuthService /// bypass) is carried into the factory so each /// connection is built with it. /// - public LdapAuthService(LdapOptions options) + /// The directory connection and search settings. + /// + /// Optional. When supplied, endpoint failover is made visible: a Warning per failed + /// endpoint hop and an Information line when the sticky preference moves. Omitting it + /// (the default) leaves the service silent — hosts without logging still work. + /// + public LdapAuthService(LdapOptions options, ILogger? logger = null) : this(options, new NovellLdapConnectionFactory( - options.AllowInsecure, options.ServerCertificateValidationCallback)) + options.AllowInsecure, options.ServerCertificateValidationCallback), logger) { } @@ -42,10 +77,33 @@ public sealed class LdapAuthService : ILdapAuthService /// so the bind/search logic can be exercised without a live directory. Internal /// because the connection seam is an implementation detail. /// - internal LdapAuthService(LdapOptions options, ILdapConnectionFactory connectionFactory) + internal LdapAuthService( + LdapOptions options, + ILdapConnectionFactory connectionFactory, + ILogger? logger = null) { _options = options; _connectionFactory = connectionFactory; + _logger = logger; + _endpoints = BuildEndpoints(options); + } + + /// + /// Primary first, then fallbacks in declared order. The validator rejects malformed + /// entries at boot; unparseable entries are skipped defensively here so a service + /// constructed without options validation degrades to the parseable subset rather + /// than throwing from a constructor. + /// + private static IReadOnlyList BuildEndpoints(LdapOptions options) + { + var endpoints = new List { new(options.Server, options.Port) }; + foreach (var entry in options.FallbackServers) + { + if (LdapEndpointParser.TryParse(entry, options.Port, out var endpoint, out _)) + endpoints.Add(endpoint); + } + + return endpoints; } /// @@ -54,7 +112,10 @@ public sealed class LdapAuthService : ILdapAuthService /// bind-then-search-then-bind flow is wrapped so that any error — expected or unexpected — /// is mapped to a structured . A /// Succeeded == true result is only ever returned when the user resolved to exactly - /// one entry, their password verified, AND at least one group was extracted. + /// one entry, their password verified, AND at least one group was extracted. With + /// configured the flow may run against more than + /// one endpoint before answering — see the type-level remarks for the walk and its + /// system-side-only failover rule. /// public Task AuthenticateAsync(string username, string password, CancellationToken ct) { @@ -80,25 +141,90 @@ public sealed class LdapAuthService : ILdapAuthService // filter and the returned result (avoids two identities for one person). username = username.Trim(); + // 4. Walk the endpoint list, preferred one first. A single-endpoint configuration + // (no FallbackServers) collapses to exactly one attempt — unchanged behaviour. + var preferred = Volatile.Read(ref _preferredIndex); + if ((uint)preferred >= (uint)_endpoints.Count) + preferred = 0; // defensive: the list is immutable, but never index out of it + + // Seeded so an exhausted walk still returns the conservative system-side bucket + // even in the impossible case of an empty endpoint list. + var lastSystemSideFailure = LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed); + + foreach (var index in WalkOrder(_endpoints.Count, preferred)) + { + var result = AuthenticateAgainst(_endpoints[index], username, password); + + // ServiceAccountBindFailed is the (sole) system-side bucket: this endpoint gave + // no authoritative answer, so try the next one. Every other outcome — success, or + // a user-side failure like UserNotFound/BadCredentials — came from a healthy + // directory and is FINAL: re-asking a second DC about a rejected password would + // multiply one bad-password event into several against the account lockout policy. + if (result.Failure == LdapAuthFailure.ServiceAccountBindFailed) + { + lastSystemSideFailure = result; + continue; + } + + // Sticky preference: any authoritative answer proves this endpoint healthy. + // Updating only on Succeeded would re-pay the dead primary's connect timeout on + // every mistyped password for the whole outage. + if (index != preferred) + { + Volatile.Write(ref _preferredIndex, index); + _logger?.LogInformation("LDAP sticky preference moved to {Endpoint}.", _endpoints[index]); + } + + return result; + } + + // Every endpoint failed system-side. The preference is deliberately left where it was: + // a total outage says nothing about which endpoint to prefer once one recovers. + return lastSystemSideFailure; + } + + /// Preferred endpoint first, then every other endpoint in declared order. + private static IEnumerable WalkOrder(int count, int preferred) + { + yield return preferred; + for (var i = 0; i < count; i++) + { + if (i != preferred) + yield return i; + } + } + + /// + /// Runs one full bind-then-search-then-bind attempt against a single endpoint on a fresh + /// connection. The fail-closed contract holds per attempt: this never throws, and it + /// returns for — and only for — + /// system-side faults, which is what the caller reads as "no authoritative answer, try + /// the next endpoint". + /// + private LdapAuthResult AuthenticateAgainst(LdapEndpoint endpoint, string username, string password) + { // The whole flow runs inside an outer fail-closed guard: a StageFailure carries an // already-mapped failure out of a stage, and any OTHER unexpected exception defaults to // the most conservative system-side bucket. Either way the caller gets a structured result. try { + // A fresh connection per attempt: a Novell connection that has failed to connect + // or bind is not safely reusable against another host. using var conn = _connectionFactory.Create(); - // 4. Open the connection (transport/TLS handling lives in the adapter). The + // 5. Open the connection (transport/TLS handling lives in the adapter). The // per-operation timeout (ConnectionTimeoutMs) is applied by the adapter here. - // A failure to connect/upgrade means the directory is unreachable — a - // system-side fault, not the user's, so map it to ServiceAccountBindFailed. + // A failure to connect/upgrade means this endpoint is unreachable — a + // system-side fault, not the user's, so map it to ServiceAccountBindFailed + // and let the caller try the next endpoint. // NOTE: the LdapAuthFailure enum has no dedicated DirectoryUnavailable value; // ServiceAccountBindFailed is the closest system-side bucket. A future // Abstractions change could add DirectoryUnavailable to disambiguate. try { conn.Connect( - _options.Server, - _options.Port, + endpoint.Host, + endpoint.Port, _options.Transport, _options.AllowInsecure, _options.ConnectionTimeoutMs, @@ -106,10 +232,10 @@ public sealed class LdapAuthService : ILdapAuthService } catch (LdapException) { - throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed); + throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "connect"); } - // 5. Service-account bind so we can search for the user's DN. A bind failure + // 6. Service-account bind so we can search for the user's DN. A bind failure // here is a service-account misconfiguration — DISTINCT from a user-credential // failure — so it maps to ServiceAccountBindFailed. try @@ -118,10 +244,10 @@ public sealed class LdapAuthService : ILdapAuthService } catch (LdapException) { - throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed); + throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "service-bind"); } - // 6. Search for the user entry by the configured username attribute. A search + // 7. Search for the user entry by the configured username attribute. A search // failure is infrastructure (directory unreachable / unhealthy) — system-side. // NOTE: same enum limitation as Connect above; ServiceAccountBindFailed is the // closest system-side bucket until a DirectoryUnavailable value exists. @@ -133,10 +259,11 @@ public sealed class LdapAuthService : ILdapAuthService } catch (LdapException) { - throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed); + throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "search"); } - // 7. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser. + // 8. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser. + // Both are authoritative answers from a healthy directory, so they end the walk. // We never attempt a user bind against an ambiguous DN. if (entries.Count == 0) return LdapAuthResult.Fail(LdapAuthFailure.UserNotFound); @@ -145,19 +272,21 @@ public sealed class LdapAuthService : ILdapAuthService var entry = entries[0]; - // 8. User bind: re-bind as the resolved DN to verify the password. A bind failure - // here is the end user's bad credentials. + // 9. User bind: re-bind as the resolved DN to verify the password. A bind failure + // here is the end user's bad credentials — the directory answered, so this is + // returned directly and never retried against another endpoint. try { conn.Bind(entry.Dn, password); } catch (LdapException) { - throw new StageFailure(LdapAuthFailure.BadCredentials); + return LdapAuthResult.Fail(LdapAuthFailure.BadCredentials); } - // 9. Group extraction. Fail closed: an empty/missing group set, or any error while - // extracting groups, is a GroupLookupFailed — never a zero-group success. + // 10. Group extraction. Fail closed: an empty/missing group set, or any error while + // extracting groups, is a GroupLookupFailed — never a zero-group success. This + // too is an answer about the user, not about the endpoint, so it ends the walk. IReadOnlyList groups; try { @@ -165,25 +294,31 @@ public sealed class LdapAuthService : ILdapAuthService } catch { - throw new StageFailure(LdapAuthFailure.GroupLookupFailed); + return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed); } if (groups.Count == 0) return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed); - // 10. Success — and only here, with a verified password and >= 1 group. + // 11. Success — and only here, with a verified password and >= 1 group. var displayName = ExtractDisplayName(entry, username); return LdapAuthResult.Success(username, displayName, groups); } catch (StageFailure stage) { - // A stage mapped its own failure; surface it as a structured result. + // A stage mapped its own failure; surface it as a structured result. Log at Warning + // so a silent failover — the failure mode this feature exists to survive — is + // visible in the host's log rather than only in a latency graph. + _logger?.LogWarning( + "LDAP endpoint {Endpoint} failed system-side at {Stage}; treating as no answer.", + endpoint, stage.Stage); return LdapAuthResult.Fail(stage.Failure); } - catch + catch (Exception ex) { // Belt-and-braces: ANY unexpected exception fails closed to the most conservative // system-side bucket rather than propagating to the caller. + _logger?.LogWarning(ex, "LDAP endpoint {Endpoint} failed system-side with an unexpected error.", endpoint); return LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed); } } @@ -194,9 +329,16 @@ public sealed class LdapAuthService : ILdapAuthService /// private sealed class StageFailure : Exception { - public StageFailure(LdapAuthFailure failure) => Failure = failure; + public StageFailure(LdapAuthFailure failure, string stage) + { + Failure = failure; + Stage = stage; + } public LdapAuthFailure Failure { get; } + + /// Which stage faulted (connect / service-bind / search), for diagnostics. + public string Stage { get; } } /// diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapOptionsValidator.cs b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapOptionsValidator.cs index 71102b0..c5fdf08 100644 --- a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapOptionsValidator.cs +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapOptionsValidator.cs @@ -10,7 +10,7 @@ namespace ZB.MOM.WW.Auth.Ldap; /// /// /// Validation is skipped entirely when is false -/// (a disabled provider's connection fields are inert). When enabled, four conditions +/// (a disabled provider's connection fields are inert). When enabled, five conditions /// are enforced: /// /// plaintext transport () is rejected unless @@ -19,7 +19,10 @@ namespace ZB.MOM.WW.Auth.Ldap; /// must be specified (the DN root every /// search runs against); /// must be specified — an empty value -/// would bind anonymously, defeating the search-then-bind authentication flow. +/// would bind anonymously, defeating the search-then-bind authentication flow; +/// every entry must parse as +/// "host" or "host:port" — a typo'd backup DC must not lie dormant +/// until the primary outage it exists to survive. /// /// public sealed class LdapOptionsValidator : IValidateOptions @@ -68,6 +71,21 @@ public sealed class LdapOptionsValidator : IValidateOptions "(e.g. \"cn=svc,dc=example,dc=com\")."); } + // Fallback endpoints are walked by LdapAuthService on system-side failures. A + // malformed entry would be silently skipped there (the walk degrades to the + // parseable subset rather than throwing), so it must be caught at boot instead — + // otherwise an operator's typo'd backup DC would only surface as a failed + // failover during the primary outage it was meant to survive. + foreach (var entry in options.FallbackServers) + { + if (!Internal.LdapEndpointParser.TryParse(entry, options.Port, out _, out var error)) + { + return ValidateOptionsResult.Fail( + $"{nameof(LdapOptions.FallbackServers)} contains an invalid entry — {error}. " + + "Each entry must be \"host\" or \"host:port\" (port 1–65535)."); + } + } + return ValidateOptionsResult.Success; } } diff --git a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/ZB.MOM.WW.Auth.Ldap.csproj b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/ZB.MOM.WW.Auth.Ldap.csproj index 4c58f23..25af340 100644 --- a/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/ZB.MOM.WW.Auth.Ldap.csproj +++ b/ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/ZB.MOM.WW.Auth.Ldap.csproj @@ -22,6 +22,7 @@ + diff --git a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/Integration/GLAuthIntegrationTests.cs b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/Integration/GLAuthIntegrationTests.cs index fe94dee..be86a05 100644 --- a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/Integration/GLAuthIntegrationTests.cs +++ b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/Integration/GLAuthIntegrationTests.cs @@ -99,4 +99,70 @@ public sealed class GLAuthIntegrationTests $"Authentication failed: {result.Failure} (server={server}:{port}, user={user})"); Assert.NotEmpty(result.Groups); } + + /// + /// Failover: primary endpoint points at a closed local port (refused immediately), + /// fallback at the real GLAuth. The login must succeed via the fallback, and a second + /// login must also succeed (sticky preference — no re-walk through the dead primary). + /// Skipped unless ZB_LDAP_IT=1 is set; skipped again if the server is unreachable. + /// + [SkippableFact] + public async Task Authenticate_FailsOverFromDeadPrimary_ToRealGLAuth() + { + // ------------------------------------------------------------------ opt-in gate + Skip.IfNot( + Environment.GetEnvironmentVariable("ZB_LDAP_IT") == "1", + "Set ZB_LDAP_IT=1 and a reachable GLAuth to run."); + + // ------------------------------------------------------------------ read config + var server = Environment.GetEnvironmentVariable("ZB_LDAP_SERVER") ?? "localhost"; + var port = int.TryParse(Environment.GetEnvironmentVariable("ZB_LDAP_PORT"), out var p) ? p : 3893; + var baseDn = Environment.GetEnvironmentVariable("ZB_LDAP_BASE") ?? "dc=lmxopcua,dc=local"; + var svcDn = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_DN") ?? ""; + var svcPw = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_PW") ?? ""; + var user = Environment.GetEnvironmentVariable("ZB_LDAP_USER") ?? ""; + var pw = Environment.GetEnvironmentVariable("ZB_LDAP_PW") ?? ""; + var userAttr = Environment.GetEnvironmentVariable("ZB_LDAP_USERATTR") ?? "cn"; + + // ------------------------------------------------------------------ reachability probe + try + { + using var tcp = new TcpClient(); + // 3-second connect timeout to keep the test suite snappy when the server is absent + var connectTask = tcp.ConnectAsync(server, port); + if (!connectTask.Wait(TimeSpan.FromSeconds(3))) + Skip.If(true, $"GLAuth not reachable at {server}:{port} (connect timed out)."); + } + catch (Exception ex) + { + Skip.If(true, $"GLAuth not reachable at {server}:{port}: {ex.Message}"); + } + + // ------------------------------------------------------------------ build options + var options = new LdapOptions + { + Enabled = true, + Server = "127.0.0.1", + Port = 39999, // closed port -> immediate refusal + Transport = LdapTransport.None, + AllowInsecure = true, + SearchBase = baseDn, + ServiceAccountDn = svcDn, + ServiceAccountPassword = svcPw, + UserNameAttribute = userAttr, + GroupAttribute = "memberOf", + ConnectionTimeoutMs = 3000, + FallbackServers = new[] { $"{server}:{port}" }, + }; + + // ------------------------------------------------------------------ exercise the real service + var svc = new LdapAuthService(options); + + var first = await svc.AuthenticateAsync(user, pw, CancellationToken.None); + Assert.True(first.Succeeded, $"first login failed: {first.Failure}"); + + // Second login exercises the sticky preference: it must not re-walk the dead primary. + var second = await svc.AuthenticateAsync(user, pw, CancellationToken.None); + Assert.True(second.Succeeded, $"second login failed: {second.Failure}"); + } } diff --git a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapAuthServiceFailoverTests.cs b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapAuthServiceFailoverTests.cs new file mode 100644 index 0000000..e8804f1 --- /dev/null +++ b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapAuthServiceFailoverTests.cs @@ -0,0 +1,297 @@ +using ZB.MOM.WW.Auth.Abstractions.Ldap; + +namespace ZB.MOM.WW.Auth.Ldap.Tests; + +/// +/// Covers the multi-endpoint failover walk: which failures fail over (system-side only), +/// which are final (any authoritative answer from a healthy directory), how the sticky +/// endpoint preference moves, and that single-endpoint behaviour is unchanged. +/// +public class LdapAuthServiceFailoverTests +{ + private static LdapOptions Opts(params string[] fallbacks) => new() + { + Enabled = true, + Server = "dc1", + Port = 636, + Transport = LdapTransport.Ldaps, + SearchBase = "dc=x", + ServiceAccountDn = "cn=svc,dc=x", + ServiceAccountPassword = "svcpw", + FallbackServers = fallbacks, + }; + + private static FakeLdapConnection HealthyWithUser() => + new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", new[] { "cn=ops,dc=x" }); + + // ---- failover triggers (system-side) ---- + + [Fact] + public async Task ConnectFailure_FailsOverToFallback_AndSucceeds() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2:3894"), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.True(result.Succeeded); + Assert.Equal(2, factory.Created.Count); + Assert.Equal("dc1", factory.Created[0].ConnectArgs!.Value.Host); + Assert.Equal("dc2", factory.Created[1].ConnectArgs!.Value.Host); + Assert.Equal(3894, factory.Created[1].ConnectArgs!.Value.Port); + } + + [Fact] + public async Task ServiceBindFailure_FailsOverToFallback() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.True(result.Succeeded); + Assert.Equal(636, factory.Created[1].ConnectArgs!.Value.Port); // bare host adopts Port + } + + // ---- user-side results NEVER fail over ---- + + [Fact] + public async Task BadUserCredentials_DoesNotFailOver() + { + var factory = new SequenceLdapConnectionFactory( + HealthyWithUser().ThrowOnUserBind()); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("alice", "wrong", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.BadCredentials, result.Failure); + Assert.Single(factory.Created); + } + + [Fact] + public async Task UserNotFound_DoesNotFailOver() + { + var factory = new SequenceLdapConnectionFactory(new FakeLdapConnection()); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.UserNotFound, result.Failure); + Assert.Single(factory.Created); + } + + [Fact] + public async Task GroupLookupFailed_DoesNotFailOver() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", Array.Empty())); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.GroupLookupFailed, result.Failure); + Assert.Single(factory.Created); + } + + [Fact] + public async Task AmbiguousUser_DoesNotFailOver() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().WithDuplicateMatch()); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.AmbiguousUser, result.Failure); + Assert.Single(factory.Created); + } + + // ---- exhaustion ---- + + [Fact] + public async Task AllEndpointsDown_ReturnsServiceAccountBindFailed() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), + new FakeLdapConnection().ThrowOnConnect()); + var svc = new LdapAuthService(Opts("dc2"), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure); + Assert.Equal(2, factory.Created.Count); + } + + // ---- stickiness ---- + + [Fact] + public async Task StickyPreference_SecondLoginStartsAtLastGoodEndpoint() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead + HealthyWithUser(), // login 1 @ dc2: answers + HealthyWithUser()); // login 2: should go straight to dc2 + var svc = new LdapAuthService(Opts("dc2"), factory); + + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + + Assert.Equal(3, factory.Created.Count); + Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host); + } + + [Fact] + public async Task StickyPreference_DriftsBackToPrimary_WhenPreferredFails() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead + HealthyWithUser(), // login 1 @ dc2: answers -> preferred + new FakeLdapConnection().ThrowOnConnect(), // login 2 @ dc2: now dead + HealthyWithUser(), // login 2 @ dc1 (declared order): recovered + HealthyWithUser()); // login 3: should start at dc1 again + var svc = new LdapAuthService(Opts("dc2"), factory); + + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + + Assert.Equal(5, factory.Created.Count); + Assert.Equal("dc1", factory.Created[3].ConnectArgs!.Value.Host); + Assert.Equal("dc1", factory.Created[4].ConnectArgs!.Value.Host); + } + + [Fact] + public async Task StickyPreference_UpdatesOnUserSideAnswer_NotOnlySuccess() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead + new FakeLdapConnection(), // login 1 @ dc2: answers UserNotFound + HealthyWithUser()); // login 2: should start at dc2 + var svc = new LdapAuthService(Opts("dc2"), factory); + + var first = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None); + Assert.Equal(LdapAuthFailure.UserNotFound, first.Failure); + + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host); + } + + [Fact] + public async Task AllEndpointsDown_LeavesPreferenceUnchanged() + { + // A total outage teaches nothing about which endpoint is healthy, so the walk + // must not move the preference — the next login still starts at the primary. + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1 + new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc2 + HealthyWithUser()); // login 2: must start at dc1 again + var svc = new LdapAuthService(Opts("dc2"), factory); + + Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, + (await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Failure); + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + + Assert.Equal("dc1", factory.Created[2].ConnectArgs!.Value.Host); + } + + // ---- back-compat ---- + + [Fact] + public async Task NoFallbacks_SingleAttempt_UnchangedBehavior() + { + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect()); + var svc = new LdapAuthService(Opts(/* no fallbacks */), factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure); + Assert.Single(factory.Created); + } + + [Fact] + public async Task Disabled_NeverOpensAConnection_EvenWithFallbacks() + { + var factory = new SequenceLdapConnectionFactory(HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2") with { Enabled = false }, factory); + + var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Equal(LdapAuthFailure.Disabled, result.Failure); + Assert.Empty(factory.Created); + } + + [Fact] + public async Task MalformedFallbackEntry_IsSkippedDefensively() + { + // The validator rejects this at boot; a service constructed without validation + // must degrade to the parseable subset, not throw. + var factory = new SequenceLdapConnectionFactory(HealthyWithUser()); + var svc = new LdapAuthService(Opts(":::"), factory); + + Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded); + Assert.Single(factory.Created); + } + + // ---- observability ---- + + /// Captures emitted log entries so failover visibility can be asserted. + private sealed class RecordingLogger : Microsoft.Extensions.Logging.ILogger + { + public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = new(); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + public void Log(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId, + TState state, Exception? exception, Func formatter) + => Entries.Add((logLevel, formatter(state, exception))); + } + + [Fact] + public async Task Failover_LogsWarningWithEndpointAndStage_AndInfoOnStickyChange() + { + var logger = new RecordingLogger(); + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2:3894"), factory, logger); + + await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Contains(logger.Entries, e => + e.Level == Microsoft.Extensions.Logging.LogLevel.Warning + && e.Message.Contains("dc1:636") && e.Message.Contains("connect")); + Assert.Contains(logger.Entries, e => + e.Level == Microsoft.Extensions.Logging.LogLevel.Information + && e.Message.Contains("dc2:3894")); + // Never log secrets: + Assert.DoesNotContain(logger.Entries, e => e.Message.Contains("svcpw") || e.Message.Contains("pw\"")); + } + + [Fact] + public async Task HappyPath_LogsNothing() + { + var logger = new RecordingLogger(); + var factory = new SequenceLdapConnectionFactory(HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2"), factory, logger); + + await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Empty(logger.Entries); + } + + [Fact] + public async Task ServiceBindFailure_LogsTheServiceBindStage_NotConnect() + { + var logger = new RecordingLogger(); + var factory = new SequenceLdapConnectionFactory( + new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser()); + var svc = new LdapAuthService(Opts("dc2"), factory, logger); + + await svc.AuthenticateAsync("alice", "pw", CancellationToken.None); + + Assert.Contains(logger.Entries, e => + e.Level == Microsoft.Extensions.Logging.LogLevel.Warning + && e.Message.Contains("dc1:636") && e.Message.Contains("service-bind")); + } +} diff --git a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapEndpointParserTests.cs b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapEndpointParserTests.cs new file mode 100644 index 0000000..67aeedf --- /dev/null +++ b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapEndpointParserTests.cs @@ -0,0 +1,35 @@ +using ZB.MOM.WW.Auth.Ldap.Internal; + +namespace ZB.MOM.WW.Auth.Ldap.Tests; + +public class LdapEndpointParserTests +{ + [Theory] + [InlineData("dc2.example.com", 636, "dc2.example.com", 636)] // bare host -> default port + [InlineData("dc2.example.com:3894", 636, "dc2.example.com", 3894)] // explicit port wins + [InlineData(" dc2.example.com : 3894 ", 636, "dc2.example.com", 3894)] // whitespace tolerated + public void TryParse_ValidEntry_Succeeds(string entry, int defaultPort, string host, int port) + { + Assert.True(LdapEndpointParser.TryParse(entry, defaultPort, out var ep, out _)); + Assert.Equal(host, ep.Host); + Assert.Equal(port, ep.Port); + } + + [Theory] + [InlineData("")] // empty + [InlineData(" ")] // whitespace + [InlineData(":636")] // empty host + [InlineData("dc2:0")] // port below range + [InlineData("dc2:65536")] // port above range + [InlineData("dc2:abc")] // non-numeric port + [InlineData("2001:db8::1:636")] // IPv6 literal (unsupported, >1 colon) + public void TryParse_InvalidEntry_Fails(string entry) + { + Assert.False(LdapEndpointParser.TryParse(entry, 636, out _, out var error)); + Assert.NotEmpty(error); + } + + [Fact] + public void Endpoint_ToString_IsHostColonPort() => + Assert.Equal("dc2:3894", new LdapEndpoint("dc2", 3894).ToString()); +} diff --git a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapOptionsValidatorTests.cs b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapOptionsValidatorTests.cs index 6535bff..144a64e 100644 --- a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapOptionsValidatorTests.cs +++ b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapOptionsValidatorTests.cs @@ -88,4 +88,32 @@ public class LdapOptionsValidatorTests ServiceAccountDn = "", }) .Failed); + + [Fact] + public void Validator_Succeeds_WithValidFallbackServers() => + Assert.False(new LdapOptionsValidator() + .Validate(null, Opts() with { FallbackServers = new[] { "dc2", "dc3:3894" } }) + .Failed); + + [Theory] + [InlineData("")] + [InlineData(":636")] + [InlineData("dc2:0")] + [InlineData("dc2:notaport")] + public void Validator_Fails_OnMalformedFallbackServer(string entry) + { + var result = new LdapOptionsValidator() + .Validate(null, Opts() with { FallbackServers = new[] { entry } }); + + Assert.True(result.Failed); + Assert.Contains(nameof(LdapOptions.FallbackServers), result.FailureMessage); + } + + [Fact] + public void Validator_Skips_FallbackServers_WhenDisabled() => + // Same inert-when-disabled rule as every other connection field: a disabled + // provider never opens a connection, so its fallback list is never walked. + Assert.False(new LdapOptionsValidator() + .Validate(null, Opts() with { Enabled = false, FallbackServers = new[] { ":::" } }) + .Failed); } diff --git a/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/SequenceLdapConnectionFactory.cs b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/SequenceLdapConnectionFactory.cs new file mode 100644 index 0000000..cf0233c --- /dev/null +++ b/ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/SequenceLdapConnectionFactory.cs @@ -0,0 +1,51 @@ +using ZB.MOM.WW.Auth.Ldap.Internal; + +namespace ZB.MOM.WW.Auth.Ldap.Tests; + +/// +/// Test double for that hands out pre-scripted +/// instances in order, one per call — +/// exactly matching the failover walk, which opens a fresh connection per endpoint attempt. +/// Records every created connection for inspection. When the script runs out, returns a +/// bare unscripted fake (which yields user-not-found) so a miscounted test fails loudly +/// rather than reusing a stale fake. +/// +internal sealed class SequenceLdapConnectionFactory : ILdapConnectionFactory +{ + private readonly Queue _script; + + /// Scripts the connections handed out, in the order the walk will request them. + public SequenceLdapConnectionFactory(params FakeLdapConnection[] fakes) + => _script = new Queue(fakes); + + /// Every connection handed out, in creation order (scripted or overflow). + public List Created { get; } = new(); + + /// + public ILdapConnection Create() + { + var fake = _script.Count > 0 ? _script.Dequeue() : new FakeLdapConnection(); + Created.Add(fake); + return fake; + } +} + +// --------------------------------------------------------------------------- +// Smoke test: verifies the sequencing and overflow behaviour the failover tests rely on. +// --------------------------------------------------------------------------- + +public class SequenceLdapConnectionFactorySmokeTests +{ + [Fact] + public void Create_ReturnsScriptedFakesInOrder_ThenOverflows() + { + var first = new FakeLdapConnection().ThrowOnConnect(); + var second = new FakeLdapConnection(); + var factory = new SequenceLdapConnectionFactory(first, second); + + Assert.Same(first, factory.Create()); + Assert.Same(second, factory.Create()); + Assert.NotSame(second, factory.Create()); // overflow: fresh bare fake + Assert.Equal(3, factory.Created.Count); + } +} diff --git a/docs/plans/2026-08-10-ldap-backup-dc-failover-design.md b/docs/plans/2026-08-10-ldap-backup-dc-failover-design.md index 7550ba1..04c529b 100644 --- a/docs/plans/2026-08-10-ldap-backup-dc-failover-design.md +++ b/docs/plans/2026-08-10-ldap-backup-dc-failover-design.md @@ -94,6 +94,13 @@ Inside `LdapAuthService.Authenticate`, an endpoint walk over an internal preferred fails; when it does, the walk tries declared order again, so preference drifts back to the primary naturally. +> **Refinement (Task 4):** sticky preference updates on ANY authoritative answer +> (success or user-side failure), not only on success — a DC that answers is healthy, +> and success-only stickiness would re-pay the dead primary's connect timeout on every +> mistyped password during an outage. Corollary, also implemented: an *exhausted* walk +> (every endpoint system-side-down) leaves the preference untouched, since a total +> outage says nothing about which endpoint to prefer once one recovers. + ## Stickiness state `volatile int _preferredIndex` on the singleton `LdapAuthService` (registered diff --git a/docs/plans/2026-08-10-ldap-backup-dc-failover.md.tasks.json b/docs/plans/2026-08-10-ldap-backup-dc-failover.md.tasks.json index a4c03a2..267aecc 100644 --- a/docs/plans/2026-08-10-ldap-backup-dc-failover.md.tasks.json +++ b/docs/plans/2026-08-10-ldap-backup-dc-failover.md.tasks.json @@ -1,14 +1,66 @@ { "planPath": "docs/plans/2026-08-10-ldap-backup-dc-failover.md", "tasks": [ - {"id": 1, "subject": "Task 1: LdapEndpoint + LdapEndpointParser", "status": "pending"}, - {"id": 2, "subject": "Task 2: LdapOptions.FallbackServers + validator rules", "status": "pending", "blockedBy": [1]}, - {"id": 3, "subject": "Task 3: SequenceLdapConnectionFactory test double", "status": "pending"}, - {"id": 4, "subject": "Task 4: Endpoint walk + sticky preference in LdapAuthService", "status": "pending", "blockedBy": [1, 2, 3]}, - {"id": 5, "subject": "Task 5: Failover observability (ILogger) + DI wiring", "status": "pending", "blockedBy": [4]}, - {"id": 6, "subject": "Task 6: GLAuth failover integration test", "status": "pending", "blockedBy": [5]}, - {"id": 7, "subject": "Task 7: Version 0.2.0, pack, publish, restore-verify", "status": "pending", "blockedBy": [6]}, - {"id": 8, "subject": "Task 8: Documentation + index truth", "status": "pending", "blockedBy": [7]} + { + "id": 1, + "subject": "Task 1: LdapEndpoint + LdapEndpointParser", + "status": "completed" + }, + { + "id": 2, + "subject": "Task 2: LdapOptions.FallbackServers + validator rules", + "status": "completed", + "blockedBy": [ + 1 + ] + }, + { + "id": 3, + "subject": "Task 3: SequenceLdapConnectionFactory test double", + "status": "completed" + }, + { + "id": 4, + "subject": "Task 4: Endpoint walk + sticky preference in LdapAuthService", + "status": "pending", + "blockedBy": [ + 1, + 2, + 3 + ] + }, + { + "id": 5, + "subject": "Task 5: Failover observability (ILogger) + DI wiring", + "status": "pending", + "blockedBy": [ + 4 + ] + }, + { + "id": 6, + "subject": "Task 6: GLAuth failover integration test", + "status": "pending", + "blockedBy": [ + 5 + ] + }, + { + "id": 7, + "subject": "Task 7: Version 0.2.0, pack, publish, restore-verify", + "status": "pending", + "blockedBy": [ + 6 + ] + }, + { + "id": 8, + "subject": "Task 8: Documentation + index truth", + "status": "pending", + "blockedBy": [ + 7 + ] + } ], "lastUpdated": "2026-08-10" -} +} \ No newline at end of file