dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(auth): LDAP backup-DC sticky failover — family 0.2.0

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-10 16:41:18 -04:00
parent 0c1702acbc 3ce9cc3657
commit 84af475683
15 changed files with 815 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.Auth/Directory.Build.props
+1 -1
View File
@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<LangVersion>latest</LangVersion>
<Version>0.1.5</Version>
<Version>0.2.0</Version>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
ZB.MOM.WW.Auth/Directory.Packages.props
+1
View File
@@ -24,6 +24,7 @@
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.7" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.7" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.7" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.7" />
<!-- ASP.NET Core Authentication / Authorization -->
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.7" />
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Abstractions/Ldap/LdapContracts.cs
+11
View File
@@ -19,6 +19,17 @@ public sealed record LdapOptions
public string GroupAttribute { get; init; } = "memberOf";
public int ConnectionTimeoutMs { get; init; } = 10_000;
/// <summary>
/// Ordered fallback LDAP endpoints tried when the primary (and any earlier fallback)
/// fails with a system-side error (connect/TLS, service-account bind, or search).
/// Each entry is <c>"host"</c> (adopts <see cref="Port"/>) or <c>"host:port"</c>.
/// Empty (the default) disables failover entirely — single-endpoint behavior is
/// unchanged. The TLS validation policy (<see cref="ServerCertificateValidationCallback"/>
/// / <see cref="AllowInsecure"/>) applies to every endpoint. Endpoint preference is
/// sticky: the service keeps using the last endpoint that answered until it fails.
/// </summary>
public IReadOnlyList<string> FallbackServers { get; init; } = Array.Empty<string>();
/// <summary>
/// Optional hook to harden (or, in dev, relax) TLS server-certificate validation for the
/// <see cref="LdapTransport.Ldaps"/> / <see cref="LdapTransport.StartTls"/> transports. When
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.AspNetCore/ServiceCollectionExtensions.cs
+9 -4
View File
@@ -49,11 +49,16 @@ public static class ServiceCollectionExtensions
// Fail fast at startup on a misconfigured directory rather than on first login.
services.AddSingleton<IValidateOptions<LdapOptions>, LdapOptionsValidator>();
// LdapAuthService is stateless: it holds only a snapshot of LdapOptions and a stateless
// connection factory, and opens/disposes a connection per call. It is not IDisposable.
// Singleton is correct; TryAdd mirrors the pattern in AddZbApiKeyAuth (idempotency).
// LdapAuthService opens/disposes a connection per call and is not IDisposable. Its only
// mutable state is the sticky endpoint preference used by FallbackServers failover —
// which is precisely why the instance must be shared, so a primary-DC outage is paid
// for once rather than on every login. Singleton is correct; TryAdd mirrors the pattern
// in AddZbApiKeyAuth (idempotency). The logger is resolved with GetService, not
// GetRequiredService, so a host without logging configured still resolves the service.
services.TryAddSingleton<ILdapAuthService>(sp =>
new LdapAuthService(sp.GetRequiredService<IOptions<LdapOptions>>().Value));
new LdapAuthService(
sp.GetRequiredService<IOptions<LdapOptions>>().Value,
sp.GetService<Microsoft.Extensions.Logging.ILogger<LdapAuthService>>()));
return services;
}
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/Internal/LdapEndpoint.cs
+60
View File
@@ -0,0 +1,60 @@
namespace ZB.MOM.WW.Auth.Ldap.Internal;
/// <summary>A single LDAP server endpoint resolved from configuration.</summary>
internal sealed record LdapEndpoint(string Host, int Port)
{
/// <summary>Renders the endpoint as <c>host:port</c> — the form used in failover log messages.</summary>
public override string ToString() => $"{Host}:{Port}";
}
/// <summary>
/// Parses <c>FallbackServers</c> entries: <c>"host"</c> (adopts the caller's default port)
/// or <c>"host:port"</c>. IPv6 literals are not supported — use a DNS name.
/// </summary>
internal static class LdapEndpointParser
{
/// <summary>
/// Attempts to parse a single <c>FallbackServers</c> entry.
/// </summary>
/// <param name="entry">The configured entry, <c>"host"</c> or <c>"host:port"</c>. Surrounding whitespace is tolerated.</param>
/// <param name="defaultPort">Port adopted when the entry carries no explicit one.</param>
/// <param name="endpoint">The parsed endpoint on success; undefined otherwise.</param>
/// <param name="error">A human-readable reason on failure; empty on success.</param>
/// <returns><see langword="true"/> when the entry parsed.</returns>
public static bool TryParse(string entry, int defaultPort, out LdapEndpoint endpoint, out string error)
{
endpoint = null!;
error = "";
if (string.IsNullOrWhiteSpace(entry))
{
error = "entry is empty or whitespace";
return false;
}
var parts = entry.Trim().Split(':');
if (parts.Length > 2)
{
error = $"entry \"{entry.Trim()}\" contains more than one ':' " +
"(IPv6 literals are not supported — use a DNS name)";
return false;
}
var host = parts[0].Trim();
if (host.Length == 0)
{
error = $"entry \"{entry.Trim()}\" has an empty host";
return false;
}
var port = defaultPort;
if (parts.Length == 2 && (!int.TryParse(parts[1].Trim(), out port) || port is < 1 or > 65535))
{
error = $"entry \"{entry.Trim()}\" has an invalid port (must be 1–65535)";
return false;
}
endpoint = new LdapEndpoint(host, port);
return true;
}
}
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapAuthService.cs
+167 -25
View File
@@ -1,3 +1,4 @@
using Microsoft.Extensions.Logging;
using Novell.Directory.Ldap;
using ZB.MOM.WW.Auth.Abstractions.Ldap;
using ZB.MOM.WW.Auth.Ldap.Internal;
@@ -11,6 +12,7 @@ namespace ZB.MOM.WW.Auth.Ldap;
/// <see cref="ILdapConnection"/> so the logic is unit-testable without a live server.
/// </summary>
/// <remarks>
/// <para>
/// Fully fail-closed: authentication never throws to the caller — every error, expected
/// or unexpected, is mapped to a structured <see cref="LdapAuthResult.Fail(LdapAuthFailure)"/>.
/// A success is only returned for a user that resolved to exactly one entry, whose password
@@ -18,11 +20,38 @@ namespace ZB.MOM.WW.Auth.Ldap;
/// Service-account bind failures (<see cref="LdapAuthFailure.ServiceAccountBindFailed"/>) are
/// kept distinct from end-user bind failures (<see cref="LdapAuthFailure.BadCredentials"/>) so
/// a system misconfiguration is not mistaken for bad user input.
/// </para>
/// <para>
/// When <see cref="LdapOptions.FallbackServers"/> is non-empty the flow above runs against
/// an ordered endpoint list (primary first, then fallbacks in declared order) and moves on
/// to the next endpoint ONLY on a system-side failure — connect/TLS, service-account bind,
/// or search — i.e. when the endpoint gave no authoritative answer. Any answer from a healthy
/// directory (success, or a user-side failure such as
/// <see cref="LdapAuthFailure.BadCredentials"/> / <see cref="LdapAuthFailure.UserNotFound"/>)
/// is final and is never retried elsewhere: retrying a rejected password against a second DC
/// would turn one bad-password event into several, tripping account lockouts. Endpoint
/// preference is sticky — the last endpoint that answered is tried first next time — so a
/// primary outage costs its connect timeout once, not once per login.
/// </para>
/// <para>
/// The service is safe to share (it is registered as a singleton). The sticky index is the
/// only mutable state; concurrent logins may race on it, which is benign — the worst outcome
/// is one login starting at a stale endpoint and walking on.
/// </para>
/// </remarks>
public sealed class LdapAuthService : ILdapAuthService
{
private readonly LdapOptions _options;
private readonly ILdapConnectionFactory _connectionFactory;
private readonly IReadOnlyList<LdapEndpoint> _endpoints;
private readonly ILogger<LdapAuthService>? _logger;
/// <summary>
/// Index into <see cref="_endpoints"/> of the endpoint tried first. Written on any
/// authoritative answer from a non-preferred endpoint. Accessed via
/// <see cref="Volatile"/> only; benign races are accepted (see the type remarks).
/// </summary>
private int _preferredIndex;
/// <summary>
/// Production constructor: binds against a live directory via the real
@@ -31,9 +60,15 @@ public sealed class LdapAuthService : ILdapAuthService
/// <see cref="LdapOptions.AllowInsecure"/> bypass) is carried into the factory so each
/// connection is built with it.
/// </summary>
public LdapAuthService(LdapOptions options)
/// <param name="options">The directory connection and search settings.</param>
/// <param name="logger">
/// Optional. When supplied, endpoint failover is made visible: a Warning per failed
/// endpoint hop and an Information line when the sticky preference moves. Omitting it
/// (the default) leaves the service silent — hosts without logging still work.
/// </param>
public LdapAuthService(LdapOptions options, ILogger<LdapAuthService>? logger = null)
: this(options, new NovellLdapConnectionFactory(
options.AllowInsecure, options.ServerCertificateValidationCallback))
options.AllowInsecure, options.ServerCertificateValidationCallback), logger)
{
}
@@ -42,10 +77,33 @@ public sealed class LdapAuthService : ILdapAuthService
/// so the bind/search logic can be exercised without a live directory. Internal
/// because the connection seam is an implementation detail.
/// </summary>
internal LdapAuthService(LdapOptions options, ILdapConnectionFactory connectionFactory)
internal LdapAuthService(
LdapOptions options,
ILdapConnectionFactory connectionFactory,
ILogger<LdapAuthService>? logger = null)
{
_options = options;
_connectionFactory = connectionFactory;
_logger = logger;
_endpoints = BuildEndpoints(options);
}
/// <summary>
/// Primary first, then fallbacks in declared order. The validator rejects malformed
/// entries at boot; unparseable entries are skipped defensively here so a service
/// constructed without options validation degrades to the parseable subset rather
/// than throwing from a constructor.
/// </summary>
private static IReadOnlyList<LdapEndpoint> BuildEndpoints(LdapOptions options)
{
var endpoints = new List<LdapEndpoint> { new(options.Server, options.Port) };
foreach (var entry in options.FallbackServers)
{
if (LdapEndpointParser.TryParse(entry, options.Port, out var endpoint, out _))
endpoints.Add(endpoint);
}
return endpoints;
}
/// <inheritdoc />
@@ -54,7 +112,10 @@ public sealed class LdapAuthService : ILdapAuthService
/// bind-then-search-then-bind flow is wrapped so that any error — expected or unexpected —
/// is mapped to a structured <see cref="LdapAuthResult.Fail(LdapAuthFailure)"/>. A
/// <c>Succeeded == true</c> result is only ever returned when the user resolved to exactly
/// one entry, their password verified, AND at least one group was extracted.
/// one entry, their password verified, AND at least one group was extracted. With
/// <see cref="LdapOptions.FallbackServers"/> configured the flow may run against more than
/// one endpoint before answering — see the type-level remarks for the walk and its
/// system-side-only failover rule.
/// </remarks>
public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct)
{
@@ -80,25 +141,90 @@ public sealed class LdapAuthService : ILdapAuthService
// filter and the returned result (avoids two identities for one person).
username = username.Trim();
// 4. Walk the endpoint list, preferred one first. A single-endpoint configuration
// (no FallbackServers) collapses to exactly one attempt — unchanged behaviour.
var preferred = Volatile.Read(ref _preferredIndex);
if ((uint)preferred >= (uint)_endpoints.Count)
preferred = 0; // defensive: the list is immutable, but never index out of it
// Seeded so an exhausted walk still returns the conservative system-side bucket
// even in the impossible case of an empty endpoint list.
var lastSystemSideFailure = LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed);
foreach (var index in WalkOrder(_endpoints.Count, preferred))
{
var result = AuthenticateAgainst(_endpoints[index], username, password);
// ServiceAccountBindFailed is the (sole) system-side bucket: this endpoint gave
// no authoritative answer, so try the next one. Every other outcome — success, or
// a user-side failure like UserNotFound/BadCredentials — came from a healthy
// directory and is FINAL: re-asking a second DC about a rejected password would
// multiply one bad-password event into several against the account lockout policy.
if (result.Failure == LdapAuthFailure.ServiceAccountBindFailed)
{
lastSystemSideFailure = result;
continue;
}
// Sticky preference: any authoritative answer proves this endpoint healthy.
// Updating only on Succeeded would re-pay the dead primary's connect timeout on
// every mistyped password for the whole outage.
if (index != preferred)
{
Volatile.Write(ref _preferredIndex, index);
_logger?.LogInformation("LDAP sticky preference moved to {Endpoint}.", _endpoints[index]);
}
return result;
}
// Every endpoint failed system-side. The preference is deliberately left where it was:
// a total outage says nothing about which endpoint to prefer once one recovers.
return lastSystemSideFailure;
}
/// <summary>Preferred endpoint first, then every other endpoint in declared order.</summary>
private static IEnumerable<int> WalkOrder(int count, int preferred)
{
yield return preferred;
for (var i = 0; i < count; i++)
{
if (i != preferred)
yield return i;
}
}
/// <summary>
/// Runs one full bind-then-search-then-bind attempt against a single endpoint on a fresh
/// connection. The fail-closed contract holds per attempt: this never throws, and it
/// returns <see cref="LdapAuthFailure.ServiceAccountBindFailed"/> for — and only for —
/// system-side faults, which is what the caller reads as "no authoritative answer, try
/// the next endpoint".
/// </summary>
private LdapAuthResult AuthenticateAgainst(LdapEndpoint endpoint, string username, string password)
{
// The whole flow runs inside an outer fail-closed guard: a StageFailure carries an
// already-mapped failure out of a stage, and any OTHER unexpected exception defaults to
// the most conservative system-side bucket. Either way the caller gets a structured result.
try
{
// A fresh connection per attempt: a Novell connection that has failed to connect
// or bind is not safely reusable against another host.
using var conn = _connectionFactory.Create();
// 4. Open the connection (transport/TLS handling lives in the adapter). The
// 5. Open the connection (transport/TLS handling lives in the adapter). The
// per-operation timeout (ConnectionTimeoutMs) is applied by the adapter here.
// A failure to connect/upgrade means the directory is unreachable — a
// system-side fault, not the user's, so map it to ServiceAccountBindFailed.
// A failure to connect/upgrade means this endpoint is unreachable — a
// system-side fault, not the user's, so map it to ServiceAccountBindFailed
// and let the caller try the next endpoint.
// NOTE: the LdapAuthFailure enum has no dedicated DirectoryUnavailable value;
// ServiceAccountBindFailed is the closest system-side bucket. A future
// Abstractions change could add DirectoryUnavailable to disambiguate.
try
{
conn.Connect(
_options.Server,
_options.Port,
endpoint.Host,
endpoint.Port,
_options.Transport,
_options.AllowInsecure,
_options.ConnectionTimeoutMs,
@@ -106,10 +232,10 @@ public sealed class LdapAuthService : ILdapAuthService
}
catch (LdapException)
{
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "connect");
}
// 5. Service-account bind so we can search for the user's DN. A bind failure
// 6. Service-account bind so we can search for the user's DN. A bind failure
// here is a service-account misconfiguration — DISTINCT from a user-credential
// failure — so it maps to ServiceAccountBindFailed.
try
@@ -118,10 +244,10 @@ public sealed class LdapAuthService : ILdapAuthService
}
catch (LdapException)
{
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "service-bind");
}
// 6. Search for the user entry by the configured username attribute. A search
// 7. Search for the user entry by the configured username attribute. A search
// failure is infrastructure (directory unreachable / unhealthy) — system-side.
// NOTE: same enum limitation as Connect above; ServiceAccountBindFailed is the
// closest system-side bucket until a DirectoryUnavailable value exists.
@@ -133,10 +259,11 @@ public sealed class LdapAuthService : ILdapAuthService
}
catch (LdapException)
{
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "search");
}
// 7. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser.
// 8. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser.
// Both are authoritative answers from a healthy directory, so they end the walk.
// We never attempt a user bind against an ambiguous DN.
if (entries.Count == 0)
return LdapAuthResult.Fail(LdapAuthFailure.UserNotFound);
@@ -145,19 +272,21 @@ public sealed class LdapAuthService : ILdapAuthService
var entry = entries[0];
// 8. User bind: re-bind as the resolved DN to verify the password. A bind failure
// here is the end user's bad credentials.
// 9. User bind: re-bind as the resolved DN to verify the password. A bind failure
// here is the end user's bad credentials — the directory answered, so this is
// returned directly and never retried against another endpoint.
try
{
conn.Bind(entry.Dn, password);
}
catch (LdapException)
{
throw new StageFailure(LdapAuthFailure.BadCredentials);
return LdapAuthResult.Fail(LdapAuthFailure.BadCredentials);
}
// 9. Group extraction. Fail closed: an empty/missing group set, or any error while
// extracting groups, is a GroupLookupFailed — never a zero-group success.
// 10. Group extraction. Fail closed: an empty/missing group set, or any error while
// extracting groups, is a GroupLookupFailed — never a zero-group success. This
// too is an answer about the user, not about the endpoint, so it ends the walk.
IReadOnlyList<string> groups;
try
{
@@ -165,25 +294,31 @@ public sealed class LdapAuthService : ILdapAuthService
}
catch
{
throw new StageFailure(LdapAuthFailure.GroupLookupFailed);
return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed);
}
if (groups.Count == 0)
return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed);
// 10. Success — and only here, with a verified password and >= 1 group.
// 11. Success — and only here, with a verified password and >= 1 group.
var displayName = ExtractDisplayName(entry, username);
return LdapAuthResult.Success(username, displayName, groups);
}
catch (StageFailure stage)
{
// A stage mapped its own failure; surface it as a structured result.
// A stage mapped its own failure; surface it as a structured result. Log at Warning
// so a silent failover — the failure mode this feature exists to survive — is
// visible in the host's log rather than only in a latency graph.
_logger?.LogWarning(
"LDAP endpoint {Endpoint} failed system-side at {Stage}; treating as no answer.",
endpoint, stage.Stage);
return LdapAuthResult.Fail(stage.Failure);
}
catch
catch (Exception ex)
{
// Belt-and-braces: ANY unexpected exception fails closed to the most conservative
// system-side bucket rather than propagating to the caller.
_logger?.LogWarning(ex, "LDAP endpoint {Endpoint} failed system-side with an unexpected error.", endpoint);
return LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed);
}
}
@@ -194,9 +329,16 @@ public sealed class LdapAuthService : ILdapAuthService
/// </summary>
private sealed class StageFailure : Exception
{
public StageFailure(LdapAuthFailure failure) => Failure = failure;
public StageFailure(LdapAuthFailure failure, string stage)
{
Failure = failure;
Stage = stage;
}
public LdapAuthFailure Failure { get; }
/// <summary>Which stage faulted (<c>connect</c> / <c>service-bind</c> / <c>search</c>), for diagnostics.</summary>
public string Stage { get; }
}
/// <summary>
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/LdapOptionsValidator.cs
+20 -2
View File
@@ -10,7 +10,7 @@ namespace ZB.MOM.WW.Auth.Ldap;
/// </summary>
/// <remarks>
/// Validation is skipped entirely when <see cref="LdapOptions.Enabled"/> is <c>false</c>
/// (a disabled provider's connection fields are inert). When enabled, four conditions
/// (a disabled provider's connection fields are inert). When enabled, five conditions
/// are enforced:
/// <list type="bullet">
/// <item>plaintext transport (<see cref="LdapTransport.None"/>) is rejected unless
@@ -19,7 +19,10 @@ namespace ZB.MOM.WW.Auth.Ldap;
/// <item><see cref="LdapOptions.SearchBase"/> must be specified (the DN root every
/// search runs against);</item>
/// <item><see cref="LdapOptions.ServiceAccountDn"/> must be specified — an empty value
/// would bind anonymously, defeating the search-then-bind authentication flow.</item>
/// would bind anonymously, defeating the search-then-bind authentication flow;</item>
/// <item>every <see cref="LdapOptions.FallbackServers"/> entry must parse as
/// <c>"host"</c> or <c>"host:port"</c> — a typo'd backup DC must not lie dormant
/// until the primary outage it exists to survive.</item>
/// </list>
/// </remarks>
public sealed class LdapOptionsValidator : IValidateOptions<LdapOptions>
@@ -68,6 +71,21 @@ public sealed class LdapOptionsValidator : IValidateOptions<LdapOptions>
"(e.g. \"cn=svc,dc=example,dc=com\").");
}
// Fallback endpoints are walked by LdapAuthService on system-side failures. A
// malformed entry would be silently skipped there (the walk degrades to the
// parseable subset rather than throwing), so it must be caught at boot instead —
// otherwise an operator's typo'd backup DC would only surface as a failed
// failover during the primary outage it was meant to survive.
foreach (var entry in options.FallbackServers)
{
if (!Internal.LdapEndpointParser.TryParse(entry, options.Port, out _, out var error))
{
return ValidateOptionsResult.Fail(
$"{nameof(LdapOptions.FallbackServers)} contains an invalid entry — {error}. " +
"Each entry must be \"host\" or \"host:port\" (port 1–65535).");
}
}
return ValidateOptionsResult.Success;
}
}
ZB.MOM.WW.Auth/src/ZB.MOM.WW.Auth.Ldap/ZB.MOM.WW.Auth.Ldap.csproj
+1
View File
@@ -22,6 +22,7 @@
<ItemGroup>
<PackageReference Include="Novell.Directory.Ldap.NETStandard" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
</ItemGroup>
<ItemGroup>
ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/Integration/GLAuthIntegrationTests.cs
+66
View File
@@ -99,4 +99,70 @@ public sealed class GLAuthIntegrationTests
$"Authentication failed: {result.Failure} (server={server}:{port}, user={user})");
Assert.NotEmpty(result.Groups);
}
/// <summary>
/// Failover: primary endpoint points at a closed local port (refused immediately),
/// fallback at the real GLAuth. The login must succeed via the fallback, and a second
/// login must also succeed (sticky preference — no re-walk through the dead primary).
/// Skipped unless <c>ZB_LDAP_IT=1</c> is set; skipped again if the server is unreachable.
/// </summary>
[SkippableFact]
public async Task Authenticate_FailsOverFromDeadPrimary_ToRealGLAuth()
{
// ------------------------------------------------------------------ opt-in gate
Skip.IfNot(
Environment.GetEnvironmentVariable("ZB_LDAP_IT") == "1",
"Set ZB_LDAP_IT=1 and a reachable GLAuth to run.");
// ------------------------------------------------------------------ read config
var server = Environment.GetEnvironmentVariable("ZB_LDAP_SERVER") ?? "localhost";
var port = int.TryParse(Environment.GetEnvironmentVariable("ZB_LDAP_PORT"), out var p) ? p : 3893;
var baseDn = Environment.GetEnvironmentVariable("ZB_LDAP_BASE") ?? "dc=lmxopcua,dc=local";
var svcDn = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_DN") ?? "";
var svcPw = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_PW") ?? "";
var user = Environment.GetEnvironmentVariable("ZB_LDAP_USER") ?? "";
var pw = Environment.GetEnvironmentVariable("ZB_LDAP_PW") ?? "";
var userAttr = Environment.GetEnvironmentVariable("ZB_LDAP_USERATTR") ?? "cn";
// ------------------------------------------------------------------ reachability probe
try
{
using var tcp = new TcpClient();
// 3-second connect timeout to keep the test suite snappy when the server is absent
var connectTask = tcp.ConnectAsync(server, port);
if (!connectTask.Wait(TimeSpan.FromSeconds(3)))
Skip.If(true, $"GLAuth not reachable at {server}:{port} (connect timed out).");
}
catch (Exception ex)
{
Skip.If(true, $"GLAuth not reachable at {server}:{port}: {ex.Message}");
}
// ------------------------------------------------------------------ build options
var options = new LdapOptions
{
Enabled = true,
Server = "127.0.0.1",
Port = 39999, // closed port -> immediate refusal
Transport = LdapTransport.None,
AllowInsecure = true,
SearchBase = baseDn,
ServiceAccountDn = svcDn,
ServiceAccountPassword = svcPw,
UserNameAttribute = userAttr,
GroupAttribute = "memberOf",
ConnectionTimeoutMs = 3000,
FallbackServers = new[] { $"{server}:{port}" },
};
// ------------------------------------------------------------------ exercise the real service
var svc = new LdapAuthService(options);
var first = await svc.AuthenticateAsync(user, pw, CancellationToken.None);
Assert.True(first.Succeeded, $"first login failed: {first.Failure}");
// Second login exercises the sticky preference: it must not re-walk the dead primary.
var second = await svc.AuthenticateAsync(user, pw, CancellationToken.None);
Assert.True(second.Succeeded, $"second login failed: {second.Failure}");
}
}
ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapAuthServiceFailoverTests.cs
+297
View File
@@ -0,0 +1,297 @@
using ZB.MOM.WW.Auth.Abstractions.Ldap;
namespace ZB.MOM.WW.Auth.Ldap.Tests;
/// <summary>
/// Covers the multi-endpoint failover walk: which failures fail over (system-side only),
/// which are final (any authoritative answer from a healthy directory), how the sticky
/// endpoint preference moves, and that single-endpoint behaviour is unchanged.
/// </summary>
public class LdapAuthServiceFailoverTests
{
private static LdapOptions Opts(params string[] fallbacks) => new()
{
Enabled = true,
Server = "dc1",
Port = 636,
Transport = LdapTransport.Ldaps,
SearchBase = "dc=x",
ServiceAccountDn = "cn=svc,dc=x",
ServiceAccountPassword = "svcpw",
FallbackServers = fallbacks,
};
private static FakeLdapConnection HealthyWithUser() =>
new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", new[] { "cn=ops,dc=x" });
// ---- failover triggers (system-side) ----
[Fact]
public async Task ConnectFailure_FailsOverToFallback_AndSucceeds()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2:3894"), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.True(result.Succeeded);
Assert.Equal(2, factory.Created.Count);
Assert.Equal("dc1", factory.Created[0].ConnectArgs!.Value.Host);
Assert.Equal("dc2", factory.Created[1].ConnectArgs!.Value.Host);
Assert.Equal(3894, factory.Created[1].ConnectArgs!.Value.Port);
}
[Fact]
public async Task ServiceBindFailure_FailsOverToFallback()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.True(result.Succeeded);
Assert.Equal(636, factory.Created[1].ConnectArgs!.Value.Port); // bare host adopts Port
}
// ---- user-side results NEVER fail over ----
[Fact]
public async Task BadUserCredentials_DoesNotFailOver()
{
var factory = new SequenceLdapConnectionFactory(
HealthyWithUser().ThrowOnUserBind());
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("alice", "wrong", CancellationToken.None);
Assert.Equal(LdapAuthFailure.BadCredentials, result.Failure);
Assert.Single(factory.Created);
}
[Fact]
public async Task UserNotFound_DoesNotFailOver()
{
var factory = new SequenceLdapConnectionFactory(new FakeLdapConnection());
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.UserNotFound, result.Failure);
Assert.Single(factory.Created);
}
[Fact]
public async Task GroupLookupFailed_DoesNotFailOver()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", Array.Empty<string>()));
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.GroupLookupFailed, result.Failure);
Assert.Single(factory.Created);
}
[Fact]
public async Task AmbiguousUser_DoesNotFailOver()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().WithDuplicateMatch());
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.AmbiguousUser, result.Failure);
Assert.Single(factory.Created);
}
// ---- exhaustion ----
[Fact]
public async Task AllEndpointsDown_ReturnsServiceAccountBindFailed()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(),
new FakeLdapConnection().ThrowOnConnect());
var svc = new LdapAuthService(Opts("dc2"), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure);
Assert.Equal(2, factory.Created.Count);
}
// ---- stickiness ----
[Fact]
public async Task StickyPreference_SecondLoginStartsAtLastGoodEndpoint()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
HealthyWithUser(), // login 1 @ dc2: answers
HealthyWithUser()); // login 2: should go straight to dc2
var svc = new LdapAuthService(Opts("dc2"), factory);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.Equal(3, factory.Created.Count);
Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host);
}
[Fact]
public async Task StickyPreference_DriftsBackToPrimary_WhenPreferredFails()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
HealthyWithUser(), // login 1 @ dc2: answers -> preferred
new FakeLdapConnection().ThrowOnConnect(), // login 2 @ dc2: now dead
HealthyWithUser(), // login 2 @ dc1 (declared order): recovered
HealthyWithUser()); // login 3: should start at dc1 again
var svc = new LdapAuthService(Opts("dc2"), factory);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.Equal(5, factory.Created.Count);
Assert.Equal("dc1", factory.Created[3].ConnectArgs!.Value.Host);
Assert.Equal("dc1", factory.Created[4].ConnectArgs!.Value.Host);
}
[Fact]
public async Task StickyPreference_UpdatesOnUserSideAnswer_NotOnlySuccess()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
new FakeLdapConnection(), // login 1 @ dc2: answers UserNotFound
HealthyWithUser()); // login 2: should start at dc2
var svc = new LdapAuthService(Opts("dc2"), factory);
var first = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.UserNotFound, first.Failure);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host);
}
[Fact]
public async Task AllEndpointsDown_LeavesPreferenceUnchanged()
{
// A total outage teaches nothing about which endpoint is healthy, so the walk
// must not move the preference — the next login still starts at the primary.
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc2
HealthyWithUser()); // login 2: must start at dc1 again
var svc = new LdapAuthService(Opts("dc2"), factory);
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed,
(await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Failure);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.Equal("dc1", factory.Created[2].ConnectArgs!.Value.Host);
}
// ---- back-compat ----
[Fact]
public async Task NoFallbacks_SingleAttempt_UnchangedBehavior()
{
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect());
var svc = new LdapAuthService(Opts(/* no fallbacks */), factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure);
Assert.Single(factory.Created);
}
[Fact]
public async Task Disabled_NeverOpensAConnection_EvenWithFallbacks()
{
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2") with { Enabled = false }, factory);
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Equal(LdapAuthFailure.Disabled, result.Failure);
Assert.Empty(factory.Created);
}
[Fact]
public async Task MalformedFallbackEntry_IsSkippedDefensively()
{
// The validator rejects this at boot; a service constructed without validation
// must degrade to the parseable subset, not throw.
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
var svc = new LdapAuthService(Opts(":::"), factory);
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
Assert.Single(factory.Created);
}
// ---- observability ----
/// <summary>Captures emitted log entries so failover visibility can be asserted.</summary>
private sealed class RecordingLogger : Microsoft.Extensions.Logging.ILogger<LdapAuthService>
{
public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = new();
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
TState state, Exception? exception, Func<TState, Exception?, string> formatter)
=> Entries.Add((logLevel, formatter(state, exception)));
}
[Fact]
public async Task Failover_LogsWarningWithEndpointAndStage_AndInfoOnStickyChange()
{
var logger = new RecordingLogger();
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2:3894"), factory, logger);
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Contains(logger.Entries, e =>
e.Level == Microsoft.Extensions.Logging.LogLevel.Warning
&& e.Message.Contains("dc1:636") && e.Message.Contains("connect"));
Assert.Contains(logger.Entries, e =>
e.Level == Microsoft.Extensions.Logging.LogLevel.Information
&& e.Message.Contains("dc2:3894"));
// Never log secrets:
Assert.DoesNotContain(logger.Entries, e => e.Message.Contains("svcpw") || e.Message.Contains("pw\""));
}
[Fact]
public async Task HappyPath_LogsNothing()
{
var logger = new RecordingLogger();
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2"), factory, logger);
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Empty(logger.Entries);
}
[Fact]
public async Task ServiceBindFailure_LogsTheServiceBindStage_NotConnect()
{
var logger = new RecordingLogger();
var factory = new SequenceLdapConnectionFactory(
new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser());
var svc = new LdapAuthService(Opts("dc2"), factory, logger);
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
Assert.Contains(logger.Entries, e =>
e.Level == Microsoft.Extensions.Logging.LogLevel.Warning
&& e.Message.Contains("dc1:636") && e.Message.Contains("service-bind"));
}
}
ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapEndpointParserTests.cs
+35
View File
@@ -0,0 +1,35 @@
using ZB.MOM.WW.Auth.Ldap.Internal;
namespace ZB.MOM.WW.Auth.Ldap.Tests;
public class LdapEndpointParserTests
{
[Theory]
[InlineData("dc2.example.com", 636, "dc2.example.com", 636)] // bare host -> default port
[InlineData("dc2.example.com:3894", 636, "dc2.example.com", 3894)] // explicit port wins
[InlineData(" dc2.example.com : 3894 ", 636, "dc2.example.com", 3894)] // whitespace tolerated
public void TryParse_ValidEntry_Succeeds(string entry, int defaultPort, string host, int port)
{
Assert.True(LdapEndpointParser.TryParse(entry, defaultPort, out var ep, out _));
Assert.Equal(host, ep.Host);
Assert.Equal(port, ep.Port);
}
[Theory]
[InlineData("")] // empty
[InlineData(" ")] // whitespace
[InlineData(":636")] // empty host
[InlineData("dc2:0")] // port below range
[InlineData("dc2:65536")] // port above range
[InlineData("dc2:abc")] // non-numeric port
[InlineData("2001:db8::1:636")] // IPv6 literal (unsupported, >1 colon)
public void TryParse_InvalidEntry_Fails(string entry)
{
Assert.False(LdapEndpointParser.TryParse(entry, 636, out _, out var error));
Assert.NotEmpty(error);
}
[Fact]
public void Endpoint_ToString_IsHostColonPort() =>
Assert.Equal("dc2:3894", new LdapEndpoint("dc2", 3894).ToString());
}
ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/LdapOptionsValidatorTests.cs
+28
View File
@@ -88,4 +88,32 @@ public class LdapOptionsValidatorTests
ServiceAccountDn = "",
})
.Failed);
[Fact]
public void Validator_Succeeds_WithValidFallbackServers() =>
Assert.False(new LdapOptionsValidator()
.Validate(null, Opts() with { FallbackServers = new[] { "dc2", "dc3:3894" } })
.Failed);
[Theory]
[InlineData("")]
[InlineData(":636")]
[InlineData("dc2:0")]
[InlineData("dc2:notaport")]
public void Validator_Fails_OnMalformedFallbackServer(string entry)
{
var result = new LdapOptionsValidator()
.Validate(null, Opts() with { FallbackServers = new[] { entry } });
Assert.True(result.Failed);
Assert.Contains(nameof(LdapOptions.FallbackServers), result.FailureMessage);
}
[Fact]
public void Validator_Skips_FallbackServers_WhenDisabled() =>
// Same inert-when-disabled rule as every other connection field: a disabled
// provider never opens a connection, so its fallback list is never walked.
Assert.False(new LdapOptionsValidator()
.Validate(null, Opts() with { Enabled = false, FallbackServers = new[] { ":::" } })
.Failed);
}
ZB.MOM.WW.Auth/tests/ZB.MOM.WW.Auth.Ldap.Tests/SequenceLdapConnectionFactory.cs
+51
View File
@@ -0,0 +1,51 @@
using ZB.MOM.WW.Auth.Ldap.Internal;
namespace ZB.MOM.WW.Auth.Ldap.Tests;
/// <summary>
/// Test double for <see cref="ILdapConnectionFactory"/> that hands out pre-scripted
/// <see cref="FakeLdapConnection"/> instances in order, one per <see cref="Create"/> call —
/// exactly matching the failover walk, which opens a fresh connection per endpoint attempt.
/// Records every created connection for inspection. When the script runs out, returns a
/// bare unscripted fake (which yields user-not-found) so a miscounted test fails loudly
/// rather than reusing a stale fake.
/// </summary>
internal sealed class SequenceLdapConnectionFactory : ILdapConnectionFactory
{
private readonly Queue<FakeLdapConnection> _script;
/// <summary>Scripts the connections handed out, in the order the walk will request them.</summary>
public SequenceLdapConnectionFactory(params FakeLdapConnection[] fakes)
=> _script = new Queue<FakeLdapConnection>(fakes);
/// <summary>Every connection handed out, in creation order (scripted or overflow).</summary>
public List<FakeLdapConnection> Created { get; } = new();
/// <inheritdoc />
public ILdapConnection Create()
{
var fake = _script.Count > 0 ? _script.Dequeue() : new FakeLdapConnection();
Created.Add(fake);
return fake;
}
}
// ---------------------------------------------------------------------------
// Smoke test: verifies the sequencing and overflow behaviour the failover tests rely on.
// ---------------------------------------------------------------------------
public class SequenceLdapConnectionFactorySmokeTests
{
[Fact]
public void Create_ReturnsScriptedFakesInOrder_ThenOverflows()
{
var first = new FakeLdapConnection().ThrowOnConnect();
var second = new FakeLdapConnection();
var factory = new SequenceLdapConnectionFactory(first, second);
Assert.Same(first, factory.Create());
Assert.Same(second, factory.Create());
Assert.NotSame(second, factory.Create()); // overflow: fresh bare fake
Assert.Equal(3, factory.Created.Count);
}
}
docs/plans/2026-08-10-ldap-backup-dc-failover-design.md
+7
View File
@@ -94,6 +94,13 @@ Inside `LdapAuthService.Authenticate`, an endpoint walk over an internal
preferred fails; when it does, the walk tries declared order again, so
preference drifts back to the primary naturally.
> **Refinement (Task 4):** sticky preference updates on ANY authoritative answer
> (success or user-side failure), not only on success — a DC that answers is healthy,
> and success-only stickiness would re-pay the dead primary's connect timeout on every
> mistyped password during an outage. Corollary, also implemented: an *exhausted* walk
> (every endpoint system-side-down) leaves the preference untouched, since a total
> outage says nothing about which endpoint to prefer once one recovers.
## Stickiness state
`volatile int _preferredIndex` on the singleton `LdapAuthService` (registered
docs/plans/2026-08-10-ldap-backup-dc-failover.md.tasks.json
+61 -9
View File
@@ -1,14 +1,66 @@
{
"planPath": "docs/plans/2026-08-10-ldap-backup-dc-failover.md",
"tasks": [
{"id": 1, "subject": "Task 1: LdapEndpoint + LdapEndpointParser", "status": "pending"},
{"id": 2, "subject": "Task 2: LdapOptions.FallbackServers + validator rules", "status": "pending", "blockedBy": [1]},
{"id": 3, "subject": "Task 3: SequenceLdapConnectionFactory test double", "status": "pending"},
{"id": 4, "subject": "Task 4: Endpoint walk + sticky preference in LdapAuthService", "status": "pending", "blockedBy": [1, 2, 3]},
{"id": 5, "subject": "Task 5: Failover observability (ILogger) + DI wiring", "status": "pending", "blockedBy": [4]},
{"id": 6, "subject": "Task 6: GLAuth failover integration test", "status": "pending", "blockedBy": [5]},
{"id": 7, "subject": "Task 7: Version 0.2.0, pack, publish, restore-verify", "status": "pending", "blockedBy": [6]},
{"id": 8, "subject": "Task 8: Documentation + index truth", "status": "pending", "blockedBy": [7]}
{
"id": 1,
"subject": "Task 1: LdapEndpoint + LdapEndpointParser",
"status": "completed"
},
{
"id": 2,
"subject": "Task 2: LdapOptions.FallbackServers + validator rules",
"status": "completed",
"blockedBy": [
1
]
},
{
"id": 3,
"subject": "Task 3: SequenceLdapConnectionFactory test double",
"status": "completed"
},
{
"id": 4,
"subject": "Task 4: Endpoint walk + sticky preference in LdapAuthService",
"status": "pending",
"blockedBy": [
1,
2,
3
]
},
{
"id": 5,
"subject": "Task 5: Failover observability (ILogger) + DI wiring",
"status": "pending",
"blockedBy": [
4
]
},
{
"id": 6,
"subject": "Task 6: GLAuth failover integration test",
"status": "pending",
"blockedBy": [
5
]
},
{
"id": 7,
"subject": "Task 7: Version 0.2.0, pack, publish, restore-verify",
"status": "pending",
"blockedBy": [
6
]
},
{
"id": 8,
"subject": "Task 8: Documentation + index truth",
"status": "pending",
"blockedBy": [
7
]
}
],
"lastUpdated": "2026-08-10"
}
}