From 4d2dcff5df0815b6a489258a2a65bbcd80fa5d21 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 15 Jul 2026 07:45:07 -0400 Subject: [PATCH] test(ldap): live GLAuth outage/recovery gate (R2-08 03/S2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the R2-08 S2 live leg. The offline LdapAuthResilienceTests prove the directory-outage circuit with fakes + an unroutable-host blackhole; this drives the SAME circuit through the production auth path (OtOpcUaLdapAuthService + LdapOpcUaUserAuthenticator) against a REAL GLAuth that is paused mid-run and unpaused — and doubles as the first end-to-end verification that the PR #451 GLAuth swap (retired bitnami/openldap) binds correctly. Triple-gated (GLAUTH_LIVE_HOST + GLAUTH_OUTAGE_START_CMD/STOP_CMD), skips clean offline (verified: 1 skipped, 8 ms). try/finally always unpauses GLAuth. Verified GREEN against 10.100.0.35:3894 (12 s): healthy binds (alice+bob, mapped roles) -> docker pause -> 3 bounded failures -> circuit opens -> sub-second "unavailable" fast-deny -> docker unpause -> half-open probe closes -> binds succeed again. A boundary AuthTimeout feeds the circuit (authenticator :123-130), so the docker-pause timeout shape is a faithful outage. --- .../LdapGlauthOutageLiveTests.cs | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapGlauthOutageLiveTests.cs diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapGlauthOutageLiveTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapGlauthOutageLiveTests.cs new file mode 100644 index 00000000..eeb52391 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapGlauthOutageLiveTests.cs @@ -0,0 +1,190 @@ +using System.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Auth.Abstractions.Roles; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.Security.Ldap; +using LdapTransport = ZB.MOM.WW.Auth.Abstractions.Ldap.LdapTransport; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// R2-08 03/S2 LIVE gate — the directory-outage circuit exercised against a REAL GLAuth server +/// (the image + config PR #451 swapped in for the retired bitnami/openldap), driven through the +/// production auth path ( + ). +/// The offline prove the circuit logic with fakes + an +/// unroutable-host blackhole; this proves the same behaviour against an actual directory that is +/// paused mid-run (a real outage) and unpaused (recovery), and doubles as the first end-to-end +/// verification that the GLAuth swap binds correctly. +/// +/// +/// Triple-gated so it never runs by accident and always restores the shared GLAuth: +/// +/// GLAUTH_LIVE_HOST — the GLAuth host (e.g. 10.100.0.35); absent ⇒ skip; +/// GLAUTH_OUTAGE_START_CMD — pauses the GLAuth container (a real outage); and +/// GLAUTH_OUTAGE_STOP_CMD — unpauses it. +/// +/// A try/finally ALWAYS runs the stop command once the pause succeeded. Operator recipe: +/// +/// ssh dohertj2@10.100.0.35 "cd ~/otopcua-harness && docker compose up -d ldap" +/// export GLAUTH_LIVE_HOST=10.100.0.35 +/// export GLAUTH_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-harness-ldap-1"' +/// export GLAUTH_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-harness-ldap-1"' +/// dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests --filter "FullyQualifiedName~LdapGlauthOutageLive" +/// +/// +[Trait("Category", "Integration")] +[Trait("Category", "LiveLdap")] +public sealed class LdapGlauthOutageLiveTests +{ + private const string HostEnvVar = "GLAUTH_LIVE_HOST"; + private const string PortEnvVar = "GLAUTH_LIVE_PORT"; + private const string StartCmdEnvVar = "GLAUTH_OUTAGE_START_CMD"; + private const string StopCmdEnvVar = "GLAUTH_OUTAGE_STOP_CMD"; + + private const int OutageThreshold = 3; + private const int CooldownSeconds = 2; + private static readonly TimeSpan AuthTimeout = TimeSpan.FromMilliseconds(2_500); + + /// + /// Healthy real binds succeed; pausing GLAuth makes consecutive binds fail bounded (never a + /// multi-second SDK stall) and opens the circuit into a sub-second fast-deny; unpausing + + /// cooldown lets a half-open probe close the circuit and binds succeed again. + /// + [Fact] + public async Task Real_glauth_outage_opens_circuit_then_recovers_on_unpause() + { + var host = Environment.GetEnvironmentVariable(HostEnvVar); + if (string.IsNullOrWhiteSpace(host)) + Assert.Skip($"Set {HostEnvVar} (+ {StartCmdEnvVar}/{StopCmdEnvVar}) to run the live GLAuth outage cycle."); + var port = int.TryParse(Environment.GetEnvironmentVariable(PortEnvVar), out var p) ? p : 3894; + var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar); + var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar); + if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd)) + Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (docker pause/unpause) to run this destructive live test."); + + var ct = TestContext.Current.CancellationToken; + var options = new LdapOptions + { + Enabled = true, + DevStubMode = false, + Server = host!, + Port = port, + Transport = LdapTransport.None, + AllowInsecure = true, + SearchBase = "dc=zb,dc=local", + ServiceAccountDn = "cn=serviceaccount,dc=zb,dc=local", + ServiceAccountPassword = "serviceaccount123", + ConnectionTimeoutMs = 2_000, + AuthTimeoutMs = (int)AuthTimeout.TotalMilliseconds, + OutageFailureThreshold = OutageThreshold, + OutageCooldownSeconds = CooldownSeconds, + }; + var authenticator = BuildAuthenticator(options); + + // ── 1. Healthy directory: real binds succeed, wrong creds deny (user-side, not an outage). ── + var alice = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct); + alice.Success.ShouldBeTrue("alice should bind against the live GLAuth"); + alice.Roles.ShouldNotBeEmpty("a healthy bind returns alice's mapped group roles"); + + var bob = await authenticator.AuthenticateUserNameAsync("bob", "bob123", ct); + bob.Success.ShouldBeTrue("bob should bind against the live GLAuth"); + + var wrong = await authenticator.AuthenticateUserNameAsync("alice", "wrong-password", ct); + wrong.Success.ShouldBeFalse("a wrong password is a user-side deny"); + + // ── 2. Outage: pause GLAuth. Consecutive binds fail BOUNDED, then the circuit fast-denies. ── + var paused = false; + try + { + await RunShellCommandAsync(startCmd!, ct); + paused = true; + + for (var i = 0; i < OutageThreshold; i++) + { + var sw = Stopwatch.StartNew(); + var outage = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct); + sw.Stop(); + + outage.Success.ShouldBeFalse($"bind #{i + 1} during the outage must fail closed"); + // The whole flow is wall-clock bounded — never the SDK's default ~10-20 s stall. + sw.Elapsed.ShouldBeLessThan(AuthTimeout + TimeSpan.FromSeconds(2), + "the authenticator must bound a hung directory call, not stall on it"); + } + + // Circuit is now open — the next call fast-denies WITHOUT a directory round-trip (sub-second). + var blockedSw = Stopwatch.StartNew(); + var blocked = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct); + blockedSw.Stop(); + + blocked.Success.ShouldBeFalse(); + blocked.Error.ShouldNotBeNull(); + blocked.Error!.ShouldContain("unavailable", Case.Insensitive, + "an open outage circuit fast-denies 'backend unavailable'"); + blockedSw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1), + "an open circuit fast-denies without a directory call"); + } + finally + { + if (paused) + await RunShellCommandAsync(stopCmd!, CancellationToken.None); // ALWAYS restore GLAuth + } + + // ── 3. Recovery: after the cooldown the half-open probe reaches the (restored) directory. ── + await Task.Delay(TimeSpan.FromSeconds(CooldownSeconds) + TimeSpan.FromSeconds(1), ct); + + var recovered = false; + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + var probe = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct); + if (probe.Success) { recovered = true; break; } + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + } + recovered.ShouldBeTrue("the circuit must half-open and close once GLAuth is unpaused"); + } + + private static LdapOpcUaUserAuthenticator BuildAuthenticator(LdapOptions options) + { + var ldap = new OtOpcUaLdapAuthService(Options.Create(options), NullLogger.Instance); + return new LdapOpcUaUserAuthenticator( + ldap, ScopeFactoryWith(new PassthroughMapper()), Options.Create(options), + NullLogger.Instance); + } + + /// Builds an whose scopes resolve the supplied mapper. + private static IServiceScopeFactory ScopeFactoryWith(IGroupRoleMapper mapper) + { + var services = new ServiceCollection(); + services.AddScoped(_ => mapper); + return services.BuildServiceProvider().GetRequiredService(); + } + + /// Runs an SSH/docker one-liner through the login shell (verbatim, as in the S7 outage gate). + private static async Task RunShellCommandAsync(string command, CancellationToken ct) + { + using var proc = new Process + { + StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }, + }; + proc.Start(); + await proc.WaitForExitAsync(ct); + proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}"); + await Task.Delay(TimeSpan.FromSeconds(1), ct); // let the pause/unpause take effect + } + + /// Passes LDAP groups straight through as roles — the live gate asserts on the bind, not the mapping. + private sealed class PassthroughMapper : IGroupRoleMapper + { + public Task> MapAsync(IReadOnlyList groups, CancellationToken ct) + => Task.FromResult(new GroupRoleMapping(groups, Scope: null)); + } +} -- 2.52.0