Merge pull request 'test(ldap): live GLAuth outage/recovery gate (R2-08 03/S2)' (#455) from test/r2-08-glauth-outage-live-gate into master

tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LdapGlauthOutageLiveTests.cs

@@ -0,0 +1,190 @@
+using System.Diagnostics;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Auth.Abstractions.Roles;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Security.Ldap;
+using LdapTransport = ZB.MOM.WW.Auth.Abstractions.Ldap.LdapTransport;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
+
+/// <summary>
+///     R2-08 03/S2 LIVE gate — the directory-outage circuit exercised against a REAL GLAuth server
+///     (the image + config PR #451 swapped in for the retired bitnami/openldap), driven through the
+///     production auth path (<see cref="OtOpcUaLdapAuthService"/> + <see cref="LdapOpcUaUserAuthenticator"/>).
+///     The offline <see cref="LdapAuthResilienceTests"/> prove the circuit logic with fakes + an
+///     unroutable-host blackhole; this proves the same behaviour against an actual directory that is
+///     paused mid-run (a real outage) and unpaused (recovery), and doubles as the first end-to-end
+///     verification that the GLAuth swap binds correctly.
+/// </summary>
+/// <remarks>
+///     Triple-gated so it never runs by accident and always restores the shared GLAuth:
+///     <list type="number">
+///         <item><c>GLAUTH_LIVE_HOST</c> — the GLAuth host (e.g. <c>10.100.0.35</c>); absent ⇒ skip;</item>
+///         <item><c>GLAUTH_OUTAGE_START_CMD</c> — pauses the GLAuth container (a real outage); and</item>
+///         <item><c>GLAUTH_OUTAGE_STOP_CMD</c> — unpauses it.</item>
+///     </list>
+///     A <c>try/finally</c> ALWAYS runs the stop command once the pause succeeded. Operator recipe:
+///     <code>
+///     ssh dohertj2@10.100.0.35 "cd ~/otopcua-harness &amp;&amp; docker compose up -d ldap"
+///     export GLAUTH_LIVE_HOST=10.100.0.35
+///     export GLAUTH_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-harness-ldap-1"'
+///     export GLAUTH_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-harness-ldap-1"'
+///     dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests --filter "FullyQualifiedName~LdapGlauthOutageLive"
+///     </code>
+/// </remarks>
+[Trait("Category", "Integration")]
+[Trait("Category", "LiveLdap")]
+public sealed class LdapGlauthOutageLiveTests
+{
+    private const string HostEnvVar = "GLAUTH_LIVE_HOST";
+    private const string PortEnvVar = "GLAUTH_LIVE_PORT";
+    private const string StartCmdEnvVar = "GLAUTH_OUTAGE_START_CMD";
+    private const string StopCmdEnvVar = "GLAUTH_OUTAGE_STOP_CMD";
+
+    private const int OutageThreshold = 3;
+    private const int CooldownSeconds = 2;
+    private static readonly TimeSpan AuthTimeout = TimeSpan.FromMilliseconds(2_500);
+
+    /// <summary>
+    ///     Healthy real binds succeed; pausing GLAuth makes consecutive binds fail bounded (never a
+    ///     multi-second SDK stall) and opens the circuit into a sub-second fast-deny; unpausing +
+    ///     cooldown lets a half-open probe close the circuit and binds succeed again.
+    /// </summary>
+    [Fact]
+    public async Task Real_glauth_outage_opens_circuit_then_recovers_on_unpause()
+    {
+        var host = Environment.GetEnvironmentVariable(HostEnvVar);
+        if (string.IsNullOrWhiteSpace(host))
+            Assert.Skip($"Set {HostEnvVar} (+ {StartCmdEnvVar}/{StopCmdEnvVar}) to run the live GLAuth outage cycle.");
+        var port = int.TryParse(Environment.GetEnvironmentVariable(PortEnvVar), out var p) ? p : 3894;
+        var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar);
+        var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar);
+        if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd))
+            Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (docker pause/unpause) to run this destructive live test.");
+
+        var ct = TestContext.Current.CancellationToken;
+        var options = new LdapOptions
+        {
+            Enabled = true,
+            DevStubMode = false,
+            Server = host!,
+            Port = port,
+            Transport = LdapTransport.None,
+            AllowInsecure = true,
+            SearchBase = "dc=zb,dc=local",
+            ServiceAccountDn = "cn=serviceaccount,dc=zb,dc=local",
+            ServiceAccountPassword = "serviceaccount123",
+            ConnectionTimeoutMs = 2_000,
+            AuthTimeoutMs = (int)AuthTimeout.TotalMilliseconds,
+            OutageFailureThreshold = OutageThreshold,
+            OutageCooldownSeconds = CooldownSeconds,
+        };
+        var authenticator = BuildAuthenticator(options);
+
+        // ── 1. Healthy directory: real binds succeed, wrong creds deny (user-side, not an outage). ──
+        var alice = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
+        alice.Success.ShouldBeTrue("alice should bind against the live GLAuth");
+        alice.Roles.ShouldNotBeEmpty("a healthy bind returns alice's mapped group roles");
+
+        var bob = await authenticator.AuthenticateUserNameAsync("bob", "bob123", ct);
+        bob.Success.ShouldBeTrue("bob should bind against the live GLAuth");
+
+        var wrong = await authenticator.AuthenticateUserNameAsync("alice", "wrong-password", ct);
+        wrong.Success.ShouldBeFalse("a wrong password is a user-side deny");
+
+        // ── 2. Outage: pause GLAuth. Consecutive binds fail BOUNDED, then the circuit fast-denies. ──
+        var paused = false;
+        try
+        {
+            await RunShellCommandAsync(startCmd!, ct);
+            paused = true;
+
+            for (var i = 0; i < OutageThreshold; i++)
+            {
+                var sw = Stopwatch.StartNew();
+                var outage = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
+                sw.Stop();
+
+                outage.Success.ShouldBeFalse($"bind #{i + 1} during the outage must fail closed");
+                // The whole flow is wall-clock bounded — never the SDK's default ~10-20 s stall.
+                sw.Elapsed.ShouldBeLessThan(AuthTimeout + TimeSpan.FromSeconds(2),
+                    "the authenticator must bound a hung directory call, not stall on it");
+            }
+
+            // Circuit is now open — the next call fast-denies WITHOUT a directory round-trip (sub-second).
+            var blockedSw = Stopwatch.StartNew();
+            var blocked = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
+            blockedSw.Stop();
+
+            blocked.Success.ShouldBeFalse();
+            blocked.Error.ShouldNotBeNull();
+            blocked.Error!.ShouldContain("unavailable", Case.Insensitive,
+                "an open outage circuit fast-denies 'backend unavailable'");
+            blockedSw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1),
+                "an open circuit fast-denies without a directory call");
+        }
+        finally
+        {
+            if (paused)
+                await RunShellCommandAsync(stopCmd!, CancellationToken.None); // ALWAYS restore GLAuth
+        }
+
+        // ── 3. Recovery: after the cooldown the half-open probe reaches the (restored) directory. ──
+        await Task.Delay(TimeSpan.FromSeconds(CooldownSeconds) + TimeSpan.FromSeconds(1), ct);
+
+        var recovered = false;
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+        while (DateTime.UtcNow < deadline)
+        {
+            var probe = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
+            if (probe.Success) { recovered = true; break; }
+            await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+        }
+        recovered.ShouldBeTrue("the circuit must half-open and close once GLAuth is unpaused");
+    }
+
+    private static LdapOpcUaUserAuthenticator BuildAuthenticator(LdapOptions options)
+    {
+        var ldap = new OtOpcUaLdapAuthService(Options.Create(options), NullLogger<OtOpcUaLdapAuthService>.Instance);
+        return new LdapOpcUaUserAuthenticator(
+            ldap, ScopeFactoryWith(new PassthroughMapper()), Options.Create(options),
+            NullLogger<LdapOpcUaUserAuthenticator>.Instance);
+    }
+
+    /// <summary>Builds an <see cref="IServiceScopeFactory"/> whose scopes resolve the supplied mapper.</summary>
+    private static IServiceScopeFactory ScopeFactoryWith(IGroupRoleMapper<string> mapper)
+    {
+        var services = new ServiceCollection();
+        services.AddScoped(_ => mapper);
+        return services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>();
+    }
+
+    /// <summary>Runs an SSH/docker one-liner through the login shell (verbatim, as in the S7 outage gate).</summary>
+    private static async Task RunShellCommandAsync(string command, CancellationToken ct)
+    {
+        using var proc = new Process
+        {
+            StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            },
+        };
+        proc.Start();
+        await proc.WaitForExitAsync(ct);
+        proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}");
+        await Task.Delay(TimeSpan.FromSeconds(1), ct); // let the pause/unpause take effect
+    }
+
+    /// <summary>Passes LDAP groups straight through as roles — the live gate asserts on the bind, not the mapping.</summary>
+    private sealed class PassthroughMapper : IGroupRoleMapper<string>
+    {
+        public Task<GroupRoleMapping<string>> MapAsync(IReadOnlyList<string> groups, CancellationToken ct)
+            => Task.FromResult(new GroupRoleMapping<string>(groups, Scope: null));
+    }
+}