test(ldap): live GLAuth outage/recovery gate (R2-08 03/S2)
Closes the R2-08 S2 live leg. The offline LdapAuthResilienceTests prove the directory-outage circuit with fakes + an unroutable-host blackhole; this drives the SAME circuit through the production auth path (OtOpcUaLdapAuthService + LdapOpcUaUserAuthenticator) against a REAL GLAuth that is paused mid-run and unpaused — and doubles as the first end-to-end verification that the PR #451 GLAuth swap (retired bitnami/openldap) binds correctly. Triple-gated (GLAUTH_LIVE_HOST + GLAUTH_OUTAGE_START_CMD/STOP_CMD), skips clean offline (verified: 1 skipped, 8 ms). try/finally always unpauses GLAuth. Verified GREEN against 10.100.0.35:3894 (12 s): healthy binds (alice+bob, mapped roles) -> docker pause -> 3 bounded failures -> circuit opens -> sub-second "unavailable" fast-deny -> docker unpause -> half-open probe closes -> binds succeed again. A boundary AuthTimeout feeds the circuit (authenticator :123-130), so the docker-pause timeout shape is a faithful outage.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,190 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Shouldly;
|
||||||
|
using Xunit;
|
||||||
|
using ZB.MOM.WW.Auth.Abstractions.Roles;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Security.Ldap;
|
||||||
|
using LdapTransport = ZB.MOM.WW.Auth.Abstractions.Ldap.LdapTransport;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// R2-08 03/S2 LIVE gate — the directory-outage circuit exercised against a REAL GLAuth server
|
||||||
|
/// (the image + config PR #451 swapped in for the retired bitnami/openldap), driven through the
|
||||||
|
/// production auth path (<see cref="OtOpcUaLdapAuthService"/> + <see cref="LdapOpcUaUserAuthenticator"/>).
|
||||||
|
/// The offline <see cref="LdapAuthResilienceTests"/> prove the circuit logic with fakes + an
|
||||||
|
/// unroutable-host blackhole; this proves the same behaviour against an actual directory that is
|
||||||
|
/// paused mid-run (a real outage) and unpaused (recovery), and doubles as the first end-to-end
|
||||||
|
/// verification that the GLAuth swap binds correctly.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Triple-gated so it never runs by accident and always restores the shared GLAuth:
|
||||||
|
/// <list type="number">
|
||||||
|
/// <item><c>GLAUTH_LIVE_HOST</c> — the GLAuth host (e.g. <c>10.100.0.35</c>); absent ⇒ skip;</item>
|
||||||
|
/// <item><c>GLAUTH_OUTAGE_START_CMD</c> — pauses the GLAuth container (a real outage); and</item>
|
||||||
|
/// <item><c>GLAUTH_OUTAGE_STOP_CMD</c> — unpauses it.</item>
|
||||||
|
/// </list>
|
||||||
|
/// A <c>try/finally</c> ALWAYS runs the stop command once the pause succeeded. Operator recipe:
|
||||||
|
/// <code>
|
||||||
|
/// ssh dohertj2@10.100.0.35 "cd ~/otopcua-harness && docker compose up -d ldap"
|
||||||
|
/// export GLAUTH_LIVE_HOST=10.100.0.35
|
||||||
|
/// export GLAUTH_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-harness-ldap-1"'
|
||||||
|
/// export GLAUTH_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-harness-ldap-1"'
|
||||||
|
/// dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests --filter "FullyQualifiedName~LdapGlauthOutageLive"
|
||||||
|
/// </code>
|
||||||
|
/// </remarks>
|
||||||
|
[Trait("Category", "Integration")]
|
||||||
|
[Trait("Category", "LiveLdap")]
|
||||||
|
public sealed class LdapGlauthOutageLiveTests
|
||||||
|
{
|
||||||
|
private const string HostEnvVar = "GLAUTH_LIVE_HOST";
|
||||||
|
private const string PortEnvVar = "GLAUTH_LIVE_PORT";
|
||||||
|
private const string StartCmdEnvVar = "GLAUTH_OUTAGE_START_CMD";
|
||||||
|
private const string StopCmdEnvVar = "GLAUTH_OUTAGE_STOP_CMD";
|
||||||
|
|
||||||
|
private const int OutageThreshold = 3;
|
||||||
|
private const int CooldownSeconds = 2;
|
||||||
|
private static readonly TimeSpan AuthTimeout = TimeSpan.FromMilliseconds(2_500);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Healthy real binds succeed; pausing GLAuth makes consecutive binds fail bounded (never a
|
||||||
|
/// multi-second SDK stall) and opens the circuit into a sub-second fast-deny; unpausing +
|
||||||
|
/// cooldown lets a half-open probe close the circuit and binds succeed again.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Real_glauth_outage_opens_circuit_then_recovers_on_unpause()
|
||||||
|
{
|
||||||
|
var host = Environment.GetEnvironmentVariable(HostEnvVar);
|
||||||
|
if (string.IsNullOrWhiteSpace(host))
|
||||||
|
Assert.Skip($"Set {HostEnvVar} (+ {StartCmdEnvVar}/{StopCmdEnvVar}) to run the live GLAuth outage cycle.");
|
||||||
|
var port = int.TryParse(Environment.GetEnvironmentVariable(PortEnvVar), out var p) ? p : 3894;
|
||||||
|
var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar);
|
||||||
|
var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar);
|
||||||
|
if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd))
|
||||||
|
Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (docker pause/unpause) to run this destructive live test.");
|
||||||
|
|
||||||
|
var ct = TestContext.Current.CancellationToken;
|
||||||
|
var options = new LdapOptions
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
DevStubMode = false,
|
||||||
|
Server = host!,
|
||||||
|
Port = port,
|
||||||
|
Transport = LdapTransport.None,
|
||||||
|
AllowInsecure = true,
|
||||||
|
SearchBase = "dc=zb,dc=local",
|
||||||
|
ServiceAccountDn = "cn=serviceaccount,dc=zb,dc=local",
|
||||||
|
ServiceAccountPassword = "serviceaccount123",
|
||||||
|
ConnectionTimeoutMs = 2_000,
|
||||||
|
AuthTimeoutMs = (int)AuthTimeout.TotalMilliseconds,
|
||||||
|
OutageFailureThreshold = OutageThreshold,
|
||||||
|
OutageCooldownSeconds = CooldownSeconds,
|
||||||
|
};
|
||||||
|
var authenticator = BuildAuthenticator(options);
|
||||||
|
|
||||||
|
// ── 1. Healthy directory: real binds succeed, wrong creds deny (user-side, not an outage). ──
|
||||||
|
var alice = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
|
||||||
|
alice.Success.ShouldBeTrue("alice should bind against the live GLAuth");
|
||||||
|
alice.Roles.ShouldNotBeEmpty("a healthy bind returns alice's mapped group roles");
|
||||||
|
|
||||||
|
var bob = await authenticator.AuthenticateUserNameAsync("bob", "bob123", ct);
|
||||||
|
bob.Success.ShouldBeTrue("bob should bind against the live GLAuth");
|
||||||
|
|
||||||
|
var wrong = await authenticator.AuthenticateUserNameAsync("alice", "wrong-password", ct);
|
||||||
|
wrong.Success.ShouldBeFalse("a wrong password is a user-side deny");
|
||||||
|
|
||||||
|
// ── 2. Outage: pause GLAuth. Consecutive binds fail BOUNDED, then the circuit fast-denies. ──
|
||||||
|
var paused = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await RunShellCommandAsync(startCmd!, ct);
|
||||||
|
paused = true;
|
||||||
|
|
||||||
|
for (var i = 0; i < OutageThreshold; i++)
|
||||||
|
{
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
var outage = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
outage.Success.ShouldBeFalse($"bind #{i + 1} during the outage must fail closed");
|
||||||
|
// The whole flow is wall-clock bounded — never the SDK's default ~10-20 s stall.
|
||||||
|
sw.Elapsed.ShouldBeLessThan(AuthTimeout + TimeSpan.FromSeconds(2),
|
||||||
|
"the authenticator must bound a hung directory call, not stall on it");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Circuit is now open — the next call fast-denies WITHOUT a directory round-trip (sub-second).
|
||||||
|
var blockedSw = Stopwatch.StartNew();
|
||||||
|
var blocked = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
|
||||||
|
blockedSw.Stop();
|
||||||
|
|
||||||
|
blocked.Success.ShouldBeFalse();
|
||||||
|
blocked.Error.ShouldNotBeNull();
|
||||||
|
blocked.Error!.ShouldContain("unavailable", Case.Insensitive,
|
||||||
|
"an open outage circuit fast-denies 'backend unavailable'");
|
||||||
|
blockedSw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1),
|
||||||
|
"an open circuit fast-denies without a directory call");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (paused)
|
||||||
|
await RunShellCommandAsync(stopCmd!, CancellationToken.None); // ALWAYS restore GLAuth
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 3. Recovery: after the cooldown the half-open probe reaches the (restored) directory. ──
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(CooldownSeconds) + TimeSpan.FromSeconds(1), ct);
|
||||||
|
|
||||||
|
var recovered = false;
|
||||||
|
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||||
|
while (DateTime.UtcNow < deadline)
|
||||||
|
{
|
||||||
|
var probe = await authenticator.AuthenticateUserNameAsync("alice", "alice123", ct);
|
||||||
|
if (probe.Success) { recovered = true; break; }
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||||
|
}
|
||||||
|
recovered.ShouldBeTrue("the circuit must half-open and close once GLAuth is unpaused");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static LdapOpcUaUserAuthenticator BuildAuthenticator(LdapOptions options)
|
||||||
|
{
|
||||||
|
var ldap = new OtOpcUaLdapAuthService(Options.Create(options), NullLogger<OtOpcUaLdapAuthService>.Instance);
|
||||||
|
return new LdapOpcUaUserAuthenticator(
|
||||||
|
ldap, ScopeFactoryWith(new PassthroughMapper()), Options.Create(options),
|
||||||
|
NullLogger<LdapOpcUaUserAuthenticator>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Builds an <see cref="IServiceScopeFactory"/> whose scopes resolve the supplied mapper.</summary>
|
||||||
|
private static IServiceScopeFactory ScopeFactoryWith(IGroupRoleMapper<string> mapper)
|
||||||
|
{
|
||||||
|
var services = new ServiceCollection();
|
||||||
|
services.AddScoped(_ => mapper);
|
||||||
|
return services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Runs an SSH/docker one-liner through the login shell (verbatim, as in the S7 outage gate).</summary>
|
||||||
|
private static async Task RunShellCommandAsync(string command, CancellationToken ct)
|
||||||
|
{
|
||||||
|
using var proc = new Process
|
||||||
|
{
|
||||||
|
StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"")
|
||||||
|
{
|
||||||
|
RedirectStandardOutput = true,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
UseShellExecute = false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
proc.Start();
|
||||||
|
await proc.WaitForExitAsync(ct);
|
||||||
|
proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}");
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(1), ct); // let the pause/unpause take effect
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Passes LDAP groups straight through as roles — the live gate asserts on the bind, not the mapping.</summary>
|
||||||
|
private sealed class PassthroughMapper : IGroupRoleMapper<string>
|
||||||
|
{
|
||||||
|
public Task<GroupRoleMapping<string>> MapAsync(IReadOnlyList<string> groups, CancellationToken ct)
|
||||||
|
=> Task.FromResult(new GroupRoleMapping<string>(groups, Scope: null));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user