lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttConnectionTests.cs

using System.Diagnostics;
using System.Net;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using MQTTnet;
using Shouldly;
using Xunit;

namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests;

/// <summary>
///     Covers <see cref="MqttConnection"/>'s two Task-3 responsibilities: the pure
///     <see cref="MqttConnection.BuildClientOptions"/> options assembly (TLS / CA pin /
///     credentials / protocol knobs) and the bounded <see cref="MqttConnection.ConnectAsync"/>
///     deadline.
/// </summary>
public sealed class MqttConnectionTests
{
    private const string Host = "broker.example.test";

    // ---------------------------------------------------------------------------------
    // Bounded connect (the R2-01 frozen-peer lesson: no unbounded wait on a dead broker)
    // ---------------------------------------------------------------------------------

    /// <summary>
    ///     A broker that completes the TCP handshake but never answers CONNACK is the shape that
    ///     actually wedges a client — a refused port returns ECONNREFUSED instantly and would pass
    ///     whether or not any deadline logic existed.
    /// </summary>
    [Fact]
    public async Task ConnectAsync_BlackholeBroker_FailsWithinDeadline_NoHang()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1", Port = blackhole.Port, UseTls = false, ConnectTimeoutSeconds = 2,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        var sw = Stopwatch.StartNew();
        await Should.ThrowAsync<Exception>(async () => await conn.ConnectAsync(CancellationToken.None));
        sw.Stop();

        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10));
        conn.IsConnected.ShouldBeFalse();
    }

    /// <summary>
    ///     Falsifiability control for the linked-CTS wiring specifically: the configured connect
    ///     deadline is 60 s, so the only thing that can end this call inside 10 s is the caller's
    ///     token actually reaching MQTTnet. Drop the <c>CreateLinkedTokenSource</c> and this test
    ///     hangs for a minute.
    /// </summary>
    [Fact]
    public async Task ConnectAsync_CallerCancellation_ObservedLongBeforeConnectDeadline()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1", Port = blackhole.Port, UseTls = false, ConnectTimeoutSeconds = 60,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);
        using var caller = new CancellationTokenSource(TimeSpan.FromMilliseconds(300));

        var sw = Stopwatch.StartNew();
        await Should.ThrowAsync<OperationCanceledException>(async () => await conn.ConnectAsync(caller.Token));
        sw.Stop();

        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10));
    }

    /// <summary>
    ///     The deadline leg is distinguishable from a caller cancellation — a timed-out connect
    ///     surfaces <see cref="TimeoutException"/>, not a bare <see cref="OperationCanceledException"/>,
    ///     so an operator can tell "broker never answered" from "we were asked to stop".
    /// </summary>
    [Fact]
    public async Task ConnectAsync_DeadlineElapsed_ThrowsTimeoutException()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1", Port = blackhole.Port, UseTls = false, ConnectTimeoutSeconds = 1,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await Should.ThrowAsync<TimeoutException>(async () => await conn.ConnectAsync(CancellationToken.None));
    }

    /// <summary>
    ///     Task 4's reconnect loop retries connect on the SAME instance, reusing the
    ///     <c>IMqttClient</c> held across failures. A failed attempt must therefore leave the
    ///     connection usable rather than poisoned.
    /// </summary>
    [Fact]
    public async Task ConnectAsync_CanBeRetriedOnTheSameInstanceAfterAFailedAttempt()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1", Port = blackhole.Port, UseTls = false, ConnectTimeoutSeconds = 1,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await Should.ThrowAsync<TimeoutException>(async () => await conn.ConnectAsync(CancellationToken.None));

        // Second attempt reaches the broker again and fails the same way — not ObjectDisposed,
        // not a null client, not a silently-swallowed no-op.
        await Should.ThrowAsync<TimeoutException>(async () => await conn.ConnectAsync(CancellationToken.None));
        conn.IsConnected.ShouldBeFalse();
    }

    /// <summary>
    ///     Connecting a disposed instance is an <see cref="ObjectDisposedException"/> — the same
    ///     classification the in-flight dispose race folds into. Dispose is idempotent.
    /// </summary>
    [Fact]
    public async Task ConnectAsync_AfterDispose_ThrowsObjectDisposed_AndDisposeIsIdempotent()
    {
        var opts = new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false, ConnectTimeoutSeconds = 1 };
        var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await conn.DisposeAsync();
        await conn.DisposeAsync();

        await Should.ThrowAsync<ObjectDisposedException>(async () => await conn.ConnectAsync(CancellationToken.None));
    }

    /// <summary>Never log or throw credentials — the plan's cross-cutting secrets rule.</summary>
    [Fact]
    public async Task ConnectAsync_Failure_DoesNotLeakPasswordIntoExceptionMessage()
    {
        const string secret = "sup3r-s3cret-broker-pw";
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1",
            Port = blackhole.Port,
            UseTls = false,
            ConnectTimeoutSeconds = 1,
            Username = "svc",
            Password = secret,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        var ex = await Should.ThrowAsync<Exception>(async () => await conn.ConnectAsync(CancellationToken.None));

        ex.ToString().ShouldNotContain(secret);
    }

    // ---------------------------------------------------------------------------------
    // BuildClientOptions — pure assembly, no I/O
    // ---------------------------------------------------------------------------------

    [Fact]
    public void BuildClientOptions_UseTlsWithCaPin_SetsTlsAndInstallsChainValidator()
    {
        var opts = new MqttDriverOptions
        {
            Host = Host,
            Port = 8883,
            UseTls = true,
            AllowUntrustedServerCertificate = false,
            CaCertificatePath = "/tmp/ca.pem",
        };

        var tls = BuildTls(opts);

        tls.UseTls.ShouldBeTrue();
        tls.TargetHost.ShouldBe(Host);
        tls.AllowUntrustedCertificates.ShouldBeFalse();
        tls.IgnoreCertificateChainErrors.ShouldBeFalse();
        tls.CertificateValidationHandler.ShouldNotBeNull();
    }

    /// <summary>
    ///     The CA file is read lazily inside the validation callback, never by the builder — so a
    ///     path that does not exist yet must not throw at build time, and must fail the *connection*
    ///     closed when the handshake finally asks.
    /// </summary>
    [Fact]
    public void BuildClientOptions_CaPinPathMissing_BuildsWithoutIo_AndValidatorFailsClosed()
    {
        var missing = Path.Combine(Path.GetTempPath(), $"otopcua-mqtt-no-such-ca-{Guid.NewGuid():N}.pem");
        File.Exists(missing).ShouldBeFalse();

        var opts = new MqttDriverOptions { Host = Host, Port = 8883, UseTls = true, CaCertificatePath = missing };

        var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);
        var tls = built.ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>().TlsOptions;

        tls.CertificateValidationHandler.ShouldNotBeNull();
        using var stranger = SelfSignedLeaf(Host);
        tls.CertificateValidationHandler(ValidationArgs(built, stranger)).ShouldBeFalse();
    }

    /// <summary>An untrusted, unpinned server certificate is rejected — the pin actually validates.</summary>
    [Fact]
    public void BuildClientOptions_CaPin_RejectsCertificateNotIssuedByThePinnedCa()
    {
        using var ca = IssueCa(issuer: null, "otopcua-test-ca", validDays: 30);
        var caPath = WritePem(ca);
        try
        {
            var opts = new MqttDriverOptions { Host = Host, Port = 8883, UseTls = true, CaCertificatePath = caPath };
            var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);
            var tls = built.ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>().TlsOptions;

            using var stranger = SelfSignedLeaf(Host);
            tls.CertificateValidationHandler!(ValidationArgs(built, stranger)).ShouldBeFalse();
        }
        finally
        {
            File.Delete(caPath);
        }
    }

    /// <summary>
    ///     The accept path. Without this every certificate test asserts rejection, so deleting the
    ///     <c>return true</c> — or getting <c>TrustMode</c> / <c>CustomTrustStore</c> wrong — would
    ///     leave the suite green while no broker could ever connect.
    /// </summary>
    [Fact]
    public void BuildClientOptions_CaPin_AcceptsLeafIssuedByThePinnedCa()
    {
        using var ca = IssueCa(issuer: null, "otopcua-test-ca", validDays: 30);
        using var leaf = IssueLeaf(ca, Host, validDays: 20);
        var caPath = WritePem(ca);
        try
        {
            var opts = new MqttDriverOptions { Host = Host, Port = 8883, UseTls = true, CaCertificatePath = caPath };
            var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);
            var tls = built.ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>().TlsOptions;

            tls.CertificateValidationHandler!(ValidationArgs(built, leaf)).ShouldBeTrue();
        }
        finally
        {
            File.Delete(caPath);
        }
    }

    /// <summary>
    ///     The common real-world broker topology: the leaf is signed by an intermediate that chains
    ///     to the pinned root, and the intermediate is delivered in the handshake rather than being
    ///     installed locally. Only the pinned ROOT is written to the CA file — the intermediate must
    ///     be picked up from what the broker presented.
    /// </summary>
    [Fact]
    public void BuildClientOptions_CaPin_AcceptsLeafBehindAnIntermediateThatTheBrokerPresented()
    {
        using var root = IssueCa(issuer: null, "otopcua-test-root", validDays: 30);
        using var intermediate = IssueCa(root, "otopcua-test-intermediate", validDays: 25);
        using var leaf = IssueLeaf(intermediate, Host, validDays: 20);
        var caPath = WritePem(root);
        try
        {
            var opts = new MqttDriverOptions { Host = Host, Port = 8883, UseTls = true, CaCertificatePath = caPath };
            var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);
            var tls = built.ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>().TlsOptions;

            tls.CertificateValidationHandler!(ValidationArgs(built, leaf, intermediate)).ShouldBeTrue();
        }
        finally
        {
            File.Delete(caPath);
        }
    }

    /// <summary>Default posture: OS trust store, no custom handler, escape hatch off.</summary>
    [Fact]
    public void BuildClientOptions_TlsWithoutCaPin_LeavesOsTrustStoreValidationInPlace()
    {
        var opts = new MqttDriverOptions { Host = Host, Port = 8883, UseTls = true };

        var tls = BuildTls(opts);

        tls.UseTls.ShouldBeTrue();
        tls.CertificateValidationHandler.ShouldBeNull();
        tls.AllowUntrustedCertificates.ShouldBeFalse();
        tls.IgnoreCertificateChainErrors.ShouldBeFalse();
        tls.IgnoreCertificateRevocationErrors.ShouldBeFalse();
    }

    /// <summary>The dev/on-prem escape hatch only opens when explicitly asked for.</summary>
    [Fact]
    public void BuildClientOptions_AllowUntrustedServerCertificate_OpensEscapeHatch()
    {
        var opts = new MqttDriverOptions
        {
            Host = Host, Port = 8883, UseTls = true, AllowUntrustedServerCertificate = true,
        };

        var tls = BuildTls(opts);

        tls.AllowUntrustedCertificates.ShouldBeTrue();
        tls.IgnoreCertificateChainErrors.ShouldBeTrue();
        tls.CertificateValidationHandler.ShouldNotBeNull();
    }

    [Fact]
    public void BuildClientOptions_UseTlsFalse_KeepsTlsOffAndInstallsNoValidator()
    {
        var opts = new MqttDriverOptions
        {
            Host = Host, Port = 1883, UseTls = false, AllowUntrustedServerCertificate = true, CaCertificatePath = "/tmp/ca.pem",
        };

        var tls = BuildTls(opts);

        tls.UseTls.ShouldBeFalse();
        tls.CertificateValidationHandler.ShouldBeNull();
        tls.AllowUntrustedCertificates.ShouldBeFalse();
    }

    [Fact]
    public void BuildClientOptions_MapsProtocolAndSessionKnobs()
    {
        var opts = new MqttDriverOptions
        {
            Host = Host,
            Port = 8883,
            ClientId = "otopcua-node-1",
            ProtocolVersion = MqttProtocolVersion.V311,
            CleanSession = false,
            KeepAliveSeconds = 45,
            ConnectTimeoutSeconds = 7,
        };

        var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);

        built.ProtocolVersion.ShouldBe(MQTTnet.Formatter.MqttProtocolVersion.V311);
        built.ClientId.ShouldBe("otopcua-node-1");
        built.CleanSession.ShouldBeFalse();
        built.KeepAlivePeriod.ShouldBe(TimeSpan.FromSeconds(45));
        built.Timeout.ShouldBe(TimeSpan.FromSeconds(7));
        var tcp = built.ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>();
        tcp.RemoteEndpoint.ShouldBeOfType<DnsEndPoint>().Host.ShouldBe(Host);
        tcp.RemoteEndpoint.ShouldBeOfType<DnsEndPoint>().Port.ShouldBe(8883);
    }

    [Fact]
    public void BuildClientOptions_ClientIdSuffix_IsAppended()
    {
        var opts = new MqttDriverOptions { Host = Host, ClientId = "otopcua" };

        MqttConnection.BuildClientOptions(opts, clientIdSuffix: "-browse").ClientId.ShouldBe("otopcua-browse");
    }

    [Fact]
    public void BuildClientOptions_NoUsername_SendsNoCredentials()
        => MqttConnection.BuildClientOptions(new MqttDriverOptions { Host = Host }, clientIdSuffix: null)
            .Credentials.ShouldBeNull();

    [Fact]
    public void BuildClientOptions_Username_SendsCredentials()
    {
        var opts = new MqttDriverOptions { Host = Host, Username = "svc", Password = "pw" };

        var built = MqttConnection.BuildClientOptions(opts, clientIdSuffix: null);

        built.Credentials.ShouldNotBeNull();
        built.Credentials.GetUserName(built).ShouldBe("svc");
    }

    /// <summary>The ctor must not touch the network — the universal-browser throwaway-instance pattern.</summary>
    [Fact]
    public void Ctor_IsConnectionFree()
    {
        var opts = new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false };

        var conn = new MqttConnection(opts, driverId: "t", logger: null);

        conn.IsConnected.ShouldBeFalse();
    }

    // ---------------------------------------------------------------------------------
    // Task 4 — NextBackoff (pure): bounded, monotone, cannot hot-loop, cannot overflow
    // ---------------------------------------------------------------------------------

    [Theory]
    [InlineData(0, 1)]
    [InlineData(1, 2)]
    [InlineData(2, 4)]
    [InlineData(3, 8)]
    [InlineData(4, 16)]
    [InlineData(5, 30)] // 32 → clamped
    [InlineData(10, 30)]
    public void NextBackoff_IsExponentialClampedToMax(int attempt, int expectedSeconds)
        => MqttConnection.NextBackoff(attempt, minSeconds: 1, maxSeconds: 30)
            .ShouldBe(TimeSpan.FromSeconds(expectedSeconds));

    /// <summary>
    ///     A shift-based implementation (<c>min &lt;&lt; attempt</c>) wraps: <c>1 &lt;&lt; 32</c> is
    ///     <c>1</c>, not 4294967296, so a broker that has been down for half an hour would suddenly
    ///     be hammered once a second. The calculator must saturate, not wrap.
    /// </summary>
    [Fact]
    public void NextBackoff_HugeAttemptCount_SaturatesAtMax_NeverWrapsBackToMin()
    {
        foreach (var attempt in new[] { 31, 32, 33, 40, 62, 63, 64, 1_000, int.MaxValue })
        {
            MqttConnection.NextBackoff(attempt, minSeconds: 1, maxSeconds: 30)
                .ShouldBe(TimeSpan.FromSeconds(30), $"attempt {attempt}");
        }
    }

    [Fact]
    public void NextBackoff_IsMonotoneNonDecreasing_AndAlwaysWithinMinMax()
    {
        var previous = TimeSpan.Zero;
        for (var attempt = 0; attempt <= 64; attempt++)
        {
            var backoff = MqttConnection.NextBackoff(attempt, minSeconds: 2, maxSeconds: 30);

            backoff.ShouldBeGreaterThanOrEqualTo(previous, $"attempt {attempt} went backwards");
            backoff.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromSeconds(2), $"attempt {attempt} below min");
            backoff.ShouldBeLessThanOrEqualTo(TimeSpan.FromSeconds(30), $"attempt {attempt} above max");
            previous = backoff;
        }
    }

    /// <summary>
    ///     A misconfigured <c>max &lt; min</c> must not collapse the floor — the floor is the only
    ///     thing standing between a dead broker and a hot reconnect loop.
    /// </summary>
    [Fact]
    public void NextBackoff_MaxBelowMin_HonoursMin_SoTheLoopCannotGoHot()
        => MqttConnection.NextBackoff(0, minSeconds: 10, maxSeconds: 5).ShouldBe(TimeSpan.FromSeconds(10));

    [Fact]
    public void NextBackoff_NonPositiveAttempt_IsTheFirstAttemptDelay()
    {
        MqttConnection.NextBackoff(0, minSeconds: 3, maxSeconds: 30).ShouldBe(TimeSpan.FromSeconds(3));
        MqttConnection.NextBackoff(-5, minSeconds: 3, maxSeconds: 30).ShouldBe(TimeSpan.FromSeconds(3));
    }

    // ---------------------------------------------------------------------------------
    // Task 4 — the Reconnected callback (the load-bearing re-subscribe hook)
    // ---------------------------------------------------------------------------------

    [Fact]
    public async Task FireReconnected_InvokesEverySubscribedCallback()
    {
        await using var conn = new MqttConnection(LoopbackOpts(), driverId: "t", logger: null);
        var first = 0;
        var second = 0;
        conn.Reconnected += () => { Interlocked.Increment(ref first); return Task.CompletedTask; };
        conn.Reconnected += () => { Interlocked.Increment(ref second); return Task.CompletedTask; };

        await conn.FireReconnectedAsync();

        first.ShouldBe(1);
        second.ShouldBe(1);
    }

    /// <summary>
    ///     Awaiting a multicast <c>Func&lt;Task&gt;</c> directly returns only the LAST handler's task
    ///     and abandons the earlier ones — so one throwing subscriber would silently skip every
    ///     subscriber behind it. The fan-out must walk the invocation list and still surface the
    ///     failure, because a re-subscribe that failed is a connection that has gone dark.
    /// </summary>
    [Fact]
    public async Task FireReconnected_OneHandlerThrows_StillInvokesTheRest_AndSurfacesTheFailure()
    {
        await using var conn = new MqttConnection(LoopbackOpts(), driverId: "t", logger: null);
        var behindTheThrower = 0;
        conn.Reconnected += () => throw new InvalidOperationException("SUBACK refused");
        conn.Reconnected += () => { Interlocked.Increment(ref behindTheThrower); return Task.CompletedTask; };

        await Should.ThrowAsync<Exception>(async () => await conn.FireReconnectedAsync());

        behindTheThrower.ShouldBe(1);
    }

    [Fact]
    public async Task FireReconnected_NoSubscribers_IsANoOp()
    {
        await using var conn = new MqttConnection(LoopbackOpts(), driverId: "t", logger: null);

        await Should.NotThrowAsync(async () => await conn.FireReconnectedAsync());
    }

    // ---------------------------------------------------------------------------------
    // Task 4 — State + LastMessage
    // ---------------------------------------------------------------------------------

    [Fact]
    public void State_AfterCtor_IsDisconnected()
        => new MqttConnection(LoopbackOpts(), driverId: "t", logger: null)
            .State.ShouldBe(MqttConnectionState.Disconnected);

    [Fact]
    public async Task State_AfterDispose_IsDisposed()
    {
        var conn = new MqttConnection(LoopbackOpts(), driverId: "t", logger: null);

        await conn.DisposeAsync();

        conn.State.ShouldBe(MqttConnectionState.Disposed);
    }

    /// <summary>
    ///     Faulted is reserved for config that cannot succeed no matter how long we retry. A broker
    ///     that is merely down is <see cref="MqttConnectionState.Reconnecting"/>, indefinitely.
    /// </summary>
    [Fact]
    public async Task State_UnrecoverableProtocolConfig_IsFaulted()
    {
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1",
            Port = 1,
            UseTls = false,
            ConnectTimeoutSeconds = 1,
            ProtocolVersion = (MqttProtocolVersion)99,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await Should.ThrowAsync<ArgumentOutOfRangeException>(async () => await conn.ConnectAsync(CancellationToken.None));

        conn.State.ShouldBe(MqttConnectionState.Faulted);
    }

    /// <summary>A broker that is simply unreachable is retryable, never <c>Faulted</c>.</summary>
    [Fact]
    public async Task State_UnreachableBroker_IsNotFaulted()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1", Port = blackhole.Port, UseTls = false, ConnectTimeoutSeconds = 1,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await Should.ThrowAsync<TimeoutException>(async () => await conn.ConnectAsync(CancellationToken.None));

        conn.State.ShouldNotBe(MqttConnectionState.Faulted);
    }

    [Fact]
    public void LastMessage_BeforeAnyTraffic_IsNull()
    {
        var conn = new MqttConnection(LoopbackOpts(), driverId: "t", logger: null);

        conn.LastMessageUtc.ShouldBeNull();
        conn.LastMessageAge.ShouldBeNull();
    }

    /// <summary>
    ///     A failed FIRST connect is the caller's problem (driver-host resilience retries
    ///     <c>InitializeAsync</c>) — it must not silently leave a background reconnect worker
    ///     hammering an endpoint the caller has already given up on.
    /// </summary>
    [Fact]
    public async Task ConnectAsync_InitialAttemptFails_LeavesNoBackgroundReconnectStorm()
    {
        using var blackhole = new BlackholeBroker();
        var opts = new MqttDriverOptions
        {
            Host = "127.0.0.1",
            Port = blackhole.Port,
            UseTls = false,
            ConnectTimeoutSeconds = 1,
            ReconnectMinBackoffSeconds = 1,
            ReconnectMaxBackoffSeconds = 1,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await Should.ThrowAsync<TimeoutException>(async () => await conn.ConnectAsync(CancellationToken.None));
        var acceptedAfterTheAttempt = blackhole.AcceptedCount;

        await Task.Delay(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);

        blackhole.AcceptedCount.ShouldBe(acceptedAfterTheAttempt);
    }

    // ---------------------------------------------------------------------------------
    // Task 4 — the reconnect supervisor, against a real (minimal) broker
    // ---------------------------------------------------------------------------------

    /// <summary>
    ///     THE load-bearing invariant. MQTT subscriptions do not survive a clean session, so a
    ///     reconnect that completes without firing <c>Reconnected</c> leaves a healthy-looking
    ///     connection that receives nothing, forever, with no error to show for it. Driven through
    ///     the real socket → real MQTTnet <c>DisconnectedAsync</c> → real re-CONNECT path, not a
    ///     test seam.
    /// </summary>
    [Fact]
    public async Task BrokerDropsTheConnection_ReconnectsAndFiresReconnected()
    {
        using var broker = new MiniBroker();
        await using var conn = new MqttConnection(BrokerOpts(broker.Port), driverId: "t", logger: null);
        var resubscribes = 0;
        conn.Reconnected += () => { Interlocked.Increment(ref resubscribes); return Task.CompletedTask; };

        await conn.ConnectAsync(CancellationToken.None);

        conn.State.ShouldBe(MqttConnectionState.Connected);
        conn.IsConnected.ShouldBeTrue();
        Volatile.Read(ref resubscribes).ShouldBe(0, "the initial connect is not a reconnect");

        broker.DropAll();

        (await WaitUntilAsync(() => Volatile.Read(ref resubscribes) >= 1, TimeSpan.FromSeconds(30)))
            .ShouldBeTrue("the reconnect never fired Reconnected — subscriptions would be silently gone");
        (await WaitUntilAsync(() => conn.IsConnected, TimeSpan.FromSeconds(10))).ShouldBeTrue();
        conn.State.ShouldBe(MqttConnectionState.Connected);
        broker.AcceptedCount.ShouldBeGreaterThanOrEqualTo(2);
    }

    /// <summary>
    ///     A broker that stays down keeps the supervisor in <c>Reconnecting</c> indefinitely (never
    ///     <c>Faulted</c>) — and the backoff keeps the retry rate low enough that a handful of
    ///     attempts, not hundreds, land in the observation window.
    /// </summary>
    [Fact]
    public async Task BrokerStaysDown_RetriesUnderBackoff_WithoutHotLooping()
    {
        using var broker = new MiniBroker();
        var opts = BrokerOpts(broker.Port) with
        {
            ConnectTimeoutSeconds = 1, ReconnectMinBackoffSeconds = 1, ReconnectMaxBackoffSeconds = 2,
        };
        await using var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await conn.ConnectAsync(CancellationToken.None);
        var acceptedWhileHealthy = broker.AcceptedCount;

        broker.Blackhole = true; // accepts TCP, never answers CONNACK again
        broker.DropAll();

        (await WaitUntilAsync(() => conn.State == MqttConnectionState.Reconnecting, TimeSpan.FromSeconds(10)))
            .ShouldBeTrue();
        await Task.Delay(TimeSpan.FromSeconds(8), TestContext.Current.CancellationToken);

        conn.State.ShouldBe(MqttConnectionState.Reconnecting, "a down broker is retryable, not Faulted");
        var retries = broker.AcceptedCount - acceptedWhileHealthy;
        retries.ShouldBeGreaterThanOrEqualTo(1, "the supervisor gave up on a down broker");
        retries.ShouldBeLessThan(15, $"backoff is not bounding the retry rate ({retries} attempts in ~8s)");
    }

    /// <summary>
    ///     Dispose during a reconnect backoff must stop the supervisor dead — no leaked task, no
    ///     later attempt resurrecting a connection the caller has torn down.
    /// </summary>
    [Fact]
    public async Task Dispose_DuringReconnectBackoff_StopsTheSupervisor_NoFurtherAttempts()
    {
        using var broker = new MiniBroker();
        var opts = BrokerOpts(broker.Port) with
        {
            ConnectTimeoutSeconds = 1, ReconnectMinBackoffSeconds = 1, ReconnectMaxBackoffSeconds = 2,
        };
        var conn = new MqttConnection(opts, driverId: "t", logger: null);

        await conn.ConnectAsync(CancellationToken.None);
        broker.Blackhole = true;
        broker.DropAll();
        (await WaitUntilAsync(() => conn.State == MqttConnectionState.Reconnecting, TimeSpan.FromSeconds(10)))
            .ShouldBeTrue();

        await conn.DisposeAsync();
        var acceptedAtDispose = broker.AcceptedCount;

        conn.State.ShouldBe(MqttConnectionState.Disposed);
        conn.IsConnected.ShouldBeFalse();
        await Task.Delay(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
        broker.AcceptedCount.ShouldBe(acceptedAtDispose, "the supervisor outlived DisposeAsync");
    }

    /// <summary>
    ///     The T3 connect/dispose leak, reproduced at the exact interleaving the reviewer traced:
    ///     the client has already CONNECTED when dispose flags the instance and blocks on the
    ///     lifecycle gate this connect is holding — so dispose disposes nothing. Without the
    ///     post-connect re-check the client stays live with an open socket that no later
    ///     <c>DisposeAsync</c> can ever reach; the broker sees the leak as a connection that never
    ///     closes.
    /// </summary>
    [Fact]
    public async Task DisposeRacingAnInFlightConnect_LeavesNoLiveBrokerConnection()
    {
        using var broker = new MiniBroker();
        var conn = new MqttConnection(BrokerOpts(broker.Port), driverId: "t", logger: null);
        conn.AfterConnectHookForTests = async () =>
        {
            _ = Task.Run(async () => await conn.DisposeAsync());
            (await WaitUntilAsync(() => conn.State == MqttConnectionState.Disposed, TimeSpan.FromSeconds(5)))
                .ShouldBeTrue("dispose never flagged the instance — the race was not reproduced");
        };

        await Should.ThrowAsync<ObjectDisposedException>(async () => await conn.ConnectAsync(CancellationToken.None));

        (await WaitUntilAsync(() => broker.LiveConnections == 0, TimeSpan.FromSeconds(15)))
            .ShouldBeTrue($"a live broker connection leaked ({broker.LiveConnections} still open)");
        conn.IsConnected.ShouldBeFalse();
    }

    // ---------------------------------------------------------------------------------
    // helpers
    // ---------------------------------------------------------------------------------

    private static MqttDriverOptions LoopbackOpts()
        => new() { Host = "127.0.0.1", Port = 1, UseTls = false, ConnectTimeoutSeconds = 1 };

    /// <summary>Options aimed at <see cref="MiniBroker"/>: plaintext v3.1.1, keep-alive parked out of the way.</summary>
    private static MqttDriverOptions BrokerOpts(int port)
        => new()
        {
            Host = "127.0.0.1",
            Port = port,
            UseTls = false,
            ProtocolVersion = MqttProtocolVersion.V311,
            CleanSession = true,
            KeepAliveSeconds = 300,
            ConnectTimeoutSeconds = 5,
            ReconnectMinBackoffSeconds = 1,
            ReconnectMaxBackoffSeconds = 2,
        };

    private static async Task<bool> WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
    {
        var deadline = Stopwatch.StartNew();
        while (deadline.Elapsed < timeout)
        {
            if (condition())
            {
                return true;
            }

            await Task.Delay(25);
        }

        return condition();
    }

    private static MqttClientTlsOptions BuildTls(MqttDriverOptions opts)
        => MqttConnection.BuildClientOptions(opts, clientIdSuffix: null)
            .ChannelOptions.ShouldBeOfType<MqttClientTcpOptions>().TlsOptions;

    private static readonly DateTimeOffset NotBefore = DateTimeOffset.UtcNow.AddDays(-1);

    /// <summary>A CA certificate, self-signed when <paramref name="issuer"/> is null.</summary>
    private static X509Certificate2 IssueCa(X509Certificate2? issuer, string commonName, int validDays)
    {
        using var key = RSA.Create(2048);
        var request = new CertificateRequest($"CN={commonName}", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
        request.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true));
        request.CertificateExtensions.Add(
            new X509KeyUsageExtension(X509KeyUsageFlags.KeyCertSign | X509KeyUsageFlags.CrlSign, true));
        request.CertificateExtensions.Add(new X509SubjectKeyIdentifierExtension(request.PublicKey, false));

        if (issuer is null)
        {
            return request.CreateSelfSigned(NotBefore, NotBefore.AddDays(validDays));
        }

        // Create() drops the private key; an intermediate needs it back to sign its own children.
        using var issued = request.Create(issuer, NotBefore, NotBefore.AddDays(validDays), NextSerial());
        return issued.CopyWithPrivateKey(key);
    }

    /// <summary>A server (end-entity) certificate signed by <paramref name="issuer"/>.</summary>
    private static X509Certificate2 IssueLeaf(X509Certificate2 issuer, string commonName, int validDays)
    {
        using var key = RSA.Create(2048);
        var request = new CertificateRequest($"CN={commonName}", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
        request.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, true));
        request.CertificateExtensions.Add(
            new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, true));
        request.CertificateExtensions.Add(
            new X509EnhancedKeyUsageExtension([new Oid("1.3.6.1.5.5.7.3.1")], false)); // serverAuth
        request.CertificateExtensions.Add(new X509SubjectKeyIdentifierExtension(request.PublicKey, false));

        return request.Create(issuer, NotBefore, NotBefore.AddDays(validDays), NextSerial());
    }

    /// <summary>A self-signed leaf that chains to nothing — the "stranger" a pin must reject.</summary>
    private static X509Certificate2 SelfSignedLeaf(string commonName)
    {
        using var key = RSA.Create(2048);
        var request = new CertificateRequest($"CN={commonName}", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
        return request.CreateSelfSigned(NotBefore, NotBefore.AddDays(20));
    }

    private static byte[] NextSerial() => RandomNumberGenerator.GetBytes(8);

    private static string WritePem(params X509Certificate2[] certificates)
    {
        var path = Path.Combine(Path.GetTempPath(), $"otopcua-mqtt-ca-{Guid.NewGuid():N}.pem");
        File.WriteAllText(path, string.Join(Environment.NewLine, certificates.Select(c => c.ExportCertificatePem())));
        return path;
    }

    /// <summary>
    ///     Mimics what <c>SslStream</c> hands a validation callback: the presented leaf plus a chain
    ///     carrying whatever intermediates the peer supplied. The chain is built (and allowed to
    ///     fail) purely so <c>ChainElements</c> is populated the way the platform would populate it.
    /// </summary>
    private static MqttClientCertificateValidationEventArgs ValidationArgs(
        MqttClientOptions built,
        X509Certificate2 leaf,
        params X509Certificate2[] presentedIntermediates)
    {
        var chain = new X509Chain();
        chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
        chain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllFlags;
        foreach (var intermediate in presentedIntermediates)
        {
            chain.ChainPolicy.ExtraStore.Add(intermediate);
        }

        chain.Build(leaf);

        return new MqttClientCertificateValidationEventArgs(
            leaf,
            chain,
            SslPolicyErrors.RemoteCertificateChainErrors,
            built.ChannelOptions);
    }

    /// <summary>
    ///     Accepts the TCP connection and then says nothing — the client sends CONNECT and waits
    ///     forever for a CONNACK that never comes. This is the frozen-peer shape; a closed port is
    ///     not (it refuses instantly and proves nothing about a deadline).
    /// </summary>
    private sealed class BlackholeBroker : IDisposable
    {
        private readonly List<TcpClient> _accepted = [];
        private readonly CancellationTokenSource _cts = new();
        private readonly TcpListener _listener;
        private int _acceptedCount;

        public BlackholeBroker()
        {
            _listener = new TcpListener(IPAddress.Loopback, 0);
            _listener.Start();
            Port = ((IPEndPoint)_listener.LocalEndpoint).Port;
            _ = Task.Run(AcceptLoopAsync);
        }

        public int Port { get; }

        /// <summary>Total TCP connections accepted — how a caller detects a background retry storm.</summary>
        public int AcceptedCount => Volatile.Read(ref _acceptedCount);

        public void Dispose()
        {
            _cts.Cancel();
            _listener.Stop();
            lock (_accepted)
            {
                foreach (var client in _accepted)
                {
                    client.Dispose();
                }

                _accepted.Clear();
            }

            _cts.Dispose();
        }

        private async Task AcceptLoopAsync()
        {
            try
            {
                while (!_cts.IsCancellationRequested)
                {
                    var client = await _listener.AcceptTcpClientAsync(_cts.Token);
                    Interlocked.Increment(ref _acceptedCount);
                    lock (_accepted)
                    {
                        _accepted.Add(client);
                    }
                }
            }
            catch (Exception)
            {
                // Listener stopped / token cancelled — the fixture is going away.
            }
        }
    }

    /// <summary>
    ///     The smallest broker that can complete an MQTT 3.1.1 handshake: accept TCP, answer CONNECT
    ///     with a success CONNACK, answer PINGREQ with PINGRESP, and nothing else. That is enough for
    ///     MQTTnet to report a real connection — which is what makes a real drop
    ///     (<see cref="DropAll"/>) drive a real <c>DisconnectedAsync</c> and a real reconnect, rather
    ///     than a test seam standing in for one.
    /// </summary>
    private sealed class MiniBroker : IDisposable
    {
        private static readonly byte[] ConnAckAccepted = [0x20, 0x02, 0x00, 0x00];
        private static readonly byte[] PingResp = [0xD0, 0x00];

        private readonly CancellationTokenSource _cts = new();
        private readonly HashSet<TcpClient> _live = [];
        private readonly TcpListener _listener;
        private int _acceptedCount;

        public MiniBroker()
        {
            _listener = new TcpListener(IPAddress.Loopback, 0);
            _listener.Start();
            Port = ((IPEndPoint)_listener.LocalEndpoint).Port;
            _ = Task.Run(AcceptLoopAsync);
        }

        public int Port { get; }

        /// <summary>When set, TCP is still accepted but CONNECT is never answered (frozen peer).</summary>
        public bool Blackhole { get; set; }

        /// <summary>Total TCP connections accepted since construction — counts reconnect attempts.</summary>
        public int AcceptedCount => Volatile.Read(ref _acceptedCount);

        /// <summary>
        ///     Sockets the broker still has open. Drops to zero only once the peer actually closes,
        ///     so a leaked-but-live client keeps this above zero.
        /// </summary>
        public int LiveConnections
        {
            get
            {
                lock (_live)
                {
                    return _live.Count;
                }
            }
        }

        /// <summary>Kills every established session — the "broker went away" event.</summary>
        public void DropAll()
        {
            lock (_live)
            {
                foreach (var client in _live)
                {
                    try
                    {
                        client.Close();
                    }
                    catch (Exception)
                    {
                        // Already gone.
                    }
                }

                _live.Clear();
            }
        }

        public void Dispose()
        {
            _cts.Cancel();
            _listener.Stop();
            DropAll();
            _cts.Dispose();
        }

        private async Task AcceptLoopAsync()
        {
            try
            {
                while (!_cts.IsCancellationRequested)
                {
                    var client = await _listener.AcceptTcpClientAsync(_cts.Token);
                    Interlocked.Increment(ref _acceptedCount);
                    lock (_live)
                    {
                        _live.Add(client);
                    }

                    _ = Task.Run(() => ServeAsync(client));
                }
            }
            catch (Exception)
            {
                // Listener stopped / token cancelled — the fixture is going away.
            }
        }

        private async Task ServeAsync(TcpClient client)
        {
            var buffer = new byte[1024];
            try
            {
                var stream = client.GetStream();
                while (!_cts.IsCancellationRequested)
                {
                    var read = await stream.ReadAsync(buffer, _cts.Token);
                    if (read == 0)
                    {
                        break; // peer closed — this is how a disposed client stops counting as live
                    }

                    if (Blackhole)
                    {
                        continue;
                    }

                    var packetType = buffer[0] & 0xF0;
                    if (packetType == 0x10)
                    {
                        await stream.WriteAsync(ConnAckAccepted, _cts.Token);
                    }
                    else if (packetType == 0xC0)
                    {
                        await stream.WriteAsync(PingResp, _cts.Token);
                    }
                }
            }
            catch (Exception)
            {
                // Socket torn down by either side.
            }
            finally
            {
                lock (_live)
                {
                    _live.Remove(client);
                }

                try
                {
                    client.Dispose();
                }
                catch (Exception)
                {
                    // Already gone.
                }
            }
        }
    }
}