fix(modbus): transport self-heals with one throttled in-gate connect; PDU length check

P3: a torn-down socket is re-dialled inside the gate (one attempt per backoff
window) instead of failing fast with 'Transport not connected' until the 90 s
comms-loss watchdog re-initialised the driver.
L3: ConnectAsync disposes the previous/failed TcpClient, DNS runs under the
per-op timeout, and the reconnect never sleeps inside the gate (the old
10-attempt backoff held it for ~81 s).
H9: an MBAP response with no PDU, or an exception PDU with no code, is a
TruncatedFrame desync rather than an IndexOutOfRangeException.

Refs #741
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs

@@ -91,7 +91,7 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport
     public async Task<byte[]> SendAsync(byte unitId, byte[] pdu, CancellationToken ct)
     {
         if (_disposed) throw new ObjectDisposedException(nameof(ModbusRtuOverTcpTransport));
-        if (CurrentStream is null) throw new InvalidOperationException("Transport not connected");
+        if (_life is null && _testStream is null) throw new InvalidOperationException("Transport not connected");
 
         // Single-flight: RTU carries no transaction id, so at most one transaction may be on the bus
         // at a time — serialise every send/receive under the gate.
@@ -101,11 +101,14 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport
             // Proactive idle-disconnect (real socket only): if the socket has been quiet longer than
             // the configured threshold, tear it down + reconnect before this PDU lands. Defends
             // against silent NAT / firewall reaping.
-            if (_life is not null && _life.ShouldReconnectForIdle())
-            {
+            if (_life is not null && _life.Stream is not null && _life.ShouldReconnectForIdle())
                 await _life.TearDownAsync().ConfigureAwait(false);
-                await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false);
-            }
+
+            // Self-heal (real socket only): a socket torn down by an earlier failure is re-dialled here,
+            // once, throttled by the backoff window — same contract as ModbusTcpTransport.
+            var freshlyConnected = _life is not null && _life.Stream is null;
+            if (freshlyConnected)
+                await _life!.EnsureConnectedAsync(ct).ConfigureAwait(false);
 
             try
             {
@@ -113,14 +116,16 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport
                 _life?.MarkSuccess();
                 return result;
             }
-            catch (Exception ex) when (_life is not null && _life.AutoReconnect && ModbusSocketLifecycle.IsSocketLevelFailure(ex))
+            catch (Exception ex) when (!freshlyConnected && _life is not null && _life.AutoReconnect
+                                       && !ct.IsCancellationRequested
+                                       && ModbusSocketLifecycle.IsSocketLevelFailure(ex))
             {
-                // Mid-transaction drop: tear down the dead socket, reconnect (with backoff if
-                // configured), resend. Single retry — a second failure propagates so health/status
-                // reflect reality. Never reached in the ForTest seam (a raw injected stream has no
-                // lifecycle to reconnect).
+                // Mid-transaction drop: tear down the dead socket, make ONE throttled reconnect attempt
+                // (never a sleep inside the gate), resend. Single retry — a second failure propagates so
+                // health/status reflect reality. Never reached in the ForTest seam (a raw injected stream
+                // has no lifecycle to reconnect).
                 await _life.TearDownAsync().ConfigureAwait(false);
-                await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false);
+                await _life.EnsureConnectedAsync(ct).ConfigureAwait(false);
                 var result = await SendOnceAsync(unitId, pdu, ct).ConfigureAwait(false);
                 _life.MarkSuccess();
                 return result;

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs

@@ -4,8 +4,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus;
 
 /// <summary>
 ///     Owns the raw TCP socket lifecycle shared by every Modbus-over-TCP transport (MBAP and
-///     RTU-over-TCP alike): the IPv4-preference connect, OS-level keep-alive, geometric-backoff
-///     reconnect, idle-disconnect tracking, and teardown. It exposes the current
+///     RTU-over-TCP alike): the IPv4-preference connect, OS-level keep-alive, throttled
+///     reconnect (one attempt per geometric-backoff window), idle-disconnect tracking, and teardown. It exposes the current
 ///     <see cref="NetworkStream"/> so the composing transport can drive its own framing on top —
 ///     the lifecycle knows nothing about MBAP / RTU wire layout.
 /// </summary>
@@ -14,7 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus;
 ///         Extracted verbatim from <see cref="ModbusTcpTransport"/> so the connect / reconnect /
 ///         keep-alive / idle behaviour is written once and composed by both transports. The
 ///         constructor is connection-free — all socket I/O happens in <see cref="ConnectAsync"/> /
-///         <see cref="ConnectWithBackoffAsync"/>.
+///         <see cref="EnsureConnectedAsync"/> (one throttled attempt per call — never a sleep loop).
 ///     </para>
 ///     <para>
 ///         Why keep-alive matters for DL205/DL260: the AutomationDirect H2-ECOM100 does NOT send
@@ -37,6 +37,9 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable
     private NetworkStream? _stream;
     private DateTime _lastSuccessUtc = DateTime.UtcNow;
 
+    /// <summary>Monotonic millisecond clock for the reconnect throttle (test seam).</summary>
+    internal Func<long> NowMs { get; set; } = () => Environment.TickCount64;
+
     /// <summary>Initializes a new instance of the <see cref="ModbusSocketLifecycle"/> class.</summary>
     /// <param name="host">The host address or hostname of the Modbus server.</param>
     /// <param name="port">The TCP port of the Modbus server.</param>
@@ -71,63 +74,107 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable
     /// <returns>A task representing the asynchronous connection operation.</returns>
     public async Task ConnectAsync(CancellationToken ct)
     {
-        // Resolve the host explicitly + prefer IPv4. .NET's TcpClient default-constructor is
-        // dual-stack (IPv6 first, fallback to IPv4) — but most Modbus TCP devices (PLCs and
-        // simulators like pymodbus) bind 0.0.0.0 only, so the IPv6 attempt times out and we
-        // burn the entire ConnectAsync budget before even trying IPv4. Resolving first +
-        // dialing the IPv4 address directly sidesteps that.
-        var addresses = await System.Net.Dns.GetHostAddressesAsync(_host, ct).ConfigureAwait(false);
-        var ipv4 = System.Linq.Enumerable.FirstOrDefault(addresses,
-            a => a.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork);
-        var target = ipv4 ?? (addresses.Length > 0 ? addresses[0] : System.Net.IPAddress.Loopback);
-
-        _client = new TcpClient(target.AddressFamily);
-        EnableKeepAlive(_client, _keepAlive);
+        // A previous socket (failed, torn down or idle-reaped) is released FIRST: overwriting _client
+        // used to strand the old TcpClient for the finalizer — up to nine per backoff loop.
+        await TearDownAsync().ConfigureAwait(false);
 
+        // DNS resolution runs under the same per-op deadline as the TCP connect: an unresolvable or
+        // slow resolver must not hold the transport gate for the OS resolver's own (much longer) timeout.
         using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
         cts.CancelAfter(_timeout);
-        await _client.ConnectAsync(target, _port, cts.Token).ConfigureAwait(false);
-        _stream = _client.GetStream();
-        _lastSuccessUtc = DateTime.UtcNow;
+        TcpClient? client = null;
+        try
+        {
+            // Resolve the host explicitly + prefer IPv4. .NET's TcpClient default-constructor is
+            // dual-stack (IPv6 first, fallback to IPv4) — but most Modbus TCP devices (PLCs and
+            // simulators like pymodbus) bind 0.0.0.0 only, so the IPv6 attempt times out and we
+            // burn the entire ConnectAsync budget before even trying IPv4. Resolving first +
+            // dialing the IPv4 address directly sidesteps that.
+            var addresses = await System.Net.Dns.GetHostAddressesAsync(_host, cts.Token).ConfigureAwait(false);
+            var ipv4 = System.Linq.Enumerable.FirstOrDefault(addresses,
+                a => a.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork);
+            var target = ipv4 ?? (addresses.Length > 0 ? addresses[0] : System.Net.IPAddress.Loopback);
+
+            client = new TcpClient(target.AddressFamily);
+            EnableKeepAlive(client, _keepAlive);
+            await client.ConnectAsync(target, _port, cts.Token).ConfigureAwait(false);
+            _stream = client.GetStream();
+            _client = client;
+            client = null; // ownership transferred
+            _lastSuccessUtc = DateTime.UtcNow;
+        }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            // The per-op deadline, not the caller: normalise to a socket-level failure so every
+            // classifier treats a connect timeout as a comms fault.
+            throw new ModbusTransportUnavailableException(
+                $"Modbus connect to {_host}:{_port} timed out after {_timeout.TotalMilliseconds:0}ms");
+        }
+        finally
+        {
+            // A failed attempt disposes its own client — never leave it for the finalizer.
+            client?.Dispose();
+        }
     }
 
+    // Reconnect throttle state. Guarded by the composing transport's single-flight gate (every caller of
+    // EnsureConnectedAsync holds it), so no extra lock is needed.
+    private long _connectNotBeforeMs;
+    private TimeSpan _failureDelay = TimeSpan.Zero;
+
     /// <summary>
-    ///     Connect attempt with the configured geometric backoff. The first attempt fires after
-    ///     <see cref="ModbusReconnectOptions.InitialDelay"/> (default zero — immediate); each
-    ///     subsequent attempt sleeps for the previous delay times <c>BackoffMultiplier</c>,
-    ///     capped at <c>MaxDelay</c>. Caller's cancellation token aborts the loop.
+    ///     Ensures a live socket for the next transaction with AT MOST ONE connect attempt, throttled by the
+    ///     configured geometric backoff: after a failed attempt, calls inside the backoff window fail fast
+    ///     with <see cref="ModbusTransportUnavailableException"/> and never dial. The window starts at
+    ///     <see cref="ModbusReconnectOptions.InitialDelay"/> (floored at 1 s) and grows by
+    ///     <c>BackoffMultiplier</c> per consecutive failure up to <c>MaxDelay</c>; a successful connect
+    ///     resets it. Never sleeps — the composing transport calls it inside its gate, and a sleep there
+    ///     used to queue every read, write and probe behind one dead socket for up to ~81 s.
     /// </summary>
     /// <param name="ct">A cancellation token to observe for cancellation.</param>
-    /// <returns>A task representing the asynchronous reconnect operation.</returns>
-    public async Task ConnectWithBackoffAsync(CancellationToken ct)
+    /// <returns>A task that completes when a socket is live.</returns>
+    /// <exception cref="ModbusTransportUnavailableException">Auto-reconnect is off, or the call is inside the backoff window.</exception>
+    public async Task EnsureConnectedAsync(CancellationToken ct)
     {
-        var delay = _reconnect.InitialDelay;
-        var attempt = 0;
-        while (true)
+        if (_stream is not null) return;
+        if (!_autoReconnect)
+            throw new ModbusTransportUnavailableException(
+                $"Modbus transport to {_host}:{_port} is not connected and auto-reconnect is disabled");
+
+        var now = NowMs();
+        if (now < _connectNotBeforeMs)
+            throw new ModbusTransportUnavailableException(
+                $"Modbus reconnect to {_host}:{_port} throttled for another {_connectNotBeforeMs - now}ms after a failed connect");
+
+        try
         {
-            if (delay > TimeSpan.Zero)
-                await Task.Delay(delay, ct).ConfigureAwait(false);
-            try
-            {
-                await ConnectAsync(ct).ConfigureAwait(false);
-                return;
-            }
-            catch (Exception ex) when (IsSocketLevelFailure(ex) && _autoReconnect)
-            {
-                attempt++;
-                // Geometric growth, capped. Use Math.Min on ticks so we don't overflow with
-                // pathological multipliers / long deployments.
-                var nextTicks = (long)(Math.Max(delay.Ticks, TimeSpan.FromMilliseconds(100).Ticks) * _reconnect.BackoffMultiplier);
-                delay = TimeSpan.FromTicks(Math.Min(nextTicks, _reconnect.MaxDelay.Ticks));
-                if (attempt >= 10)
-                {
-                    // Bail after 10 attempts to surface persistent failure to the caller. With
-                    // the default backoff (1s base, 2.0x mult, 30s cap) this is roughly 4 minutes
-                    // of attempts; with InitialDelay=0 it's immediate up to the same cap.
-                    throw;
-                }
-            }
+            await ConnectAsync(ct).ConfigureAwait(false);
+            _failureDelay = TimeSpan.Zero;
+            _connectNotBeforeMs = 0;
         }
+        catch (Exception) when (!ct.IsCancellationRequested)
+        {
+            _failureDelay = NextDelay(_failureDelay);
+            _connectNotBeforeMs = NowMs() + (long)_failureDelay.TotalMilliseconds;
+            throw;
+        }
+    }
+
+    /// <summary>Minimum reconnect window after a failed connect — a dead device is dialled at most once per second.</summary>
+    private static readonly TimeSpan MinReconnectWindow = TimeSpan.FromSeconds(1);
+
+    private TimeSpan NextDelay(TimeSpan previous)
+    {
+        if (previous <= TimeSpan.Zero)
+        {
+            var first = _reconnect.InitialDelay > MinReconnectWindow ? _reconnect.InitialDelay : MinReconnectWindow;
+            return first < _reconnect.MaxDelay ? first : _reconnect.MaxDelay > TimeSpan.Zero ? _reconnect.MaxDelay : first;
+        }
+        // Geometric growth, capped. Math.Min on ticks so a pathological multiplier cannot overflow.
+        var multiplier = _reconnect.BackoffMultiplier >= 1.0 ? _reconnect.BackoffMultiplier : 1.0;
+        var nextTicks = previous.Ticks * multiplier;
+        var cap = _reconnect.MaxDelay > TimeSpan.Zero ? _reconnect.MaxDelay.Ticks : long.MaxValue;
+        return TimeSpan.FromTicks(nextTicks >= cap ? cap : (long)nextTicks);
     }
 
     /// <summary>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs

@@ -56,6 +56,9 @@ public sealed class ModbusTcpTransport : IModbusTransport
         _life = new ModbusSocketLifecycle(host, port, timeout, autoReconnect, keepAlive, idleDisconnect, reconnect);
     }
 
+    /// <summary>The socket lifecycle this transport composes (test seam for its clock).</summary>
+    internal ModbusSocketLifecycle Lifecycle => _life;
+
     /// <inheritdoc />
     public Task ConnectAsync(CancellationToken ct) => _life.ConnectAsync(ct);
 
@@ -63,7 +66,6 @@ public sealed class ModbusTcpTransport : IModbusTransport
     public async Task<byte[]> SendAsync(byte unitId, byte[] pdu, CancellationToken ct)
     {
         if (_disposed) throw new ObjectDisposedException(nameof(ModbusTcpTransport));
-        if (_life.Stream is null) throw new InvalidOperationException("Transport not connected");
 
         await _gate.WaitAsync(ct).ConfigureAwait(false);
         try
@@ -72,11 +74,16 @@ public sealed class ModbusTcpTransport : IModbusTransport
             // threshold, tear it down + reconnect before this PDU lands. Defends against silent
             // NAT / firewall reaping where the socket looks alive locally but the upstream side
             // dropped it minutes ago.
-            if (_life.ShouldReconnectForIdle())
-            {
+            if (_life.Stream is not null && _life.ShouldReconnectForIdle())
                 await _life.TearDownAsync().ConfigureAwait(false);
-                await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false);
-            }
+
+            // Self-heal: a socket torn down by an earlier failure (two lost replies in a row, a refused
+            // reconnect) is re-dialled HERE, once, throttled by the backoff window. This used to throw
+            // "Transport not connected" before the gate, so the transport never recovered on its own and
+            // every call failed fast until the 90 s comms-loss watchdog re-initialised the whole driver.
+            var freshlyConnected = _life.Stream is null;
+            if (freshlyConnected)
+                await _life.EnsureConnectedAsync(ct).ConfigureAwait(false);
 
             try
             {
@@ -84,13 +91,16 @@ public sealed class ModbusTcpTransport : IModbusTransport
                 _life.MarkSuccess();
                 return result;
             }
-            catch (Exception ex) when (_life.AutoReconnect && ModbusSocketLifecycle.IsSocketLevelFailure(ex))
+            catch (Exception ex) when (!freshlyConnected && _life.AutoReconnect
+                                       && !ct.IsCancellationRequested
+                                       && ModbusSocketLifecycle.IsSocketLevelFailure(ex))
             {
-                // Mid-transaction drop: tear down the dead socket, reconnect (with backoff if
-                // configured), resend. Single retry — if it fails again, let it propagate so
-                // health/status reflect reality.
+                // Mid-transaction drop on a socket that had been live: tear down the dead socket, make ONE
+                // throttled reconnect attempt (never a sleep inside the gate), resend. Single retry — if it
+                // fails again, let it propagate so health/status reflect reality. A socket this call just
+                // dialled is not retried: it proved nothing about a stale connection.
                 await _life.TearDownAsync().ConfigureAwait(false);
-                await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false);
+                await _life.EnsureConnectedAsync(ct).ConfigureAwait(false);
                 var result = await SendOnceAsync(unitId, pdu, ct).ConfigureAwait(false);
                 _life.MarkSuccess();
                 return result;
@@ -162,10 +172,22 @@ public sealed class ModbusTcpTransport : IModbusTransport
             var respPdu = new byte[respLen - 1];
             await ReadExactlyAsync(stream, respPdu, cts.Token).ConfigureAwait(false);
 
+            // Length check before indexing: a PDU must carry at least its function code, and an exception
+            // PDU its exception code too. Indexing past the end used to throw IndexOutOfRangeException —
+            // not an IOException, so no teardown/retry, and a malformed reply read as a driver bug.
+            if (respPdu.Length < 1)
+                throw new ModbusTransportDesyncException(
+                    ModbusTransportDesyncException.DesyncReason.TruncatedFrame,
+                    "Modbus response carried no PDU (MBAP length 1)");
+
             // Exception PDU: function code has high bit set. This is a well-formed protocol-level
             // error — the socket is still coherent, so it MUST propagate without teardown.
             if ((respPdu[0] & 0x80) != 0)
             {
+                if (respPdu.Length < 2)
+                    throw new ModbusTransportDesyncException(
+                        ModbusTransportDesyncException.DesyncReason.TruncatedFrame,
+                        $"Modbus exception response fc=0x{respPdu[0]:X2} carried no exception code");
                 var fc = (byte)(respPdu[0] & 0x7F);
                 var ex = respPdu[1];
                 throw new ModbusException(fc, ex, $"Modbus exception fc={fc} code={ex}");

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTransportUnavailableException.cs

@@ -0,0 +1,23 @@
+namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus;
+
+/// <summary>
+///     Raised when a Modbus transport has no live socket and cannot get one for this call: the one
+///     in-gate connect attempt failed, the reconnect is inside its backoff window, or auto-reconnect is
+///     disabled. It says the ENDPOINT is unreachable, so the driver fails the rest of the batch fast
+///     instead of paying the same failure once per tag.
+/// </summary>
+/// <remarks>
+///     Derives from <see cref="IOException"/> so every existing socket-level classifier
+///     (<c>ModbusSocketLifecycle.IsSocketLevelFailure</c>, the driver's comms-fault mapping) treats it as
+///     a communication failure without extra plumbing.
+/// </remarks>
+public sealed class ModbusTransportUnavailableException : IOException
+{
+    /// <summary>Initializes a new instance of the <see cref="ModbusTransportUnavailableException"/> class.</summary>
+    /// <param name="message">A human-readable description.</param>
+    /// <param name="inner">The originating exception, if any.</param>
+    public ModbusTransportUnavailableException(string message, Exception? inner = null)
+        : base(message, inner)
+    {
+    }
+}