fix(modbus): transport self-heals with one throttled in-gate connect; PDU length check
P3: a torn-down socket is re-dialled inside the gate (one attempt per backoff window) instead of failing fast with 'Transport not connected' until the 90 s comms-loss watchdog re-initialised the driver. L3: ConnectAsync disposes the previous/failed TcpClient, DNS runs under the per-op timeout, and the reconnect never sleeps inside the gate (the old 10-attempt backoff held it for ~81 s). H9: an MBAP response with no PDU, or an exception PDU with no code, is a TruncatedFrame desync rather than an IndexOutOfRangeException. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,88 @@
|
||||
using System.Diagnostics;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Comms-loss → recovery against the real pymodbus <c>standard</c> fixture, with a loopback
|
||||
/// <see cref="TcpRelay"/> in front of it so the device can disappear and come back without touching the
|
||||
/// shared container. Covers the review 07 High findings P3 (transport self-heal) and H2 (deadband
|
||||
/// recovery publish).
|
||||
/// </summary>
|
||||
[Collection(ModbusSimulatorCollection.Name)]
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Device", "Standard")]
|
||||
public sealed class CommsRecoveryTests(ModbusSimulatorFixture sim)
|
||||
{
|
||||
private const uint StatusGood = 0u;
|
||||
|
||||
private void SkipUnlessStandardLive()
|
||||
{
|
||||
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
|
||||
var profile = Environment.GetEnvironmentVariable("MODBUS_SIM_PROFILE");
|
||||
if (profile is not null && !string.Equals(profile, "standard", StringComparison.OrdinalIgnoreCase))
|
||||
Assert.Skip($"MODBUS_SIM_PROFILE={profile} — these tests need the standard profile (HR[0..31] = address).");
|
||||
}
|
||||
|
||||
private ModbusDriverOptions Options(TcpRelay relay, params ModbusTagDefinition[] tags) => new()
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = relay.Port,
|
||||
UnitId = 1,
|
||||
Timeout = TimeSpan.FromSeconds(2),
|
||||
RawTags = ModbusRawTags.Entries(tags),
|
||||
Probe = new ModbusProbeOptions { Enabled = false },
|
||||
};
|
||||
|
||||
private static async Task<T> WaitFor<T>(Func<Task<T>> probe, Func<T, bool> done, TimeSpan budget)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
T last;
|
||||
do
|
||||
{
|
||||
last = await probe();
|
||||
if (done(last)) return last;
|
||||
await Task.Delay(200, TestContext.Current.CancellationToken);
|
||||
} while (sw.Elapsed < budget);
|
||||
return last;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// P3: after the device drops (two failed calls in a row, socket torn down, reconnect refused) the
|
||||
/// transport must reconnect ON ITS OWN once the device is back — before this fix every later call
|
||||
/// failed fast with "Transport not connected" until the 90 s comms-loss watchdog re-initialised the
|
||||
/// driver.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Driver_reads_again_after_the_device_drops_and_returns_without_reinit()
|
||||
{
|
||||
SkipUnlessStandardLive();
|
||||
await using var relay = new TcpRelay(sim.Host, sim.Port);
|
||||
var tag = new ModbusTagDefinition("HR5", ModbusRegion.HoldingRegisters, 5, ModbusDataType.UInt16, Writable: false);
|
||||
await using var driver = new ModbusDriver(Options(relay, tag), "modbus-p3");
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
|
||||
var good = await driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken);
|
||||
good[0].StatusCode.ShouldBe(StatusGood);
|
||||
good[0].Value.ShouldBe((ushort)5);
|
||||
|
||||
relay.Cut();
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
var bad = await driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken);
|
||||
bad[0].StatusCode.ShouldNotBe(StatusGood);
|
||||
}
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||||
|
||||
relay.Restore();
|
||||
var recovered = await WaitFor(
|
||||
() => driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken),
|
||||
r => r[0].StatusCode == StatusGood,
|
||||
TimeSpan.FromSeconds(20));
|
||||
recovered[0].StatusCode.ShouldBe(StatusGood, "the transport must reconnect by itself once the device is back");
|
||||
recovered[0].Value.ShouldBe((ushort)5);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Loopback TCP relay in front of the real Modbus fixture, so a test can make the device "disappear"
|
||||
/// (<see cref="Cut"/>: every relayed connection is closed and new connects are refused) and "come back"
|
||||
/// (<see cref="Restore"/>: listen again on the SAME port) without touching the shared container.
|
||||
/// </summary>
|
||||
internal sealed class TcpRelay : IAsyncDisposable
|
||||
{
|
||||
private readonly string _upstreamHost;
|
||||
private readonly int _upstreamPort;
|
||||
private readonly object _lock = new();
|
||||
private readonly List<TcpClient> _live = [];
|
||||
private TcpListener? _listener;
|
||||
private CancellationTokenSource _stop = new();
|
||||
private int _accepted;
|
||||
|
||||
public TcpRelay(string upstreamHost, int upstreamPort)
|
||||
{
|
||||
_upstreamHost = upstreamHost;
|
||||
_upstreamPort = upstreamPort;
|
||||
var probe = new TcpListener(IPAddress.Loopback, 0);
|
||||
probe.Start();
|
||||
Port = ((IPEndPoint)probe.LocalEndpoint).Port;
|
||||
probe.Stop();
|
||||
Restore();
|
||||
}
|
||||
|
||||
/// <summary>Gets the loopback port clients dial.</summary>
|
||||
public int Port { get; }
|
||||
|
||||
/// <summary>Gets the number of client connections the relay has accepted.</summary>
|
||||
public int AcceptedConnections => Volatile.Read(ref _accepted);
|
||||
|
||||
/// <summary>Starts (or resumes) relaying on <see cref="Port"/>.</summary>
|
||||
public void Restore()
|
||||
{
|
||||
_stop = new CancellationTokenSource();
|
||||
var listener = new TcpListener(IPAddress.Loopback, Port);
|
||||
listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
listener.Start();
|
||||
_listener = listener;
|
||||
var token = _stop.Token;
|
||||
_ = Task.Run(() => AcceptLoopAsync(listener, token));
|
||||
}
|
||||
|
||||
/// <summary>Closes every relayed connection and refuses new ones until <see cref="Restore"/>.</summary>
|
||||
public void Cut()
|
||||
{
|
||||
try { _stop.Cancel(); } catch { }
|
||||
try { _listener?.Stop(); } catch { }
|
||||
_listener = null;
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var c in _live) { try { c.Close(); } catch { } }
|
||||
_live.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(TcpListener listener, CancellationToken token)
|
||||
{
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
TcpClient downstream;
|
||||
try { downstream = await listener.AcceptTcpClientAsync(token); }
|
||||
catch { return; }
|
||||
Interlocked.Increment(ref _accepted);
|
||||
_ = Task.Run(() => RelayAsync(downstream, token));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RelayAsync(TcpClient downstream, CancellationToken token)
|
||||
{
|
||||
var upstream = new TcpClient(AddressFamily.InterNetwork);
|
||||
lock (_lock) { _live.Add(downstream); _live.Add(upstream); }
|
||||
try
|
||||
{
|
||||
var addr = (await Dns.GetHostAddressesAsync(_upstreamHost, token))
|
||||
.First(a => a.AddressFamily == AddressFamily.InterNetwork);
|
||||
await upstream.ConnectAsync(addr, _upstreamPort, token);
|
||||
var a = downstream.GetStream();
|
||||
var b = upstream.GetStream();
|
||||
await Task.WhenAny(a.CopyToAsync(b, token), b.CopyToAsync(a, token));
|
||||
}
|
||||
catch { /* cut or upstream gone */ }
|
||||
finally
|
||||
{
|
||||
try { downstream.Close(); } catch { }
|
||||
try { upstream.Close(); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Cut();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user