09e0fe1841
N1: linkDown is set only when NO reply arrived, i.e. a timeout, a transport exception or a connect failure. A status the CNC returned proves it answered. EW_REJECT and every unlisted rc map to BadCommunicationError, so a deterministic rc on one tag used to fail every later tag of a reachable control on every poll. The comms-classed status still degrades health. The coalescer already treated a returned refusal this way; it is unchanged. Tests: a returned EW_REJECT on tag 1 leaves tags 2 and 3 read and Good (red before). A timeout still fails the rest without reading them (positive control). Minors: - m1: the _lifecycleLock doc now covers the health transitions it also guards. - m2: the NoteFixedTreeFailed summary says a refusal credits the answer. - m3: ReadAsync stamps a failed batch's exception with the generation the batch started under, and HandlePollError degrades that generation only. - m4: the fixed-tree "blip" absence check runs unconditionally under a wide budget, and asserts that both ticks landed inside it. - m5: direct unit cases for the coalescer link-down paths. A transport fault on the first range fails the second range and the per-tag tag without reading them. A connect failure costs one connect per batch. CncRelay.Up() retries the re-bind. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
219 lines
9.4 KiB
C#
219 lines
9.4 KiB
C#
using System.Net;
|
||
using System.Net.Sockets;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests.Series;
|
||
|
||
/// <summary>
|
||
/// #740 fix round 1 (review I2) — wire-level cases for H15 (fixed-tree health) and H20 (alarm record layout)
|
||
/// against the focas-mock fixture.
|
||
/// </summary>
|
||
[Collection(FocasSimCollection.Name)]
|
||
public sealed class WireBackendHealthAndLayoutTests
|
||
{
|
||
private readonly FocasSimFixture _fx;
|
||
|
||
/// <summary>Initializes a new instance of the test class with the FOCAS simulation fixture.</summary>
|
||
/// <param name="fx">The FOCAS simulation fixture.</param>
|
||
public WireBackendHealthAndLayoutTests(FocasSimFixture fx) => _fx = fx;
|
||
|
||
/// <summary>
|
||
/// A local TCP relay in front of the shared mock, so ONE test can take "the CNC" away and bring it back
|
||
/// without stopping a fixture other suites use: <see cref="Down"/> closes every relayed session and stops
|
||
/// listening (new connects are refused); <see cref="Up"/> listens again on the same port.
|
||
/// </summary>
|
||
private sealed class CncRelay : IAsyncDisposable
|
||
{
|
||
private readonly string _targetHost;
|
||
private readonly int _targetPort;
|
||
private readonly List<TcpClient> _live = [];
|
||
private TcpListener? _listener;
|
||
private CancellationTokenSource? _cts;
|
||
|
||
public CncRelay(string targetHost, int targetPort)
|
||
{
|
||
_targetHost = targetHost;
|
||
_targetPort = targetPort;
|
||
Port = Listen(0);
|
||
}
|
||
|
||
public int Port { get; private set; }
|
||
|
||
private int Listen(int port)
|
||
{
|
||
_cts = new CancellationTokenSource();
|
||
_listener = new TcpListener(IPAddress.Loopback, port);
|
||
_listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||
_listener.Start();
|
||
var ct = _cts.Token;
|
||
var listener = _listener;
|
||
_ = Task.Run(() => AcceptLoopAsync(listener, ct), CancellationToken.None);
|
||
return ((IPEndPoint)_listener.LocalEndpoint).Port;
|
||
}
|
||
|
||
private async Task AcceptLoopAsync(TcpListener listener, CancellationToken ct)
|
||
{
|
||
while (!ct.IsCancellationRequested)
|
||
{
|
||
TcpClient inbound;
|
||
try { inbound = await listener.AcceptTcpClientAsync(ct); }
|
||
catch { return; }
|
||
var outbound = new TcpClient();
|
||
try { await outbound.ConnectAsync(_targetHost, _targetPort, ct); }
|
||
catch { inbound.Dispose(); outbound.Dispose(); continue; }
|
||
lock (_live) { _live.Add(inbound); _live.Add(outbound); }
|
||
_ = Pump(inbound, outbound, ct);
|
||
_ = Pump(outbound, inbound, ct);
|
||
}
|
||
}
|
||
|
||
private static async Task Pump(TcpClient from, TcpClient to, CancellationToken ct)
|
||
{
|
||
try { await from.GetStream().CopyToAsync(to.GetStream(), ct); }
|
||
catch { /* the relay was taken down, or a side closed */ }
|
||
finally { from.Dispose(); to.Dispose(); }
|
||
}
|
||
|
||
public void Down()
|
||
{
|
||
_cts?.Cancel();
|
||
_listener?.Stop();
|
||
lock (_live)
|
||
{
|
||
foreach (var c in _live) c.Dispose();
|
||
_live.Clear();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Listens again on the SAME port (the driver's host string names it). The re-bind is retried briefly: the
|
||
/// OS can hold the port for a moment after <see cref="Down"/>, or hand it to another process — the latter
|
||
/// surfaces as a clear failure after the retries rather than a random flake.
|
||
/// </summary>
|
||
public void Up()
|
||
{
|
||
for (var attempt = 1; ; attempt++)
|
||
{
|
||
try { Listen(Port); return; }
|
||
catch (SocketException) when (attempt < 20) { Thread.Sleep(100); }
|
||
}
|
||
}
|
||
|
||
public ValueTask DisposeAsync()
|
||
{
|
||
Down();
|
||
return ValueTask.CompletedTask;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// H15 over the wire: a driver serving only <c>FIXED:</c> tags is Healthy with a moving LastSuccessfulRead
|
||
/// while its CNC answers; when the CNC disappears it degrades with LastSuccessfulRead frozen (the #527
|
||
/// watchdog's conjunction); and it recovers to Healthy when the CNC returns (positive control).
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task Fixed_only_driver_degrades_on_cnc_loss_and_recovers_over_the_wire()
|
||
{
|
||
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
|
||
var ct = TestContext.Current.CancellationToken;
|
||
await _fx.LoadProfileAsync("FWLIB64", ct);
|
||
|
||
await using var relay = new CncRelay(_fx.Host, _fx.Port);
|
||
var host = $"focas://127.0.0.1:{relay.Port}";
|
||
var drv = new FocasDriver(new FocasDriverOptions
|
||
{
|
||
Devices = [new FocasDeviceOptions(host)],
|
||
RawTags = FocasRawTags.Entries(
|
||
new FocasTagDefinition("Series", host, "FIXED:Identity/SeriesNumber", FocasDataType.String, Writable: false)),
|
||
Probe = new FocasProbeOptions { Enabled = false },
|
||
Timeout = TimeSpan.FromSeconds(1),
|
||
FixedTree = new FocasFixedTreeOptions
|
||
{
|
||
PollInterval = TimeSpan.FromMilliseconds(50),
|
||
ProgramPollInterval = TimeSpan.Zero,
|
||
TimerPollInterval = TimeSpan.Zero,
|
||
},
|
||
}, driverInstanceId: "wire-fixed-health", clientFactory: new WireFocasClientFactory())
|
||
{
|
||
FixedTreeStaleBudgetOverride = TimeSpan.FromMilliseconds(500),
|
||
};
|
||
|
||
await using (drv)
|
||
{
|
||
await drv.InitializeAsync("{}", ct);
|
||
await WaitFor(() => drv.GetDeviceState(host)?.LastFixedPollUtc is not null, TimeSpan.FromSeconds(10));
|
||
var first = drv.GetHealth().LastSuccessfulRead!.Value;
|
||
await WaitFor(() => drv.GetHealth().LastSuccessfulRead > first, TimeSpan.FromSeconds(5));
|
||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||
|
||
relay.Down();
|
||
await WaitFor(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(15));
|
||
var frozen = drv.GetHealth().LastSuccessfulRead;
|
||
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
||
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "nothing answered, so LastSuccessfulRead stays frozen");
|
||
(await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError,
|
||
"the stale fixed-tree value reads Bad on the same budget that degrades the driver");
|
||
|
||
// Positive control: the CNC returns ⇒ Healthy again, LastSuccessfulRead moves. The connect backoff
|
||
// (1 s doubling) bounds how soon the loop re-dials.
|
||
relay.Up();
|
||
await WaitFor(() => drv.GetHealth().State == DriverState.Healthy, TimeSpan.FromSeconds(40));
|
||
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
|
||
(await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// H20 over the wire: the mock answers <c>cnc_rdalmmsg2</c> in the 80-byte LEGACY layout. Nineteen active
|
||
/// alarms make a 1,520-byte payload — also exactly 20 × 76 — which the old length test decoded as twenty
|
||
/// garbage vendor records. Positive control: the unambiguous 3-alarm reply decodes the same alarms.
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task Nineteen_legacy_alarms_decode_as_nineteen_over_the_wire()
|
||
{
|
||
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
|
||
var ct = TestContext.Current.CancellationToken;
|
||
await _fx.LoadProfileAsync("FWLIB64", ct);
|
||
|
||
var client = new WireFocasClientFactory().Create();
|
||
try
|
||
{
|
||
await client.ConnectAsync(FocasHostAddress.TryParse(_fx.DeviceUri)!, TimeSpan.FromSeconds(2), ct);
|
||
|
||
await _fx.PatchStateAsync(new { alarms = Alarms(3) }, ct);
|
||
var three = await client.ReadAlarmsAsync(ct);
|
||
three.Count.ShouldBe(3);
|
||
three[2].AlarmNumber.ShouldBe(402);
|
||
|
||
await _fx.PatchStateAsync(new { alarms = Alarms(19) }, ct);
|
||
var nineteen = await client.ReadAlarmsAsync(ct);
|
||
nineteen.Count.ShouldBe(19, "a 1,520-byte legacy reply is 19 records, not 20 vendor ones");
|
||
nineteen.Select(a => a.AlarmNumber).ShouldBe(Enumerable.Range(400, 19));
|
||
nineteen[18].Message.ShouldBe("SERVO ALARM 418");
|
||
nineteen[18].Axis.ShouldBe((short)(1 + (18 % 4)));
|
||
}
|
||
finally
|
||
{
|
||
client.Dispose();
|
||
await _fx.PatchStateAsync(new { alarms = Array.Empty<object>() }, ct);
|
||
}
|
||
|
||
static object[] Alarms(int n) => [.. Enumerable.Range(0, n)
|
||
.Select(i => (object)new { alm_no = 400 + i, type = 2 + (i % 3), axis = 1 + (i % 4), msg = $"SERVO ALARM {400 + i}" })];
|
||
}
|
||
|
||
private static async Task WaitFor(Func<bool> condition, TimeSpan timeout)
|
||
{
|
||
var deadline = DateTime.UtcNow + timeout;
|
||
while (!condition())
|
||
{
|
||
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
|
||
await Task.Delay(25);
|
||
}
|
||
}
|
||
}
|