Joseph Doherty
374 lines
18 KiB
C#
374 lines
18 KiB
C#
using System.Collections.Concurrent;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
|
|
|
|
/// <summary>
|
|
/// Unit tests for the STAB-2 fix: native ADS notifications are re-registered onto a fresh
|
|
/// client after a reconnect, instead of being silently orphaned. There is <b>no TwinCAT docker
|
|
/// fixture</b> (the 13 integration tests need a real TC3 XAR target), so these fake-client tests
|
|
/// are the <i>authoritative</i> automated coverage for the reconnect path.
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class TwinCATReconnectReplayTests
|
|
{
|
|
private const string Host = "ads://5.23.91.23.1.1:851";
|
|
|
|
private static (TwinCATDriver drv, FakeTwinCATClientFactory factory) NewNativeDriver(
|
|
bool probe = false, TimeSpan? probeInterval = null, params TwinCATTagDefinition[] tags)
|
|
{
|
|
var factory = new FakeTwinCATClientFactory();
|
|
var drv = new TwinCATDriver(new TwinCATDriverOptions
|
|
{
|
|
Devices = [new TwinCATDeviceOptions(Host)],
|
|
Tags = tags,
|
|
Probe = new TwinCATProbeOptions
|
|
{
|
|
Enabled = probe,
|
|
Interval = probeInterval ?? TimeSpan.FromMilliseconds(30),
|
|
Timeout = TimeSpan.FromMilliseconds(100),
|
|
},
|
|
UseNativeNotifications = true,
|
|
}, "drv-1", factory);
|
|
return (drv, factory);
|
|
}
|
|
|
|
/// <summary>
|
|
/// After a client drop, the next connection (driven here through a read) re-registers the
|
|
/// native notification with the same symbol / type / interval, a push on the fresh client
|
|
/// reaches OnDataChange, and the old handle is disposed.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Reconnect_replays_native_notifications_onto_the_fresh_client()
|
|
{
|
|
var (drv, factory) = NewNativeDriver(
|
|
tags: new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt));
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
|
drv.OnDataChange += (_, e) => events.Enqueue(e);
|
|
|
|
var interval = TimeSpan.FromMilliseconds(250);
|
|
_ = await drv.SubscribeAsync(["Speed"], interval, TestContext.Current.CancellationToken);
|
|
|
|
var client1 = factory.Clients[0];
|
|
client1.Notifications.Count.ShouldBe(1);
|
|
var oldHandle = client1.Notifications[0];
|
|
|
|
// Simulate a wire-level drop, then trigger a reconnect via a read.
|
|
client1.SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken);
|
|
|
|
// A brand-new client was built and the notification replayed onto it — same intent.
|
|
factory.Clients.Count.ShouldBe(2);
|
|
var client2 = factory.Clients[1];
|
|
client2.Notifications.Count.ShouldBe(1);
|
|
var replayed = client2.Notifications[0];
|
|
replayed.SymbolPath.ShouldBe("MAIN.Speed");
|
|
replayed.Type.ShouldBe(TwinCATDataType.DInt);
|
|
|
|
// The old handle was disposed; a push on the FRESH client reaches OnDataChange.
|
|
oldHandle.Disposed.ShouldBeTrue();
|
|
client2.FireNotification("MAIN.Speed", 7777);
|
|
events.Count.ShouldBe(1);
|
|
events.Last().Snapshot.Value.ShouldBe(7777);
|
|
events.Last().FullReference.ShouldBe("Speed");
|
|
}
|
|
|
|
/// <summary>Every subscribed tag on the device is replayed, not just the first.</summary>
|
|
[Fact]
|
|
public async Task Reconnect_replays_all_registered_tags()
|
|
{
|
|
var (drv, factory) = NewNativeDriver(
|
|
tags:
|
|
[
|
|
new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt),
|
|
new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.Real),
|
|
]);
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
_ = await drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
|
|
|
|
factory.Clients[0].SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["A"], TestContext.Current.CancellationToken);
|
|
|
|
factory.Clients.Count.ShouldBe(2);
|
|
factory.Clients[1].Notifications.Select(n => n.SymbolPath)
|
|
.ShouldBe(["MAIN.A", "MAIN.B"], ignoreOrder: true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// After a reconnect + replay, unsubscribing disposes the CURRENT (replayed) handle and the
|
|
/// registration no longer replays on a subsequent reconnect.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Unsubscribe_after_reconnect_disposes_the_current_handle_and_stops_replaying()
|
|
{
|
|
var (drv, factory) = NewNativeDriver(
|
|
tags: new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt));
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
var handle = await drv.SubscribeAsync(["X"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
|
|
|
|
// First reconnect → replayed onto client2.
|
|
factory.Clients[0].SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["X"], TestContext.Current.CancellationToken);
|
|
var client2 = factory.Clients[1];
|
|
client2.Notifications.Count.ShouldBe(1);
|
|
|
|
// Unsubscribe disposes the current (client2) handle.
|
|
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
|
|
client2.Notifications.ShouldBeEmpty();
|
|
|
|
// A second reconnect must NOT resurrect the unsubscribed notification.
|
|
client2.SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["X"], TestContext.Current.CancellationToken);
|
|
factory.Clients.Count.ShouldBe(3);
|
|
factory.Clients[2].Notifications.ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>
|
|
/// A wire-level probe failure on a locally-"connected" client forces a recycle so the next
|
|
/// tick rebuilds the client — the compounding half of STAB-2 (the IsConnected fast-path
|
|
/// keys on port state, not wire liveness).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Probe_failure_recycles_the_client_and_rebuilds()
|
|
{
|
|
// First client's wire probe fails; the recycle nulls it and the next tick builds a second
|
|
// client whose probe succeeds.
|
|
var buildCount = 0;
|
|
var factory = new FakeTwinCATClientFactory
|
|
{
|
|
Customise = () =>
|
|
{
|
|
buildCount++;
|
|
// Client #1 reports a failed wire probe; #2+ succeed.
|
|
return new FakeTwinCATClient { ProbeResult = buildCount > 1 };
|
|
},
|
|
};
|
|
var drv = new TwinCATDriver(new TwinCATDriverOptions
|
|
{
|
|
Devices = [new TwinCATDeviceOptions(Host)],
|
|
Tags = [new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt)],
|
|
Probe = new TwinCATProbeOptions
|
|
{
|
|
Enabled = true,
|
|
Interval = TimeSpan.FromMilliseconds(30),
|
|
Timeout = TimeSpan.FromMilliseconds(100),
|
|
},
|
|
UseNativeNotifications = true,
|
|
}, "drv-probe", factory);
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
// The probe loop should recycle the wire-dead client and rebuild a healthy one.
|
|
await WaitForAsync(() => factory.Clients.Count >= 2, TimeSpan.FromSeconds(3));
|
|
factory.Clients.Count.ShouldBeGreaterThanOrEqualTo(2);
|
|
factory.Clients[0].DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
|
|
|
|
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
|
|
}
|
|
|
|
// ---- STAB-16 / STAB-17 / STAB-12 replay hardening ----
|
|
|
|
/// <summary>
|
|
/// STAB-16: a replay failure must not be silent — the registration is marked dead, a Bad
|
|
/// snapshot is pushed to subscribers (instead of the frozen-Good-stale value), and health
|
|
/// degrades preserving the last successful read.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ReplayFailure_EmitsBadQuality_AndDegradesHealth()
|
|
{
|
|
var build = 0;
|
|
var factory = new FakeTwinCATClientFactory
|
|
{
|
|
// client1 normal; client2 (the reconnect target) fails AddNotification (replay) + reads.
|
|
Customise = () => new FakeTwinCATClient
|
|
{
|
|
ThrowOnAddNotification = ++build >= 2,
|
|
ThrowOnRead = build >= 2,
|
|
Exception = new InvalidOperationException("ADS notification quota exceeded"),
|
|
},
|
|
};
|
|
var drv = new TwinCATDriver(new TwinCATDriverOptions
|
|
{
|
|
Devices = [new TwinCATDeviceOptions(Host)],
|
|
Tags = [new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)],
|
|
Probe = new TwinCATProbeOptions { Enabled = false },
|
|
UseNativeNotifications = true,
|
|
}, "drv-1", factory);
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
|
drv.OnDataChange += (_, e) => events.Enqueue(e);
|
|
|
|
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(250), TestContext.Current.CancellationToken);
|
|
// Establish a last-successful-read on the healthy client1.
|
|
_ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken);
|
|
var lastGoodRead = drv.GetHealth().LastSuccessfulRead;
|
|
lastGoodRead.ShouldNotBeNull();
|
|
|
|
// Drop the wire; the next read reconnects onto client2, whose replay AddNotification fails.
|
|
factory.Clients[0].SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken);
|
|
|
|
// STAB-16: subscribers see the tag go Bad (not frozen Good), and health degraded while
|
|
// preserving the last successful read.
|
|
var bad = events.SingleOrDefault(e => e.FullReference == "Speed"
|
|
&& e.Snapshot.StatusCode == TwinCATStatusMapper.BadCommunicationError);
|
|
bad.ShouldNotBeNull("a failed replay must publish a Bad snapshot for the affected reference");
|
|
var health = drv.GetHealth();
|
|
health.State.ShouldBe(DriverState.Degraded);
|
|
health.LastSuccessfulRead.ShouldBe(lastGoodRead);
|
|
}
|
|
|
|
/// <summary>
|
|
/// STAB-16: a registration whose replay failed (dead handle) is re-registered on the next
|
|
/// successful probe tick — a subsequent push then reaches OnDataChange with Good, and no
|
|
/// duplicate registration is created.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ReplayFailure_RetriedOnNextSuccessfulProbeTick()
|
|
{
|
|
var build = 0;
|
|
var factory = new FakeTwinCATClientFactory
|
|
{
|
|
// Only the 2nd client (the reconnect target) fails its first replay; flipped off below.
|
|
Customise = () => new FakeTwinCATClient { ThrowOnAddNotification = ++build == 2 },
|
|
};
|
|
var drv = new TwinCATDriver(new TwinCATDriverOptions
|
|
{
|
|
Devices = [new TwinCATDeviceOptions(Host)],
|
|
Tags = [new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)],
|
|
Probe = new TwinCATProbeOptions
|
|
{
|
|
Enabled = true,
|
|
Interval = TimeSpan.FromMilliseconds(40),
|
|
Timeout = TimeSpan.FromMilliseconds(100),
|
|
},
|
|
UseNativeNotifications = true,
|
|
}, "drv-1", factory);
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
|
drv.OnDataChange += (_, e) => events.Enqueue(e);
|
|
|
|
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(250), TestContext.Current.CancellationToken);
|
|
await WaitForAsync(() => factory.Clients.Count >= 1, TimeSpan.FromSeconds(2));
|
|
|
|
// Drop the wire; the reconnect target (client2) fails its replay, leaving a dead registration.
|
|
factory.Clients[0].SimulateWireDrop();
|
|
_ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken);
|
|
await WaitForAsync(() => factory.Clients.Count >= 2, TimeSpan.FromSeconds(2));
|
|
var client2 = factory.Clients[1];
|
|
client2.Notifications.ShouldBeEmpty(); // replay failed — no live handle
|
|
|
|
// Device now accepts registrations again — the probe-tick retry must re-register the intent.
|
|
client2.ThrowOnAddNotification = false;
|
|
await WaitForAsync(() => client2.Notifications.Count == 1, TimeSpan.FromSeconds(3));
|
|
client2.Notifications.Count.ShouldBe(1); // re-registered exactly once (no duplicate)
|
|
|
|
// A fresh push on the recovered registration reaches OnDataChange with Good.
|
|
client2.FireNotification("MAIN.Speed", 4242);
|
|
await WaitForAsync(() => events.Any(e => e.FullReference == "Speed"
|
|
&& e.Snapshot.StatusCode == TwinCATStatusMapper.Good), TimeSpan.FromSeconds(2));
|
|
events.Any(e => e.FullReference == "Speed"
|
|
&& e.Snapshot.StatusCode == TwinCATStatusMapper.Good
|
|
&& Equals(e.Snapshot.Value, 4242)).ShouldBeTrue();
|
|
|
|
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// STAB-17: unsubscribing WHILE a replay is mid-<c>AddNotificationAsync</c> must not leak a
|
|
/// live ADS notification. After the fresh handle is installed, an ownership re-check finds the
|
|
/// registration was unsubscribed and retracts (disposes) the fresh handle — no orphan.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task UnsubscribeDuringReplay_DisposesFreshHandle_NoOrphan()
|
|
{
|
|
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var build = 0;
|
|
var factory = new FakeTwinCATClientFactory
|
|
{
|
|
Customise = () =>
|
|
{
|
|
var c = new FakeTwinCATClient();
|
|
if (++build == 2) c.AddNotificationGate = gate; // client2's replay blocks on the gate
|
|
return c;
|
|
},
|
|
};
|
|
var drv = new TwinCATDriver(new TwinCATDriverOptions
|
|
{
|
|
Devices = [new TwinCATDeviceOptions(Host)],
|
|
Tags = [new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt)],
|
|
Probe = new TwinCATProbeOptions { Enabled = false },
|
|
UseNativeNotifications = true,
|
|
}, "drv-1", factory);
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
var handle = await drv.SubscribeAsync(["X"], TimeSpan.FromMilliseconds(250), TestContext.Current.CancellationToken);
|
|
|
|
// Drop the wire; a read reconnects onto client2, whose replay AddNotification blocks on the gate.
|
|
factory.Clients[0].SimulateWireDrop();
|
|
var readTask = Task.Run(() => drv.ReadAsync(["X"], CancellationToken.None));
|
|
await WaitForAsync(() => factory.Clients.Count >= 2, TimeSpan.FromSeconds(2));
|
|
await Task.Delay(50); // let the replay reach the blocked AddNotification
|
|
|
|
// Unsubscribe mid-replay (gate-free) — removes the registration from the device registry.
|
|
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
|
|
|
|
// Release the gate: AddNotification completes, SwapHandle installs the fresh handle, and the
|
|
// ownership re-check must retract it (no owner) rather than leave a live orphan notification.
|
|
gate.SetResult();
|
|
await readTask;
|
|
|
|
await WaitForAsync(() => factory.Clients[1].Notifications.Count == 0, TimeSpan.FromSeconds(2));
|
|
factory.Clients[1].Notifications.ShouldBeEmpty("the fresh handle must be disposed — no orphan ADS notification");
|
|
}
|
|
|
|
/// <summary>
|
|
/// STAB-12 (compounded part): <c>RecycleClientAsync</c> must honor cancellation — it must
|
|
/// take the probe token into <c>ConnectGate.WaitAsync(ct)</c> rather than
|
|
/// <c>CancellationToken.None</c>, so a shutdown mid-recycle (or a connect wedged under the
|
|
/// gate) can't hang the probe loop uncancellably. Verified white-box: with the gate held and
|
|
/// a cancelled token, the call must throw promptly rather than block forever.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task RecycleClient_HonorsCancellation()
|
|
{
|
|
var (drv, _) = NewNativeDriver(
|
|
tags: new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt));
|
|
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
|
|
|
// Reach the single configured device + wedge its ConnectGate so a recycle must wait on it.
|
|
var devicesField = typeof(TwinCATDriver).GetField("_devices",
|
|
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!;
|
|
var devicesObj = devicesField.GetValue(drv)!;
|
|
var values = (System.Collections.IEnumerable)devicesObj.GetType().GetProperty("Values")!.GetValue(devicesObj)!;
|
|
var device = (TwinCATDriver.DeviceState)values.Cast<object>().First();
|
|
await device.ConnectGate.WaitAsync(TestContext.Current.CancellationToken);
|
|
|
|
var recycle = typeof(TwinCATDriver).GetMethod("RecycleClientAsync",
|
|
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!;
|
|
|
|
using var cts = new CancellationTokenSource();
|
|
await cts.CancelAsync();
|
|
var task = (Task)recycle.Invoke(drv, [device, cts.Token])!;
|
|
|
|
// Must throw promptly (cancelled token), NOT hang on the held gate. The 2 s bound turns an
|
|
// uncancellable-wait regression into a failure rather than a hung suite.
|
|
await Should.ThrowAsync<OperationCanceledException>(async () => await task.WaitAsync(TimeSpan.FromSeconds(2)));
|
|
|
|
device.ConnectGate.Release();
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (!condition() && DateTime.UtcNow < deadline)
|
|
await Task.Delay(20);
|
|
}
|
|
}
|