using System.Collections.Concurrent; using System.Diagnostics; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; /// /// #739 L8 + the Lane A runtime contract (final stop = ShutdownAsync then DisposeAsync, /// bounded 5 s + 250 ms): a probe wedged in the blocking ADS connect cannot hang teardown; dispose may run /// while a shutdown is still running; an abandoned initialise overlapping the next shutdown never installs /// its generation; a value queued before unsubscribe is not delivered after it; re-init swaps its tables /// and retracts the previous generation's notifications; queue drops are reported. /// [Trait("Category", "Unit")] public sealed class TwinCATLifecycleTests { private const string Host = "ads://5.23.91.23.1.1:851"; private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, bool probe = true, bool native = true, TwinCATCapturingLogger? log = null, params TwinCATTagDefinition[] tags) => new(new TwinCATDriverOptions { Devices = [new TwinCATDeviceOptions(Host)], RawTags = TwinCATRawTags.Entries(tags.Length > 0 ? tags : [new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]), Probe = probe ? new TwinCATProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(30), Timeout = TimeSpan.FromMilliseconds(100) } : new TwinCATProbeOptions { Enabled = false }, UseNativeNotifications = native, }, "drv-1", factory, log); private static async Task WaitForAsync(Func condition, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (!condition() && DateTime.UtcNow < deadline) await Task.Delay(10); } /// /// A probe wedged in the (uncancellable) connect no longer hangs ShutdownAsync — the join is bounded — /// and the client the wedged connect eventually produces is disposed, not installed on a dead device. /// [Fact] public async Task Shutdown_is_bounded_when_a_probe_is_wedged_in_connect() { var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } }; var drv = NewDriver(factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); factory.Clients.Count.ShouldBe(1); // the probe is inside the wedged connect var sw = Stopwatch.StartNew(); await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); wedge.SetResult(); await WaitForAsync(() => factory.Clients[0].DisposeCount == 1, TimeSpan.FromSeconds(2)); factory.Clients[0].DisposeCount.ShouldBe(1); drv.DeviceCount.ShouldBe(0); } /// /// DisposeAsync while a ShutdownAsync is still running (joining a wedged probe) returns promptly and /// both complete without throwing; the device is released exactly once. /// [Fact] public async Task Dispose_concurrent_with_a_running_shutdown_completes_promptly() { var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } }; var drv = NewDriver(factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); var shutdown = drv.ShutdownAsync(CancellationToken.None); var sw = Stopwatch.StartNew(); await drv.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); // Ordering is the real assertion: dispose finished while the first shutdown was still joining the // wedged probe. The elapsed bound is generous (the runtime allows 250 ms; a loaded CI box gets 1 s). shutdown.IsCompleted.ShouldBeFalse("the positive control: the first shutdown really was still joining the probe"); sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1)); wedge.SetResult(); await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); drv.Dispose(); // and a late sync Dispose is a no-op drv.GetHealth().State.ShouldBe(DriverState.Unknown); } /// /// An InitializeAsync overtaken by a ShutdownAsync before it commits must not install its devices or /// start probe loops when it resumes. Positive control: a fresh initialise afterwards installs normally. /// [Fact] public async Task An_initialise_overtaken_by_shutdown_before_commit_installs_nothing() { var factory = new FakeTwinCATClientFactory(); var drv = NewDriver(factory); var paused = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); drv.BeforeCommitForTest = async () => { paused.TrySetResult(); await resume.Task; }; var abandoned = drv.InitializeAsync("{}", CancellationToken.None); await paused.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); await drv.ShutdownAsync(TestContext.Current.CancellationToken); resume.SetResult(); await Should.ThrowAsync(() => abandoned.WaitAsync(TimeSpan.FromSeconds(5))); drv.DeviceCount.ShouldBe(0); factory.Clients.ShouldBeEmpty(); drv.GetHealth().State.ShouldBe(DriverState.Unknown); // Positive control + measured window: a fresh generation's probe creates exactly one client and // ticks several times; an overtaken-generation probe loop (same interval, started earlier) would have // created a second client within that window. drv.BeforeCommitForTest = null; await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); drv.DeviceCount.ShouldBe(1); await WaitForAsync(() => factory.Clients.Count >= 1 && factory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3)); factory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4); factory.Clients.Count.ShouldBe(1, "no probe loop of the overtaken generation may be running"); await drv.ShutdownAsync(TestContext.Current.CancellationToken); } /// /// The Lane A shape: an InitializeAsync the runtime abandoned while it is still releasing the previous /// generation (joining a probe wedged in connect) overlaps the next ShutdownAsync. Shutdown stays /// bounded and, once everything settles, no device, probe loop or live client survives. /// [Fact] public async Task An_abandoned_initialise_overlapping_shutdown_leaves_nothing_running() { var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var first = true; var factory = new FakeTwinCATClientFactory { Customise = () => { var c = new FakeTwinCATClient { BlockConnect = first ? wedge : null }; first = false; return c; }, }; var drv = NewDriver(factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); var abandoned = drv.InitializeAsync("{}", CancellationToken.None); // joins the wedged generation-1 probe await WaitForAsync(() => drv.DeviceCount == 1 && factory.Clients.Count >= 2, TimeSpan.FromSeconds(3)); // gen 2 committed + probing var sw = Stopwatch.StartNew(); await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); wedge.SetResult(); try { await abandoned.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } catch (OperationCanceledException) when (!TestContext.Current.CancellationToken.IsCancellationRequested) { } drv.DeviceCount.ShouldBe(0); await WaitForAsync(() => factory.Clients.All(c => c.DisposeCount > 0 || !c.IsConnected), TimeSpan.FromSeconds(2)); var settled = factory.Clients.Count; // Measured window: a sibling driver on the same probe interval ticks 4 times; any surviving probe // loop of the torn-down driver would have built a new client in that time. var witnessFactory = new FakeTwinCATClientFactory(); var witness = NewDriver(witnessFactory); await witness.InitializeAsync("{}", TestContext.Current.CancellationToken); await WaitForAsync(() => witnessFactory.Clients.Count >= 1 && witnessFactory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3)); witnessFactory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4); await witness.ShutdownAsync(TestContext.Current.CancellationToken); factory.Clients.Count.ShouldBe(settled, "no probe loop may still be creating clients"); factory.Clients.ShouldAllBe(c => c.DisposeCount > 0 || !c.IsConnected); } /// /// A value the client queued before the unsubscribe is dropped if dispatched after it. /// Positive control: the same callback delivers while subscribed. /// [Fact] public async Task A_notification_dispatched_after_unsubscribe_is_dropped() { var factory = new FakeTwinCATClientFactory(); var drv = NewDriver(factory, probe: false); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); var events = new ConcurrentQueue(); drv.OnDataChange += (_, e) => events.Enqueue(e); var handle = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); var callback = factory.Clients[0].Notifications.Single().OnChange; // what the client's queue holds callback("MAIN.Speed", 1); events.Count.ShouldBe(1); await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); callback("MAIN.Speed", 2); events.Count.ShouldBe(1); } /// /// A direct re-initialise with a changed config swaps the tag table (the removed tag is unknown, the /// added one resolves) and retracts the previous generation's live notifications. /// [Fact] public async Task Reinitialise_swaps_tables_and_retracts_the_previous_generations_notifications() { var factory = new FakeTwinCATClientFactory(); var drv = NewDriver(factory, probe: false); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); _ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); factory.Clients[0].Notifications.Count.ShouldBe(1); var next = System.Text.Json.JsonSerializer.Serialize(new { devices = new[] { new { hostAddress = Host } }, rawTags = new[] { new { rawPath = "Torque", tagConfig = TwinCATTagDefinitionFactory.ToTagConfig( new TwinCATTagDefinition("Torque", Host, "MAIN.Torque", TwinCATDataType.DInt)), writeIdempotent = false, deviceName = Host, }, }, probe = new { enabled = false }, }); await drv.InitializeAsync(next, TestContext.Current.CancellationToken); factory.Clients[0].Notifications.ShouldBeEmpty(); factory.Clients[0].DisposeCount.ShouldBe(1); (await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken)).Single().StatusCode .ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); (await drv.ReadAsync(["Torque"], TestContext.Current.CancellationToken)).Single().StatusCode .ShouldBe(TwinCATStatusMapper.Good); } /// Reads racing a stream of re-initialises never throw (no table is ever mid-Clear). [Fact] public async Task Reads_racing_reinitialise_never_throw() { var factory = new FakeTwinCATClientFactory(); var drv = NewDriver(factory, probe: false); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // Run for at least 300 ms of overlap AND until the reader has seen a Good read mid-churn (positive // control), bounded at 10 s — a condition, not a fixed window, so a loaded machine cannot starve it. var good = 0; var reads = 0; var inits = 0; var minUntil = DateTime.UtcNow.AddMilliseconds(300); var deadline = DateTime.UtcNow.AddSeconds(10); bool Done() => (DateTime.UtcNow > minUntil && Volatile.Read(ref good) > 0 && Volatile.Read(ref inits) > 10) || DateTime.UtcNow > deadline; var reader = Task.Run(async () => { while (!Done()) { var r = await drv.ReadAsync(["Speed"], CancellationToken.None); Interlocked.Increment(ref reads); if (r[0].StatusCode == TwinCATStatusMapper.Good) Interlocked.Increment(ref good); } }, TestContext.Current.CancellationToken); while (!Done()) { await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); Interlocked.Increment(ref inits); await Task.Yield(); // let the reader in between generations } await reader; // throws if any read threw good.ShouldBeGreaterThan(0); inits.ShouldBeGreaterThan(10); } /// Notification-queue drops are reported once per probe tick that sees new drops. [Fact] public async Task Notification_queue_drops_are_reported() { var fake = new FakeTwinCATClient(); var log = new TwinCATCapturingLogger(); var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, log: log); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); await WaitForAsync(() => fake.ProbeCount >= 3, TimeSpan.FromSeconds(3)); log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(0); // 3 ticks, no drops, no warning fake.DroppedNotificationCount = 5; await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 5")), TimeSpan.FromSeconds(2)); var ticks = fake.ProbeCount; await WaitForAsync(() => fake.ProbeCount >= ticks + 3, TimeSpan.FromSeconds(3)); log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(1); // 3 more ticks, no new drops, no repeat fake.DroppedNotificationCount = 8; await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 3")), TimeSpan.FromSeconds(2)); log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(2); await drv.ShutdownAsync(TestContext.Current.CancellationToken); } }