using Opc.Ua; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Gitea #676 item 1 — the IN-PLACE blip (#670): the upstream stops answering but the TCP connection and the /// session survive, keep-alives fail, then traffic resumes before the session times out. Until now only the live /// gate (docker pause of opc-plc) covered it. reproduces it on loopback: the driver /// connects THROUGH the proxy (which works only because of the #676 endpoint rewrite — the server advertises its own /// port, which would bypass the proxy), and pausing the proxy stalls every byte without closing anything. /// /// /// Writing this test found a real defect: a change the server published right after the stall ended rode a Publish /// response that had been in flight across the stall; the SDK timed that request out, dropped the response and never /// republished it, so the node stayed Good on the PRE-change value indefinitely. The driver's trailing reconcile read /// (ReconcileAfterRecoveryAsync) is the fix; the "40.0 arrives" leg below goes red without it. /// [Trait("Category", "Integration")] public sealed class OpcUaClientDriverStallWireTests { private const uint BadNoCommunication = 0x80310000u; private static async Task WaitAsync(Func condition, string because, TimeSpan? timeout = null) { var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(30)); while (DateTime.UtcNow < deadline) { if (condition()) return; await Task.Delay(25); } condition().ShouldBeTrue(because); } private sealed record Seen(uint Status, object? Value, DateTime? Source); [Fact] public async Task A_stalled_connection_goes_Bad_once_and_recovers_in_place_on_the_same_session_without_regressing() { var ct = TestContext.Current.CancellationToken; await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-stall", ct); await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); // A 3 s operation timeout keeps the stall short: a keep-alive is declared failed when its request times out. await using var drv = OpcUaClientDriverWireTests.NewDriver( rig, "wire-stall", endpointUrl: proxy.Through(rig.EndpointUrl), timeout: TimeSpan.FromSeconds(3)); var connectionLost = 0; drv.ConnectionLost += (_, _) => Interlocked.Increment(ref connectionLost); await drv.InitializeAsync("{}", ct); proxy.AcceptedConnections.ShouldBeGreaterThan(0, "the session must run THROUGH the proxy, or pausing it proves nothing"); var seen = new List(); drv.OnDataChange += (_, e) => { if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef) lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc)); }; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value"); var session = drv.Session.ShouldNotBeNull(); var connectionsBefore = proxy.AcceptedConnections; var badKeepAlives = 0; session.KeepAlive += (_, e) => { if (ServiceResult.IsBad(e.Status)) Interlocked.Increment(ref badKeepAlives); }; // The SDK's late backlog flush: a data-carrying NotificationMessage whose sequence number is BELOW one already // seen (keep-alive messages carry the next number). It is the positive control for the no-regression check. uint maxSequence = 0; var lateMessages = 0; session.Notification += (_, e) => { var msg = e.NotificationMessage; if (msg is null) return; lock (seen) { if (msg.SequenceNumber < maxSequence && msg.NotificationData.Count > 0) lateMessages++; maxSequence = Math.Max(maxSequence, msg.SequenceNumber); } }; // ── Stall ──────────────────────────────────────────────────────────────────────────────────────────────── proxy.Pause(); await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); }, "a stalled upstream must publish BadNoCommunication for the subscribed ref"); lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f, "the Bad retains the value"); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); var frozen = drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); // The server's value moves on while the client cannot see it: away and back to the pre-outage value, so the // recovered value is IDENTICAL to the one published Bad — the (value, status) dedup trap. OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 20.0f); await Task.Delay(300, ct); // > one publishing interval, so the server queues 20.0 as its own notification OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); // Stays Bad and frozen across a further failed keep-alive (counted, not slept for). var badAtDetection = Volatile.Read(ref badKeepAlives); await WaitAsync(() => Volatile.Read(ref badKeepAlives) >= badAtDetection + 2, "further keep-alives must fail while stalled"); drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "the read stamp is the #527 watchdog's silence measure"); lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive"); // ── Resume well inside the 30 s session timeout ────────────────────────────────────────────────────────── proxy.Resume(); await WaitAsync(() => drv.GetHealth().State == DriverState.Healthy, "a resumed stall must recover in place"); int badIndex; lock (seen) badIndex = seen.FindIndex(s => s.Status == BadNoCommunication); await WaitAsync(() => { lock (seen) return seen.Skip(badIndex + 1).Any(s => s is { Status: 0u, Value: 10.0f }); }, "the identical value must be republished Good after the outage"); drv.Session.ShouldBeSameAs(session, "an in-place blip must not cost the session"); proxy.AcceptedConnections.ShouldBe(connectionsBefore, "and not the TCP connection either"); // A change published straight after recovery — while the Publish requests held across the stall are still // being answered or timed out — must still reach the node. OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 40.0f); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 40.0f }); }, "a change published right after recovery must reach the node (via the subscription or the trailing re-read)"); await drv.ReconcileTaskForTest; // The SDK flushes the notifications held during the stall seconds after recovery (the #670 F2 shape). Wait for // that flush before judging ordering, or the check below would pass on a run where nothing late arrived. await WaitAsync(() => { lock (seen) return lateMessages > 0; }, "the SDK's late backlog flush did not arrive — the no-regression check below would be vacuous"); lock (seen) { var good = seen.Where(s => s.Status == 0u && s.Source is not null).ToList(); for (var i = 1; i < good.Count; i++) good[i].Source!.Value.ShouldBeGreaterThanOrEqualTo(good[i - 1].Source!.Value, $"published Good value #{i} ({good[i].Value}) regressed in source time — a late replay reached the " + $"node. Sequence: [{string.Join(", ", good.Select(s => $"{s.Value}@{s.Source:HH:mm:ss.fff}"))}]"); seen.Last().Value.ShouldBe(40.0f, "nothing older may land after the latest value"); seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1); } drv.GetHealth().State.ShouldBe(DriverState.Healthy); drv.Session.ShouldBeSameAs(session); Volatile.Read(ref connectionLost).ShouldBe(0, "a stall is AMBIGUOUS (the session survived it) — it must go through the #527 watchdog, never the fast path"); } }