using Opc.Ua; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Gitea #676 item 1 — the IN-PLACE blip (#670): the upstream stops answering but the TCP connection and the /// session survive, keep-alives fail, then traffic resumes before the session times out. Until now only the live /// gate (docker pause of opc-plc) covered it. reproduces it on loopback: the driver /// connects THROUGH the proxy (which works only because of the #676 endpoint rewrite — the server advertises its own /// port, which would bypass the proxy), and pausing the proxy stalls every byte without closing anything. /// /// /// Writing this test found a real defect: a change the server published right after the stall ended rode a Publish /// response that had been in flight across the stall; the SDK timed that request out, dropped the response and never /// republished it, so the node stayed Good on the PRE-change value indefinitely. The driver's trailing reconcile read /// (ReconcileAfterRecoveryAsync) is the fix; the "40.0 arrives" leg below goes red without it. /// [Trait("Category", "Integration")] public sealed class OpcUaClientDriverStallWireTests { private const uint BadNoCommunication = 0x80310000u; private static async Task WaitAsync(Func condition, string because, TimeSpan? timeout = null) { var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(30)); while (DateTime.UtcNow < deadline) { if (condition()) return; await Task.Delay(25); } condition().ShouldBeTrue(because); } private sealed record Seen(uint Status, object? Value, DateTime? Source); [Fact] public async Task A_stalled_connection_goes_Bad_once_and_recovers_in_place_on_the_same_session_without_regressing() { var ct = TestContext.Current.CancellationToken; await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-stall", ct); await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); // A 3 s operation timeout keeps the stall short (a keep-alive is declared failed when its request times out); a // 120 s session timeout and a 3000-cycle (300 s at 100 ms) subscription lifetime keep both far outside the // stall, so the test exercises the in-place recovery and nothing else. await using var drv = OpcUaClientDriverWireTests.NewDriver( rig, "wire-stall", endpointUrl: proxy.Through(rig.EndpointUrl), timeout: TimeSpan.FromSeconds(3), sessionTimeout: TimeSpan.FromSeconds(120), subscriptionLifetimeCount: 3000); var connectionLost = 0; drv.ConnectionLost += (_, _) => Interlocked.Increment(ref connectionLost); await drv.InitializeAsync("{}", ct); proxy.AcceptedConnections.ShouldBeGreaterThan(0, "the session must run THROUGH the proxy, or pausing it proves nothing"); var seen = new List(); drv.OnDataChange += (_, e) => { if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef) lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc)); }; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value"); var session = drv.Session.ShouldNotBeNull(); var connectionsBefore = proxy.AcceptedConnections; // Bad keep-alive TICKS: the SDK reports a failed tick as two events at the same instant (the request's status and // BadNoCommunication), so events closer than half a keep-alive interval are one tick. var badKeepAlives = 0; var lastBadTick = DateTime.MinValue; session.KeepAlive += (_, e) => { if (!ServiceResult.IsBad(e.Status)) return; lock (seen) { if (DateTime.UtcNow - lastBadTick < TimeSpan.FromMilliseconds(500)) return; lastBadTick = DateTime.UtcNow; badKeepAlives++; } }; // The SDK's late backlog flush: a data-carrying NotificationMessage whose sequence number is BELOW one already // seen (keep-alive messages carry the next number). It is the positive control for the no-regression check. uint maxSequence = 0; var lateMessages = 0; session.Notification += (_, e) => { var msg = e.NotificationMessage; if (msg is null) return; lock (seen) { if (msg.SequenceNumber < maxSequence && msg.NotificationData.Count > 0) lateMessages++; maxSequence = Math.Max(maxSequence, msg.SequenceNumber); } }; // ── Stall ──────────────────────────────────────────────────────────────────────────────────────────────── proxy.Pause(); await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); }, "a stalled upstream must publish BadNoCommunication for the subscribed ref"); lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f, "the Bad retains the value"); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); var frozen = drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); // The server's value moves on while the client cannot see it: away and back to the pre-outage value, so the // recovered value is IDENTICAL to the one published Bad — the (value, status) dedup trap. OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 20.0f); await Task.Delay(300, ct); // > one publishing interval, so the server queues 20.0 as its own notification OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); // Stays Bad and frozen across one further failed keep-alive tick (counted, not slept for). int badAtDetection; lock (seen) badAtDetection = badKeepAlives; await WaitAsync(() => { lock (seen) return badKeepAlives > badAtDetection; }, "a further keep-alive must fail while stalled"); drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "the read stamp is the #527 watchdog's silence measure"); lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive"); // ── Resume well inside the 30 s session timeout ────────────────────────────────────────────────────────── proxy.Resume(); await WaitAsync(() => drv.GetHealth().State == DriverState.Healthy, "a resumed stall must recover in place"); int badIndex; lock (seen) badIndex = seen.FindIndex(s => s.Status == BadNoCommunication); await WaitAsync(() => { lock (seen) return seen.Skip(badIndex + 1).Any(s => s is { Status: 0u, Value: 10.0f }); }, "the identical value must be republished Good after the outage"); drv.Session.ShouldBeSameAs(session, "an in-place blip must not cost the session"); proxy.AcceptedConnections.ShouldBe(connectionsBefore, "and not the TCP connection either"); // A change published straight after recovery — while the Publish requests held across the stall are still // being answered or timed out — must still reach the node. OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 40.0f); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 40.0f }); }, "a change published right after recovery must reach the node (via the subscription or the trailing re-read)", TimeSpan.FromSeconds(60)); await drv.ReconcileTaskForTest; // The SDK flushes the notifications held during the stall seconds after recovery (the #670 F2 shape). Wait for // that flush before judging ordering, or the check below would pass on a run where nothing late arrived. await WaitAsync(() => { lock (seen) return lateMessages > 0; }, "the SDK's late backlog flush did not arrive — the no-regression check below would be vacuous", TimeSpan.FromSeconds(60)); lock (seen) { var good = seen.Where(s => s.Status == 0u && s.Source is not null).ToList(); for (var i = 1; i < good.Count; i++) good[i].Source!.Value.ShouldBeGreaterThanOrEqualTo(good[i - 1].Source!.Value, $"published Good value #{i} ({good[i].Value}) regressed in source time — a late replay reached the " + $"node. Sequence: [{string.Join(", ", good.Select(s => $"{s.Value}@{s.Source:HH:mm:ss.fff}"))}]"); seen.Last().Value.ShouldBe(40.0f, "nothing older may land after the latest value"); seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1); } drv.GetHealth().State.ShouldBe(DriverState.Healthy); drv.Session.ShouldBeSameAs(session); Volatile.Read(ref connectionLost).ShouldBe(0, "a stall is AMBIGUOUS (the session survived it) — it must go through the #527 watchdog, never the fast path"); } /// /// #676 live gate A1 — a real upstream process stop is a CLOSED connection, not a stall: the keep-alives report /// BadSecureChannelClosed, then BadConnectionClosed (with BadNoCommunication) for the rest of /// the outage, and the SDK never opens a new connection on its own — not even when the server (here, with the /// session still alive behind the proxy) is reachable again. The driver must therefore raise /// ConnectionLost once the closed channel is sustained, so the host re-initialises in seconds instead of /// waiting ~90 s for the watchdog; the re-initialise brings the identical value back Good. /// [Fact] public async Task A_hard_closed_connection_raises_ConnectionLost_once_and_the_reinit_recovers_in_seconds() { var ct = TestContext.Current.CancellationToken; await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-hardclose", ct); await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); await using var drv = OpcUaClientDriverWireTests.NewDriver( rig, "wire-hardclose", endpointUrl: proxy.Through(rig.EndpointUrl), sessionTimeout: TimeSpan.FromSeconds(120)); var lost = new List(); drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); }; await drv.InitializeAsync("{}", ct); var seen = new List(); drv.OnDataChange += (_, e) => { if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef) lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc)); }; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value"); var session = drv.Session.ShouldNotBeNull(); var keepAlives = new List(); session.KeepAlive += (_, e) => { lock (keepAlives) keepAlives.Add(e.Status?.StatusCode.Code ?? 0); }; // ── The upstream process goes away: every connection reset, new ones refused ────────────────────────────── var killedAt = DateTime.UtcNow; proxy.HardClose(); await WaitAsync(() => { lock (lost) return lost.Count > 0; }, "a sustained closed channel must raise ConnectionLost", TimeSpan.FromSeconds(15)); (DateTime.UtcNow - killedAt).ShouldBeLessThan(TimeSpan.FromSeconds(15), "seconds, not the watchdog's ~90 s"); lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "the outage was published Bad first"); lock (keepAlives) { keepAlives.ShouldContain(0x80860000u, "BadSecureChannelClosed — the live-gate sequence"); keepAlives.ShouldContain(0x80AE0000u, "BadConnectionClosed — the live-gate sequence"); } // The server is back — the session still exists on it — yet the SDK never reconnects by itself: no new // connection reaches the proxy while further keep-alives fail. That is what makes the closed channel definitive. proxy.Reopen(); var accepted = proxy.AcceptedConnections; int kaAtReopen; lock (keepAlives) kaAtReopen = keepAlives.Count; await WaitAsync(() => { lock (keepAlives) return keepAlives.Count >= kaAtReopen + 4; }, "keep-alives keep failing"); proxy.AcceptedConnections.ShouldBe(accepted, "the SDK session never reopens a closed connection on its own"); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); lock (lost) lost.Count.ShouldBe(1, "once per session, however many closed-channel keep-alives follow"); // ── The host's response (DriverInstanceActor: Reconnecting → ShutdownAsync + InitializeAsync → re-subscribe) ── await drv.ShutdownAsync(ct); await drv.InitializeAsync("{}", ct); int before; lock (seen) before = seen.Count; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is { Status: 0u, Value: 10.0f }); }, "the identical value must come back Good after the re-initialise"); (DateTime.UtcNow - killedAt).ShouldBeLessThan(TimeSpan.FromSeconds(60), "well under the watchdog's 90 s"); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 77.0f); await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 77.0f }); }, "a later change arrives"); drv.GetHealth().State.ShouldBe(DriverState.Healthy); lock (lost) lost.Count.ShouldBe(1, "the healthy re-initialised session reports no loss"); } }