lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/OpcUaClientDriverStallWireTests.cs

using Opc.Ua;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

/// <summary>
///     Gitea #676 item 1 — the IN-PLACE blip (#670): the upstream stops answering but the TCP connection and the
///     session survive, keep-alives fail, then traffic resumes before the session times out. Until now only the live
///     gate (<c>docker pause</c> of opc-plc) covered it. <see cref="TcpStallProxy"/> reproduces it on loopback: the driver
///     connects THROUGH the proxy (which works only because of the #676 endpoint rewrite — the server advertises its own
///     port, which would bypass the proxy), and pausing the proxy stalls every byte without closing anything.
/// </summary>
/// <remarks>
///     Writing this test found a real defect: a change the server published right after the stall ended rode a Publish
///     response that had been in flight across the stall; the SDK timed that request out, dropped the response and never
///     republished it, so the node stayed Good on the PRE-change value indefinitely. The driver's trailing reconcile read
///     (<c>ReconcileAfterRecoveryAsync</c>) is the fix; the "40.0 arrives" leg below goes red without it.
/// </remarks>
[Trait("Category", "Integration")]
public sealed class OpcUaClientDriverStallWireTests
{
    private const uint BadNoCommunication = 0x80310000u;

    private static async Task WaitAsync(Func<bool> condition, string because, TimeSpan? timeout = null)
    {
        var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(30));
        while (DateTime.UtcNow < deadline)
        {
            if (condition()) return;
            await Task.Delay(25);
        }
        condition().ShouldBeTrue(because);
    }

    private sealed record Seen(uint Status, object? Value, DateTime? Source);

    [Fact]
    public async Task A_stalled_connection_goes_Bad_once_and_recovers_in_place_on_the_same_session_without_regressing()
    {
        var ct = TestContext.Current.CancellationToken;
        await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-stall", ct);
        await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port);
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);

        // A 3 s operation timeout keeps the stall short: a keep-alive is declared failed when its request times out.
        await using var drv = OpcUaClientDriverWireTests.NewDriver(
            rig, "wire-stall", endpointUrl: proxy.Through(rig.EndpointUrl), timeout: TimeSpan.FromSeconds(3));
        var connectionLost = 0;
        drv.ConnectionLost += (_, _) => Interlocked.Increment(ref connectionLost);
        await drv.InitializeAsync("{}", ct);
        proxy.AcceptedConnections.ShouldBeGreaterThan(0, "the session must run THROUGH the proxy, or pausing it proves nothing");

        var seen = new List<Seen>();
        drv.OnDataChange += (_, e) =>
        {
            if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
                lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc));
        };
        await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
        await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value");

        var session = drv.Session.ShouldNotBeNull();
        var connectionsBefore = proxy.AcceptedConnections;
        var badKeepAlives = 0;
        session.KeepAlive += (_, e) =>
        {
            if (ServiceResult.IsBad(e.Status)) Interlocked.Increment(ref badKeepAlives);
        };
        // The SDK's late backlog flush: a data-carrying NotificationMessage whose sequence number is BELOW one already
        // seen (keep-alive messages carry the next number). It is the positive control for the no-regression check.
        uint maxSequence = 0;
        var lateMessages = 0;
        session.Notification += (_, e) =>
        {
            var msg = e.NotificationMessage;
            if (msg is null) return;
            lock (seen)
            {
                if (msg.SequenceNumber < maxSequence && msg.NotificationData.Count > 0) lateMessages++;
                maxSequence = Math.Max(maxSequence, msg.SequenceNumber);
            }
        };

        // ── Stall ────────────────────────────────────────────────────────────────────────────────────────────────
        proxy.Pause();
        await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
            "a stalled upstream must publish BadNoCommunication for the subscribed ref");
        lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f, "the Bad retains the value");
        drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
        var frozen = drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull();

        // The server's value moves on while the client cannot see it: away and back to the pre-outage value, so the
        // recovered value is IDENTICAL to the one published Bad — the (value, status) dedup trap.
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 20.0f);
        await Task.Delay(300, ct); // > one publishing interval, so the server queues 20.0 as its own notification
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);

        // Stays Bad and frozen across a further failed keep-alive (counted, not slept for).
        var badAtDetection = Volatile.Read(ref badKeepAlives);
        await WaitAsync(() => Volatile.Read(ref badKeepAlives) >= badAtDetection + 2, "further keep-alives must fail while stalled");
        drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "the read stamp is the #527 watchdog's silence measure");
        lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");

        // ── Resume well inside the 30 s session timeout ──────────────────────────────────────────────────────────
        proxy.Resume();
        await WaitAsync(() => drv.GetHealth().State == DriverState.Healthy, "a resumed stall must recover in place");
        int badIndex;
        lock (seen) badIndex = seen.FindIndex(s => s.Status == BadNoCommunication);
        await WaitAsync(() => { lock (seen) return seen.Skip(badIndex + 1).Any(s => s is { Status: 0u, Value: 10.0f }); },
            "the identical value must be republished Good after the outage");

        drv.Session.ShouldBeSameAs(session, "an in-place blip must not cost the session");
        proxy.AcceptedConnections.ShouldBe(connectionsBefore, "and not the TCP connection either");

        // A change published straight after recovery — while the Publish requests held across the stall are still
        // being answered or timed out — must still reach the node.
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 40.0f);
        await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 40.0f }); },
            "a change published right after recovery must reach the node (via the subscription or the trailing re-read)");
        await drv.ReconcileTaskForTest;

        // The SDK flushes the notifications held during the stall seconds after recovery (the #670 F2 shape). Wait for
        // that flush before judging ordering, or the check below would pass on a run where nothing late arrived.
        await WaitAsync(() => { lock (seen) return lateMessages > 0; },
            "the SDK's late backlog flush did not arrive — the no-regression check below would be vacuous");

        lock (seen)
        {
            var good = seen.Where(s => s.Status == 0u && s.Source is not null).ToList();
            for (var i = 1; i < good.Count; i++)
                good[i].Source!.Value.ShouldBeGreaterThanOrEqualTo(good[i - 1].Source!.Value,
                    $"published Good value #{i} ({good[i].Value}) regressed in source time — a late replay reached the " +
                    $"node. Sequence: [{string.Join(", ", good.Select(s => $"{s.Value}@{s.Source:HH:mm:ss.fff}"))}]");
            seen.Last().Value.ShouldBe(40.0f, "nothing older may land after the latest value");
            seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1);
        }
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
        drv.Session.ShouldBeSameAs(session);
        Volatile.Read(ref connectionLost).ShouldBe(0,
            "a stall is AMBIGUOUS (the session survived it) — it must go through the #527 watchdog, never the fast path");
    }
}