lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/OpcUaClientDriverReconnectWireTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

/// <summary>
///     Outage + recovery leg of the OpcUaClient driver's wire coverage (Gitea #622, reworked for #670): the in-process
///     server is stopped and a fresh one is booted on the SAME port / application URI / PKI store — a process restart
///     as the driver sees it, with an empty address space the test then re-creates. The driver must REPORT the loss
///     (subscribed value BadNoCommunication with the value retained, health non-Healthy — what the host's #527
///     comms-loss watchdog trips on). Recovery is the watchdog's sequence, replayed here as <c>DriverInstanceActor</c>
///     runs it: ShutdownAsync, InitializeAsync, re-subscribe — after which the value comes back Good. There is no
///     in-driver reconnect any more (#670: the SDK reconnect handler kept everything Good and Healthy forever).
///     Since #676 definitive session loss also raises <c>ConnectionLost</c>, so the host runs that sequence at once
///     instead of after the watchdog's silence window. Shutdown timing determines the SDK's loss status.
/// </summary>
[Trait("Category", "Integration")]
public sealed class OpcUaClientDriverReconnectWireTests
{
    private const uint BadNoCommunication = 0x80310000u;

    private static async Task WaitAsync(Func<bool> condition, string because, TimeSpan? timeout = null)
    {
        var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(60));
        while (DateTime.UtcNow < deadline)
        {
            if (condition()) return;
            await Task.Delay(50);
        }
        condition().ShouldBeTrue(because);
    }

    [Fact]
    public async Task A_server_restart_goes_Bad_then_the_watchdog_reinit_brings_the_value_back_Good()
    {
        var ct = TestContext.Current.CancellationToken;
        var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-reconnect", ct);
        await using var _rig = rig;
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);

        await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-reconnect");
        var lost = new List<string>();
        drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); };
        await drv.InitializeAsync("{}", ct);

        var seen = new List<(uint Status, object? Value)>();
        drv.OnDataChange += (_, e) =>
        {
            if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
                lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value));
        };

        await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
        await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 10.0f)); },
            "initial value must arrive before the outage");

        // Outage: the loss must be REPORTED while the server is genuinely down — Bad on the value with the value
        // retained, and a health state the watchdog can act on.
        await rig.RestartAsync(ct, whileDown: async () =>
        {
            await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
                "a lost session must publish BadNoCommunication for the subscribed ref", TimeSpan.FromSeconds(30));
            lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f);
            drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
            drv.GetHostStatuses().Single().State.ShouldNotBe(HostState.Running);

            // The read stamp must stay frozen for the whole outage — it is the watchdog's silence measure.
            var frozen = drv.GetHealth().LastSuccessfulRead;
            frozen.ShouldNotBeNull();
            await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals
            drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);

            // #676/#702: shutdown can report a halted server, an invalid/closed session, or a sustained closed
            // channel. Each proves definitive loss; #702 reproduced BadSessionIdInvalid during this graceful stop.
            // Deterministic ConnectionLossSignal tests cover the individual statuses and the sustained-channel gate.
            await WaitAsync(() => { lock (lost) return lost.Count > 0; },
                "a halted upstream must raise ConnectionLost", TimeSpan.FromSeconds(10));
            lock (lost)
            {
                var reason = lost.Single();
                string[] definitiveStatuses =
                [
                    "BadSessionIdInvalid", "BadSessionClosed", "BadSessionNotActivated", "BadServerHalted",
                    "BadSecureChannelClosed", "BadConnectionClosed",
                ];
                definitiveStatuses.ShouldContain(status => reason.EndsWith(status, StringComparison.Ordinal),
                    $"the restart must report definitive loss, but reported: {reason}");
            }
        });

        // The restarted server knows nothing of the old session, so the driver stays down on its own …
        await Task.Delay(TimeSpan.FromSeconds(3), ct);
        drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "nothing in the driver may paper over a dead session");
        lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");
        lock (lost) lost.Count.ShouldBe(1, "once per session — the restarted server's continued refusals raise nothing more");

        // … until the host's comms-loss watchdog re-initialises it (DriverInstanceActor: Reconnecting → ShutdownAsync +
        // InitializeAsync → Connected → re-subscribe). The new server is empty: redeploy the nodes first, holding the
        // SAME value as before the outage.
        rig.EnsureRawVariable("Speed", "Float", writable: true);
        rig.EnsureRawVariable("Counter", "Int32", writable: false);
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);

        await drv.ShutdownAsync(ct);
        await drv.InitializeAsync("{}", ct);
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
        int before;
        lock (seen) before = seen.Count;
        await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);

        // The IDENTICAL value comes back Good — a (value, status) dedup downstream lets it through.
        await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 10.0f)); },
            "the re-subscribed ref must come back Good with its value");

        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 88.0f);
        await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 88.0f)); },
            "a later change must also reach the new subscription");
        drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running);

        var back = await drv.ReadAsync([OpcUaClientDriverWireTests.SpeedRef], ct);
        back[0].StatusCode.ShouldBe(0u);
        Convert.ToSingle(back[0].Value).ShouldBe(88.0f);
        lock (lost) lost.Count.ShouldBe(1, "the healthy re-initialised session reports no loss");
    }

    /// <summary>
    ///     #670 review M1 — the subscription dies while the SESSION lives (in production: an outage longer than the
    ///     subscription lifetime but shorter than the session timeout; here: the server deletes it). Keep-alives stay
    ///     Good throughout, so a driver that only watched them would end Healthy with fresh read stamps and a frozen
    ///     value forever. It must go Bad, stay non-Healthy with the stamp frozen (so the #527 watchdog fires), and the
    ///     watchdog's re-initialise must bring the value back.
    /// </summary>
    [Fact]
    public async Task A_subscription_lost_while_the_session_lives_is_reported_and_recovered_by_reinit()
    {
        var ct = TestContext.Current.CancellationToken;
        await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-sublost", ct);
        OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 5.0f);

        await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-sublost");
        var lost = new List<string>();
        drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); };
        await drv.InitializeAsync("{}", ct);
        var seen = new List<(uint Status, object? Value)>();
        drv.OnDataChange += (_, e) =>
        {
            if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
                lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value));
        };
        await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
        await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 5.0f)); }, "initial value");

        // Delete the subscription on the server behind the SDK's back; the session itself stays healthy.
        var session = drv.Session.ShouldNotBeNull();
        var subId = session.Subscriptions.Single(s => s.Created).Id;
        await session.DeleteSubscriptionsAsync(null, [subId], ct);

        await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
            "a subscription the server dropped must be reported as comms loss", TimeSpan.FromSeconds(30));
        drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
        var frozen = drv.GetHealth().LastSuccessfulRead;
        await Task.Delay(TimeSpan.FromSeconds(3), ct); // good keep-alives keep arriving …
        drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "… and must not revive a driver whose data plane is dead");
        drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
        drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running, "the session itself is fine");
        // #676: a subscription the server dropped never comes back on this session — a definitive loss.
        lock (lost) lost.ShouldHaveSingleItem().ShouldContain("subscription lost");

        // The watchdog's re-initialise recreates the subscription and the value comes back Good.
        await drv.ShutdownAsync(ct);
        await drv.InitializeAsync("{}", ct);
        int before;
        lock (seen) before = seen.Count;
        await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
        await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 5.0f)); },
            "the re-created subscription must deliver the value Good");
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
    }
}