using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Outage + recovery leg of the OpcUaClient driver's wire coverage (Gitea #622, reworked for #670): the in-process /// server is stopped and a fresh one is booted on the SAME port / application URI / PKI store — a process restart /// as the driver sees it, with an empty address space the test then re-creates. The driver must REPORT the loss /// (subscribed value BadNoCommunication with the value retained, health non-Healthy — what the host's #527 /// comms-loss watchdog trips on). Recovery is the watchdog's sequence, replayed here as DriverInstanceActor /// runs it: ShutdownAsync, InitializeAsync, re-subscribe — after which the value comes back Good. There is no /// in-driver reconnect any more (#670: the SDK reconnect handler kept everything Good and Healthy forever). /// Since #676 a gracefully stopped server is a DEFINITIVE loss (BadServerHalted), so the driver also raises /// ConnectionLost and the host runs that sequence at once instead of after the watchdog's silence window. /// [Trait("Category", "Integration")] public sealed class OpcUaClientDriverReconnectWireTests { private const uint BadNoCommunication = 0x80310000u; private static async Task WaitAsync(Func condition, string because, TimeSpan? timeout = null) { var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(60)); while (DateTime.UtcNow < deadline) { if (condition()) return; await Task.Delay(50); } condition().ShouldBeTrue(because); } [Fact] public async Task A_server_restart_goes_Bad_then_the_watchdog_reinit_brings_the_value_back_Good() { var ct = TestContext.Current.CancellationToken; var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-reconnect", ct); await using var _rig = rig; OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-reconnect"); var lost = new List(); drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); }; await drv.InitializeAsync("{}", ct); var seen = new List<(uint Status, object? Value)>(); drv.OnDataChange += (_, e) => { if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef) lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value)); }; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 10.0f)); }, "initial value must arrive before the outage"); // Outage: the loss must be REPORTED while the server is genuinely down — Bad on the value with the value // retained, and a health state the watchdog can act on. await rig.RestartAsync(ct, whileDown: async () => { await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); }, "a lost session must publish BadNoCommunication for the subscribed ref", TimeSpan.FromSeconds(30)); lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); drv.GetHostStatuses().Single().State.ShouldNotBe(HostState.Running); // The read stamp must stay frozen for the whole outage — it is the watchdog's silence measure. var frozen = drv.GetHealth().LastSuccessfulRead; frozen.ShouldNotBeNull(); await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen); // #676: a server that HALTED (graceful stop) is a definitive loss — measured here, the SDK reports // BadServerHalted on every keep-alive and never re-opens the channel, so the session can never recover in // place. The driver says so at once, so the host re-initialises it instead of waiting out the ~90 s watchdog. await WaitAsync(() => { lock (lost) return lost.Count > 0; }, "a halted upstream must raise ConnectionLost", TimeSpan.FromSeconds(10)); lock (lost) lost.Single().ShouldContain("BadServerHalted"); }); // The restarted server knows nothing of the old session, so the driver stays down on its own … await Task.Delay(TimeSpan.FromSeconds(3), ct); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "nothing in the driver may paper over a dead session"); lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive"); lock (lost) lost.Count.ShouldBe(1, "once per session — the restarted server's continued refusals raise nothing more"); // … until the host's comms-loss watchdog re-initialises it (DriverInstanceActor: Reconnecting → ShutdownAsync + // InitializeAsync → Connected → re-subscribe). The new server is empty: redeploy the nodes first, holding the // SAME value as before the outage. rig.EnsureRawVariable("Speed", "Float", writable: true); rig.EnsureRawVariable("Counter", "Int32", writable: false); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f); await drv.ShutdownAsync(ct); await drv.InitializeAsync("{}", ct); drv.GetHealth().State.ShouldBe(DriverState.Healthy); int before; lock (seen) before = seen.Count; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); // The IDENTICAL value comes back Good — a (value, status) dedup downstream lets it through. await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 10.0f)); }, "the re-subscribed ref must come back Good with its value"); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 88.0f); await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 88.0f)); }, "a later change must also reach the new subscription"); drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running); var back = await drv.ReadAsync([OpcUaClientDriverWireTests.SpeedRef], ct); back[0].StatusCode.ShouldBe(0u); Convert.ToSingle(back[0].Value).ShouldBe(88.0f); lock (lost) lost.Count.ShouldBe(1, "the healthy re-initialised session reports no loss"); } /// /// #670 review M1 — the subscription dies while the SESSION lives (in production: an outage longer than the /// subscription lifetime but shorter than the session timeout; here: the server deletes it). Keep-alives stay /// Good throughout, so a driver that only watched them would end Healthy with fresh read stamps and a frozen /// value forever. It must go Bad, stay non-Healthy with the stamp frozen (so the #527 watchdog fires), and the /// watchdog's re-initialise must bring the value back. /// [Fact] public async Task A_subscription_lost_while_the_session_lives_is_reported_and_recovered_by_reinit() { var ct = TestContext.Current.CancellationToken; await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-sublost", ct); OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 5.0f); await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-sublost"); var lost = new List(); drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); }; await drv.InitializeAsync("{}", ct); var seen = new List<(uint Status, object? Value)>(); drv.OnDataChange += (_, e) => { if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef) lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value)); }; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 5.0f)); }, "initial value"); // Delete the subscription on the server behind the SDK's back; the session itself stays healthy. var session = drv.Session.ShouldNotBeNull(); var subId = session.Subscriptions.Single(s => s.Created).Id; await session.DeleteSubscriptionsAsync(null, [subId], ct); await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); }, "a subscription the server dropped must be reported as comms loss", TimeSpan.FromSeconds(30)); drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); var frozen = drv.GetHealth().LastSuccessfulRead; await Task.Delay(TimeSpan.FromSeconds(3), ct); // good keep-alives keep arriving … drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "… and must not revive a driver whose data plane is dead"); drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen); drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running, "the session itself is fine"); // #676: a subscription the server dropped never comes back on this session — a definitive loss. lock (lost) lost.ShouldHaveSingleItem().ShouldContain("subscription lost"); // The watchdog's re-initialise recreates the subscription and the value comes back Good. await drv.ShutdownAsync(ct); await drv.InitializeAsync("{}", ct); int before; lock (seen) before = seen.Count; await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct); await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 5.0f)); }, "the re-created subscription must deliver the value Good"); drv.GetHealth().State.ShouldBe(DriverState.Healthy); } }