fix(opcuaclient): #676 live gate A1 — a sustained closed connection is a definitive loss

Live gate A1 (docker-dev, opc-plc restart / SIGINT / kill): the fast path never fired — a real process stop delivers no session status; the keep-alives report BadSecureChannelClosed (0x80860000) once, then BadConnectionClosed (0x80AE0000) interleaved with BadNoCommunication for the whole outage. Reproduced in-process with TcpStallProxy.HardClose (reset every connection, refuse new ones; server and session alive behind it): the identical sequence, and the SDK 1.5.378 session never opens a new connection on its own (48 s with the proxy accepting again, zero attempts) — a closed channel cannot recover in place. The driver now treats BadSecureChannelClosed/BadConnectionClosed as definitive on the SECOND closed-channel keep-alive failure in one outage (a good keep-alive resets the count), raising ConnectionLost ~1 keep-alive after the drop. BadNoCommunication / BadRequestTimeout / BadNotConnected (stall/pause) stay with the watchdog; the stall test still sees no ConnectionLost. New wire test (hard close + reopen ⇒ raised once, SDK never reconnects, reinit brings the identical value back Good) and unit tests (sustained raises once; a single blip that recovers does not) — all red on the previous code. Docs updated.
2026-10-11 12:34:52 -04:00
parent 945ec2c069
commit beb1c583b1
7 changed files with 218 additions and 17 deletions
@@ -298,6 +298,14 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
    /// cleared when the session is released, so a re-initialised driver reports its next loss.</summary>
    private int _connectionLostRaised;

+    /// <summary>#676 live gate A1 — closed-channel keep-alive failures (<see cref="IsClosedChannel"/>) in the current
+    /// outage; reset by a good keep-alive and when the session is released. Two make a closed channel "sustained".</summary>
+    private int _closedChannelFailures;
+
+    /// <summary>Closed-channel keep-alive failures in one outage that make the loss definitive (a single one may be a
+    /// transient that the next good keep-alive contradicts).</summary>
+    private const int SustainedClosedChannelFailures = 2;
+
    /// <summary>#676 — the trailing reconcile read scheduled by the last in-place recovery. Test seam.</summary>
    private Task _reconcileTask = Task.CompletedTask;

@@ -1081,6 +1089,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
        Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
        // #676: re-arm the definitive-loss signal for the next session.
        Interlocked.Exchange(ref _connectionLostRaised, 0);
+        Interlocked.Exchange(ref _closedChannelFailures, 0);
        // Review of 3b88056f: the released session's post-recovery timers must not outlive it.
        CancelSessionWork();
    }
@@ -2869,6 +2878,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit

        if (!ServiceResult.IsBad(e.Status))
        {
+            Interlocked.Exchange(ref _closedChannelFailures, 0);
            TransitionTo(HostState.Running);
            // M1: the session answers but a subscription is dead — no stamp, no recovery; the watchdog reinit is the fix.
            if (Volatile.Read(ref _subscriptionLost) == 1)
@@ -2913,7 +2923,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
        // #676: the first bad keep-alive is often ambiguous and a definitive one follows, so this is checked on EVERY
        // bad keep-alive — but only after the outage below has been published, so the Bad reaches subscribers first.
        var keepAliveCode = e.Status?.StatusCode ?? StatusCodes.Good;
-        var definitive = IsDefinitiveSessionLoss(keepAliveCode);
+        var definitive = IsDefinitiveSessionLoss(keepAliveCode)
+                         || (IsClosedChannel(keepAliveCode)
+                             && Interlocked.Increment(ref _closedChannelFailures) >= SustainedClosedChannelFailures);
        // A subscription loss already published Bad for this connection.
        if (Interlocked.Exchange(ref _sessionLost, 1) == 0 && Volatile.Read(ref _subscriptionLost) == 0)
        {
@@ -2927,6 +2939,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
            RaiseConnectionLost($"upstream session keep-alive reported {StatusCodes.GetBrowseName(keepAliveCode.Code)}");
    }

+    /// <summary>
+    ///     #676 live gate A1 — the connection under the session is CLOSED (not merely silent). A real upstream process stop
+    ///     (docker restart, SIGINT, docker kill) never produced a session status on the docker-dev gate: the keep-alives
+    ///     reported <c>BadSecureChannelClosed</c> once, then <c>BadConnectionClosed</c> interleaved with
+    ///     <c>BadNoCommunication</c> for the whole outage. Reproduced in-process (a proxy that resets every connection
+    ///     while the server and its session stay alive): the SDK 1.5.378 session never opens a new connection on its
+    ///     own — over 48 s with the server reachable again, not one connection attempt — so a closed channel can never
+    ///     recover in place. It is treated as definitive once SUSTAINED (<see cref="SustainedClosedChannelFailures"/> in
+    ///     one outage), so a single transient contradicted by the next good keep-alive does not count. A stall (a paused
+    ///     upstream, a congested link) produces <c>BadRequestTimeout</c>/<c>BadNoCommunication</c> instead and keeps
+    ///     its in-place recovery.
+    /// </summary>
+    /// <param name="status">A keep-alive status.</param>
+    /// <returns>True for a closed secure channel or connection.</returns>
+    internal static bool IsClosedChannel(StatusCode status) =>
+        status == StatusCodes.BadSecureChannelClosed || status == StatusCodes.BadConnectionClosed;
+
    /// <summary>
    ///     #676 — keep-alive statuses that prove the session can NEVER recover in place, so waiting out the #527
    ///     watchdog would only delay the inevitable re-initialise. The server forgot the session
@@ -2934,9 +2963,10 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
    ///     (<c>BadSessionClosed</c>) or never activated it (<c>BadSessionNotActivated</c>); or the server HALTED
    ///     (<c>BadServerHalted</c>) — measured on the #676 restart wire test, the SDK never re-opens a channel the
    ///     server halted: 15 s after the server was back the same session still failed every request with
-    ///     <c>BadServerHalted</c>, so a halted session is dead for good. Everything else — <c>BadNoCommunication</c>,
-    ///     <c>BadConnectionClosed</c>, <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that
-    ///     resumes keeps its session, see the stall wire test) and stays with the watchdog.
+    ///     <c>BadServerHalted</c>, so a halted session is dead for good. A closed connection is definitive too, but only
+    ///     once sustained — see <see cref="IsClosedChannel"/>. Everything else — <c>BadNoCommunication</c>,
+    ///     <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that resumes keeps its session, see the
+    ///     stall wire test) and stays with the watchdog.
    /// </summary>
    /// <param name="status">A keep-alive status.</param>
    /// <returns>True when the session is definitively gone.</returns>