Merge R2-01 S7 fault-path hardening (arch-review round 2) [PR #429]

Findings 05/STAB-14 (High regression — connect-timeout OCE no longer silently kills S7 subscriptions), STAB-15 (framing/desync + mid-PDU cancel fatal classification), STAB-8 seam note. All prod changes in S7Driver.cs; T11 live outage gate deferred. STATUS.md conflict resolved additively. Build clean.
2026-07-13 10:14:59 -04:00
parent 12b3206555 bdb7ed412d
commit fea6477f45
7 changed files with 654 additions and 42 deletions
@@ -0,0 +1,138 @@
+using System.Diagnostics;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests.S7_1500;
+
+/// <summary>
+///     Live gate for STAB-14 — the connect-TIMEOUT outage class the ordinary bounce test
+///     (<see cref="S7_1500ReconnectTests"/>) structurally cannot produce. A <c>docker restart</c>
+///     yields connection-<i>refused</i> (an immediate <c>SocketException</c>); STAB-14 only fires
+///     on connect-<i>timeout</i> — the SYN-blackhole shape (packets DROPPED, not rejected) of a
+///     pulled cable / powered-off PLC / firewall DROP. Before the fix, a subscription poll loop
+///     that ticked during such an outage was silently and permanently killed (the timeout-born OCE
+///     read as teardown); this test would hang at the recovery assertion.
+/// </summary>
+/// <remarks>
+///     <para>
+///         Triple-gated so it never runs (or leaves the shared sim blackholed) by accident:
+///         <list type="number">
+///             <item><see cref="Snap7ServerFixture.SkipReason"/> — the sim must be reachable;</item>
+///             <item><c>S7_TIMEOUT_OUTAGE_START_CMD</c> — begins DROPPING traffic to the sim port
+///                 (a blackhole, not a reset); and</item>
+///             <item><c>S7_TIMEOUT_OUTAGE_STOP_CMD</c> — removes the block.</item>
+///         </list>
+///         Any absent gate ⇒ clean <see cref="Assert.Skip(string)"/> (safe offline on macOS). A
+///         <c>try/finally</c> ALWAYS runs the stop command once the start succeeded, so a failure
+///         mid-test never leaves the shared sim unreachable for other suites.
+///     </para>
+///     <para>
+///         Operator run recipe (also drives 10.100.0.35 via <c>lmxopcua-fix up s7 s7_1500</c>):
+///         <code>
+///         lmxopcua-fix up s7 s7_1500
+///         # docker pause = ACKless blackhole for established flows + unanswered SYNs for new connects:
+///         export S7_TIMEOUT_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-python-snap7-s7_1500"'
+///         export S7_TIMEOUT_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-python-snap7-s7_1500"'
+///         # iptables DROP variant (more faithful to a firewall blackhole):
+///         #   START: ssh dohertj2@10.100.0.35 "sudo iptables -I INPUT -p tcp --dport 1102 -j DROP"
+///         #   STOP:  ssh dohertj2@10.100.0.35 "sudo iptables -D INPUT -p tcp --dport 1102 -j DROP"
+///         dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests --filter "FullyQualifiedName~ConnectTimeoutOutage"
+///         </code>
+///     </para>
+/// </remarks>
+[Collection(Snap7ServerCollection.Name)]
+[Trait("Category", "Integration")]
+[Trait("Category", "Reconnect")]
+[Trait("Device", "S7_1500")]
+public sealed class S7_1500ConnectTimeoutOutageTests(Snap7ServerFixture sim)
+{
+    private const string StartCmdEnvVar = "S7_TIMEOUT_OUTAGE_START_CMD";
+    private const string StopCmdEnvVar = "S7_TIMEOUT_OUTAGE_STOP_CMD";
+
+    /// <summary>
+    ///     Subscribes against the live sim, blackholes its traffic (each reconnect now TIMES OUT
+    ///     rather than being refused), observes at least one Bad tick, then restores traffic and
+    ///     asserts a Good <c>OnDataChange</c> resumes on the SAME subscription — the poll loop
+    ///     survived the connect-timeout outage.
+    /// </summary>
+    [Fact]
+    public async Task Subscription_survives_a_live_connect_timeout_outage_ConnectTimeoutOutage()
+    {
+        if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
+        var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar);
+        var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar);
+        if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd))
+            Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (SYN-blackhole start/stop) to run this destructive live test.");
+
+        var ct = TestContext.Current.CancellationToken;
+        var options = S7_1500Profile.BuildOptions(sim.Host, sim.Port);
+        await using var drv = new S7Driver(options, driverInstanceId: "s7-connect-timeout-live");
+        await drv.InitializeAsync("{}", ct);
+
+        var sawBad = false;
+        var recoveredTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        drv.OnDataChange += (_, e) =>
+        {
+            if (e.Snapshot.StatusCode != 0u) sawBad = true;
+            else if (sawBad) recoveredTcs.TrySetResult();
+        };
+
+        await drv.SubscribeAsync([S7_1500Profile.ProbeTag], TimeSpan.FromMilliseconds(250), ct);
+
+        // Good baseline first — wait for at least one Good tick before the outage.
+        var baselineTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        void Baseline(object? _, DataChangeEventArgs e)
+        {
+            if (e.Snapshot.StatusCode == 0u) baselineTcs.TrySetResult();
+        }
+        drv.OnDataChange += Baseline;
+        await baselineTcs.Task.WaitAsync(TimeSpan.FromSeconds(15), ct);
+        drv.OnDataChange -= Baseline;
+
+        var blackholed = false;
+        try
+        {
+            // Begin dropping traffic — every reconnect attempt now TIMES OUT (SYN blackhole),
+            // the exact class STAB-14 killed. docker restart could not produce this.
+            await RunShellCommandAsync(startCmd!, ct);
+            blackholed = true;
+
+            // Observe at least one Bad tick during the outage.
+            var badDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+            while (!sawBad && DateTime.UtcNow < badDeadline)
+                await Task.Delay(TimeSpan.FromMilliseconds(250), ct);
+            sawBad.ShouldBeTrue("the subscription should have observed the blackhole outage as a Bad tick");
+        }
+        finally
+        {
+            if (blackholed)
+            {
+                // ALWAYS restore traffic once the blackhole started — never leave the shared sim dead.
+                await RunShellCommandAsync(stopCmd!, CancellationToken.None);
+            }
+        }
+
+        // The loop must have SURVIVED the timeout outage and recovered on the same subscription.
+        await recoveredTcs.Task.WaitAsync(TimeSpan.FromSeconds(90), ct);
+    }
+
+    private static async Task RunShellCommandAsync(string command, CancellationToken ct)
+    {
+        // Run through the login shell so an SSH/docker one-liner in the env var works verbatim.
+        using var proc = new Process
+        {
+            StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            },
+        };
+        proc.Start();
+        await proc.WaitForExitAsync(ct);
+        proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}");
+        // Give the network rule / pause a moment to take effect before the next poll tick.
+        await Task.Delay(TimeSpan.FromSeconds(1), ct);
+    }
+}
@@ -133,6 +133,301 @@ public sealed class S7DriverReconnectTests
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
    }

+    // ---- STAB-14: connect-timeout must degrade, not escape as OCE ----
+
+    /// <summary>
+    ///     T1 (STAB-14 regression). When the reopen HANGS and is cut by the internal
+    ///     <c>CancelAfter(_options.Timeout)</c> (the unreachable-host outage — SYNs dropped, not
+    ///     refused), the read batch must degrade to a communication error with Degraded health,
+    ///     NOT throw the timeout-born <see cref="TaskCanceledException"/> up to the caller.
+    ///     Before the fix the OCE escaped <see cref="S7Driver.ReadAsync"/>.
+    /// </summary>
+    [Fact]
+    public async Task Read_degrades_batch_on_connect_timeout_instead_of_throwing()
+    {
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(Options(), "s7-read-timeout", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        // Drop connection #1 so the next read reopens; every subsequent open hangs until the
+        // internal connect-timeout CTS fires.
+        factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped");
+        await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead
+        factory.OpenHangsUntilCancelledCount = int.MaxValue;
+
+        // Must NOT throw — the connect timeout degrades the batch.
+        var timedOut = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken);
+        timedOut[0].StatusCode.ShouldNotBe(0u);
+        drv.GetHealth().State.ShouldBe(DriverState.Degraded);
+    }
+
+    /// <summary>
+    ///     T3 (STAB-14 regression). The same connect-timeout degrade through
+    ///     <see cref="S7Driver.WriteAsync"/> — <c>EnsureConnectedAsync</c> is called before the
+    ///     per-item loop, so a hanging reopen must degrade the whole write batch, not throw OCE.
+    /// </summary>
+    [Fact]
+    public async Task Write_degrades_batch_on_connect_timeout_instead_of_throwing()
+    {
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(Options(), "s7-write-timeout", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped");
+        await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead
+        factory.OpenHangsUntilCancelledCount = int.MaxValue;
+
+        var timedOut = await drv.WriteAsync(
+            [new WriteRequest("W0", (ushort)1)], TestContext.Current.CancellationToken);
+        timedOut[0].StatusCode.ShouldNotBe(0u);
+        drv.GetHealth().State.ShouldBe(DriverState.Degraded);
+    }
+
+    /// <summary>
+    ///     T2 (STAB-14 regression). A subscription poll loop that ticks during an
+    ///     unreachable-host outage (each reconnect times out) must SURVIVE and recover: a Good
+    ///     <c>OnDataChange</c> must arrive after a Bad one once the host returns. Before the fix
+    ///     the timeout-born OCE escaped <c>ReadAsync</c>, the poll loop's bare
+    ///     <c>catch (OperationCanceledException) { return; }</c> read it as teardown, and the loop
+    ///     died permanently — no recovery event ever fired (10 s deadline trips).
+    /// </summary>
+    [Fact]
+    public async Task Subscription_poll_loop_survives_a_connect_timeout_outage_and_recovers()
+    {
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(Options(), "s7-poll-survive", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var sawBad = false;
+        drv.OnDataChange += (_, e) =>
+        {
+            if (e.Snapshot.StatusCode != 0u) sawBad = true;
+            else if (sawBad) tcs.TrySetResult();
+        };
+
+        // First poll tick drops the socket (marks dead); the next two reconnect attempts hang
+        // until the internal connect-timeout CTS fires; then opens succeed again.
+        factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped");
+        factory.OpenHangsUntilCancelledCount = 2;
+
+        await drv.SubscribeAsync(["W0"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
+
+        // Must observe a Good change AFTER a Bad one within the deadline — proves the loop lived.
+        await tcs.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
+        sawBad.ShouldBeTrue();
+    }
+
+    /// <summary>
+    ///     T4 (pinning). Real CALLER cancellation during a connect must still propagate as
+    ///     <see cref="OperationCanceledException"/> — the STAB-14 fix must not over-rotate into
+    ///     swallowing genuine cancellation. Passes before and after the fix.
+    /// </summary>
+    [Fact]
+    public async Task Read_still_propagates_caller_cancellation_during_connect()
+    {
+        var opts = new S7DriverOptions
+        {
+            Host = "192.0.2.1",
+            Timeout = TimeSpan.FromSeconds(5), // long, so the CALLER token fires first, not the internal timeout
+            Probe = new S7ProbeOptions { Enabled = false },
+            Tags = [new S7TagDefinition("W0", "DB1.DBW0", S7DataType.UInt16, Writable: false)],
+        };
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(opts, "s7-caller-cancel", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped");
+        await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead
+        factory.OpenHangsUntilCancelledCount = 1;
+
+        using var cts = new CancellationTokenSource();
+        cts.CancelAfter(50);
+        await Should.ThrowAsync<OperationCanceledException>(
+            async () => await drv.ReadAsync(["W0"], cts.Token));
+    }
+
+    // ---- STAB-15a: ISO-on-TCP framing/desync faults must reopen ----
+
+    /// <summary>
+    ///     T5 (STAB-15a). A framing/desync fault on ISO-on-TCP (S7.Net's
+    ///     <see cref="TPKTInvalidException"/> / <see cref="TPDUInvalidException"/> /
+    ///     <see cref="WrongNumberOfBytesException"/>, or a <see cref="PlcException"/> carrying
+    ///     <see cref="ErrorCode.WrongNumberReceivedBytes"/>) leaves the stream position
+    ///     untrustworthy — only a reopen recovers. The next read must dispose connection #1 and
+    ///     open a fresh #2. Before the fix these were not classified fatal, so the desynced handle
+    ///     was reused forever (<c>Created.Count</c> pinned at 1).
+    /// </summary>
+    [Theory]
+    [InlineData("tpkt")]
+    [InlineData("tpdu")]
+    [InlineData("wrongbytes")]
+    [InlineData("plc-wrongnumber")]
+    public async Task Framing_faults_are_classified_connection_fatal_and_reopen(string kind)
+    {
+        Exception framing = kind switch
+        {
+            "tpkt" => new TPKTInvalidException(),
+            "tpdu" => new TPDUInvalidException(),
+            "wrongbytes" => new WrongNumberOfBytesException(),
+            "plc-wrongnumber" => new PlcException(ErrorCode.WrongNumberReceivedBytes, "short read"),
+            _ => throw new ArgumentOutOfRangeException(nameof(kind)),
+        };
+
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(Options(), "s7-framing", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        // Read #1 hits the framing fault — Bad status, handle marked dead, but no reopen yet.
+        factory.Created[0].NextReadThrows = framing;
+        var bad = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken);
+        bad[0].StatusCode.ShouldNotBe(0u);
+        factory.Created.Count.ShouldBe(1);
+
+        // Read #2 reopens a fresh connection and reads Good.
+        var recovered = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken);
+        recovered[0].StatusCode.ShouldBe(0u);
+        factory.Created.Count.ShouldBe(2);
+        factory.Created[0].Disposed.ShouldBeTrue();
+    }
+
+    // ---- STAB-15a: cancellation observed mid-I/O must mark the handle dead ----
+
+    /// <summary>
+    ///     T6 (STAB-15a). Cancellation observed WHILE a read is in flight abandons a half-read
+    ///     ISO-on-TCP response on the single gated stream — the handle can't be reused. The OCE
+    ///     must propagate (caller cancelled; nobody consumes the batch) AND mark the handle dead so
+    ///     the next read reopens. Before the fix the read path converted the OCE to a Bad snapshot
+    ///     and kept the desynced handle.
+    /// </summary>
+    [Fact]
+    public async Task Cancelled_read_marks_handle_dead_and_next_read_reopens()
+    {
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(Options(), "s7-cancel-read", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        factory.Created[0].ReadHangsUntilCancelled = true;
+        using var cts = new CancellationTokenSource();
+        cts.CancelAfter(50);
+        await Should.ThrowAsync<OperationCanceledException>(
+            async () => await drv.ReadAsync(["W0"], cts.Token));
+
+        // Fresh-token read reopens a fresh connection.
+        var recovered = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken);
+        recovered[0].StatusCode.ShouldBe(0u);
+        factory.Created.Count.ShouldBe(2);
+        factory.Created[0].Disposed.ShouldBeTrue();
+    }
+
+    /// <summary>
+    ///     T7 (STAB-15a). Cancellation observed while a WRITE is in flight likewise marks the
+    ///     handle dead before propagating (the OCE already propagated pre-fix; the reopen is what
+    ///     was missing).
+    /// </summary>
+    [Fact]
+    public async Task Cancelled_write_marks_handle_dead_and_next_read_reopens()
+    {
+        var opts = new S7DriverOptions
+        {
+            Host = "192.0.2.1",
+            Timeout = TimeSpan.FromMilliseconds(250),
+            Probe = new S7ProbeOptions { Enabled = false },
+            Tags = [new S7TagDefinition("WW0", "DB1.DBW0", S7DataType.UInt16, Writable: true)],
+        };
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(opts, "s7-cancel-write", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        factory.Created[0].WriteHangsUntilCancelled = true;
+        using var cts = new CancellationTokenSource();
+        cts.CancelAfter(50);
+        await Should.ThrowAsync<OperationCanceledException>(
+            async () => await drv.WriteAsync([new WriteRequest("WW0", (ushort)1)], cts.Token));
+
+        var recovered = await drv.ReadAsync(["WW0"], TestContext.Current.CancellationToken);
+        recovered[0].StatusCode.ShouldBe(0u);
+        factory.Created.Count.ShouldBe(2);
+        factory.Created[0].Disposed.ShouldBeTrue();
+    }
+
+    // ---- STAB-15b: the probe must mark a lying/hung handle dead so the next tick reopens ----
+
+    /// <summary>
+    ///     T8 (STAB-15b). A probe failure on a handle whose <c>TcpClient.Connected</c> still lies
+    ///     true (a wire-dead or desynced socket) must mark the handle dead under the gate, so the
+    ///     NEXT probe tick's <c>EnsureConnectedAsync</c> disposes it and reopens. Before the fix the
+    ///     probe swallowed the failure without marking dead, so the lying handle was re-probed
+    ///     forever (<c>Created.Count</c> pinned at 1).
+    /// </summary>
+    [Fact]
+    public async Task Probe_failure_on_connected_socket_marks_dead_and_next_tick_reopens()
+    {
+        var opts = ProbeEnabledOptions();
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(opts, "s7-probe-fail", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        // Arm connection #1 to fail EVERY probe with a connection-fatal fault (persistent). Later
+        // connections stay healthy (default ReadStatusThrows == null).
+        factory.Created[0].ReadStatusThrows = new System.Net.Sockets.SocketException();
+
+        await WaitUntilAsync(() => factory.Created.Count >= 2, TimeSpan.FromSeconds(5));
+        factory.Created.Count.ShouldBeGreaterThanOrEqualTo(2);
+        factory.Created[0].Disposed.ShouldBeTrue();
+
+        await WaitUntilAsync(() => drv.GetHostStatuses()[0].State == HostState.Running, TimeSpan.FromSeconds(5));
+        drv.GetHostStatuses()[0].State.ShouldBe(HostState.Running);
+    }
+
+    /// <summary>
+    ///     T9 (STAB-15b). A probe that TIMES OUT mid-<c>ReadStatusAsync</c> (the CPU-status PDU may
+    ///     still be in flight ⇒ desync) likewise marks the handle dead under the gate so the next
+    ///     tick reopens. Before the fix the probe-deadline OCE fell to the bare swallow and the
+    ///     handle was reused forever.
+    /// </summary>
+    [Fact]
+    public async Task Probe_timeout_marks_dead()
+    {
+        var opts = ProbeEnabledOptions();
+        var factory = new FakeS7PlcFactory();
+        using var drv = new S7Driver(opts, "s7-probe-timeout", factory);
+        await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
+
+        factory.Created[0].ReadStatusHangsUntilCancelled = true;
+
+        await WaitUntilAsync(() => factory.Created.Count >= 2, TimeSpan.FromSeconds(5));
+        factory.Created.Count.ShouldBeGreaterThanOrEqualTo(2);
+        factory.Created[0].Disposed.ShouldBeTrue();
+
+        await WaitUntilAsync(() => drv.GetHostStatuses()[0].State == HostState.Running, TimeSpan.FromSeconds(5));
+        drv.GetHostStatuses()[0].State.ShouldBe(HostState.Running);
+    }
+
+    private static S7DriverOptions ProbeEnabledOptions() => new()
+    {
+        Host = "192.0.2.1",
+        Timeout = TimeSpan.FromMilliseconds(250),
+        Probe = new S7ProbeOptions
+        {
+            Enabled = true,
+            Interval = TimeSpan.FromMilliseconds(50),
+            Timeout = TimeSpan.FromMilliseconds(250),
+        },
+        Tags = [new S7TagDefinition("W0", "DB1.DBW0", S7DataType.UInt16, Writable: false)],
+    };
+
+    /// <summary>Polls <paramref name="condition"/> until it holds or <paramref name="timeout"/> elapses (returns either way).</summary>
+    private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
+    {
+        var deadline = DateTime.UtcNow + timeout;
+        while (DateTime.UtcNow < deadline)
+        {
+            if (condition()) return;
+            await Task.Delay(25).ConfigureAwait(false);
+        }
+    }
+
    // ---- fakes ----

    private sealed class FakeS7PlcFactory : IS7PlcFactory
@@ -143,6 +438,21 @@ public sealed class S7DriverReconnectTests
        /// <summary>When set, the NEXT <see cref="Create"/>d connection throws this on OpenAsync (once).</summary>
        public Exception? NextOpenThrows { get; set; }

+        /// <summary>
+        ///     Number of subsequently-created connections whose <see cref="FakeS7Plc.OpenAsync"/>
+        ///     hangs until its token is cancelled (simulating the unreachable-but-not-refusing host
+        ///     — pulled cable / firewall DROP — that STAB-14 needs and <c>docker restart</c> can't
+        ///     produce). Each <see cref="Create"/> decrements it while positive.
+        /// </summary>
+        public int OpenHangsUntilCancelledCount { get; set; }
+
+        /// <summary>
+        ///     When set, copied onto every created connection's persistent
+        ///     <see cref="FakeS7Plc.ReadStatusThrows"/> so a probe fails on every tick. Clear it
+        ///     after init to arm connection #1 only.
+        /// </summary>
+        public Exception? ReadStatusThrowsForNewConnections { get; set; }
+
        public IS7Plc Create(S7CpuType cpuType, string host, int port, short rack, short slot, TimeSpan timeout)
        {
            var plc = new FakeS7Plc();
@@ -151,6 +461,13 @@ public sealed class S7DriverReconnectTests
                plc.OpenThrowsOnce = ex;
                NextOpenThrows = null;
            }
+            if (OpenHangsUntilCancelledCount > 0)
+            {
+                plc.OpenHangsUntilCancelled = true;
+                OpenHangsUntilCancelledCount--;
+            }
+            if (ReadStatusThrowsForNewConnections is { } rex)
+                plc.ReadStatusThrows = rex;
            Created.Add(plc);
            return plc;
        }
@@ -164,45 +481,76 @@ public sealed class S7DriverReconnectTests
        /// <summary>Thrown once by <see cref="OpenAsync"/>, then cleared.</summary>
        public Exception? OpenThrowsOnce { get; set; }

+        /// <summary>
+        ///     When true, <see cref="OpenAsync"/> hangs until its token is cancelled, then throws
+        ///     <see cref="TaskCanceledException"/> — the connect-timeout shape (an unreachable host
+        ///     whose SYNs are dropped) the live rig can't cheaply produce (STAB-14).
+        /// </summary>
+        public bool OpenHangsUntilCancelled { get; set; }
+
        /// <summary>Thrown once by the next <see cref="ReadAsync"/>, then cleared.</summary>
        public Exception? NextReadThrows { get; set; }

+        /// <summary>When true, <see cref="ReadAsync"/> hangs until its token is cancelled (STAB-15 cancel-mid-I/O).</summary>
+        public bool ReadHangsUntilCancelled { get; set; }
+
+        /// <summary>When true, <see cref="WriteAsync"/> hangs until its token is cancelled (STAB-15 cancel-mid-I/O).</summary>
+        public bool WriteHangsUntilCancelled { get; set; }
+
+        /// <summary>Thrown by EVERY <see cref="ReadStatusAsync"/> (persistent, not one-shot) — arms a probe fault (STAB-15b).</summary>
+        public Exception? ReadStatusThrows { get; set; }
+
+        /// <summary>When true, <see cref="ReadStatusAsync"/> hangs until its token is cancelled (probe-timeout, STAB-15b).</summary>
+        public bool ReadStatusHangsUntilCancelled { get; set; }
+
        /// <summary>Boxed value returned by a good read (UInt16 tag → ushort).</summary>
        public object? ReadValue { get; set; } = (ushort)42;

-        public Task OpenAsync(CancellationToken cancellationToken)
+        public async Task OpenAsync(CancellationToken cancellationToken)
        {
+            if (OpenHangsUntilCancelled)
+                await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false);
            if (OpenThrowsOnce is { } ex)
            {
                OpenThrowsOnce = null;
-                return Task.FromException(ex);
+                throw ex;
            }
            IsConnected = true;
-            return Task.CompletedTask;
        }

        public void Close() => IsConnected = false;

-        public Task<object?> ReadAsync(string address, CancellationToken cancellationToken)
+        public async Task<object?> ReadAsync(string address, CancellationToken cancellationToken)
        {
+            if (ReadHangsUntilCancelled)
+                await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false);
            if (NextReadThrows is { } ex)
            {
                NextReadThrows = null;
-                return Task.FromException<object?>(ex);
+                throw ex;
            }
-            return Task.FromResult(ReadValue);
+            return ReadValue;
        }

        public Task<byte[]> ReadBytesAsync(S7NetDataType area, int db, int startByteAdr, int count, CancellationToken cancellationToken) =>
            throw new NotSupportedException("FakeS7Plc: block reads not needed for the reconnect tests");

-        public Task WriteAsync(string address, object value, CancellationToken cancellationToken) =>
-            Task.CompletedTask;
+        public async Task WriteAsync(string address, object value, CancellationToken cancellationToken)
+        {
+            if (WriteHangsUntilCancelled)
+                await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false);
+        }

        public Task WriteBytesAsync(S7NetDataType area, int db, int startByteAdr, byte[] value, CancellationToken cancellationToken) =>
            Task.CompletedTask;

-        public Task ReadStatusAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+        public async Task ReadStatusAsync(CancellationToken cancellationToken)
+        {
+            if (ReadStatusHangsUntilCancelled)
+                await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false);
+            if (ReadStatusThrows is { } ex)
+                throw ex;
+        }

        public void Dispose()
        {