test(r2-01): env-gated live connect-timeout outage test — the STAB-14 outage class the bounce test cannot produce (task 11)

archreview/plans/R2-01-s7-fault-hardening-plan.md.tasks.json

@@ -70,7 +70,8 @@
     {
       "id": 11,
       "subject": "Env-gated live connect-timeout outage test (S7_TIMEOUT_OUTAGE_START_CMD/STOP_CMD blackhole design; skips cleanly offline)",
-      "status": "pending",
+      "status": "deferred-live",
+      "note": "env-gated; runs in serial live pass (integration suite; serialized heavy pass). Test authored + committed; verified it SKIPs cleanly offline.",
       "blockedBy": [5]
     },
     {

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests/S7_1500/S7_1500ConnectTimeoutOutageTests.cs

@@ -0,0 +1,138 @@
+using System.Diagnostics;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests.S7_1500;
+
+/// <summary>
+///     Live gate for STAB-14 — the connect-TIMEOUT outage class the ordinary bounce test
+///     (<see cref="S7_1500ReconnectTests"/>) structurally cannot produce. A <c>docker restart</c>
+///     yields connection-<i>refused</i> (an immediate <c>SocketException</c>); STAB-14 only fires
+///     on connect-<i>timeout</i> — the SYN-blackhole shape (packets DROPPED, not rejected) of a
+///     pulled cable / powered-off PLC / firewall DROP. Before the fix, a subscription poll loop
+///     that ticked during such an outage was silently and permanently killed (the timeout-born OCE
+///     read as teardown); this test would hang at the recovery assertion.
+/// </summary>
+/// <remarks>
+///     <para>
+///         Triple-gated so it never runs (or leaves the shared sim blackholed) by accident:
+///         <list type="number">
+///             <item><see cref="Snap7ServerFixture.SkipReason"/> — the sim must be reachable;</item>
+///             <item><c>S7_TIMEOUT_OUTAGE_START_CMD</c> — begins DROPPING traffic to the sim port
+///                 (a blackhole, not a reset); and</item>
+///             <item><c>S7_TIMEOUT_OUTAGE_STOP_CMD</c> — removes the block.</item>
+///         </list>
+///         Any absent gate ⇒ clean <see cref="Assert.Skip(string)"/> (safe offline on macOS). A
+///         <c>try/finally</c> ALWAYS runs the stop command once the start succeeded, so a failure
+///         mid-test never leaves the shared sim unreachable for other suites.
+///     </para>
+///     <para>
+///         Operator run recipe (also drives 10.100.0.35 via <c>lmxopcua-fix up s7 s7_1500</c>):
+///         <code>
+///         lmxopcua-fix up s7 s7_1500
+///         # docker pause = ACKless blackhole for established flows + unanswered SYNs for new connects:
+///         export S7_TIMEOUT_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-python-snap7-s7_1500"'
+///         export S7_TIMEOUT_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-python-snap7-s7_1500"'
+///         # iptables DROP variant (more faithful to a firewall blackhole):
+///         #   START: ssh dohertj2@10.100.0.35 "sudo iptables -I INPUT -p tcp --dport 1102 -j DROP"
+///         #   STOP:  ssh dohertj2@10.100.0.35 "sudo iptables -D INPUT -p tcp --dport 1102 -j DROP"
+///         dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests --filter "FullyQualifiedName~ConnectTimeoutOutage"
+///         </code>
+///     </para>
+/// </remarks>
+[Collection(Snap7ServerCollection.Name)]
+[Trait("Category", "Integration")]
+[Trait("Category", "Reconnect")]
+[Trait("Device", "S7_1500")]
+public sealed class S7_1500ConnectTimeoutOutageTests(Snap7ServerFixture sim)
+{
+    private const string StartCmdEnvVar = "S7_TIMEOUT_OUTAGE_START_CMD";
+    private const string StopCmdEnvVar = "S7_TIMEOUT_OUTAGE_STOP_CMD";
+
+    /// <summary>
+    ///     Subscribes against the live sim, blackholes its traffic (each reconnect now TIMES OUT
+    ///     rather than being refused), observes at least one Bad tick, then restores traffic and
+    ///     asserts a Good <c>OnDataChange</c> resumes on the SAME subscription — the poll loop
+    ///     survived the connect-timeout outage.
+    /// </summary>
+    [Fact]
+    public async Task Subscription_survives_a_live_connect_timeout_outage_ConnectTimeoutOutage()
+    {
+        if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
+        var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar);
+        var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar);
+        if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd))
+            Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (SYN-blackhole start/stop) to run this destructive live test.");
+
+        var ct = TestContext.Current.CancellationToken;
+        var options = S7_1500Profile.BuildOptions(sim.Host, sim.Port);
+        await using var drv = new S7Driver(options, driverInstanceId: "s7-connect-timeout-live");
+        await drv.InitializeAsync("{}", ct);
+
+        var sawBad = false;
+        var recoveredTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        drv.OnDataChange += (_, e) =>
+        {
+            if (e.Snapshot.StatusCode != 0u) sawBad = true;
+            else if (sawBad) recoveredTcs.TrySetResult();
+        };
+
+        await drv.SubscribeAsync([S7_1500Profile.ProbeTag], TimeSpan.FromMilliseconds(250), ct);
+
+        // Good baseline first — wait for at least one Good tick before the outage.
+        var baselineTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        void Baseline(object? _, DataChangeEventArgs e)
+        {
+            if (e.Snapshot.StatusCode == 0u) baselineTcs.TrySetResult();
+        }
+        drv.OnDataChange += Baseline;
+        await baselineTcs.Task.WaitAsync(TimeSpan.FromSeconds(15), ct);
+        drv.OnDataChange -= Baseline;
+
+        var blackholed = false;
+        try
+        {
+            // Begin dropping traffic — every reconnect attempt now TIMES OUT (SYN blackhole),
+            // the exact class STAB-14 killed. docker restart could not produce this.
+            await RunShellCommandAsync(startCmd!, ct);
+            blackholed = true;
+
+            // Observe at least one Bad tick during the outage.
+            var badDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+            while (!sawBad && DateTime.UtcNow < badDeadline)
+                await Task.Delay(TimeSpan.FromMilliseconds(250), ct);
+            sawBad.ShouldBeTrue("the subscription should have observed the blackhole outage as a Bad tick");
+        }
+        finally
+        {
+            if (blackholed)
+            {
+                // ALWAYS restore traffic once the blackhole started — never leave the shared sim dead.
+                await RunShellCommandAsync(stopCmd!, CancellationToken.None);
+            }
+        }
+
+        // The loop must have SURVIVED the timeout outage and recovered on the same subscription.
+        await recoveredTcs.Task.WaitAsync(TimeSpan.FromSeconds(90), ct);
+    }
+
+    private static async Task RunShellCommandAsync(string command, CancellationToken ct)
+    {
+        // Run through the login shell so an SSH/docker one-liner in the env var works verbatim.
+        using var proc = new Process
+        {
+            StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            },
+        };
+        proc.Start();
+        await proc.WaitForExitAsync(ct);
+        proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}");
+        // Give the network rule / pause a moment to take effect before the next poll tick.
+        await Task.Delay(TimeSpan.FromSeconds(1), ct);
+    }
+}