diff --git a/archreview/plans/R2-01-s7-fault-hardening-plan.md.tasks.json b/archreview/plans/R2-01-s7-fault-hardening-plan.md.tasks.json index 7070e3ab..93f78ee9 100644 --- a/archreview/plans/R2-01-s7-fault-hardening-plan.md.tasks.json +++ b/archreview/plans/R2-01-s7-fault-hardening-plan.md.tasks.json @@ -70,7 +70,8 @@ { "id": 11, "subject": "Env-gated live connect-timeout outage test (S7_TIMEOUT_OUTAGE_START_CMD/STOP_CMD blackhole design; skips cleanly offline)", - "status": "pending", + "status": "deferred-live", + "note": "env-gated; runs in serial live pass (integration suite; serialized heavy pass). Test authored + committed; verified it SKIPs cleanly offline.", "blockedBy": [5] }, { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests/S7_1500/S7_1500ConnectTimeoutOutageTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests/S7_1500/S7_1500ConnectTimeoutOutageTests.cs new file mode 100644 index 00000000..07793557 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests/S7_1500/S7_1500ConnectTimeoutOutageTests.cs @@ -0,0 +1,138 @@ +using System.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests.S7_1500; + +/// +/// Live gate for STAB-14 — the connect-TIMEOUT outage class the ordinary bounce test +/// () structurally cannot produce. A docker restart +/// yields connection-refused (an immediate SocketException); STAB-14 only fires +/// on connect-timeout — the SYN-blackhole shape (packets DROPPED, not rejected) of a +/// pulled cable / powered-off PLC / firewall DROP. Before the fix, a subscription poll loop +/// that ticked during such an outage was silently and permanently killed (the timeout-born OCE +/// read as teardown); this test would hang at the recovery assertion. +/// +/// +/// +/// Triple-gated so it never runs (or leaves the shared sim blackholed) by accident: +/// +/// — the sim must be reachable; +/// S7_TIMEOUT_OUTAGE_START_CMD — begins DROPPING traffic to the sim port +/// (a blackhole, not a reset); and +/// S7_TIMEOUT_OUTAGE_STOP_CMD — removes the block. +/// +/// Any absent gate ⇒ clean (safe offline on macOS). A +/// try/finally ALWAYS runs the stop command once the start succeeded, so a failure +/// mid-test never leaves the shared sim unreachable for other suites. +/// +/// +/// Operator run recipe (also drives 10.100.0.35 via lmxopcua-fix up s7 s7_1500): +/// +/// lmxopcua-fix up s7 s7_1500 +/// # docker pause = ACKless blackhole for established flows + unanswered SYNs for new connects: +/// export S7_TIMEOUT_OUTAGE_START_CMD='ssh dohertj2@10.100.0.35 "docker pause otopcua-python-snap7-s7_1500"' +/// export S7_TIMEOUT_OUTAGE_STOP_CMD='ssh dohertj2@10.100.0.35 "docker unpause otopcua-python-snap7-s7_1500"' +/// # iptables DROP variant (more faithful to a firewall blackhole): +/// # START: ssh dohertj2@10.100.0.35 "sudo iptables -I INPUT -p tcp --dport 1102 -j DROP" +/// # STOP: ssh dohertj2@10.100.0.35 "sudo iptables -D INPUT -p tcp --dport 1102 -j DROP" +/// dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests --filter "FullyQualifiedName~ConnectTimeoutOutage" +/// +/// +/// +[Collection(Snap7ServerCollection.Name)] +[Trait("Category", "Integration")] +[Trait("Category", "Reconnect")] +[Trait("Device", "S7_1500")] +public sealed class S7_1500ConnectTimeoutOutageTests(Snap7ServerFixture sim) +{ + private const string StartCmdEnvVar = "S7_TIMEOUT_OUTAGE_START_CMD"; + private const string StopCmdEnvVar = "S7_TIMEOUT_OUTAGE_STOP_CMD"; + + /// + /// Subscribes against the live sim, blackholes its traffic (each reconnect now TIMES OUT + /// rather than being refused), observes at least one Bad tick, then restores traffic and + /// asserts a Good OnDataChange resumes on the SAME subscription — the poll loop + /// survived the connect-timeout outage. + /// + [Fact] + public async Task Subscription_survives_a_live_connect_timeout_outage_ConnectTimeoutOutage() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + var startCmd = Environment.GetEnvironmentVariable(StartCmdEnvVar); + var stopCmd = Environment.GetEnvironmentVariable(StopCmdEnvVar); + if (string.IsNullOrWhiteSpace(startCmd) || string.IsNullOrWhiteSpace(stopCmd)) + Assert.Skip($"Set both {StartCmdEnvVar} and {StopCmdEnvVar} (SYN-blackhole start/stop) to run this destructive live test."); + + var ct = TestContext.Current.CancellationToken; + var options = S7_1500Profile.BuildOptions(sim.Host, sim.Port); + await using var drv = new S7Driver(options, driverInstanceId: "s7-connect-timeout-live"); + await drv.InitializeAsync("{}", ct); + + var sawBad = false; + var recoveredTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + drv.OnDataChange += (_, e) => + { + if (e.Snapshot.StatusCode != 0u) sawBad = true; + else if (sawBad) recoveredTcs.TrySetResult(); + }; + + await drv.SubscribeAsync([S7_1500Profile.ProbeTag], TimeSpan.FromMilliseconds(250), ct); + + // Good baseline first — wait for at least one Good tick before the outage. + var baselineTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + void Baseline(object? _, DataChangeEventArgs e) + { + if (e.Snapshot.StatusCode == 0u) baselineTcs.TrySetResult(); + } + drv.OnDataChange += Baseline; + await baselineTcs.Task.WaitAsync(TimeSpan.FromSeconds(15), ct); + drv.OnDataChange -= Baseline; + + var blackholed = false; + try + { + // Begin dropping traffic — every reconnect attempt now TIMES OUT (SYN blackhole), + // the exact class STAB-14 killed. docker restart could not produce this. + await RunShellCommandAsync(startCmd!, ct); + blackholed = true; + + // Observe at least one Bad tick during the outage. + var badDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!sawBad && DateTime.UtcNow < badDeadline) + await Task.Delay(TimeSpan.FromMilliseconds(250), ct); + sawBad.ShouldBeTrue("the subscription should have observed the blackhole outage as a Bad tick"); + } + finally + { + if (blackholed) + { + // ALWAYS restore traffic once the blackhole started — never leave the shared sim dead. + await RunShellCommandAsync(stopCmd!, CancellationToken.None); + } + } + + // The loop must have SURVIVED the timeout outage and recovered on the same subscription. + await recoveredTcs.Task.WaitAsync(TimeSpan.FromSeconds(90), ct); + } + + private static async Task RunShellCommandAsync(string command, CancellationToken ct) + { + // Run through the login shell so an SSH/docker one-liner in the env var works verbatim. + using var proc = new Process + { + StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }, + }; + proc.Start(); + await proc.WaitForExitAsync(ct); + proc.ExitCode.ShouldBe(0, $"outage command failed: {await proc.StandardError.ReadToEndAsync(ct)}"); + // Give the network rule / pause a moment to take effect before the next poll tick. + await Task.Delay(TimeSpan.FromSeconds(1), ct); + } +}