using S7.Net; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using S7NetDataType = global::S7.Net.DataType; namespace ZB.MOM.WW.OtOpcUa.Driver.S7.Tests; /// /// Unit tests for the S7 reconnect path (STAB-1). The seam lets /// these exercise the lazy reconnect logic — dispose-dead-handle + /// re-open on the next data call — without a live PLC, which was previously impossible (the /// driver new-ed a concrete S7.Net.Plc inline and had no reopen path at all). /// [Trait("Category", "Unit")] public sealed class S7DriverReconnectTests { private static S7DriverOptions Options() => new() { Host = "192.0.2.1", Timeout = TimeSpan.FromMilliseconds(250), // Probe off — these tests drive reconnect through the read path deterministically; a // background probe would race the created-connection assertions. Probe = new S7ProbeOptions { Enabled = false }, Tags = [new S7TagDefinition("W0", "DB1.DBW0", S7DataType.UInt16, Writable: false)], }; /// /// A connection-fatal fault on a read marks the handle dead; the NEXT read disposes it and /// opens a fresh connection, and the tag recovers to Good with health Degraded→Healthy. /// [Fact] public async Task Read_reopens_a_fresh_connection_after_a_connection_fatal_fault() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-reconnect", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created.Count.ShouldBe(1); // Read #1 — good value, Healthy. var r1 = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); r1[0].StatusCode.ShouldBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Healthy); // The live socket drops on its next read. IsConnected deliberately stays true (a stale // socket that hasn't noticed the drop) so the ONLY trigger for reopen is the driver's // own dead-handle flag set by the connection-fatal classification. factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "socket dropped"); // Read #2 — connection-fatal fault → Degraded, handle marked dead, but no reopen yet. var r2 = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); r2[0].StatusCode.ShouldNotBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Degraded); factory.Created.Count.ShouldBe(1); // Read #3 — EnsureConnectedAsync disposes the dead handle and opens a SECOND connection, // which reads Good; health recovers to Healthy. var r3 = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); factory.Created.Count.ShouldBe(2); r3[0].StatusCode.ShouldBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Healthy); factory.Created[0].Disposed.ShouldBeTrue(); factory.Created[1].Disposed.ShouldBeFalse(); } /// /// A protocol / data-address error (S7.Net ) is NOT a /// connection loss — the driver keeps the same connection and does not churn a reopen. /// [Fact] public async Task Read_does_not_reopen_on_a_data_address_error() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-dataerr", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // A data error — not a socket drop. factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ReadData, "bad address"); var bad = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); bad[0].StatusCode.ShouldNotBe(0u); // Next read reuses the SAME connection — no reopen for a data error. var ok = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); ok[0].StatusCode.ShouldBe(0u); factory.Created.Count.ShouldBe(1); } /// /// A raw socket exception (no S7.Net wrapper) is also classified connection-fatal and /// triggers a reopen — the driver doesn't depend on every drop arriving as a /// . /// [Fact] public async Task Read_reopens_after_a_raw_socket_exception() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-socket", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].NextReadThrows = new System.Net.Sockets.SocketException(); await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); drv.GetHealth().State.ShouldBe(DriverState.Degraded); var recovered = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); recovered[0].StatusCode.ShouldBe(0u); factory.Created.Count.ShouldBe(2); } /// /// When the reopen itself fails (PLC still down), the batch degrades to a communication /// error rather than throwing, and a later successful reopen recovers the tag. /// [Fact] public async Task Read_degrades_when_reopen_fails_then_recovers() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-reopen-fail", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // Drop the live socket, and make the NEXT-created connection fail to open. factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped"); factory.NextOpenThrows = new PlcException(ErrorCode.ConnectionError, "still down"); await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead // Reopen attempt #1 fails — whole batch is a comm error, no throw. var down = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); down[0].StatusCode.ShouldNotBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Degraded); // PLC comes back; the next call reopens cleanly and the tag reads Good. var up = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); up[0].StatusCode.ShouldBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Healthy); } // ---- STAB-14: connect-timeout must degrade, not escape as OCE ---- /// /// T1 (STAB-14 regression). When the reopen HANGS and is cut by the internal /// CancelAfter(_options.Timeout) (the unreachable-host outage — SYNs dropped, not /// refused), the read batch must degrade to a communication error with Degraded health, /// NOT throw the timeout-born up to the caller. /// Before the fix the OCE escaped . /// [Fact] public async Task Read_degrades_batch_on_connect_timeout_instead_of_throwing() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-read-timeout", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // Drop connection #1 so the next read reopens; every subsequent open hangs until the // internal connect-timeout CTS fires. factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped"); await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead factory.OpenHangsUntilCancelledCount = int.MaxValue; // Must NOT throw — the connect timeout degrades the batch. var timedOut = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); timedOut[0].StatusCode.ShouldNotBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Degraded); } /// /// T3 (STAB-14 regression). The same connect-timeout degrade through /// EnsureConnectedAsync is called before the /// per-item loop, so a hanging reopen must degrade the whole write batch, not throw OCE. /// [Fact] public async Task Write_degrades_batch_on_connect_timeout_instead_of_throwing() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-write-timeout", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped"); await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead factory.OpenHangsUntilCancelledCount = int.MaxValue; var timedOut = await drv.WriteAsync( [new WriteRequest("W0", (ushort)1)], TestContext.Current.CancellationToken); timedOut[0].StatusCode.ShouldNotBe(0u); drv.GetHealth().State.ShouldBe(DriverState.Degraded); } /// /// T2 (STAB-14 regression). A subscription poll loop that ticks during an /// unreachable-host outage (each reconnect times out) must SURVIVE and recover: a Good /// OnDataChange must arrive after a Bad one once the host returns. Before the fix /// the timeout-born OCE escaped ReadAsync, the poll loop's bare /// catch (OperationCanceledException) { return; } read it as teardown, and the loop /// died permanently — no recovery event ever fired (10 s deadline trips). /// [Fact] public async Task Subscription_poll_loop_survives_a_connect_timeout_outage_and_recovers() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-poll-survive", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var sawBad = false; drv.OnDataChange += (_, e) => { if (e.Snapshot.StatusCode != 0u) sawBad = true; else if (sawBad) tcs.TrySetResult(); }; // First poll tick drops the socket (marks dead); the next two reconnect attempts hang // until the internal connect-timeout CTS fires; then opens succeed again. factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped"); factory.OpenHangsUntilCancelledCount = 2; await drv.SubscribeAsync(["W0"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // Must observe a Good change AFTER a Bad one within the deadline — proves the loop lived. await tcs.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); sawBad.ShouldBeTrue(); } /// /// T4 (pinning). Real CALLER cancellation during a connect must still propagate as /// — the STAB-14 fix must not over-rotate into /// swallowing genuine cancellation. Passes before and after the fix. /// [Fact] public async Task Read_still_propagates_caller_cancellation_during_connect() { var opts = new S7DriverOptions { Host = "192.0.2.1", Timeout = TimeSpan.FromSeconds(5), // long, so the CALLER token fires first, not the internal timeout Probe = new S7ProbeOptions { Enabled = false }, Tags = [new S7TagDefinition("W0", "DB1.DBW0", S7DataType.UInt16, Writable: false)], }; var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(opts, "s7-caller-cancel", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].NextReadThrows = new PlcException(ErrorCode.ConnectionError, "dropped"); await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); // marks dead factory.OpenHangsUntilCancelledCount = 1; using var cts = new CancellationTokenSource(); cts.CancelAfter(50); await Should.ThrowAsync( async () => await drv.ReadAsync(["W0"], cts.Token)); } // ---- STAB-15a: ISO-on-TCP framing/desync faults must reopen ---- /// /// T5 (STAB-15a). A framing/desync fault on ISO-on-TCP (S7.Net's /// / / /// , or a carrying /// ) leaves the stream position /// untrustworthy — only a reopen recovers. The next read must dispose connection #1 and /// open a fresh #2. Before the fix these were not classified fatal, so the desynced handle /// was reused forever (Created.Count pinned at 1). /// [Theory] [InlineData("tpkt")] [InlineData("tpdu")] [InlineData("wrongbytes")] [InlineData("plc-wrongnumber")] public async Task Framing_faults_are_classified_connection_fatal_and_reopen(string kind) { Exception framing = kind switch { "tpkt" => new TPKTInvalidException(), "tpdu" => new TPDUInvalidException(), "wrongbytes" => new WrongNumberOfBytesException(), "plc-wrongnumber" => new PlcException(ErrorCode.WrongNumberReceivedBytes, "short read"), _ => throw new ArgumentOutOfRangeException(nameof(kind)), }; var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-framing", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // Read #1 hits the framing fault — Bad status, handle marked dead, but no reopen yet. factory.Created[0].NextReadThrows = framing; var bad = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); bad[0].StatusCode.ShouldNotBe(0u); factory.Created.Count.ShouldBe(1); // Read #2 reopens a fresh connection and reads Good. var recovered = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); recovered[0].StatusCode.ShouldBe(0u); factory.Created.Count.ShouldBe(2); factory.Created[0].Disposed.ShouldBeTrue(); } // ---- STAB-15a: cancellation observed mid-I/O must mark the handle dead ---- /// /// T6 (STAB-15a). Cancellation observed WHILE a read is in flight abandons a half-read /// ISO-on-TCP response on the single gated stream — the handle can't be reused. The OCE /// must propagate (caller cancelled; nobody consumes the batch) AND mark the handle dead so /// the next read reopens. Before the fix the read path converted the OCE to a Bad snapshot /// and kept the desynced handle. /// [Fact] public async Task Cancelled_read_marks_handle_dead_and_next_read_reopens() { var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(Options(), "s7-cancel-read", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].ReadHangsUntilCancelled = true; using var cts = new CancellationTokenSource(); cts.CancelAfter(50); await Should.ThrowAsync( async () => await drv.ReadAsync(["W0"], cts.Token)); // Fresh-token read reopens a fresh connection. var recovered = await drv.ReadAsync(["W0"], TestContext.Current.CancellationToken); recovered[0].StatusCode.ShouldBe(0u); factory.Created.Count.ShouldBe(2); factory.Created[0].Disposed.ShouldBeTrue(); } /// /// T7 (STAB-15a). Cancellation observed while a WRITE is in flight likewise marks the /// handle dead before propagating (the OCE already propagated pre-fix; the reopen is what /// was missing). /// [Fact] public async Task Cancelled_write_marks_handle_dead_and_next_read_reopens() { var opts = new S7DriverOptions { Host = "192.0.2.1", Timeout = TimeSpan.FromMilliseconds(250), Probe = new S7ProbeOptions { Enabled = false }, Tags = [new S7TagDefinition("WW0", "DB1.DBW0", S7DataType.UInt16, Writable: true)], }; var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(opts, "s7-cancel-write", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].WriteHangsUntilCancelled = true; using var cts = new CancellationTokenSource(); cts.CancelAfter(50); await Should.ThrowAsync( async () => await drv.WriteAsync([new WriteRequest("WW0", (ushort)1)], cts.Token)); var recovered = await drv.ReadAsync(["WW0"], TestContext.Current.CancellationToken); recovered[0].StatusCode.ShouldBe(0u); factory.Created.Count.ShouldBe(2); factory.Created[0].Disposed.ShouldBeTrue(); } // ---- STAB-15b: the probe must mark a lying/hung handle dead so the next tick reopens ---- /// /// T8 (STAB-15b). A probe failure on a handle whose TcpClient.Connected still lies /// true (a wire-dead or desynced socket) must mark the handle dead under the gate, so the /// NEXT probe tick's EnsureConnectedAsync disposes it and reopens. Before the fix the /// probe swallowed the failure without marking dead, so the lying handle was re-probed /// forever (Created.Count pinned at 1). /// [Fact] public async Task Probe_failure_on_connected_socket_marks_dead_and_next_tick_reopens() { var opts = ProbeEnabledOptions(); var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(opts, "s7-probe-fail", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // Arm connection #1 to fail EVERY probe with a connection-fatal fault (persistent). Later // connections stay healthy (default ReadStatusThrows == null). factory.Created[0].ReadStatusThrows = new System.Net.Sockets.SocketException(); await WaitUntilAsync(() => factory.Created.Count >= 2, TimeSpan.FromSeconds(5)); factory.Created.Count.ShouldBeGreaterThanOrEqualTo(2); factory.Created[0].Disposed.ShouldBeTrue(); await WaitUntilAsync(() => drv.GetHostStatuses()[0].State == HostState.Running, TimeSpan.FromSeconds(5)); drv.GetHostStatuses()[0].State.ShouldBe(HostState.Running); } /// /// T9 (STAB-15b). A probe that TIMES OUT mid-ReadStatusAsync (the CPU-status PDU may /// still be in flight ⇒ desync) likewise marks the handle dead under the gate so the next /// tick reopens. Before the fix the probe-deadline OCE fell to the bare swallow and the /// handle was reused forever. /// [Fact] public async Task Probe_timeout_marks_dead() { var opts = ProbeEnabledOptions(); var factory = new FakeS7PlcFactory(); using var drv = new S7Driver(opts, "s7-probe-timeout", factory); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); factory.Created[0].ReadStatusHangsUntilCancelled = true; await WaitUntilAsync(() => factory.Created.Count >= 2, TimeSpan.FromSeconds(5)); factory.Created.Count.ShouldBeGreaterThanOrEqualTo(2); factory.Created[0].Disposed.ShouldBeTrue(); await WaitUntilAsync(() => drv.GetHostStatuses()[0].State == HostState.Running, TimeSpan.FromSeconds(5)); drv.GetHostStatuses()[0].State.ShouldBe(HostState.Running); } private static S7DriverOptions ProbeEnabledOptions() => new() { Host = "192.0.2.1", Timeout = TimeSpan.FromMilliseconds(250), Probe = new S7ProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(50), Timeout = TimeSpan.FromMilliseconds(250), }, Tags = [new S7TagDefinition("W0", "DB1.DBW0", S7DataType.UInt16, Writable: false)], }; /// Polls until it holds or elapses (returns either way). private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (condition()) return; await Task.Delay(25).ConfigureAwait(false); } } // ---- fakes ---- private sealed class FakeS7PlcFactory : IS7PlcFactory { /// Every connection handed out, in creation order. public List Created { get; } = new(); /// When set, the NEXT d connection throws this on OpenAsync (once). public Exception? NextOpenThrows { get; set; } /// /// Number of subsequently-created connections whose /// hangs until its token is cancelled (simulating the unreachable-but-not-refusing host /// — pulled cable / firewall DROP — that STAB-14 needs and docker restart can't /// produce). Each decrements it while positive. /// public int OpenHangsUntilCancelledCount { get; set; } /// /// When set, copied onto every created connection's persistent /// so a probe fails on every tick. Clear it /// after init to arm connection #1 only. /// public Exception? ReadStatusThrowsForNewConnections { get; set; } public IS7Plc Create(S7CpuType cpuType, string host, int port, short rack, short slot, TimeSpan timeout) { var plc = new FakeS7Plc(); if (NextOpenThrows is { } ex) { plc.OpenThrowsOnce = ex; NextOpenThrows = null; } if (OpenHangsUntilCancelledCount > 0) { plc.OpenHangsUntilCancelled = true; OpenHangsUntilCancelledCount--; } if (ReadStatusThrowsForNewConnections is { } rex) plc.ReadStatusThrows = rex; Created.Add(plc); return plc; } } private sealed class FakeS7Plc : IS7Plc { public bool IsConnected { get; private set; } public bool Disposed { get; private set; } /// Thrown once by , then cleared. public Exception? OpenThrowsOnce { get; set; } /// /// When true, hangs until its token is cancelled, then throws /// — the connect-timeout shape (an unreachable host /// whose SYNs are dropped) the live rig can't cheaply produce (STAB-14). /// public bool OpenHangsUntilCancelled { get; set; } /// Thrown once by the next , then cleared. public Exception? NextReadThrows { get; set; } /// When true, hangs until its token is cancelled (STAB-15 cancel-mid-I/O). public bool ReadHangsUntilCancelled { get; set; } /// When true, hangs until its token is cancelled (STAB-15 cancel-mid-I/O). public bool WriteHangsUntilCancelled { get; set; } /// Thrown by EVERY (persistent, not one-shot) — arms a probe fault (STAB-15b). public Exception? ReadStatusThrows { get; set; } /// When true, hangs until its token is cancelled (probe-timeout, STAB-15b). public bool ReadStatusHangsUntilCancelled { get; set; } /// Boxed value returned by a good read (UInt16 tag → ushort). public object? ReadValue { get; set; } = (ushort)42; public async Task OpenAsync(CancellationToken cancellationToken) { if (OpenHangsUntilCancelled) await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false); if (OpenThrowsOnce is { } ex) { OpenThrowsOnce = null; throw ex; } IsConnected = true; } public void Close() => IsConnected = false; public async Task ReadAsync(string address, CancellationToken cancellationToken) { if (ReadHangsUntilCancelled) await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false); if (NextReadThrows is { } ex) { NextReadThrows = null; throw ex; } return ReadValue; } public Task ReadBytesAsync(S7NetDataType area, int db, int startByteAdr, int count, CancellationToken cancellationToken) => throw new NotSupportedException("FakeS7Plc: block reads not needed for the reconnect tests"); public async Task WriteAsync(string address, object value, CancellationToken cancellationToken) { if (WriteHangsUntilCancelled) await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false); } public Task WriteBytesAsync(S7NetDataType area, int db, int startByteAdr, byte[] value, CancellationToken cancellationToken) => Task.CompletedTask; public async Task ReadStatusAsync(CancellationToken cancellationToken) { if (ReadStatusHangsUntilCancelled) await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken).ConfigureAwait(false); if (ReadStatusThrows is { } ex) throw ex; } public void Dispose() { Disposed = true; IsConnected = false; } } }