Merge R2-01 S7 fault-path hardening (arch-review round 2) [PR #429]

Findings 05/STAB-14 (High regression — connect-timeout OCE no longer silently
kills S7 subscriptions), STAB-15 (framing/desync + mid-PDU cancel fatal
classification), STAB-8 seam note. All prod changes in S7Driver.cs; T11 live
outage gate deferred. STATUS.md conflict resolved additively. Build clean.

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.S7/S7Driver.cs

@@ -187,6 +187,16 @@ public sealed class S7Driver
                 cts.CancelAfter(_options.Timeout);
                 await plc.OpenAsync(cts.Token).ConfigureAwait(false);
             }
+            catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
+            {
+                // The linked CancelAfter fired — a connect TIMEOUT, not a caller cancellation.
+                // Surface it as a TimeoutException so init-time and reconnect-time timeouts report
+                // the same exception type (STAB-14 consistency); the outer catch still faults health.
+                try { plc.Dispose(); } catch { }
+                throw new TimeoutException(
+                    $"S7 connect to {_options.Host}:{_options.Port} timed out after " +
+                    $"{(int)_options.Timeout.TotalMilliseconds} ms.");
+            }
             catch
             {
                 // Dispose the partially-constructed connection so a failed init doesn't leak it
@@ -485,7 +495,10 @@ public sealed class S7Driver
             {
                 plc = await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);
             }
-            catch (OperationCanceledException) { throw; }
+            // Rethrow ONLY caller cancellation. A timeout-born OCE (none expected after the
+            // EnsureConnectedAsync conversion, but e.g. an S7.Net-internal CTS) must degrade the
+            // batch below, not escape as teardown (STAB-14).
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
             catch (Exception ex)
             {
                 for (var i = 0; i < fullReferences.Count; i++)
@@ -508,6 +521,17 @@ public sealed class S7Driver
                     results[i] = new DataValueSnapshot(value, 0u, now, now);
                     _health = new DriverHealth(DriverState.Healthy, now, null);
                 }
+                catch (OperationCanceledException)
+                {
+                    // Cancellation observed mid-PDU abandons a half-read ISO-on-TCP response on the
+                    // single gated stream — the handle must be reopened, not reused (STAB-15a). Mark
+                    // dead (we hold _gate) and propagate: the caller cancelled, nobody consumes the
+                    // batch. Marking is unconditional on OCE (not routed through the type classifier):
+                    // whether bytes moved is unknowable, and a false positive costs one cheap reopen
+                    // versus permanent desync for a false negative.
+                    _plcDead = true;
+                    throw;
+                }
                 catch (NotSupportedException)
                 {
                     results[i] = new DataValueSnapshot(null, StatusBadNotSupported, null, now);
@@ -997,7 +1021,9 @@ public sealed class S7Driver
             {
                 plc = await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);
             }
-            catch (OperationCanceledException) { throw; }
+            // Rethrow ONLY caller cancellation — a timeout-born OCE degrades the batch below rather
+            // than escaping as teardown (STAB-14 defense-in-depth, mirrors ReadAsync).
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
             catch (Exception ex)
             {
                 for (var i = 0; i < writes.Count; i++)
@@ -1042,8 +1068,12 @@ public sealed class S7Driver
                 }
                 catch (OperationCanceledException)
                 {
-                    // Let cancellation propagate rather than turning it into
-                    // a status code — the gate is still held so Release() runs in finally.
+                    // Cancellation observed mid-PDU abandons a half-written ISO-on-TCP request on
+                    // the single gated stream — the handle must be reopened, not reused (STAB-15a).
+                    // Mark dead (we hold _gate) and propagate: the caller cancelled, nobody consumes
+                    // the batch, and Release() still runs in finally. Marking is unconditional on
+                    // OCE — a false positive costs one cheap reopen versus permanent desync.
+                    _plcDead = true;
                     throw;
                 }
                 catch (NotSupportedException)
@@ -1181,6 +1211,15 @@ public sealed class S7Driver
     ///     MUST be called while holding <see cref="_gate"/> — it mutates <see cref="Plc"/> /
     ///     <see cref="_plcDead"/> and reuses the single-connection-per-PLC invariant the gate
     ///     enforces. The read / write / probe paths all take the gate before calling it.
+    ///     <para>
+    ///         While the PLC is down every data call and probe tick pays a full connect attempt
+    ///         bounded only by <c>_options.Timeout</c> (STAB-8). This method is the single
+    ///         choke-point all S7 reconnects flow through (already gate-serialized), so the
+    ///         fleet-wide connect-attempt throttle planned in R2-09 plugs in here: a
+    ///         last-failed-attempt timestamp checked at the top of the slow path. Deliberately NOT
+    ///         implemented driver-locally — see
+    ///         <c>archreview/plans/R2-01-s7-fault-hardening-plan.md</c> Finding 3.
+    ///     </para>
     /// </remarks>
     /// <param name="ct">Cancellation token for the (re)connect attempt.</param>
     /// <returns>A live <see cref="IS7Plc"/>.</returns>
@@ -1205,6 +1244,17 @@ public sealed class S7Driver
             cts.CancelAfter(_options.Timeout);
             await plc.OpenAsync(cts.Token).ConfigureAwait(false);
         }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            // The linked CancelAfter fired — a connect TIMEOUT, not a caller cancellation.
+            // Surface it as a connection FAILURE (TimeoutException) so callers route it through
+            // their degrade paths: an escaping OCE reads as teardown and permanently killed the
+            // subscription poll loops (STAB-14, the unreachable-host regression in the Crit-3 fix).
+            try { plc.Dispose(); } catch { /* best-effort */ }
+            throw new TimeoutException(
+                $"S7 connect to {_options.Host}:{_options.Port} timed out after " +
+                $"{(int)_options.Timeout.TotalMilliseconds} ms.");
+        }
         catch
         {
             try { plc.Dispose(); } catch { /* best-effort */ }
@@ -1230,10 +1280,24 @@ public sealed class S7Driver
 
     /// <summary>
     ///     True when <paramref name="ex"/> (or any inner exception) is a socket-level / connection
-    ///     loss that a reopen can repair — a <see cref="System.Net.Sockets.SocketException"/> /
-    ///     <see cref="System.IO.IOException"/> / <see cref="ObjectDisposedException"/>, or an S7.Net
-    ///     <see cref="PlcException"/> carrying <see cref="ErrorCode.ConnectionError"/>. A
-    ///     data-address / type error (S7.Net's <see cref="ErrorCode.ReadData"/> /
+    ///     loss OR an ISO-on-TCP framing/desync fault that a reopen can repair — a
+    ///     <see cref="System.Net.Sockets.SocketException"/> / <see cref="System.IO.IOException"/> /
+    ///     <see cref="ObjectDisposedException"/>; an S7.Net framing violation
+    ///     (<see cref="TPKTInvalidException"/> / <see cref="TPDUInvalidException"/> /
+    ///     <see cref="WrongNumberOfBytesException"/>); or a <see cref="PlcException"/> carrying
+    ///     <see cref="ErrorCode.ConnectionError"/> or <see cref="ErrorCode.WrongNumberReceivedBytes"/>.
+    ///     A framing violation means the single serialized stream's position is untrustworthy — a
+    ///     half-read PDU left by a timeout/cancellation makes every later response mis-frame, so only
+    ///     a reopen recovers (STAB-15a: the Modbus STAB-3 desync mode rebuilt in S7).
+    ///     <para>
+    ///         Deliberately NOT <see cref="System.IO.InvalidDataException"/>: the driver's own decode
+    ///         backstop (<see cref="ReinterpretRawValue"/> / <see cref="DecodeScalarBlock"/>) throws
+    ///         it for a declared-type/address-size CONFIG mismatch on a HEALTHY socket — classifying
+    ///         it would churn a full reopen on every read of a mis-authored tag. A genuinely desynced
+    ///         stream's very next PDU still lands in TPKT/TPDU/WrongNumber territory, which now tears
+    ///         down, so a real desync self-heals within one extra failed call.
+    ///     </para>
+    ///     A data-address / type error (S7.Net's <see cref="ErrorCode.ReadData"/> /
     ///     <see cref="ErrorCode.WriteData"/> for a bad address, PUT/GET-denied, etc.) is deliberately
     ///     NOT treated as fatal — reopening wouldn't help and would churn a healthy connection.
     /// </summary>
@@ -1245,7 +1309,13 @@ public sealed class S7Driver
         {
             if (e is System.Net.Sockets.SocketException or System.IO.IOException or ObjectDisposedException)
                 return true;
-            if (e is PlcException { ErrorCode: ErrorCode.ConnectionError })
+            // ISO-on-TCP framing/desync surface (STAB-15a): the stream position is untrustworthy —
+            // a half-read PDU left by a timeout/cancellation makes every later response mis-frame.
+            // NOTE: deliberately NOT System.IO.InvalidDataException — ReinterpretRawValue throws it
+            // for a declared-type/address-size CONFIG mismatch on a healthy socket.
+            if (e is TPKTInvalidException or TPDUInvalidException or WrongNumberOfBytesException)
+                return true;
+            if (e is PlcException { ErrorCode: ErrorCode.ConnectionError or ErrorCode.WrongNumberReceivedBytes })
                 return true;
         }
         return false;
@@ -1380,7 +1450,10 @@ public sealed class S7Driver
             await PollOnceAsync(state, forceRaise: true, ct).ConfigureAwait(false);
             consecutiveFailures = 0;
         }
-        catch (OperationCanceledException) { return; }
+        // Only a teardown OCE (the loop's own token) exits the loop; any other OCE (e.g. a
+        // connect-timeout that slipped past the wrapper filter) falls to the backoff path below
+        // so the loop lives (STAB-14 defense-in-depth).
+        catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
         catch (Exception ex)
         {
             // First-read error — polling continues; log so the operator has an event trail.
@@ -1394,14 +1467,17 @@ public sealed class S7Driver
             // ticks reset consecutiveFailures back to 0 so the cadence snaps back to Interval.
             var delay = ComputeBackoffDelay(state.Interval, consecutiveFailures);
             try { await Task.Delay(delay, ct).ConfigureAwait(false); }
-            catch (OperationCanceledException) { return; }
+            // Task.Delay can only observe ct-triggered OCE, so this filter is purely for uniformity.
+            catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
 
             try
             {
                 await PollOnceAsync(state, forceRaise: false, ct).ConfigureAwait(false);
                 consecutiveFailures = 0;
             }
-            catch (OperationCanceledException) { return; }
+            // Only teardown exits; any non-teardown OCE (e.g. a slipped-through connect timeout)
+            // backs off instead of killing the loop (STAB-14 defense-in-depth).
+            catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
             catch (Exception ex)
             {
                 // Sustained polling error — loop continues with backoff; log + update health.
@@ -1529,17 +1605,39 @@ public sealed class S7Driver
                 await _gate.WaitAsync(probeCts.Token).ConfigureAwait(false);
                 try
                 {
-                    // EnsureConnectedAsync doubles as the reconnect backstop: if a read/write
-                    // marked the handle dead (or the socket dropped between ticks), the probe
-                    // reopens it here so recovery doesn't wait for the next data call.
-                    var plc = await EnsureConnectedAsync(probeCts.Token).ConfigureAwait(false);
-                    await plc.ReadStatusAsync(probeCts.Token).ConfigureAwait(false);
-                    success = true;
+                    try
+                    {
+                        // EnsureConnectedAsync doubles as the reconnect backstop: if a read/write
+                        // marked the handle dead (or the socket dropped between ticks), the probe
+                        // reopens it here so recovery doesn't wait for the next data call.
+                        var plc = await EnsureConnectedAsync(probeCts.Token).ConfigureAwait(false);
+                        await plc.ReadStatusAsync(probeCts.Token).ConfigureAwait(false);
+                        success = true;
+                    }
+                    catch (OperationCanceledException) when (ct.IsCancellationRequested)
+                    {
+                        // Loop teardown — rethrow to the outer filter (after finally releases the gate).
+                        throw;
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        // Probe deadline fired mid-PDU: the CPU-status response may still be in
+                        // flight, so the handle can't be reused (half-read ⇒ desync). We hold _gate
+                        // here, keeping _plcDead gate-guarded (STAB-15b).
+                        _plcDead = true;
+                    }
+                    catch (Exception ex)
+                    {
+                        // STAB-15b: classify + mark while holding _gate so a wire-dead/desynced
+                        // handle whose TcpClient still claims Connected is reopened by the NEXT
+                        // tick's EnsureConnectedAsync instead of being re-probed forever.
+                        MarkConnectionDeadIfFatal(ex);
+                    }
                 }
                 finally { _gate.Release(); }
             }
             catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
-            catch { /* transport/timeout/exception — treated as Stopped below */ }
+            catch { /* gate-wait timeout / other — treated as Stopped below (gate contention is not a connection fault) */ }
 
             TransitionTo(success ? HostState.Running : HostState.Stopped);