Merge R2-09 Driver fleet batch (arch-review round 2) [PR #437]

Findings 05/STAB-3/4/5/8/12/16/17 + CONV-4 + onError (6 sub-batches, 29 tasks):
Modbus desync teardown, AbCip runtime lock, FOCAS concurrent caches, new
ConnectionBackoff + PollGroupEngine v2 (S7 poll fork deleted onto it, reconciled
with R2-01), fleet connect throttles, ResolveHost via resolver, TwinCAT replay
hardening. 29/29 tasks. S7 254/254 verified post-merge; build clean.

src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/ConnectionBackoff.cs

@@ -0,0 +1,93 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+/// <summary>
+///     Shared capped-exponential backoff for poll-based / lazy-reconnect drivers, extracted from
+///     the S7 driver's bespoke poll fork (05/STAB-8, CONV-1). Provides two surfaces:
+///     <list type="bullet">
+///         <item><see cref="ComputeDelay"/> — the pure schedule (base, 2×, 4×, … saturating at a
+///         cap) used by <c>PollGroupEngine</c> to stretch its retry cadence under sustained
+///         failure.</item>
+///         <item>The instance API (<see cref="ShouldAttempt"/> / <see cref="RecordFailure"/> /
+///         <see cref="RecordSuccess"/>) — a per-device connect-attempt throttle so a dead device
+///         is not hammered with a full reconnect on every data call / poll tick.</item>
+///     </list>
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Not internally synchronized.</b> The instance methods mutate plain fields; every
+///         intended call site already serializes on its per-device gate (the driver's
+///         <c>ConnectGate</c> / connect semaphore / probe lock), so adding lock-free interlocked
+///         state would be needless complexity. Callers MUST invoke the instance under that gate.
+///     </para>
+///     <para>
+///         <see cref="ComputeDelay"/> is pure + thread-safe and may be called from anywhere.
+///     </para>
+/// </remarks>
+public sealed class ConnectionBackoff
+{
+    private readonly TimeSpan _baseDelay;
+    private readonly TimeSpan _maxDelay;
+    private int _consecutiveFailures;
+    private DateTime _nextAttemptUtc = DateTime.MinValue;
+
+    /// <summary>Initializes a new per-device attempt throttle.</summary>
+    /// <param name="baseDelay">The initial backoff window after the first failure.</param>
+    /// <param name="maxDelay">The upper bound the exponential growth saturates at.</param>
+    public ConnectionBackoff(TimeSpan baseDelay, TimeSpan maxDelay)
+    {
+        _baseDelay = baseDelay;
+        _maxDelay = maxDelay;
+    }
+
+    /// <summary>
+    ///     Capped exponential backoff schedule. <paramref name="consecutiveFailures"/> ≤ 0 returns
+    ///     <paramref name="baseInterval"/>; each subsequent failure doubles the wait up to
+    ///     <paramref name="cap"/>. Computed in ticks with an overflow guard so a large failure
+    ///     count saturates at the cap rather than wrapping negative. Ported verbatim from the S7
+    ///     poll fork's <c>ComputeBackoffDelay</c>.
+    /// </summary>
+    /// <param name="baseInterval">The base interval (returned when there are no failures).</param>
+    /// <param name="consecutiveFailures">The number of consecutive failures observed.</param>
+    /// <param name="cap">The maximum delay the schedule saturates at.</param>
+    /// <returns>The computed backoff delay.</returns>
+    public static TimeSpan ComputeDelay(TimeSpan baseInterval, int consecutiveFailures, TimeSpan cap)
+    {
+        if (consecutiveFailures <= 0) return baseInterval;
+        // Cap the shift to avoid overflow — at 30 the result already saturates the cap for any
+        // reasonable base interval.
+        var shift = Math.Min(consecutiveFailures - 1, 30);
+        var ticks = baseInterval.Ticks << shift;
+        if (ticks <= 0 || ticks > cap.Ticks) return cap;
+        return TimeSpan.FromTicks(ticks);
+    }
+
+    /// <summary>
+    ///     Returns <c>true</c> when <paramref name="nowUtc"/> is at or past the end of the current
+    ///     backoff window (i.e. a connect / create attempt is due). A fresh instance and a
+    ///     just-reset instance always return <c>true</c>.
+    /// </summary>
+    /// <param name="nowUtc">The current UTC time.</param>
+    /// <returns><c>true</c> when an attempt is permitted.</returns>
+    public bool ShouldAttempt(DateTime nowUtc) => nowUtc >= _nextAttemptUtc;
+
+    /// <summary>
+    ///     Records a failed attempt: increments the consecutive-failure count and opens a backoff
+    ///     window ending <see cref="ComputeDelay"/> from <paramref name="nowUtc"/>.
+    /// </summary>
+    /// <param name="nowUtc">The current UTC time (the window is measured from here).</param>
+    public void RecordFailure(DateTime nowUtc)
+    {
+        _consecutiveFailures++;
+        _nextAttemptUtc = nowUtc + ComputeDelay(_baseDelay, _consecutiveFailures, _maxDelay);
+    }
+
+    /// <summary>
+    ///     Records a successful attempt: resets the failure count and clears the window so the next
+    ///     attempt is permitted immediately. Recovery is never delayed by a residual backoff window.
+    /// </summary>
+    public void RecordSuccess()
+    {
+        _consecutiveFailures = 0;
+        _nextAttemptUtc = DateTime.MinValue;
+    }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/PollGroupEngine.cs

@@ -35,6 +35,7 @@ public sealed class PollGroupEngine : IAsyncDisposable
     private readonly Action<ISubscriptionHandle, string, DataValueSnapshot> _onChange;
     private readonly Action<Exception>? _onError;
     private readonly TimeSpan _minInterval;
+    private readonly TimeSpan? _backoffCap;
     private readonly ConcurrentDictionary<long, SubscriptionState> _subscriptions = new();
     private long _nextId;
 
@@ -52,11 +53,17 @@ public sealed class PollGroupEngine : IAsyncDisposable
     ///     internal contract-violation throw) so the owning driver can route the failure to its
     ///     health surface. Defensive: an <c>onError</c> handler that
     ///     itself throws is silently absorbed so a buggy forwarder cannot crash the poll loop.</param>
+    /// <param name="backoffCap">Optional cap enabling capped-exponential failure backoff
+    ///     (05/STAB-8). When supplied, sustained reader failures stretch the retry cadence
+    ///     (interval, 2×, 4×, … saturating at this cap) instead of hammering a dead device every
+    ///     tick; a successful poll resets to the base interval. <c>null</c> (default) keeps the
+    ///     byte-identical fixed-interval cadence for consumers that have not opted in.</param>
     public PollGroupEngine(
         Func<IReadOnlyList<string>, CancellationToken, Task<IReadOnlyList<DataValueSnapshot>>> reader,
         Action<ISubscriptionHandle, string, DataValueSnapshot> onChange,
         TimeSpan? minInterval = null,
-        Action<Exception>? onError = null)
+        Action<Exception>? onError = null,
+        TimeSpan? backoffCap = null)
     {
         ArgumentNullException.ThrowIfNull(reader);
         ArgumentNullException.ThrowIfNull(onChange);
@@ -64,6 +71,7 @@ public sealed class PollGroupEngine : IAsyncDisposable
         _onChange = onChange;
         _onError = onError;
         _minInterval = minInterval ?? DefaultMinInterval;
+        _backoffCap = backoffCap;
     }
 
     /// <summary>Register a new polled subscription and start its background loop.</summary>
@@ -116,31 +124,54 @@ public sealed class PollGroupEngine : IAsyncDisposable
 
     private async Task PollLoopAsync(SubscriptionState state, CancellationToken ct)
     {
+        var consecutiveFailures = 0;
+
         // Initial-data push: every subscribed tag fires once at subscribe time regardless of
         // whether it has changed, satisfying OPC UA Part 4 initial-value semantics.
-        try { await PollOnceAsync(state, forceRaise: true, ct).ConfigureAwait(false); }
-        catch (OperationCanceledException) { return; }
+        try
+        {
+            await PollOnceAsync(state, forceRaise: true, ct).ConfigureAwait(false);
+            consecutiveFailures = 0;
+        }
+        // Only a teardown OCE (the loop's OWN token) exits the loop. A reader that throws OCE with
+        // the loop token un-cancelled — a driver-internal timeout CTS (the STAB-14/STAB-15 class) —
+        // must NOT tear the loop down: it falls through to the generic handler, is reported, and the
+        // loop retries with backoff. This is the guard that makes the S7 fork migration safe.
+        catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
         catch (Exception ex) when (!IsFatal(ex))
         {
             // first-read error tolerated — loop continues; forward to driver health surface.
+            consecutiveFailures++;
             ReportError(ex);
         }
 
         while (!ct.IsCancellationRequested)
         {
-            try { await Task.Delay(state.Interval, ct).ConfigureAwait(false); }
+            // Fixed cadence by default; when a backoff cap is configured, sustained failures
+            // stretch the delay (capped exponential) and a success snaps it back to Interval.
+            var delay = _backoffCap is null
+                ? state.Interval
+                : ConnectionBackoff.ComputeDelay(state.Interval, consecutiveFailures, _backoffCap.Value);
+            try { await Task.Delay(delay, ct).ConfigureAwait(false); }
             catch (OperationCanceledException) { return; }
             // Defensive: the CTS may be disposed by Unsubscribe/DisposeAsync between the
             // cancellation check above and the Task.Delay touching the token. Treat that race
             // as a normal cancellation rather than a fatal exception.
             catch (ObjectDisposedException) { return; }
 
-            try { await PollOnceAsync(state, forceRaise: false, ct).ConfigureAwait(false); }
-            catch (OperationCanceledException) { return; }
+            try
+            {
+                await PollOnceAsync(state, forceRaise: false, ct).ConfigureAwait(false);
+                consecutiveFailures = 0;
+            }
+            // Only teardown (the loop token) exits; a reader OCE with ct un-cancelled backs off
+            // like any other failure instead of killing the loop (STAB-14 engine guard).
+            catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
             catch (Exception ex) when (!IsFatal(ex))
             {
-                // transient poll error — loop continues, driver health surface logs it
-                // via the supplied onError callback.
+                // transient poll error — loop continues (with backoff when configured); driver
+                // health surface logs it via the supplied onError callback.
+                consecutiveFailures++;
                 ReportError(ex);
             }
         }