fix(focas): per-device connect-attempt backoff across the three loops (05/STAB-8)

archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json

@@ -129,7 +129,7 @@
     {
       "id": "B4.2",
       "subject": "B4: FOCAS per-device connect-attempt backoff across the three loops, tests FAIL->PASS \u2014 STAB-8",
-      "status": "pending",
+      "status": "completed",
       "blockedBy": [
         "B3.2"
       ]

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs

@@ -629,7 +629,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
             var success = false;
             try
             {
-                var client = await EnsureConnectedAsync(state, ct).ConfigureAwait(false);
+                // The probe bypasses the connect backoff — its own interval is the bounded recovery
+                // cadence, and a successful probe connect resets the window for the data + loop paths.
+                var client = await EnsureConnectedAsync(state, ct, bypassBackoff: true).ConfigureAwait(false);
                 // Apply Probe.Timeout so a hung CNC socket gets cancelled at the configured
                 // budget rather than blocking until the OS TCP timeout.
                 // TimeSpan.Zero / negative means "no per-probe timeout" — fall back to the loop
@@ -1120,10 +1122,24 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
         return _options.Devices.FirstOrDefault()?.HostAddress ?? DriverInstanceId;
     }
 
-    private async Task<IFocasClient> EnsureConnectedAsync(DeviceState device, CancellationToken ct)
+    /// <summary>
+    ///     Thrown when a per-device <see cref="ConnectionBackoff"/> window is still open (05/STAB-8)
+    ///     — a fail-fast that costs no wire traffic. Read/write map it to <c>BadCommunicationError</c>
+    ///     via their generic catch, exactly as a real connect failure.
+    /// </summary>
+    private sealed class ConnectionThrottledException(string message) : Exception(message);
+
+    private async Task<IFocasClient> EnsureConnectedAsync(
+        DeviceState device, CancellationToken ct, bool bypassBackoff = false)
     {
         if (device.Client is { IsConnected: true } c) return c;
 
+        // 05/STAB-8: inside an open backoff window a data-path / per-tick-loop caller fails fast — no
+        // socket create, no wire connect. The probe loop bypasses this and drives failure/success.
+        if (!bypassBackoff && !device.Backoff.ShouldAttempt(DateTime.UtcNow))
+            throw new ConnectionThrottledException(
+                $"FOCAS connect to {device.Options.HostAddress} throttled — backoff window open after a recent connect failure.");
+
         // Discard the existing client (if any) before creating a new one. A client that is
         // non-null but not connected may have been disposed by a HandleRecycle race or a prior
         // teardown — retrying ConnectAsync on a disposed FocasWireClient hits ThrowIfDisposed and
@@ -1148,8 +1164,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
         {
             device.Client.Dispose();
             device.Client = null;
+            device.Backoff.RecordFailure(DateTime.UtcNow); // 05/STAB-8: open the backoff window
             throw;
         }
+        device.Backoff.RecordSuccess(); // 05/STAB-8: recovery resets the window immediately
         return device.Client;
     }
 
@@ -1195,6 +1213,17 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
         /// <summary>Gets or sets the FOCAS client instance.</summary>
         public IFocasClient? Client { get; set; }
 
+        /// <summary>
+        ///     Per-device connect-attempt throttle (05/STAB-8). A dead CNC fails fast inside the
+        ///     backoff window instead of paying a full two-socket reconnect on every tick of each of
+        ///     the read / write / probe / fixed-tree / recycle loops; the probe loop bypasses it and a
+        ///     successful connect resets it. FOCAS's <c>EnsureConnectedAsync</c> is not gate-serialized
+        ///     (the loops already race the client swap by design), so this is best-effort — atomic
+        ///     enough on 64-bit for the int/DateTime state, and a benign extra/skipped attempt is the
+        ///     worst outcome of a race.
+        /// </summary>
+        public ConnectionBackoff Backoff { get; } = new(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
+
         /// <summary>Gets the lock object for probe synchronization.</summary>
         public object ProbeLock { get; } = new();
         /// <summary>Gets or sets the current host connectivity state.</summary>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConnectBackoffTests.cs

@@ -0,0 +1,86 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
+
+/// <summary>
+///     05/STAB-8 — a dead FOCAS CNC must not pay a full two-socket reconnect on every tick of each
+///     read / probe / fixed-tree / recycle loop. The per-device <see cref="ConnectionBackoff"/>
+///     gates the connect path; the probe bypasses it and a successful connect resets it. No FOCAS
+///     fixture exists — the fake-client suite is authoritative.
+/// </summary>
+[Trait("Category", "Unit")]
+public sealed class FocasConnectBackoffTests
+{
+    private static FocasDriverOptions Options(bool probeEnabled) => new()
+    {
+        Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")],
+        Tags = [new FocasTagDefinition("R", "focas://10.0.0.5:8193", "R100", FocasDataType.Byte)],
+        Probe = new FocasProbeOptions
+        {
+            Enabled = probeEnabled,
+            Interval = TimeSpan.FromMilliseconds(50),
+            Timeout = TimeSpan.FromMilliseconds(250),
+        },
+        // FixedTree / HandleRecycle / AlarmProjection default off — keep only the data path (+ probe).
+    };
+
+    /// <summary>Rapid data calls against a dead CNC attempt a single connect, then fail fast inside the backoff window.</summary>
+    [Fact]
+    public async Task DeadDevice_ConnectAttemptsFollowBackoffSchedule()
+    {
+        var factory = new FakeFocasClientFactory
+        {
+            Customise = () => new FakeFocasClient
+            {
+                ThrowOnConnect = true,
+                Exception = new InvalidOperationException("cnc down"),
+            },
+        };
+        var drv = new FocasDriver(Options(probeEnabled: false), "focas-1", factory);
+        await drv.InitializeAsync("{}", CancellationToken.None);
+
+        for (var i = 0; i < 6; i++)
+            await drv.ReadAsync(["R"], CancellationToken.None);
+
+        // One real connect attempt; the remaining five fail fast inside the window — no new socket
+        // is created (the fake factory records one client per attempted connect).
+        factory.Clients.Count.ShouldBe(1);
+    }
+
+    /// <summary>The probe bypasses the backoff window and a successful connect resets it so the next data call succeeds immediately.</summary>
+    [Fact]
+    public async Task ProbeBypassesBackoff_AndSuccessResets()
+    {
+        var fail = true;
+        var factory = new FakeFocasClientFactory
+        {
+            Customise = () => new FakeFocasClient
+            {
+                ThrowOnConnect = fail,
+                Exception = new InvalidOperationException("cnc down"),
+            },
+        };
+        var drv = new FocasDriver(Options(probeEnabled: true), "focas-1", factory);
+        await drv.InitializeAsync("{}", CancellationToken.None);
+
+        var down = await drv.ReadAsync(["R"], CancellationToken.None);
+        down[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
+
+        // Device recovers. The probe bypasses the still-open window, reconnects, and resets it so
+        // the following data read succeeds with no residual delay.
+        fail = false;
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(3);
+        DataValueSnapshot? good = null;
+        while (DateTime.UtcNow < deadline)
+        {
+            var r = await drv.ReadAsync(["R"], CancellationToken.None);
+            if (r[0].StatusCode == FocasStatusMapper.Good) { good = r[0]; break; }
+            await Task.Delay(25, CancellationToken.None);
+        }
+
+        good.ShouldNotBeNull("probe-driven reconnect must reset the backoff so data reads recover");
+    }
+}