diff --git a/archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json b/archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json index 62ac8427..f04d866c 100644 --- a/archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json +++ b/archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json @@ -129,7 +129,7 @@ { "id": "B4.2", "subject": "B4: FOCAS per-device connect-attempt backoff across the three loops, tests FAIL->PASS \u2014 STAB-8", - "status": "pending", + "status": "completed", "blockedBy": [ "B3.2" ] diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index 4e64e2bd..25386b28 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -629,7 +629,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, var success = false; try { - var client = await EnsureConnectedAsync(state, ct).ConfigureAwait(false); + // The probe bypasses the connect backoff — its own interval is the bounded recovery + // cadence, and a successful probe connect resets the window for the data + loop paths. + var client = await EnsureConnectedAsync(state, ct, bypassBackoff: true).ConfigureAwait(false); // Apply Probe.Timeout so a hung CNC socket gets cancelled at the configured // budget rather than blocking until the OS TCP timeout. // TimeSpan.Zero / negative means "no per-probe timeout" — fall back to the loop @@ -1120,10 +1122,24 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, return _options.Devices.FirstOrDefault()?.HostAddress ?? DriverInstanceId; } - private async Task EnsureConnectedAsync(DeviceState device, CancellationToken ct) + /// + /// Thrown when a per-device window is still open (05/STAB-8) + /// — a fail-fast that costs no wire traffic. Read/write map it to BadCommunicationError + /// via their generic catch, exactly as a real connect failure. + /// + private sealed class ConnectionThrottledException(string message) : Exception(message); + + private async Task EnsureConnectedAsync( + DeviceState device, CancellationToken ct, bool bypassBackoff = false) { if (device.Client is { IsConnected: true } c) return c; + // 05/STAB-8: inside an open backoff window a data-path / per-tick-loop caller fails fast — no + // socket create, no wire connect. The probe loop bypasses this and drives failure/success. + if (!bypassBackoff && !device.Backoff.ShouldAttempt(DateTime.UtcNow)) + throw new ConnectionThrottledException( + $"FOCAS connect to {device.Options.HostAddress} throttled — backoff window open after a recent connect failure."); + // Discard the existing client (if any) before creating a new one. A client that is // non-null but not connected may have been disposed by a HandleRecycle race or a prior // teardown — retrying ConnectAsync on a disposed FocasWireClient hits ThrowIfDisposed and @@ -1148,8 +1164,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { device.Client.Dispose(); device.Client = null; + device.Backoff.RecordFailure(DateTime.UtcNow); // 05/STAB-8: open the backoff window throw; } + device.Backoff.RecordSuccess(); // 05/STAB-8: recovery resets the window immediately return device.Client; } @@ -1195,6 +1213,17 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// Gets or sets the FOCAS client instance. public IFocasClient? Client { get; set; } + /// + /// Per-device connect-attempt throttle (05/STAB-8). A dead CNC fails fast inside the + /// backoff window instead of paying a full two-socket reconnect on every tick of each of + /// the read / write / probe / fixed-tree / recycle loops; the probe loop bypasses it and a + /// successful connect resets it. FOCAS's EnsureConnectedAsync is not gate-serialized + /// (the loops already race the client swap by design), so this is best-effort — atomic + /// enough on 64-bit for the int/DateTime state, and a benign extra/skipped attempt is the + /// worst outcome of a race. + /// + public ConnectionBackoff Backoff { get; } = new(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30)); + /// Gets the lock object for probe synchronization. public object ProbeLock { get; } = new(); /// Gets or sets the current host connectivity state. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConnectBackoffTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConnectBackoffTests.cs new file mode 100644 index 00000000..318ecd1d --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConnectBackoffTests.cs @@ -0,0 +1,86 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// 05/STAB-8 — a dead FOCAS CNC must not pay a full two-socket reconnect on every tick of each +/// read / probe / fixed-tree / recycle loop. The per-device +/// gates the connect path; the probe bypasses it and a successful connect resets it. No FOCAS +/// fixture exists — the fake-client suite is authoritative. +/// +[Trait("Category", "Unit")] +public sealed class FocasConnectBackoffTests +{ + private static FocasDriverOptions Options(bool probeEnabled) => new() + { + Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")], + Tags = [new FocasTagDefinition("R", "focas://10.0.0.5:8193", "R100", FocasDataType.Byte)], + Probe = new FocasProbeOptions + { + Enabled = probeEnabled, + Interval = TimeSpan.FromMilliseconds(50), + Timeout = TimeSpan.FromMilliseconds(250), + }, + // FixedTree / HandleRecycle / AlarmProjection default off — keep only the data path (+ probe). + }; + + /// Rapid data calls against a dead CNC attempt a single connect, then fail fast inside the backoff window. + [Fact] + public async Task DeadDevice_ConnectAttemptsFollowBackoffSchedule() + { + var factory = new FakeFocasClientFactory + { + Customise = () => new FakeFocasClient + { + ThrowOnConnect = true, + Exception = new InvalidOperationException("cnc down"), + }, + }; + var drv = new FocasDriver(Options(probeEnabled: false), "focas-1", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + for (var i = 0; i < 6; i++) + await drv.ReadAsync(["R"], CancellationToken.None); + + // One real connect attempt; the remaining five fail fast inside the window — no new socket + // is created (the fake factory records one client per attempted connect). + factory.Clients.Count.ShouldBe(1); + } + + /// The probe bypasses the backoff window and a successful connect resets it so the next data call succeeds immediately. + [Fact] + public async Task ProbeBypassesBackoff_AndSuccessResets() + { + var fail = true; + var factory = new FakeFocasClientFactory + { + Customise = () => new FakeFocasClient + { + ThrowOnConnect = fail, + Exception = new InvalidOperationException("cnc down"), + }, + }; + var drv = new FocasDriver(Options(probeEnabled: true), "focas-1", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var down = await drv.ReadAsync(["R"], CancellationToken.None); + down[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError); + + // Device recovers. The probe bypasses the still-open window, reconnects, and resets it so + // the following data read succeeds with no residual delay. + fail = false; + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(3); + DataValueSnapshot? good = null; + while (DateTime.UtcNow < deadline) + { + var r = await drv.ReadAsync(["R"], CancellationToken.None); + if (r[0].StatusCode == FocasStatusMapper.Good) { good = r[0]; break; } + await Task.Delay(25, CancellationToken.None); + } + + good.ShouldNotBeNull("probe-driven reconnect must reset the backoff so data reads recover"); + } +}