dohertj2/lmxopcua
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(focas): per-device connect-attempt backoff across the three loops (05/STAB-8)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 12:16:31 -04:00
parent 962afed241
commit e54af9b948
3 changed files with 118 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
archreview/plans/R2-09-driver-fleet-batch-plan.md.tasks.json
+1 -1
View File
@@ -129,7 +129,7 @@
{
"id": "B4.2",
"subject": "B4: FOCAS per-device connect-attempt backoff across the three loops, tests FAIL->PASS \u2014 STAB-8",
"status": "pending",
"status": "completed",
"blockedBy": [
"B3.2"
]
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs
+31 -2
View File
@@ -629,7 +629,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
var success = false;
try
{
var client = await EnsureConnectedAsync(state, ct).ConfigureAwait(false);
// The probe bypasses the connect backoff — its own interval is the bounded recovery
// cadence, and a successful probe connect resets the window for the data + loop paths.
var client = await EnsureConnectedAsync(state, ct, bypassBackoff: true).ConfigureAwait(false);
// Apply Probe.Timeout so a hung CNC socket gets cancelled at the configured
// budget rather than blocking until the OS TCP timeout.
// TimeSpan.Zero / negative means "no per-probe timeout" — fall back to the loop
@@ -1120,10 +1122,24 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
return _options.Devices.FirstOrDefault()?.HostAddress ?? DriverInstanceId;
}
private async Task<IFocasClient> EnsureConnectedAsync(DeviceState device, CancellationToken ct)
/// <summary>
/// Thrown when a per-device <see cref="ConnectionBackoff"/> window is still open (05/STAB-8)
/// — a fail-fast that costs no wire traffic. Read/write map it to <c>BadCommunicationError</c>
/// via their generic catch, exactly as a real connect failure.
/// </summary>
private sealed class ConnectionThrottledException(string message) : Exception(message);
private async Task<IFocasClient> EnsureConnectedAsync(
DeviceState device, CancellationToken ct, bool bypassBackoff = false)
{
if (device.Client is { IsConnected: true } c) return c;
// 05/STAB-8: inside an open backoff window a data-path / per-tick-loop caller fails fast — no
// socket create, no wire connect. The probe loop bypasses this and drives failure/success.
if (!bypassBackoff && !device.Backoff.ShouldAttempt(DateTime.UtcNow))
throw new ConnectionThrottledException(
$"FOCAS connect to {device.Options.HostAddress} throttled — backoff window open after a recent connect failure.");
// Discard the existing client (if any) before creating a new one. A client that is
// non-null but not connected may have been disposed by a HandleRecycle race or a prior
// teardown — retrying ConnectAsync on a disposed FocasWireClient hits ThrowIfDisposed and
@@ -1148,8 +1164,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
{
device.Client.Dispose();
device.Client = null;
device.Backoff.RecordFailure(DateTime.UtcNow); // 05/STAB-8: open the backoff window
throw;
}
device.Backoff.RecordSuccess(); // 05/STAB-8: recovery resets the window immediately
return device.Client;
}
@@ -1195,6 +1213,17 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
/// <summary>Gets or sets the FOCAS client instance.</summary>
public IFocasClient? Client { get; set; }
/// <summary>
/// Per-device connect-attempt throttle (05/STAB-8). A dead CNC fails fast inside the
/// backoff window instead of paying a full two-socket reconnect on every tick of each of
/// the read / write / probe / fixed-tree / recycle loops; the probe loop bypasses it and a
/// successful connect resets it. FOCAS's <c>EnsureConnectedAsync</c> is not gate-serialized
/// (the loops already race the client swap by design), so this is best-effort — atomic
/// enough on 64-bit for the int/DateTime state, and a benign extra/skipped attempt is the
/// worst outcome of a race.
/// </summary>
public ConnectionBackoff Backoff { get; } = new(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
/// <summary>Gets the lock object for probe synchronization.</summary>
public object ProbeLock { get; } = new();
/// <summary>Gets or sets the current host connectivity state.</summary>
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConnectBackoffTests.cs
+86
View File
@@ -0,0 +1,86 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// 05/STAB-8 — a dead FOCAS CNC must not pay a full two-socket reconnect on every tick of each
/// read / probe / fixed-tree / recycle loop. The per-device <see cref="ConnectionBackoff"/>
/// gates the connect path; the probe bypasses it and a successful connect resets it. No FOCAS
/// fixture exists — the fake-client suite is authoritative.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasConnectBackoffTests
{
private static FocasDriverOptions Options(bool probeEnabled) => new()
{
Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")],
Tags = [new FocasTagDefinition("R", "focas://10.0.0.5:8193", "R100", FocasDataType.Byte)],
Probe = new FocasProbeOptions
{
Enabled = probeEnabled,
Interval = TimeSpan.FromMilliseconds(50),
Timeout = TimeSpan.FromMilliseconds(250),
},
// FixedTree / HandleRecycle / AlarmProjection default off — keep only the data path (+ probe).
};
/// <summary>Rapid data calls against a dead CNC attempt a single connect, then fail fast inside the backoff window.</summary>
[Fact]
public async Task DeadDevice_ConnectAttemptsFollowBackoffSchedule()
{
var factory = new FakeFocasClientFactory
{
Customise = () => new FakeFocasClient
{
ThrowOnConnect = true,
Exception = new InvalidOperationException("cnc down"),
},
};
var drv = new FocasDriver(Options(probeEnabled: false), "focas-1", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
for (var i = 0; i < 6; i++)
await drv.ReadAsync(["R"], CancellationToken.None);
// One real connect attempt; the remaining five fail fast inside the window — no new socket
// is created (the fake factory records one client per attempted connect).
factory.Clients.Count.ShouldBe(1);
}
/// <summary>The probe bypasses the backoff window and a successful connect resets it so the next data call succeeds immediately.</summary>
[Fact]
public async Task ProbeBypassesBackoff_AndSuccessResets()
{
var fail = true;
var factory = new FakeFocasClientFactory
{
Customise = () => new FakeFocasClient
{
ThrowOnConnect = fail,
Exception = new InvalidOperationException("cnc down"),
},
};
var drv = new FocasDriver(Options(probeEnabled: true), "focas-1", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
var down = await drv.ReadAsync(["R"], CancellationToken.None);
down[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
// Device recovers. The probe bypasses the still-open window, reconnects, and resets it so
// the following data read succeeds with no residual delay.
fail = false;
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(3);
DataValueSnapshot? good = null;
while (DateTime.UtcNow < deadline)
{
var r = await drv.ReadAsync(["R"], CancellationToken.None);
if (r[0].StatusCode == FocasStatusMapper.Good) { good = r[0]; break; }
await Task.Delay(25, CancellationToken.None);
}
good.ShouldNotBeNull("probe-driven reconnect must reset the backoff so data reads recover");
}
}