Merge hw2-makino: hardware live-gate fixes (hw2-makino)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 21:35:10 -04:00
3 changed files with 215 additions and 40 deletions
@@ -1816,41 +1816,60 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
{
if (device.Client is { IsConnected: true } c) return c;
// 05/STAB-8: inside an open backoff window a data-path / per-tick-loop caller fails fast — no
// socket create, no wire connect. The probe loop bypasses this and drives failure/success.
if (!bypassBackoff && !device.Backoff.ShouldAttempt(DateTime.UtcNow))
throw new ConnectionThrottledException(
$"FOCAS connect to {device.Options.HostAddress} throttled — backoff window open after a recent connect failure.");
// Discard the existing client (if any) before creating a new one. A client that is
// non-null but not connected may have been disposed by a HandleRecycle race or a prior
// teardown — retrying ConnectAsync on a disposed FocasWireClient hits ThrowIfDisposed and
// returns a permanent BadCommunicationError with no recovery. Replacing it unconditionally
// ensures EnsureConnectedAsync always works with a fresh, non-disposed instance.
if (device.Client is not null)
{
device.Client.Dispose();
device.Client = null;
}
// Wrap the raw wire client so every operation on the device's single FOCAS/2 socket is
// serialized (request→response on one socket cannot interleave) and time-bounded. Without
// this, the equipment poll, fixed-tree loop, probe, and recycle loop collide on the shared
// socket and a stalled read blocks forever — leaving bound tags at BadWaitingForInitialData.
device.Client = new SynchronizedFocasClient(_clientFactory.Create(), _options.Timeout);
// Serialize the connect (hw2 Makino live gate, 2026-10-08). The alarm poll loop and the fixed-tree loop
// both reach here right after init; unserialized, the second caller saw the first caller's still-
// CONNECTING client as "not connected", disposed it and opened a second session, and the first caller's
// failure path then disposed the second caller's client — two sessions to a CNC with a small handle pool,
// a disposed client handed back, and a spurious backoff failure (observed live as the fixed-tree
// bootstrap failing with ObjectDisposedException: FocasWireClient).
await device.ConnectGate.WaitAsync(ct).ConfigureAwait(false);
try
{
await device.Client.ConnectAsync(device.ParsedAddress, _options.Timeout, ct).ConfigureAwait(false);
// Another caller may have connected while this one waited.
if (device.Client is { IsConnected: true } connected) return connected;
// 05/STAB-8: inside an open backoff window a data-path / per-tick-loop caller fails fast — no
// socket create, no wire connect. The probe loop bypasses this and drives failure/success.
if (!bypassBackoff && !device.Backoff.ShouldAttempt(DateTime.UtcNow))
throw new ConnectionThrottledException(
$"FOCAS connect to {device.Options.HostAddress} throttled — backoff window open after a recent connect failure.");
// Discard the existing client (if any) before creating a new one. A client that is
// non-null but not connected may have been disposed by a HandleRecycle race or a prior
// teardown — retrying ConnectAsync on a disposed FocasWireClient hits ThrowIfDisposed and
// returns a permanent BadCommunicationError with no recovery. Replacing it unconditionally
// ensures EnsureConnectedAsync always works with a fresh, non-disposed instance.
if (device.Client is not null)
{
device.Client.Dispose();
device.Client = null;
}
// Wrap the raw wire client so every operation on the device's single FOCAS/2 socket is
// serialized (request→response on one socket cannot interleave) and time-bounded. Without
// this, the equipment poll, fixed-tree loop, probe, and recycle loop collide on the shared
// socket and a stalled read blocks forever — leaving bound tags at BadWaitingForInitialData.
var client = new SynchronizedFocasClient(_clientFactory.Create(), _options.Timeout);
device.Client = client;
try
{
await client.ConnectAsync(device.ParsedAddress, _options.Timeout, ct).ConfigureAwait(false);
}
catch (Exception ex)
{
// Dispose only OUR client — never one another path installed meanwhile (recycle/teardown).
client.Dispose();
if (ReferenceEquals(device.Client, client)) device.Client = null;
// The caller's OWN cancellation says nothing about the CNC: do not open the backoff window, or
// every caller queued on ConnectGate behind it would be throttled for ~1 s for nothing.
if (!(ex is OperationCanceledException && ct.IsCancellationRequested))
device.Backoff.RecordFailure(DateTime.UtcNow); // 05/STAB-8: open the backoff window
throw;
}
device.Backoff.RecordSuccess(); // 05/STAB-8: recovery resets the window immediately
return client;
}
catch
{
device.Client.Dispose();
device.Client = null;
device.Backoff.RecordFailure(DateTime.UtcNow); // 05/STAB-8: open the backoff window
throw;
}
device.Backoff.RecordSuccess(); // 05/STAB-8: recovery resets the window immediately
return device.Client;
finally { device.ConnectGate.Release(); }
}
/// <summary>Disposes the driver and releases all resources synchronously.</summary>
@@ -1919,13 +1938,17 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
/// Per-device connect-attempt throttle (05/STAB-8). A dead CNC fails fast inside the
/// backoff window instead of paying a full two-socket reconnect on every tick of each of
/// the read / write / probe / fixed-tree / recycle loops; the probe loop bypasses it and a
/// successful connect resets it. FOCAS's <c>EnsureConnectedAsync</c> is not gate-serialized
/// (the loops already race the client swap by design), so this is best-effort — atomic
/// enough on 64-bit for the int/DateTime state, and a benign extra/skipped attempt is the
/// worst outcome of a race.
/// successful connect resets it. The connect itself is serialized by
/// <see cref="ConnectGate"/>; the backoff is consulted inside it.
/// </summary>
public ConnectionBackoff Backoff { get; } = new(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
/// <summary>
/// Serializes <c>EnsureConnectedAsync</c> per device, so concurrent loops (alarm poll, fixed tree, data
/// path) share ONE connect instead of racing two sessions and disposing each other's client.
/// </summary>
public SemaphoreSlim ConnectGate { get; } = new(1, 1);
/// <summary>Gets the lock object for probe synchronization.</summary>
public object ProbeLock { get; } = new();
/// <summary>Gets or sets the current host connectivity state.</summary>