fix(twincat): review round 1 - PLC-not-in-RUN degrades, per-device health, retry backoff
- IMPORTANT: a successful probe records the PLC's ADS state; a PLC not in Run
keeps the driver Degraded with 'PLC in {state}' while LastSuccessfulRead
still advances (the device answered). RecordDeviceAnswered never restores
Healthy while any device is out of Run.
- Health is per device, rolled up worst-of: a comms fault is remembered on the
device it was seen on and only that device answering clears it (device A
answering no longer clears device B's fault).
- Refused (non-comms) dead registrations back off 30 s doubling to 10 min and
are re-armed by a symbol-version change or retried by the reconnect replay;
comms-class failures keep the every-tick retry. A retry whose failure class
changes republishes the tag's status, once.
- Bit-RMW gates are kept across generations and teardowns (never cleared).
- ShutdownAsync honours its CancellationToken: it stops waiting, finishes the
bookkeeping and throws OperationCanceledException.
- Remarks on why a superseded InitializeAsync throws OperationCanceledException.
- REAL / LREAL values (scalar or array element) outside the target range are
OverflowException -> BadOutOfRange instead of being written as Infinity.
- String / array write encoding factored into AdsTwinCATClient.PlanStringWrite
/ PlanArrayWrite; the fake runs them so a driver-level test covers the real
encode path against a fake resolved symbol.
- Timing-fragile tests use condition waits / measured probe-tick windows; the
dispose-during-shutdown test asserts ordering with a 1 s bound.
Refs #739
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
@@ -478,11 +478,8 @@ internal sealed class AdsTwinCATClient : ITwinCATClient
|
||||
var (shape, _, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus;
|
||||
if (TwinCATTypeGuard.CheckRead(type, shape) is not null) return TwinCATStatusMapper.BadTypeMismatch;
|
||||
var text = Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty;
|
||||
var encoded = type == TwinCATDataType.WString
|
||||
? TwinCATStringCodec.EncodeWString(text, shape.ByteSize)
|
||||
: TwinCATStringCodec.EncodeString(text, shape.ByteSize, StringsAreUtf8());
|
||||
var (planStatus, encoded) = PlanStringWrite(type, shape, value, StringsAreUtf8());
|
||||
if (planStatus != TwinCATStatusMapper.Good) return planStatus;
|
||||
return await WriteRawAsync(symbolPath, symbol!, encoded, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -499,6 +496,47 @@ internal sealed class AdsTwinCATClient : ITwinCATClient
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pure encode plan of a STRING / WSTRING write against the RESOLVED symbol: the declared kind must
|
||||
/// match the PLC's (<see cref="TwinCATTypeGuard.CheckRead"/>, else <c>BadTypeMismatch</c>), then the value is
|
||||
/// encoded at the PLC's byte width by <see cref="TwinCATStringCodec"/>.
|
||||
/// </summary>
|
||||
/// <param name="type">The declared String / WString type.</param>
|
||||
/// <param name="shape">The resolved PLC symbol shape.</param>
|
||||
/// <param name="value">The written value.</param>
|
||||
/// <param name="utf8">True when the target reports UTF-8 STRING data.</param>
|
||||
/// <returns>Good with the bytes to write, or the refusal status with no bytes.</returns>
|
||||
/// <exception cref="OverflowException">The value does not fit the PLC string.</exception>
|
||||
internal static (uint status, byte[] bytes) PlanStringWrite(
|
||||
TwinCATDataType type, TwinCATPlcSymbolShape shape, object? value, bool utf8)
|
||||
{
|
||||
if (TwinCATTypeGuard.CheckRead(type, shape) is not null) return (TwinCATStatusMapper.BadTypeMismatch, []);
|
||||
var text = Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty;
|
||||
return (TwinCATStatusMapper.Good, type == TwinCATDataType.WString
|
||||
? TwinCATStringCodec.EncodeWString(text, shape.ByteSize)
|
||||
: TwinCATStringCodec.EncodeString(text, shape.ByteSize, utf8));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pure encode plan of an array write against the RESOLVED PLC array: the layout comes from
|
||||
/// <see cref="TwinCATTypeGuard.CheckArray"/> (the array read's rule, else <c>BadTypeMismatch</c>) and the
|
||||
/// bytes from <see cref="TwinCATArrayCodec.Encode"/>.
|
||||
/// </summary>
|
||||
/// <param name="type">The declared element type.</param>
|
||||
/// <param name="elements">The converted elements.</param>
|
||||
/// <param name="shape">The resolved PLC symbol shape.</param>
|
||||
/// <param name="array">The resolved PLC array shape, or null for a non-array symbol.</param>
|
||||
/// <param name="utf8">True when the target reports UTF-8 STRING data.</param>
|
||||
/// <returns>Good with the bytes to write, or the refusal status with no bytes.</returns>
|
||||
/// <exception cref="OverflowException">A string element does not fit its stride.</exception>
|
||||
internal static (uint status, byte[] bytes) PlanArrayWrite(TwinCATDataType type, IReadOnlyList<object> elements,
|
||||
TwinCATPlcSymbolShape shape, TwinCATPlcArrayShape? array, bool utf8)
|
||||
{
|
||||
if (TwinCATTypeGuard.CheckArray(type, elements.Count, shape, array, out var layout) is not null)
|
||||
return (TwinCATStatusMapper.BadTypeMismatch, []);
|
||||
return (TwinCATStatusMapper.Good, TwinCATArrayCodec.Encode(type, elements, layout, utf8));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<uint> WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList<object> elements,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -510,9 +548,8 @@ internal sealed class AdsTwinCATClient : ITwinCATClient
|
||||
var (shape, array, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus;
|
||||
if (TwinCATTypeGuard.CheckArray(type, elements.Count, shape, array, out var layout) is not null)
|
||||
return TwinCATStatusMapper.BadTypeMismatch;
|
||||
var bytes = TwinCATArrayCodec.Encode(type, elements, layout, StringsAreUtf8());
|
||||
var (planStatus, bytes) = PlanArrayWrite(type, elements, shape, array, StringsAreUtf8());
|
||||
if (planStatus != TwinCATStatusMapper.Good) return planStatus;
|
||||
return await WriteRawAsync(symbolPath, symbol!, bytes, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (AdsErrorException ex)
|
||||
@@ -1034,14 +1071,40 @@ internal sealed class AdsTwinCATClient : ITwinCATClient
|
||||
TwinCATDataType.UDInt => Convert.ToUInt32(value),
|
||||
TwinCATDataType.LInt => Convert.ToInt64(value),
|
||||
TwinCATDataType.ULInt => Convert.ToUInt64(value),
|
||||
TwinCATDataType.Real => Convert.ToSingle(value),
|
||||
TwinCATDataType.LReal => Convert.ToDouble(value),
|
||||
TwinCATDataType.Real => ToSingleInRange(value),
|
||||
TwinCATDataType.LReal => ToDoubleInRange(value),
|
||||
TwinCATDataType.String or TwinCATDataType.WString => Convert.ToString(value) ?? string.Empty,
|
||||
TwinCATDataType.Time or TwinCATDataType.Date
|
||||
or TwinCATDataType.DateTime or TwinCATDataType.TimeOfDay => TwinCATTemporalCodec.Encode(type, value),
|
||||
_ => throw new NotSupportedException($"TwinCATDataType {type} not writable."),
|
||||
};
|
||||
|
||||
// Convert.ToSingle(1e300) is +Infinity, and Convert.ToDouble("1e400") too: writing that to the PLC
|
||||
// would store a value nobody wrote. A finite source that overflows the target is out of range; an
|
||||
// explicit infinity (or NaN) passes through as the caller's own value.
|
||||
private static float ToSingleInRange(object? value)
|
||||
{
|
||||
var f = Convert.ToSingle(value, System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (float.IsInfinity(f) && !IsExplicitInfinity(value))
|
||||
throw new OverflowException($"{value} is outside the REAL range (±{float.MaxValue:R})");
|
||||
return f;
|
||||
}
|
||||
|
||||
private static double ToDoubleInRange(object? value)
|
||||
{
|
||||
var d = Convert.ToDouble(value, System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (double.IsInfinity(d) && !IsExplicitInfinity(value))
|
||||
throw new OverflowException($"{value} is outside the LREAL range");
|
||||
return d;
|
||||
}
|
||||
|
||||
private static bool IsExplicitInfinity(object? value) => value switch
|
||||
{
|
||||
float f => float.IsInfinity(f),
|
||||
double d => double.IsInfinity(d),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
private static bool ExtractBit(object? rawWord, int bit) => rawWord switch
|
||||
{
|
||||
short s => (s & (1 << bit)) != 0,
|
||||
|
||||
@@ -51,6 +51,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
private readonly object _lifecycleLock = new();
|
||||
private long _lifecycleGeneration;
|
||||
|
||||
/// <summary>Test accessor: the RMW gate for a device + parent word, or null.</summary>
|
||||
internal SemaphoreSlim? BitRmwGateForTest(string host, string parentPath) =>
|
||||
_bitRmwLocks.TryGetValue($"{host}|{parentPath}", out var g) ? g : null;
|
||||
|
||||
/// <summary>Test seam: awaited after the new generation is built and before it is committed, so a test can
|
||||
/// interleave a teardown deterministically. Never set in production.</summary>
|
||||
internal Func<Task>? BeforeCommitForTest { get; set; }
|
||||
@@ -73,9 +77,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// read-modify-write of the parent word (TwinCAT's symbol table doesn't expose "Word.N"),
|
||||
// so concurrent bit-writers to the same word must serialise or they lose each other's
|
||||
// updates. Keyed by device + parent symbol. (Cannot guard against the PLC program writing
|
||||
// the word between our read and write — inherent to RMW.) Teardown only CLEARS it: a
|
||||
// SemaphoreSlim owns no kernel handle unless AvailableWaitHandle is read, and disposing one an
|
||||
// in-flight write still holds turns its Release into an ObjectDisposedException.
|
||||
// the word between our read and write — inherent to RMW.) The gates are KEPT across generations
|
||||
// and teardowns (never cleared or disposed): a write still in flight on the old generation and the
|
||||
// first write of the next one must serialise on the SAME gate. A SemaphoreSlim owns no kernel handle
|
||||
// unless AvailableWaitHandle is read, and the key set is bounded by the authored bit tags.
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _bitRmwLocks =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -152,6 +157,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
public string DriverType => "TwinCAT";
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// An initialise that is overtaken before it commits — a <see cref="ShutdownAsync"/> / <see cref="Dispose"/>
|
||||
/// or a newer <see cref="InitializeAsync"/> bumped the lifecycle generation while it was building — releases
|
||||
/// what it built and throws <see cref="OperationCanceledException"/>. Cancellation is the honest outcome:
|
||||
/// the work was abandoned by a later lifecycle call, not failed, so it must not write <c>Faulted</c> over
|
||||
/// the newer state or let a caller believe this generation is live; returning normally would claim a
|
||||
/// driver that was never installed. The runtime has usually abandoned the task by then, and an
|
||||
/// overlapping retry treats it like any other cancelled attempt.
|
||||
/// </remarks>
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
// InitializeAsync is reachable on a live driver with no teardown first (the IDriver contract
|
||||
@@ -353,7 +367,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// The poll loops and the device release wind down in parallel (releasing a client can unblock a
|
||||
// poll loop stuck in a read on it); the whole teardown is bounded.
|
||||
var teardown = Task.WhenAll(_poll.DisposeAsync().AsTask(), ReleaseDevicesAsync(taken));
|
||||
try { await teardown.WaitAsync(ShutdownBudget).ConfigureAwait(false); }
|
||||
var cancelled = false;
|
||||
try { await teardown.WaitAsync(ShutdownBudget, cancellationToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller stopped waiting. The tables are already swapped out and every loop is cancelled, so
|
||||
// the rest of the teardown finishes in the background; report the cancellation after the cheap
|
||||
// bookkeeping below.
|
||||
cancelled = true;
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
@@ -364,8 +386,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
|
||||
_warnedRefs.Clear();
|
||||
_resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry
|
||||
_bitRmwLocks.Clear();
|
||||
_health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
|
||||
if (cancelled) throw new OperationCanceledException(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -448,7 +470,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
var (value, status) = await client.ReadValueAsync(
|
||||
symbolName, def.DataType, parsed?.BitIndex, def.ArrayLength, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return CompleteRead(reference, def, symbolName, value, status, now);
|
||||
return CompleteRead(device, reference, def, symbolName, value, status, now);
|
||||
}
|
||||
catch (OperationCanceledException) { throw; }
|
||||
catch (Exception ex)
|
||||
@@ -459,13 +481,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
}
|
||||
|
||||
/// <summary>Maps one read outcome to a snapshot and applies the health / log side effects.</summary>
|
||||
private DataValueSnapshot CompleteRead(
|
||||
private DataValueSnapshot CompleteRead(DeviceState device,
|
||||
string reference, TwinCATTagDefinition def, string symbolName, object? value, uint status, DateTime now)
|
||||
{
|
||||
if (status == TwinCATStatusMapper.Good)
|
||||
{
|
||||
_warnedRefs.TryRemove(reference, out _); // a later failure is a new episode and warns again
|
||||
RecordDeviceAnswered(now, force: true);
|
||||
RecordDeviceAnswered(now, device, force: true);
|
||||
return new DataValueSnapshot(value, status, now, now);
|
||||
}
|
||||
|
||||
@@ -487,7 +509,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
}
|
||||
|
||||
if (IsCommsStatus(status))
|
||||
MarkCommsFault($"ADS status {status:X8} reading {reference}");
|
||||
MarkCommsFault($"ADS status {status:X8} reading {reference}", device);
|
||||
else
|
||||
RecordDeviceAnsweredKeepState(now); // a refusal: the device answered — no state change
|
||||
return new DataValueSnapshot(value, status, now, now);
|
||||
@@ -558,7 +580,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
{
|
||||
var e = chunk[k];
|
||||
var (value, status) = k < bulk.Count ? bulk[k] : (null, TwinCATStatusMapper.BadCommunicationError);
|
||||
results[e.Index] = CompleteRead(e.Reference, e.Def, e.SymbolName, value, status, now);
|
||||
results[e.Index] = CompleteRead(device, e.Reference, e.Def, e.SymbolName, value, status, now);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { throw; }
|
||||
@@ -842,7 +864,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// A push is the device answering: stamp LastSuccessfulRead so a notification-only
|
||||
// driver does not look silent to the comms-loss watchdog.
|
||||
var now = DateTime.UtcNow;
|
||||
RecordDeviceAnswered(now);
|
||||
RecordDeviceAnswered(now, device);
|
||||
RaiseDataChange(handle, reference, new DataValueSnapshot(value, TwinCATStatusMapper.Good, now, now));
|
||||
},
|
||||
// When the handle goes dead (a failed replay) push a Bad snapshot so
|
||||
@@ -856,6 +878,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// Same device already failed to connect in this call: keep the intent dead for the
|
||||
// probe-tick retry / reconnect replay and surface Bad now.
|
||||
device.NativeRegistrations[reg.Id] = reg;
|
||||
NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow);
|
||||
RaiseDataChange(handle, reference, new DataValueSnapshot(
|
||||
null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow));
|
||||
continue;
|
||||
@@ -876,6 +899,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
_logger.LogWarning(
|
||||
"TwinCAT driver '{DriverInstanceId}' refused native notification for '{Reference}': {Reason}",
|
||||
_driverInstanceId, reference, refused.Message);
|
||||
NoteDeadFailure(reg, refused.StatusCode, refused: true, DateTime.UtcNow);
|
||||
RaiseDataChange(handle, reference, new DataValueSnapshot(
|
||||
null, refused.StatusCode, null, DateTime.UtcNow));
|
||||
}
|
||||
@@ -885,6 +909,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// Every tag of this device goes Bad + dead for the retry; other devices carry on.
|
||||
(unreachable ??= []).Add(device);
|
||||
MarkCommsFault(unavailable.InnerException?.Message ?? unavailable.Message, device);
|
||||
NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow);
|
||||
RaiseDataChange(handle, reference, new DataValueSnapshot(
|
||||
null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow));
|
||||
}
|
||||
@@ -897,6 +922,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
"'{Reference}'; it will be retried on the next healthy probe tick",
|
||||
_driverInstanceId, reference);
|
||||
MarkCommsFault(ex.Message, device);
|
||||
NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow);
|
||||
RaiseDataChange(handle, reference, new DataValueSnapshot(
|
||||
null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow));
|
||||
}
|
||||
@@ -928,7 +954,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
|
||||
/// <summary>
|
||||
/// Records a comms fault: <see cref="DriverState.Degraded"/> preserving <c>LastSuccessfulRead</c>.
|
||||
/// Never downgrades <see cref="DriverState.Faulted"/>.
|
||||
/// Never downgrades <see cref="DriverState.Faulted"/>. A fault observed on a device is remembered ON
|
||||
/// that device (<see cref="DeviceState.CommsFault"/>) and only that device answering clears it — the
|
||||
/// driver state is the worst of every device (#739 fix round 1).
|
||||
/// </summary>
|
||||
/// <param name="message">The fault description.</param>
|
||||
/// <param name="device">The device the fault was observed on; a fault on a device already swapped out of
|
||||
@@ -937,6 +965,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
private void MarkCommsFault(string message, DeviceState? device = null)
|
||||
{
|
||||
if (device?.Released == true) return;
|
||||
if (device is not null) device.CommsFault = message;
|
||||
var current = _health;
|
||||
if (current.State != DriverState.Faulted)
|
||||
_health = new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, message);
|
||||
@@ -954,29 +983,90 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
private static readonly TimeSpan AnsweredStampInterval = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Records that a device answered (a native push, a successful probe, a Good read): advances
|
||||
/// <c>LastSuccessfulRead</c> and restores <see cref="DriverState.Healthy"/> unless a device is still
|
||||
/// probe-unreachable (then the state stays Degraded, the stamp still advances). Throttled so a
|
||||
/// Why one device keeps the driver Degraded, or null when it does not: probe-unreachable, a PLC whose
|
||||
/// ADS state is not <c>Run</c> (coordinator ruling: a stopped / config-mode PLC is Degraded with a
|
||||
/// reason even though it answers), or an uncleared comms fault seen on it.
|
||||
/// </summary>
|
||||
private static string? DeviceDegradedReason(DeviceState d)
|
||||
{
|
||||
if (d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded)
|
||||
return $"TwinCAT device {d.Options.HostAddress} unreachable ({d.ConsecutiveProbeFailures} consecutive failed probes)";
|
||||
if (d.PlcState is { } st && !string.Equals(st, "Run", StringComparison.OrdinalIgnoreCase))
|
||||
return $"TwinCAT device {d.Options.HostAddress}: PLC in {st}";
|
||||
return d.CommsFault;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records that a device answered (a native push, a successful probe, a Good read): clears THAT
|
||||
/// device's comms fault, advances <c>LastSuccessfulRead</c>, and sets the state to the worst of every
|
||||
/// device — Healthy only when no device is unreachable, out of RUN or still faulted (then Degraded
|
||||
/// with that device's reason; the stamp still advances, since a device did answer). Throttled so a
|
||||
/// notification burst does not allocate a health record per value; never touches Faulted.
|
||||
/// </summary>
|
||||
/// <param name="now">The time the device answered.</param>
|
||||
/// <param name="device">The device that answered; null when not attributable.</param>
|
||||
/// <param name="force">Write even inside the throttle window (an explicit read / probe outcome).</param>
|
||||
private void RecordDeviceAnswered(DateTime now, bool force = false)
|
||||
private void RecordDeviceAnswered(DateTime now, DeviceState? device = null, bool force = false)
|
||||
{
|
||||
if (device?.Released == true) return;
|
||||
if (device?.CommsFault is not null)
|
||||
{
|
||||
device.CommsFault = null;
|
||||
force = true;
|
||||
}
|
||||
var current = _health;
|
||||
if (current.State == DriverState.Faulted) return;
|
||||
// Hot path (a notification on a healthy driver inside the window): no device scan, no allocation.
|
||||
if (!force && current.State == DriverState.Healthy && current.LastSuccessfulRead is { } recent
|
||||
&& now - recent < AnsweredStampInterval)
|
||||
return;
|
||||
var unreachable = _devices.Values.FirstOrDefault(d => d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded);
|
||||
var state = unreachable is null ? DriverState.Healthy : DriverState.Degraded;
|
||||
string? reason = null;
|
||||
foreach (var d in _devices.Values)
|
||||
if ((reason = DeviceDegradedReason(d)) is not null) break;
|
||||
var state = reason is null ? DriverState.Healthy : DriverState.Degraded;
|
||||
if (!force && current.State == state && current.LastSuccessfulRead is { } last
|
||||
&& now - last < AnsweredStampInterval)
|
||||
return;
|
||||
_health = new DriverHealth(state, now, unreachable is null
|
||||
? null
|
||||
: $"TwinCAT device {unreachable.Options.HostAddress} unreachable ({unreachable.ConsecutiveProbeFailures} consecutive failed probes)");
|
||||
_health = new DriverHealth(state, now, reason);
|
||||
}
|
||||
|
||||
/// <summary>First probe-tick retry delay of a REFUSED dead registration (the PLC answered and refused it).</summary>
|
||||
internal static readonly TimeSpan RefusedRetryInitial = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Cap of the refused-registration retry backoff.</summary>
|
||||
internal static readonly TimeSpan RefusedRetryMax = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Books a failed registration attempt. A REFUSAL is deterministic until the PLC program changes, so it
|
||||
/// backs off exponentially (30 s doubling to 10 min) and is re-armed at once by a symbol-version change
|
||||
/// or retried by the reconnect replay; a comms-class failure stays on the every-tick retry.
|
||||
/// </summary>
|
||||
/// <returns>True when the published status changes (the caller republishes the tag).</returns>
|
||||
private static bool NoteDeadFailure(NativeRegistration reg, uint status, bool refused, DateTime now)
|
||||
{
|
||||
if (refused)
|
||||
{
|
||||
reg.RefusalBackoff = reg.RefusalBackoff == TimeSpan.Zero
|
||||
? RefusedRetryInitial
|
||||
: TimeSpan.FromTicks(Math.Min(reg.RefusalBackoff.Ticks * 2, RefusedRetryMax.Ticks));
|
||||
reg.NextRetryUtc = now + reg.RefusalBackoff;
|
||||
}
|
||||
else
|
||||
{
|
||||
reg.RefusalBackoff = TimeSpan.Zero;
|
||||
reg.NextRetryUtc = DateTime.MinValue;
|
||||
}
|
||||
var changed = reg.DeadStatus != status;
|
||||
reg.DeadStatus = status;
|
||||
return changed;
|
||||
}
|
||||
|
||||
/// <summary>Clears the dead-registration bookkeeping after a successful (re-)registration.</summary>
|
||||
private static void NoteLive(NativeRegistration reg)
|
||||
{
|
||||
reg.DeadStatus = null;
|
||||
reg.RefusalBackoff = TimeSpan.Zero;
|
||||
reg.NextRetryUtc = DateTime.MinValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1159,6 +1249,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
public ITwinCATNotificationHandle? MarkHandleDead() =>
|
||||
Interlocked.Exchange(ref _handle, null);
|
||||
|
||||
/// <summary>The Bad status last published for this registration while dead; null while live.</summary>
|
||||
public uint? DeadStatus { get; set; }
|
||||
|
||||
/// <summary>Earliest time the probe-tick retry may re-attempt this dead registration.</summary>
|
||||
public DateTime NextRetryUtc { get; set; }
|
||||
|
||||
/// <summary>Current refusal backoff (zero after a comms-class failure or a success).</summary>
|
||||
public TimeSpan RefusalBackoff { get; set; }
|
||||
|
||||
private volatile bool _disposed;
|
||||
|
||||
/// <summary>Gets a value indicating whether this registration has been disposed (its callbacks are dropped).</summary>
|
||||
@@ -1195,6 +1294,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
var client = await EnsureConnectedAsync(state, ct, _options.Probe.Timeout, bypassBackoff: true)
|
||||
.ConfigureAwait(false);
|
||||
success = await client.ProbeAsync(ct).ConfigureAwait(false);
|
||||
// The probe's ReadState also says whether the PLC is in RUN; a stopped PLC answers but
|
||||
// serves no live data, so it keeps the driver Degraded (RecordDeviceAnswered below).
|
||||
if (success) state.PlcState = client.LastAdsState;
|
||||
|
||||
// A locally-connected client whose wire probe FAILS is wire-dead but port-open:
|
||||
// force a recycle so the next tick rebuilds + replays native notifications. Without
|
||||
@@ -1224,13 +1326,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
if (success)
|
||||
{
|
||||
state.ConsecutiveProbeFailures = 0;
|
||||
RecordDeviceAnswered(DateTime.UtcNow, force: true);
|
||||
RecordDeviceAnswered(DateTime.UtcNow, state, force: true);
|
||||
if (probed is not null) ReportNotificationDrops(state, probed);
|
||||
}
|
||||
else if (++state.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded)
|
||||
{
|
||||
MarkCommsFault($"TwinCAT device {state.Options.HostAddress} unreachable " +
|
||||
$"({state.ConsecutiveProbeFailures} consecutive failed probes)");
|
||||
$"({state.ConsecutiveProbeFailures} consecutive failed probes)", state);
|
||||
}
|
||||
|
||||
TransitionDeviceState(state, success ? HostState.Running : HostState.Stopped);
|
||||
@@ -1625,6 +1727,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
try { orphan?.Dispose(); } catch { /* best-effort */ }
|
||||
continue;
|
||||
}
|
||||
NoteLive(reg);
|
||||
replayed++;
|
||||
}
|
||||
catch (TwinCATNotificationRefusedException refused)
|
||||
@@ -1638,7 +1741,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
_driverInstanceId, reg.SymbolName, device.Options.HostAddress, refused.Message);
|
||||
var dead = reg.MarkHandleDead();
|
||||
try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ }
|
||||
try { reg.OnRefused(refused.StatusCode); } catch { /* never abort recovery of the rest */ }
|
||||
if (NoteDeadFailure(reg, refused.StatusCode, refused: true, DateTime.UtcNow))
|
||||
try { reg.OnRefused(refused.StatusCode); } catch { /* never abort recovery of the rest */ }
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -1652,8 +1756,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
// no quality channel), and degrade health preserving the last successful read.
|
||||
var dead = reg.MarkHandleDead();
|
||||
try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ }
|
||||
try { reg.OnUnavailable(); } catch { /* never let a subscriber callback abort recovery of the rest */ }
|
||||
MarkCommsFault($"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}");
|
||||
if (NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow))
|
||||
try { reg.OnUnavailable(); } catch { /* never let a subscriber callback abort recovery of the rest */ }
|
||||
MarkCommsFault($"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}", device);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1682,9 +1787,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
if (device.Client is not { IsConnected: true } client) return;
|
||||
|
||||
var recovered = 0;
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var reg in device.NativeRegistrations.Values)
|
||||
{
|
||||
if (reg.HasLiveHandle) continue;
|
||||
// A refused registration waits out its backoff (a symbol-version change re-arms it).
|
||||
if (now < reg.NextRetryUtc) continue;
|
||||
try
|
||||
{
|
||||
var newHandle = await client.AddNotificationAsync(
|
||||
@@ -1698,11 +1806,18 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
try { orphan?.Dispose(); } catch { /* best-effort */ }
|
||||
continue;
|
||||
}
|
||||
NoteLive(reg);
|
||||
recovered++;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Republish only when the failure CLASS changes (a comms blip that turned into a refusal,
|
||||
// or the reverse); the same failure again publishes nothing.
|
||||
var refusal = ex as TwinCATNotificationRefusedException;
|
||||
var status = refusal?.StatusCode ?? TwinCATStatusMapper.BadCommunicationError;
|
||||
if (NoteDeadFailure(reg, status, refusal is not null, now))
|
||||
try { reg.OnRefused(status); } catch { /* never abort the rest */ }
|
||||
_logger.LogDebug(ex,
|
||||
"TwinCAT driver '{DriverInstanceId}' probe-tick retry of dead notification " +
|
||||
"'{Symbol}' on device '{HostAddress}' failed; will retry next tick",
|
||||
@@ -1758,10 +1873,18 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
/// space must be rebuilt — this is the documented TwinCAT failure mode, not a transient
|
||||
/// connection error.
|
||||
/// </summary>
|
||||
private void HandleSymbolVersionChanged(object? sender, EventArgs e) =>
|
||||
private void HandleSymbolVersionChanged(object? sender, EventArgs e)
|
||||
{
|
||||
// A program change can make a refused registration valid: re-arm every backed-off dead intent of the
|
||||
// device whose client saw it, so the next probe tick re-checks them at once.
|
||||
foreach (var d in _devices.Values)
|
||||
if (ReferenceEquals(d.Client, sender))
|
||||
foreach (var reg in d.NativeRegistrations.Values)
|
||||
reg.NextRetryUtc = DateTime.MinValue;
|
||||
OnRediscoveryNeeded?.Invoke(this, new RediscoveryEventArgs(
|
||||
"TwinCAT symbol-version-changed (DeviceSymbolVersionInvalid 0x0711) — PLC program re-downloaded",
|
||||
ScopeHint: "TwinCAT"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous teardown — no <c>await</c>, no captured sync context. The OPC UA stack
|
||||
@@ -1790,7 +1913,6 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
|
||||
_warnedRefs.Clear();
|
||||
_resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry
|
||||
_bitRmwLocks.Clear();
|
||||
_health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
|
||||
}
|
||||
|
||||
@@ -1852,6 +1974,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
/// <summary>True once an unexpected sum-read chunk failure was logged for this device.</summary>
|
||||
public volatile bool SumChunkFailureWarned;
|
||||
|
||||
/// <summary>The last comms fault seen on this device; cleared when this device answers. Null = none.</summary>
|
||||
public volatile string? CommsFault;
|
||||
|
||||
/// <summary>The ADS state (<c>Run</c>, <c>Stop</c>, <c>Config</c>, …) the last successful probe read; null = unknown.</summary>
|
||||
public volatile string? PlcState;
|
||||
|
||||
/// <summary>Consecutive failed probe ticks; reset by a successful probe. Written only by this device's probe loop.</summary>
|
||||
public volatile int ConsecutiveProbeFailures;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user