fix(ablegacy): any answer restores Healthy; writes feed health and the backoff

Review r0 I2 + M1/M2/M4/M7/M10. - I2: one health writer (ApplyHealth) for reads AND writes, worst wins per call: a comms fault => Degraded; otherwise any answer (Good value, device refusal, Good or refused write) => Healthy; LastSuccessfulRead advances on any answer. A refusal-only read used to leave a recovered driver Degraded, and writes never touched health at all. - M10: a no-reply write STATUS now degrades like the exception path did; a Good/refused write records an answer on the device's backoff, a no-reply one a fault (bit RMW included — it now reports its raw status). - M2: a write inside the device's backoff window fails fast with BadCommunicationError even when its handle is cached. - M1: ShutdownAsync/Dispose write their final Unknown only while the epoch they retired is still current (a newer initialise owns health otherwise). - M4: an indirect bit write releases the previous index value's parent-word handle (the per-word RMW gate is deliberately kept: removing a gate another writer may hold would let two RMWs of one word race). - M7: backoff decisions read an injectable Clock; the isolation tests drop their stopwatch asserts and cross the window on a manual clock. Refs #738 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 15:16:49 -04:00
parent 8a80dcc787
commit 17ae2eea0b
3 changed files with 360 additions and 63 deletions
@@ -177,7 +177,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        // allows it, and a reconnect retry can overlap a running attempt). Release the previous device states' probe loops + runtimes BEFORE the
        // new table is published, or they are orphaned beyond the reach of ShutdownAsync. Deliberately NOT
        // ShutdownAsync: that also drops _poll and every subscription, which are owned by the shutdown path (the actor re-subscribes after every successful init).
-        await ReleaseDeviceStatesAsync(RetireTables(bumpEpoch: false), "re-initialise").ConfigureAwait(false);
+        await ReleaseDeviceStatesAsync(RetireTables(bumpEpoch: false).States, "re-initialise").ConfigureAwait(false);
        SetHealthIfCurrent(epoch, new DriverHealth(DriverState.Initializing, null, null));
        _logger.LogInformation(
            "AbLegacy driver {DriverInstanceId} initializing: transport={Transport} tagFactory={TagFactory}",
@@ -328,8 +328,9 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    ///     is clearing.
    /// </summary>
    /// <param name="bumpEpoch">True for a teardown: supersedes any in-flight initialise.</param>
-    /// <returns>The device states that were published.</returns>
-    private List<DeviceState> RetireTables(bool bumpEpoch)
+    /// <returns>The device states that were published, and the epoch as of the retirement (a teardown writes its final
+    /// health only while that epoch is still current — review 07 r0 M1).</returns>
+    private (List<DeviceState> States, long Epoch) RetireTables(bool bumpEpoch)
    {
        lock (_lifecycleLock)
        {
@@ -340,7 +341,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
            }
            var retired = _devices.Values.ToList();
            _devices = new Dictionary<string, DeviceState>(StringComparer.OrdinalIgnoreCase);
-            return retired;
+            return (retired, _lifecycleEpoch);
        }
    }

@@ -401,15 +402,19 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    public async Task ShutdownAsync(CancellationToken cancellationToken)
    {
        // Unpublish FIRST, so an in-flight initialise is superseded and new reads resolve nothing.
-        var retired = RetireTables(bumpEpoch: true);
+        var (retired, teardownEpoch) = RetireTables(bumpEpoch: true);
        await _poll.DisposeAsync().ConfigureAwait(false);
        // Cancel the probe loops, JOIN them (bounded), then dispose the CTSs and runtimes.
        // The loop used to be fire-and-forget, so one parked in a probe-runtime create/read could
        // outlive shutdown and keep a libplctag handle alive past retirement.
        await ReleaseDeviceStatesAsync(retired, "shutdown").ConfigureAwait(false);
        _resolver.Clear(); // drop transient equipment-tag parses so a config change re-parses
+        // A newer InitializeAsync that ran while this teardown was still joining owns health now.
        lock (_lifecycleLock)
-            _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+        {
+            if (teardownEpoch == _lifecycleEpoch)
+                _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+        }
    }

    /// <inheritdoc />
@@ -445,6 +450,13 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    /// </summary>
    internal const int FaultsToMarkDeviceDown = 2;

+    /// <summary>
+    ///     The clock every per-device backoff decision reads (window open? when does it end?). Wall-clock UTC in
+    ///     production; a test substitutes a manual clock so the window can be crossed deterministically rather than by
+    ///     sleeping (review 07 r0 M7).
+    /// </summary>
+    internal Func<DateTime> Clock { get; set; } = () => DateTime.UtcNow;
+
    /// <summary>One resolved reference queued for its device's pass.</summary>
    private sealed record ReadItem(int Index, string Reference, AbLegacyTagDefinition Def, AbLegacyAddress Parsed);

@@ -552,19 +564,30 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
            anyAnswered |= o.AnyAnswered;
            fault ??= o.FaultMessage;
        }
-        if (fault is null && !anyGood && !anyAnswered) return;
+        ApplyHealth(epoch, fault, anyGood, anyAnswered, now);
+    }

+    /// <summary>
+    ///     The single health write for a read or write call (worst wins): any comms fault ⇒ Degraded; otherwise any
+    ///     answer — a Good value OR a device refusal, both prove the device replied — ⇒ Healthy (review 07 r0 I2: a
+    ///     refusal-only pass used to leave a recovered driver Degraded). <c>LastSuccessfulRead</c> advances whenever
+    ///     any device answered. A call that reached no wire changes nothing; a Faulted (config-level) state is kept.
+    ///     Epoch-guarded so a late call cannot rewrite the health of a teardown or newer initialise.
+    /// </summary>
+    private void ApplyHealth(long epoch, string? fault, bool anyGood, bool anyAnswered, DateTime now)
+    {
+        if (fault is null && !anyGood && !anyAnswered) return;
        lock (_lifecycleLock)
        {
            if (epoch != _lifecycleEpoch) return;
            var current = _health;
            var lastRead = anyAnswered ? now : current.LastSuccessfulRead;
-            if (fault is not null)
-                _health = new DriverHealth(DriverState.Degraded, lastRead, fault);
-            else if (anyGood)
-                _health = new DriverHealth(DriverState.Healthy, lastRead, null);
-            else
+            if (current.State == DriverState.Faulted)
                _health = current with { LastSuccessfulRead = lastRead };
+            else if (fault is not null)
+                _health = new DriverHealth(DriverState.Degraded, lastRead, fault);
+            else
+                _health = new DriverHealth(DriverState.Healthy, lastRead, null);
        }
    }

@@ -577,7 +600,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        while (work.TryDequeue(out var unit))
        {
            var members = unit.Block?.Members ?? [unit.Single!];
-            if (device.IsThrottled(DateTime.UtcNow))
+            if (device.IsThrottled(Clock()))
            {
                foreach (var m in members)
                    results[m.Index] = new DataValueSnapshot(null, AbLegacyStatusMapper.BadCommunicationError, null, now);
@@ -613,7 +636,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                    break;
                case OpVerdict.Fault:
                    outcome.FaultMessage ??= message;
-                    device.RecordCommsFault(DateTime.UtcNow);
+                    device.RecordCommsFault(Clock());
                    break;
                case OpVerdict.Throttled:
                    outcome.FaultMessage ??= message;
@@ -785,6 +808,9 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        var epoch = CurrentEpoch;
        var results = new WriteResult[writes.Count];
        var devices = _devices; // one snapshot of the published table for the whole call
+        // Writes feed health too (review 07 r0 I2): a Good or device-refused write proves the device answered; a
+        // no-reply failure is a comms fault. Collected per call and written once, worst wins.
+        var outcome = new DeviceReadOutcome();

        for (var i = 0; i < writes.Count; i++)
        {
@@ -817,6 +843,15 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                    continue;
                }

+                // Inside the device's backoff window nothing is dialled — the write fails fast exactly as a read
+                // does, instead of paying the full Timeout on a handle that is still cached (review 07 r0 M2).
+                if (device.IsThrottled(Clock()))
+                {
+                    results[i] = new WriteResult(AbLegacyStatusMapper.BadCommunicationError);
+                    outcome.FaultMessage ??= $"device {device.Options.HostAddress} is in its reconnect backoff window";
+                    continue;
+                }
+
                // Indirect: resolve the index first (non-atomic with the write - the index may change
                // between the two PLC accesses), then write the concrete address through the normal path.
                if (parsed.IsIndirect)
@@ -826,8 +861,10 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                    {
                        results[i] = new WriteResult(res.Status);
                        if (res.TransportFault)
-                            SetHealthIfCurrent(epoch, new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead,
-                                $"transport fault resolving indirect address for {w.FullReference}"));
+                        {
+                            device.RecordCommsFault(Clock());
+                            outcome.FaultMessage ??= $"transport fault resolving indirect address for {w.FullReference}";
+                        }
                        continue;
                    }
                    def = res.Def;
@@ -872,8 +909,10 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                // may reject it; we surface that real PCCC status rather than pre-rejecting at the driver.
                else if (def.DataType == AbLegacyDataType.Bit && parsed.BitIndex is int bit)
                {
-                    results[i] = new WriteResult(
-                        await WriteBitInWordAsync(device, parsed, bit, w.Value, cancellationToken).ConfigureAwait(false));
+                    var (code, raw) = await WriteBitInWordAsync(device, def, parsed, bit, w.Value, cancellationToken)
+                        .ConfigureAwait(false);
+                    results[i] = new WriteResult(code);
+                    RecordWriteStatus(device, def, raw, ref outcome);
                    continue;
                }

@@ -896,36 +935,23 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                    opLock.Release();
                }

-                if (status == 0)
-                {
-                    device.RecordAnswered();
-                    results[i] = new WriteResult(AbLegacyStatusMapper.Good);
-                }
-                else if (IsDeviceRefusal((AbStatus)status))
-                {
-                    // The device answered and refused this write deterministically: keep the handle, warn once.
-                    device.RecordAnswered();
-                    WarnRefusalOnce(device, def, "write", $"device status {(AbStatus)status}");
-                    results[i] = new WriteResult(AbLegacyStatusMapper.MapStatus(status));
-                }
-                else
-                {
-                    // Evict the stale handle so the next call re-creates it (mirrors AbCip).
-                    EvictRuntime(device, def.Name);
-                    device.RecordCommsFault(DateTime.UtcNow);
-                    results[i] = new WriteResult(AbLegacyStatusMapper.MapStatus(status));
-                }
+                results[i] = new WriteResult(status == 0 ? AbLegacyStatusMapper.Good : AbLegacyStatusMapper.MapStatus(status));
+                // A no-reply status evicts the stale handle so the next call re-creates it (mirrors AbCip).
+                if (IsNoReply((AbStatus)status) && status != 0) EvictRuntime(device, def.Name);
+                RecordWriteStatus(device, def, status, ref outcome);
            }
            catch (OperationCanceledException) { throw; }
-            catch (ConnectionThrottledException)
+            catch (ConnectionThrottledException tex)
            {
-                // The device is in its reconnect backoff window: fail fast, no wire, no new evidence.
+                // The device is in its reconnect backoff window: fail fast, no wire, no new evidence for the backoff.
                results[i] = new WriteResult(AbLegacyStatusMapper.BadCommunicationError);
+                outcome.FaultMessage ??= tex.Message;
            }
            catch (AbTransportException ex) when (IsDeviceRefusal(ex.Status))
            {
                device.RecordAnswered();
-                WarnRefusalOnce(device, def, "write", ex.Message);
+                outcome.AnyAnswered = true;
+                WarnRefusalOnce(device, def, "write", ex.Message, AbLegacyStatusMapper.MapStatus(ex.Status));
                results[i] = new WriteResult(AbLegacyStatusMapper.MapStatus(ex.Status));
            }
            catch (NotSupportedException nse)
@@ -953,15 +979,38 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
            {
                // Transport exception — evict so the next write/read creates a fresh handle (mirrors AbCip).
                EvictRuntime(device, def.Name);
-                device.RecordCommsFault(DateTime.UtcNow);
+                device.RecordCommsFault(Clock());
                results[i] = new WriteResult(AbLegacyStatusMapper.BadCommunicationError);
-                SetHealthIfCurrent(epoch, new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, ex.Message));
+                outcome.FaultMessage ??= ex.Message;
            }
        }

+        ApplyHealth(epoch, outcome.FaultMessage, anyGood: outcome.AnyAnswered, anyAnswered: outcome.AnyAnswered, DateTime.UtcNow);
        return results;
    }

+    /// <summary>
+    ///     Classifies a completed write's raw status into the call's health evidence and the device's backoff:
+    ///     Good or a device refusal = answered; a no-reply status = comms fault (it used to leave health untouched on
+    ///     this path while the exception path degraded — review 07 r0 M10); Unsupported = neutral.
+    /// </summary>
+    private void RecordWriteStatus(DeviceState device, AbLegacyTagDefinition def, int status, ref DeviceReadOutcome outcome)
+    {
+        var s = (AbStatus)status;
+        if (status == 0 || IsDeviceRefusal(s))
+        {
+            device.RecordAnswered();
+            outcome.AnyAnswered = true;
+            if (status != 0)
+                WarnRefusalOnce(device, def, "write", $"device status {s}", AbLegacyStatusMapper.MapStatus(status));
+        }
+        else if (IsNoReply(s))
+        {
+            device.RecordCommsFault(Clock());
+            outcome.FaultMessage ??= $"transport status {status} writing {def.Name}";
+        }
+    }
+
    // ---- ITagDiscovery ----

    /// <inheritdoc />
@@ -1052,7 +1101,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                // The probe's verdict feeds the data path's backoff (review 07 P7): an answer resets the window so a
                // recovered device is read again at once; a comms fault counts toward marking it down.
                if (success || IsDeviceRefusal((AbStatus)status)) state.RecordAnswered();
-                else if ((AbStatus)status != AbStatus.Unsupported) state.RecordCommsFault(DateTime.UtcNow);
+                else if ((AbStatus)status != AbStatus.Unsupported) state.RecordCommsFault(Clock());
            }
            catch (OperationCanceledException) when (ct.IsCancellationRequested) { break; }
            catch (Exception ex)
@@ -1061,7 +1110,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                probeRuntime = null;
                state.ProbeInitialized = false;
                if (ex is AbTransportException ate && IsDeviceRefusal(ate.Status)) state.RecordAnswered();
-                else if (!IsLocalRefusal(ex)) state.RecordCommsFault(DateTime.UtcNow);
+                else if (!IsLocalRefusal(ex)) state.RecordCommsFault(Clock());
            }

            TransitionDeviceState(state, success ? HostState.Running : HostState.Stopped);
@@ -1155,8 +1204,10 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    ///     <see cref="SemaphoreSlim"/>. The parent word is masked to its native width before the
    ///     bit operation so a sign-extended decode never corrupts the high bits.
    /// </summary>
-    private async Task<uint> WriteBitInWordAsync(
-        AbLegacyDriver.DeviceState device, AbLegacyAddress bitAddress, int bit, object? value, CancellationToken ct)
+    /// <returns>The OPC UA status the write is answered with, and the raw transport status (0 = Good) for health.</returns>
+    private async Task<(uint Code, int Raw)> WriteBitInWordAsync(
+        AbLegacyDriver.DeviceState device, AbLegacyTagDefinition def, AbLegacyAddress bitAddress, int bit, object? value,
+        CancellationToken ct)
    {
        // The parent word width follows the file letter: an L-file element is 32-bit, every
        // other bit-addressable PCCC file (N/I/O/S/A) is a 16-bit word. bit is already
@@ -1169,6 +1220,23 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        var parentAddress = bitAddress with { BitIndex = null };
        var parentName = parentAddress.ToLibplctagName();

+        // An indirect bit write (N7:[N7:0]/3) resolves to a different parent word per index value: release the
+        // previous resolution's parent handle so a sweeping index does not keep one per word (review 07 r0 M4). The
+        // per-word RMW gate is kept — removing a gate another writer may hold would let two RMWs of one word race.
+        var arrow = def.Name.IndexOf("=>", StringComparison.Ordinal);
+        if (arrow > 0)
+        {
+            var indirectName = def.Name[..arrow];
+            var previous = device.IndirectLastParent.TryGetValue(indirectName, out var p) ? p : null;
+            device.IndirectLastParent[indirectName] = parentName;
+            if (previous is not null && !string.Equals(previous, parentName, StringComparison.OrdinalIgnoreCase))
+            {
+                var oldGate = device.GetRmwLock(previous);
+                await oldGate.WaitAsync(ct).ConfigureAwait(false);
+                try { EvictParentRuntime(device, previous); } finally { oldGate.Release(); }
+            }
+        }
+
        var rmwLock = device.GetRmwLock(parentName);
        await rmwLock.WaitAsync(ct).ConfigureAwait(false);
        try
@@ -1176,7 +1244,11 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
            var parentRuntime = await EnsureParentRuntimeAsync(device, parentName, ct).ConfigureAwait(false);
            await parentRuntime.ReadAsync(ct).ConfigureAwait(false);
            var readStatus = parentRuntime.GetStatus();
-            if (readStatus != 0) return AbLegacyStatusMapper.MapStatus(readStatus);
+            if (readStatus != 0)
+            {
+                if (IsNoReply((AbStatus)readStatus)) EvictParentRuntime(device, parentName);
+                return (AbLegacyStatusMapper.MapStatus(readStatus), readStatus);
+            }

            // Mask the decoded word to its native width: DecodeValue for a 16-bit Int returns a
            // sign-extended int, so a word with bit 15 set decodes negative. Masking pins the
@@ -1193,9 +1265,8 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
                parentRuntime.EncodeValue(AbLegacyDataType.Int, bitIndex: null, (short)updated);
            await parentRuntime.WriteAsync(ct).ConfigureAwait(false);
            var writeStatus = parentRuntime.GetStatus();
-            return writeStatus == 0
-                ? AbLegacyStatusMapper.Good
-                : AbLegacyStatusMapper.MapStatus(writeStatus);
+            if (writeStatus != 0 && IsNoReply((AbStatus)writeStatus)) EvictParentRuntime(device, parentName);
+            return (writeStatus == 0 ? AbLegacyStatusMapper.Good : AbLegacyStatusMapper.MapStatus(writeStatus), writeStatus);
        }
        finally
        {
@@ -1381,7 +1452,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    ///     (<see cref="ConnectionThrottledException"/>). The HOT path (cached handle) never touches the gate. The
    ///     caller records the outcome on the device's backoff (once — recording here as well would double-count).
    /// </summary>
-    private static async Task<IAbLegacyTagRuntime> CreateSingleFlightAsync(
+    private async Task<IAbLegacyTagRuntime> CreateSingleFlightAsync(
        DeviceState device,
        System.Collections.Concurrent.ConcurrentDictionary<string, IAbLegacyTagRuntime> table,
        string key,
@@ -1395,7 +1466,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        {
            // Re-check under the gate: a sibling caller may have connected this very key while we waited.
            if (table.TryGetValue(key, out existing)) return existing;
-            if (device.IsThrottled(DateTime.UtcNow))
+            if (device.IsThrottled(Clock()))
                throw new ConnectionThrottledException(
                    $"AbLegacy connect for '{key}' on {device.Options.HostAddress} throttled: the device is in its reconnect backoff window.");

@@ -1544,7 +1615,7 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
    /// </summary>
    public void Dispose()
    {
-        var retired = RetireTables(bumpEpoch: true);
+        var (retired, teardownEpoch) = RetireTables(bumpEpoch: true);
        // ValueTask.ConfigureAwait(false).GetAwaiter().GetResult() — drains the poll engine
        // shutdown on the current thread without capturing the SynchronizationContext. The
        // engine cancels every loop CTS up-front then either completes immediately (no
@@ -1557,8 +1628,12 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        JoinProbeLoopsSync(retired);
        foreach (var state in retired) state.Retire();
        _resolver.Clear(); // drop transient equipment-tag parses so a config change re-parses
+        // A newer InitializeAsync that ran while this teardown was still joining owns health now.
        lock (_lifecycleLock)
-            _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+        {
+            if (teardownEpoch == _lifecycleEpoch)
+                _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null);
+        }
    }

    /// <summary>
@@ -1620,6 +1695,11 @@ public sealed partial class AbLegacyDriver : IDriver, IReadable, IWritable, ITag
        public System.Collections.Concurrent.ConcurrentDictionary<string, string> IndirectLastName { get; } =
            new(StringComparer.OrdinalIgnoreCase);

+        /// <summary>Indirect bit-write tag name to the parent word it last read-modify-wrote, so a moved index releases
+        /// the previous parent handle.</summary>
+        public System.Collections.Concurrent.ConcurrentDictionary<string, string> IndirectLastParent { get; } =
+            new(StringComparer.OrdinalIgnoreCase);
+
        /// <summary>
        ///     Per-device connect gate (review 07 P7): every COLD runtime create + initialise on this device runs under
        ///     it, so connects are single-flight per device — a down device sees one attempt at a time, not one per
@@ -1,4 +1,3 @@
-using System.Diagnostics;
 using Shouldly;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
@@ -67,12 +66,9 @@ public sealed class AbLegacyDeviceIsolationTests
        var before = drv.GetHealth().LastSuccessfulRead;
        Interlocked.Exchange(ref DeadTag.WireOps, 0);

-        var sw = Stopwatch.StartNew();
        var first = await drv.ReadAsync(refs, CancellationToken.None);
-        sw.Stop();

-        // Serial-with-no-backoff would pay 20 x 150 ms = 3 s on device B alone.
-        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1.5));
+        // The proof is the wire-op count, not a stopwatch: serial-with-no-backoff would dial B 20 times.
        Volatile.Read(ref DeadTag.WireOps).ShouldBeLessThanOrEqualTo(2);
        first.Take(5).ShouldAllBe(s => s.StatusCode == AbLegacyStatusMapper.Good); // positive control: A is served
        first.Skip(5).ShouldAllBe(s => s.StatusCode == AbLegacyStatusMapper.BadCommunicationError);
@@ -83,11 +79,8 @@ public sealed class AbLegacyDeviceIsolationTests

        // Inside B's backoff window the next poll touches B's wire not at all.
        var opsAfterFirst = Volatile.Read(ref DeadTag.WireOps);
-        sw.Restart();
        var second = await drv.ReadAsync(refs, CancellationToken.None);
-        sw.Stop();
        Volatile.Read(ref DeadTag.WireOps).ShouldBe(opsAfterFirst);
-        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromMilliseconds(500));
        second.Take(5).ShouldAllBe(s => s.StatusCode == AbLegacyStatusMapper.Good);
        await drv.ShutdownAsync(CancellationToken.None);
    }
@@ -239,6 +232,8 @@ public sealed class AbLegacyDeviceIsolationTests
            },
        };
        var drv = NewDriver(factory, new AbLegacyTagDefinition("B", HostB, "N7:0", AbLegacyDataType.Int));
+        var now = new DateTime(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
+        drv.Clock = () => now;
        await drv.InitializeAsync("{}", CancellationToken.None);

        // Two consecutive comms faults mark the device down and open its first (1 s) window.
@@ -249,7 +244,7 @@ public sealed class AbLegacyDeviceIsolationTests
        // Inside the window: fails fast without a connect attempt, although the device is back.
        (await drv.ReadAsync(["B"], CancellationToken.None))[0].StatusCode.ShouldBe(AbLegacyStatusMapper.BadCommunicationError);
        Volatile.Read(ref creates).ShouldBe(attempts);
-        await Task.Delay(TimeSpan.FromMilliseconds(1100));
+        now = now.AddMilliseconds(1100); // cross the first (1 s) window on the driver's clock, no sleeping

        var recovered = await drv.ReadAsync(["B"], CancellationToken.None);
        recovered[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good);
@@ -0,0 +1,222 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Ab.Transport;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests;
+
+/// <summary>
+///     Review 07 r0 I2 + M1/M2/M4/M10: any evidence that the device answered — a Good value, a device refusal, a Good
+///     or refused write — restores Healthy and advances <c>LastSuccessfulRead</c>; a no-reply failure on either path
+///     degrades; writes respect the device's backoff window; teardown health is epoch-guarded.
+/// </summary>
+[Trait("Category", "Unit")]
+public sealed class AbLegacyHealthEvidenceTests
+{
+    private const string Host = "ab://10.0.0.5/1,0";
+
+    private sealed class SwitchableFactory : IAbLegacyTagFactory
+    {
+        public Func<AbLegacyTagCreateParams, FakeAbLegacyTag> Make { get; set; } = p => new FakeAbLegacyTag(p) { Value = 1 };
+        public System.Collections.Concurrent.ConcurrentDictionary<string, FakeAbLegacyTag> Tags { get; } = new();
+
+        public IAbLegacyTagRuntime Create(AbLegacyTagCreateParams p) => Tags[p.TagName] = Make(p);
+    }
+
+    private static FakeAbLegacyTag Dead(AbLegacyTagCreateParams p) =>
+        new(p) { ThrowOnRead = true, ThrowOnWrite = true, Exception = new System.Net.Sockets.SocketException(10060) };
+
+    private static AbLegacyDriver NewDriver(IAbLegacyTagFactory f, params AbLegacyTagDefinition[] tags) =>
+        new(new AbLegacyDriverOptions
+        {
+            Devices = [new AbLegacyDeviceOptions(Host)],
+            RawTags = AbLegacyRawTags.Entries(tags),
+            Probe = new AbLegacyProbeOptions { Enabled = false },
+            CoalesceReads = false,
+        }, "drv-health", f);
+
+    private static async Task<AbLegacyDriver> DegradedDriverAsync(SwitchableFactory f, params AbLegacyTagDefinition[] tags)
+    {
+        var drv = NewDriver(f, tags);
+        await drv.InitializeAsync("{}", CancellationToken.None);
+        f.Make = Dead;
+        (await drv.ReadAsync([tags[0].Name], CancellationToken.None))[0].StatusCode
+            .ShouldBe(AbLegacyStatusMapper.BadCommunicationError);
+        drv.GetHealth().State.ShouldBe(DriverState.Degraded); // precondition
+        return drv;
+    }
+
+    [Fact]
+    public async Task A_refusal_only_read_after_a_fault_restores_Healthy_and_advances_LastSuccessfulRead()
+    {
+        var f = new SwitchableFactory();
+        var drv = await DegradedDriverAsync(f, new AbLegacyTagDefinition("X", Host, "N7:0", AbLegacyDataType.Int));
+        var before = drv.GetHealth().LastSuccessfulRead;
+        f.Make = p => new FakeAbLegacyTag(p) { Status = (int)AbStatus.NotFound };
+        await Task.Delay(5);
+
+        var r = await drv.ReadAsync(["X"], CancellationToken.None);
+
+        r[0].StatusCode.ShouldBe(AbLegacyStatusMapper.BadNodeIdUnknown);
+        var h = drv.GetHealth();
+        h.State.ShouldBe(DriverState.Healthy);
+        h.LastSuccessfulRead!.Value.ShouldBeGreaterThan(before!.Value);
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    [Theory]
+    [InlineData(0)]                          // a Good write
+    [InlineData((int)AbStatus.NotAllowed)]   // a write the device refused
+    public async Task A_good_or_refused_write_after_a_fault_restores_Healthy(int writeStatus)
+    {
+        var f = new SwitchableFactory();
+        var drv = await DegradedDriverAsync(f,
+            new AbLegacyTagDefinition("X", Host, "N7:0", AbLegacyDataType.Int),
+            new AbLegacyTagDefinition("W", Host, "N7:5", AbLegacyDataType.Int));
+        var before = drv.GetHealth().LastSuccessfulRead;
+        f.Make = p => new FakeAbLegacyTag(p) { WriteStatusOverride = writeStatus };
+        await Task.Delay(5);
+
+        var w = await drv.WriteAsync([new WriteRequest("W", 3)], CancellationToken.None);
+
+        w[0].StatusCode.ShouldBe(writeStatus == 0 ? AbLegacyStatusMapper.Good : AbLegacyStatusMapper.BadNotWritable);
+        f.Tags["N7:5"].WriteCount.ShouldBe(1); // positive control: the write reached the device
+        var h = drv.GetHealth();
+        h.State.ShouldBe(DriverState.Healthy);
+        h.LastSuccessfulRead!.Value.ShouldBeGreaterThan(before!.Value);
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task A_no_reply_write_status_degrades_like_the_exception_path_does()
+    {
+        var f = new SwitchableFactory { Make = p => new FakeAbLegacyTag(p) { WriteStatusOverride = (int)AbStatus.Timeout } };
+        var drv = NewDriver(f, new AbLegacyTagDefinition("W", Host, "N7:5", AbLegacyDataType.Int));
+        await drv.InitializeAsync("{}", CancellationToken.None);
+
+        var w = await drv.WriteAsync([new WriteRequest("W", 3)], CancellationToken.None);
+
+        w[0].StatusCode.ShouldBe(AbLegacyStatusMapper.BadTimeout);
+        drv.GetHealth().State.ShouldBe(DriverState.Degraded);
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task One_write_call_is_worst_wins_but_still_advances_LastSuccessfulRead()
+    {
+        var f = new SwitchableFactory
+        {
+            Make = p => p.TagName == "N7:6" ? Dead(p) : new FakeAbLegacyTag(p) { Value = 1 },
+        };
+        var drv = NewDriver(f,
+            new AbLegacyTagDefinition("Good", Host, "N7:5", AbLegacyDataType.Int),
+            new AbLegacyTagDefinition("Bad", Host, "N7:6", AbLegacyDataType.Int));
+        await drv.InitializeAsync("{}", CancellationToken.None);
+        var before = drv.GetHealth().LastSuccessfulRead;
+        await Task.Delay(5);
+
+        var w = await drv.WriteAsync([new WriteRequest("Good", 1), new WriteRequest("Bad", 2)], CancellationToken.None);
+
+        w[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good);
+        w[1].StatusCode.ShouldBe(AbLegacyStatusMapper.BadCommunicationError);
+        var h = drv.GetHealth();
+        h.State.ShouldBe(DriverState.Degraded);
+        h.LastSuccessfulRead!.Value.ShouldBeGreaterThan(before!.Value);
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task A_write_inside_the_backoff_window_fails_fast_even_with_a_cached_handle()
+    {
+        var now = new DateTime(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
+        var f = new SwitchableFactory();
+        var drv = NewDriver(f,
+            new AbLegacyTagDefinition("W", Host, "N7:5", AbLegacyDataType.Int),
+            new AbLegacyTagDefinition("D1", Host, "N7:6", AbLegacyDataType.Int),
+            new AbLegacyTagDefinition("D2", Host, "N7:7", AbLegacyDataType.Int));
+        drv.Clock = () => now;
+        await drv.InitializeAsync("{}", CancellationToken.None);
+        (await drv.ReadAsync(["W"], CancellationToken.None))[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good); // cache W
+        var cached = f.Tags["N7:5"];
+        f.Make = Dead;
+        await drv.ReadAsync(["D1", "D2"], CancellationToken.None); // two consecutive no-reply faults: device down
+        drv.GetDeviceState(Host)!.IsThrottled(now).ShouldBeTrue();
+
+        var w = await drv.WriteAsync([new WriteRequest("W", 9)], CancellationToken.None);
+
+        w[0].StatusCode.ShouldBe(AbLegacyStatusMapper.BadCommunicationError);
+        cached.WriteCount.ShouldBe(0);
+
+        // Positive control: once the window has passed, the same cached handle is written.
+        now = now.AddSeconds(2);
+        (await drv.WriteAsync([new WriteRequest("W", 9)], CancellationToken.None))[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good);
+        cached.WriteCount.ShouldBe(1);
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task An_indirect_bit_write_releases_the_previous_parent_word_handle()
+    {
+        var f = new SwitchableFactory { Make = p => new FakeAbLegacyTag(p) { Value = p.TagName == "N7:0" ? (short)5 : (short)0 } };
+        var drv = NewDriver(f, new AbLegacyTagDefinition("IndBit", Host, "N7:[N7:0]/3", AbLegacyDataType.Bit));
+        await drv.InitializeAsync("{}", CancellationToken.None);
+
+        (await drv.WriteAsync([new WriteRequest("IndBit", true)], CancellationToken.None))[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good);
+        var state = drv.GetDeviceState(Host)!;
+        state.ParentRuntimes.ContainsKey("N7:5").ShouldBeTrue(); // positive control: the parent word was RMW'd
+        f.Tags["N7:0"].Value = (short)6;
+        (await drv.WriteAsync([new WriteRequest("IndBit", true)], CancellationToken.None))[0].StatusCode.ShouldBe(AbLegacyStatusMapper.Good);
+
+        state.ParentRuntimes.ContainsKey("N7:5").ShouldBeFalse();
+        state.ParentRuntimes.ContainsKey("N7:6").ShouldBeTrue();
+        f.Tags["N7:5"].Disposed.ShouldBeTrue();
+        await drv.ShutdownAsync(CancellationToken.None);
+    }
+
+    /// <summary>A probe read that parks ignoring cancellation, so ShutdownAsync is still joining.</summary>
+    private sealed class ParkingTag(AbLegacyTagCreateParams p, Task release, TaskCompletionSource entered) : FakeAbLegacyTag(p)
+    {
+        public override async Task ReadAsync(CancellationToken ct)
+        {
+            entered.TrySetResult();
+            await release.ConfigureAwait(false);
+        }
+    }
+
+    [Fact]
+    public async Task A_slow_teardown_does_not_overwrite_a_newer_initialise_health()
+    {
+        var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var parked = false;
+        var f = new SwitchableFactory
+        {
+            Make = p =>
+            {
+                if (parked) return new FakeAbLegacyTag(p) { Value = 1 };
+                parked = true;
+                return new ParkingTag(p, release.Task, entered);
+            },
+        };
+        var drv = new AbLegacyDriver(new AbLegacyDriverOptions
+        {
+            Devices = [new AbLegacyDeviceOptions(Host)],
+            Probe = new AbLegacyProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(20) },
+        }, "drv-m1", f);
+        await drv.InitializeAsync("{}", CancellationToken.None);
+        await entered.Task;
+
+        var shutdown = drv.ShutdownAsync(CancellationToken.None); // parks joining the wedged probe loop
+        await Task.Delay(50);
+        shutdown.IsCompleted.ShouldBeFalse();
+        await drv.InitializeAsync("{}", CancellationToken.None);  // a newer initialise wins
+        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
+        release.SetResult();
+        await shutdown;
+
+        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
+        // Positive control: an uncontested shutdown still writes Unknown.
+        await drv.ShutdownAsync(CancellationToken.None);
+        drv.GetHealth().State.ShouldBe(DriverState.Unknown);
+    }
+}