fix(focas): a CNC refusal is per-tag Bad, not a driver Degraded (lane health checklist)

Applies the driver-lane health contract to the value path. Every non-Good read status used
to write Degraded on every poll, so one vacant macro (BadNoData), one EW_NUMBER address
(BadOutOfRange) or one fractional macro on an integer tag made a reachable CNC look sick
forever. Now:
- a comms-class status (BadCommunicationError / BadInternalError (EW_HANDLE) / BadTimeout)
  still degrades, with LastSuccessfulRead preserved;
- any other non-Good status is a refusal: the tag reads Bad, one Warning is logged per tag
  (re-armed when it next reads Good, cleared at init), driver state is untouched, and
  LastSuccessfulRead advances because the CNC answered (CAS, since devices now read
  concurrently);
- a write refused with NotSupportedException no longer degrades the driver.

Refs #740
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 14:26:28 -04:00
parent 29b566d51b
commit 2724d7dae8
3 changed files with 193 additions and 4 deletions
@@ -235,6 +235,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
var generation = Interlocked.Increment(ref _lifecycleGeneration);
await ReleaseDeviceStatesAsync().ConfigureAwait(false);
ResetAlarmHealth();
_refusalWarned.Clear();
WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, null, null));
var devices = new Dictionary<string, DeviceState>(StringComparer.OrdinalIgnoreCase);
try
@@ -756,11 +757,20 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
results[i] = new DataValueSnapshot(value, status, now, now);
if (status == FocasStatusMapper.Good)
{
_refusalWarned.TryRemove(reference, out _);
Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null));
else
}
else if (IsCommsStatus(status))
{
Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded,
Volatile.Read(ref _health).LastSuccessfulRead,
$"FOCAS status 0x{status:X8} reading {reference}"));
}
else
{
NoteRefusal(reference, def.Address, status, now);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (OperationCanceledException)
@@ -781,6 +791,43 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
}
}
/// <summary>
/// RawPaths whose CNC refusal has already been logged — a refusal warns ONCE per tag, not once per poll,
/// and re-arms when the tag next reads Good. Cleared at initialise.
/// </summary>
private readonly ConcurrentDictionary<string, byte> _refusalWarned = new(StringComparer.Ordinal);
/// <summary>
/// True for a status that says the LINK failed (or the read never got an answer), as opposed to the CNC
/// answering with a refusal. <see cref="FocasStatusMapper.MapFocasReturn"/> maps EW_SOCKET / EW_UNEXP /
/// EW_PARITY / unknown codes to BadCommunicationError and EW_HANDLE to BadInternalError; the wire client
/// reports a dead session as BadCommunicationError.
/// </summary>
private static bool IsCommsStatus(uint status) => status is FocasStatusMapper.BadCommunicationError
or FocasStatusMapper.BadInternalError or FocasStatusMapper.BadTimeout;
/// <summary>
/// A deterministic CNC refusal (drivers.md health contract): per-tag Bad only — one Warning per tag, no
/// state change — but the CNC DID answer, so LastSuccessfulRead advances (else a reachable control whose
/// only tags are refused would look silent to the comms-loss watchdog).
/// </summary>
private void NoteRefusal(string reference, string address, uint status, DateTime now)
{
// CAS, not a plain write: devices are read concurrently, and a read-modify-write could otherwise put back
// the state another device's comms fault had just replaced.
while (true)
{
var current = Volatile.Read(ref _health);
if (current.LastSuccessfulRead is { } last && last >= now) break;
if (ReferenceEquals(Interlocked.CompareExchange(ref _health, current with { LastSuccessfulRead = now }, current), current))
break;
}
if (_refusalWarned.TryAdd(reference, 0))
_logger.LogWarning(
"FOCAS CNC refused {RawPath} ({Address}) with 0x{Status:X8}; the tag reads Bad until it is fixed (warned once). Driver={DriverInstanceId}",
reference, address, status, _driverInstanceId);
}
/// <summary>
/// PMC coalescing pre-pass for ONE device's share of a batch. Plans merged ranges
/// (<see cref="FocasPmcCoalescer"/>) and serves every multi-member range with ONE
@@ -956,9 +1003,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
}
catch (NotSupportedException nse)
{
// A refusal, not a comms fault (drivers.md, #611): nothing was wrong with the link, so health is
// left alone — degrading here handed the comms-loss watchdog a reason to reconnect a healthy CNC.
results[i] = new WriteResult(FocasStatusMapper.BadNotSupported);
Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded,
Volatile.Read(ref _health).LastSuccessfulRead, nse.Message));
_logger.LogDebug(nse, "FOCAS write to {RawPath} refused as unsupported", w.FullReference);
}
catch (Exception ex) when (ex is FormatException or InvalidCastException)
{
@@ -645,7 +645,8 @@ public sealed class FocasAlarmProjectionTests
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
// Value path Degraded, alarm path good ⇒ Degraded: an alarm success cannot mask it.
factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
// A COMMS-class status: a CNC refusal (e.g. BadOutOfRange) is not a value-path Degraded since #740.
factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadCommunicationError;
await drv.ReadAsync([valuePath], CancellationToken.None);
drv.NoteAlarmReadSucceeded(Host);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
@@ -0,0 +1,140 @@
using Microsoft.Extensions.Logging;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 driver-lane health checklist (drivers.md, #527/#611): a deterministic CNC refusal — the control
/// answered with a non-OK return code, or with a value the tag's type cannot hold (vacant macro, fractional
/// macro on an integer tag) — is per-tag Bad with ONE Warning per tag and NO health write, and it still
/// proves the device answered (LastSuccessfulRead advances). Only a comms-class status degrades the driver.
/// Before this, every non-Good status latched Degraded every poll, so one vacant macro made a reachable CNC
/// look sick forever.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasReadHealthContractTests
{
private const string Host = "focas://10.0.0.5:8193";
private sealed class WarningCounter : ILogger<FocasDriver>
{
private int _warnings;
public int Warnings => Volatile.Read(ref _warnings);
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning && formatter(state, exception).Contains("refused", StringComparison.Ordinal))
Interlocked.Increment(ref _warnings);
}
}
private static (FocasDriver drv, FakeFocasClient client, WarningCounter log) NewDriver(uint r100Status)
{
var client = new FakeFocasClient { Values = { ["R101"] = (sbyte)1 } };
client.ReadStatuses["R100"] = r100Status;
var log = new WarningCounter();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags =
[
FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/R101", Host, "R101", FocasDataType.Byte)),
],
Probe = new FocasProbeOptions { Enabled = false },
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
}, "focas-contract", new FakeFocasClientFactory { Customise = () => client }, log);
return (drv, client, log);
}
/// <summary>A refusal leaves health alone, warns once, and still credits the answer.</summary>
[Theory]
[InlineData(FocasStatusMapper.BadOutOfRange)] // EW_NUMBER
[InlineData(FocasStatusMapper.BadNotSupported)] // EW_FUNC / EW_NOOPT
[InlineData(FocasStatusMapper.BadNoData)] // vacant macro
[InlineData(FocasStatusMapper.BadTypeMismatch)] // fractional macro on an integer tag
[InlineData(FocasStatusMapper.BadDeviceFailure)] // EW_BUSY
public async Task Cnc_refusal_is_per_tag_bad_with_one_warning_and_no_health_write(uint refusal)
{
var (drv, _, log) = NewDriver(refusal);
await drv.InitializeAsync("{}", CancellationToken.None);
var atInit = drv.GetHealth().LastSuccessfulRead;
for (var i = 0; i < 3; i++)
{
await Task.Delay(5);
var snap = (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0];
snap.StatusCode.ShouldBe(refusal);
}
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Healthy, "a CNC refusal is not a comms fault");
health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(atInit!.Value, "the CNC answered — that is evidence of life");
log.Warnings.ShouldBe(1, "one Warning per refused tag, not one per poll");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A refused tag that recovers and is refused again warns again (the dedup clears on Good).</summary>
[Fact]
public async Task Refusal_warning_rearms_after_the_tag_reads_good()
{
var (drv, client, log) = NewDriver(FocasStatusMapper.BadOutOfRange);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
client.ReadStatuses.Remove("R100");
client.Values["R100"] = (sbyte)3;
(await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
log.Warnings.ShouldBe(2);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Positive control: a comms-class status still degrades, preserving LastSuccessfulRead.</summary>
[Fact]
public async Task Comms_status_still_degrades_with_last_successful_read_preserved()
{
var (drv, _, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R101"], CancellationToken.None);
var lastGood = drv.GetHealth().LastSuccessfulRead;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastSuccessfulRead.ShouldBe(lastGood);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A write the driver refuses (NotSupportedException) is not a comms fault either.</summary>
[Fact]
public async Task Write_refusal_does_not_degrade()
{
var client = new ThrowingWriteClient();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/W", Host, "R100", FocasDataType.Byte, Writable: true))],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-write-refusal", new FakeFocasClientFactory { Customise = () => client });
await drv.InitializeAsync("{}", CancellationToken.None);
var result = await drv.WriteAsync([new WriteRequest("raw/W", (sbyte)1)], CancellationToken.None);
result[0].StatusCode.ShouldBe(FocasStatusMapper.BadNotSupported);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(CancellationToken.None);
}
private sealed class ThrowingWriteClient : FakeFocasClient
{
public override Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken ct) =>
throw new NotSupportedException("bit writes are not supported on this area");
}
}