Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs
T
Joseph Doherty fd64fed217 fix(focas): a stalled device fails fast for the rest of its batch; generation-stamp health (#740 review round 1)
I1: Task.WhenAll waits for the slowest device, so concurrency alone still let a device
that accepts TCP and then stalls cost (its tag count x call timeout) per batch. After the
first timeout, transport failure or comms-class status on a device, its remaining tags in
the batch now fail BadCommunicationError immediately, the same as the coalescer's
FailMembers, which now also stops after a failed range. FIXED: tags are still served
from the cache. A malformed address is BadNodeIdUnknown and does not count as a link
failure. Test: a 10-tag stalled device beside a healthy one completes in under 3 call
timeouts with the healthy value Good. The P9 test now also asserts per-client read
counts and that two reads overlapped across devices.

Minors:
- 1: a superseded init releases its table only if it was never published (flag).
- 2: a fixed-tree refusal credits LastSuccessfulRead; a value-path refusal after a comms
  fault restores Healthy.
- 5: alarm/fixed/refusal health is reset inside the publish lock, after the generation
  check.
- 6: FocasFixedTreeHealthTests wait on served-failure/refusal counters, not wall-clock.
- 8: data-path health writes (reads, coalescer, writes, poll errors, refusals) carry the
  generation they started under and are dropped once a lifecycle call has moved on. Each
  write is a check-and-set under the lifecycle lock.
Also adds a unit case where 20 vendor records with type 0 and empty messages stay vendor.

Refs #740
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 14:44:42 -04:00

213 lines
10 KiB
C#

using Microsoft.Extensions.Logging;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 driver-lane health checklist (drivers.md, #527/#611): a deterministic CNC refusal — the control
/// answered with a non-OK return code, or with a value the tag's type cannot hold (vacant macro, fractional
/// macro on an integer tag) — is per-tag Bad with ONE Warning per tag and NO health write, and it still
/// proves the device answered (LastSuccessfulRead advances). Only a comms-class status degrades the driver.
/// Before this, every non-Good status latched Degraded every poll, so one vacant macro made a reachable CNC
/// look sick forever.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasReadHealthContractTests
{
private const string Host = "focas://10.0.0.5:8193";
private sealed class WarningCounter : ILogger<FocasDriver>
{
private int _warnings;
public int Warnings => Volatile.Read(ref _warnings);
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning && formatter(state, exception).Contains("refused", StringComparison.Ordinal))
Interlocked.Increment(ref _warnings);
}
}
private static (FocasDriver drv, FakeFocasClient client, WarningCounter log) NewDriver(uint r100Status)
{
var client = new FakeFocasClient { Values = { ["R101"] = (sbyte)1 } };
client.ReadStatuses["R100"] = r100Status;
var log = new WarningCounter();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags =
[
FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/R101", Host, "R101", FocasDataType.Byte)),
],
Probe = new FocasProbeOptions { Enabled = false },
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
}, "focas-contract", new FakeFocasClientFactory { Customise = () => client }, log);
return (drv, client, log);
}
/// <summary>A refusal leaves health alone, warns once, and still credits the answer.</summary>
[Theory]
[InlineData(FocasStatusMapper.BadOutOfRange)] // EW_NUMBER
[InlineData(FocasStatusMapper.BadNotSupported)] // EW_FUNC / EW_NOOPT
[InlineData(FocasStatusMapper.BadNoData)] // vacant macro
[InlineData(FocasStatusMapper.BadTypeMismatch)] // fractional macro on an integer tag
[InlineData(FocasStatusMapper.BadDeviceFailure)] // EW_BUSY
public async Task Cnc_refusal_is_per_tag_bad_with_one_warning_and_no_health_write(uint refusal)
{
var (drv, _, log) = NewDriver(refusal);
await drv.InitializeAsync("{}", CancellationToken.None);
var atInit = drv.GetHealth().LastSuccessfulRead;
for (var i = 0; i < 3; i++)
{
await Task.Delay(5);
var snap = (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0];
snap.StatusCode.ShouldBe(refusal);
}
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Healthy, "a CNC refusal is not a comms fault");
health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(atInit!.Value, "the CNC answered — that is evidence of life");
log.Warnings.ShouldBe(1, "one Warning per refused tag, not one per poll");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A refused tag that recovers and is refused again warns again (the dedup clears on Good).</summary>
[Fact]
public async Task Refusal_warning_rearms_after_the_tag_reads_good()
{
var (drv, client, log) = NewDriver(FocasStatusMapper.BadOutOfRange);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
client.ReadStatuses.Remove("R100");
client.Values["R100"] = (sbyte)3;
(await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
log.Warnings.ShouldBe(2);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Positive control: a comms-class status still degrades, preserving LastSuccessfulRead.</summary>
[Fact]
public async Task Comms_status_still_degrades_with_last_successful_read_preserved()
{
var (drv, _, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R101"], CancellationToken.None);
var lastGood = drv.GetHealth().LastSuccessfulRead;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastSuccessfulRead.ShouldBe(lastGood);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>
/// Review minor 2 — a device that recovers from a comms fault and then only REFUSES (every tag it serves is
/// refused) is answering, so it is restored to Healthy instead of staying Degraded forever.
/// </summary>
[Fact]
public async Task Refusal_after_a_comms_fault_restores_healthy()
{
var (drv, client, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "positive control: the comms fault degraded it");
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A read whose CNC answer is held until released, then fails at the transport.</summary>
private sealed class HeldReadClient(TaskCompletionSource entered, Task release) : FakeFocasClient
{
public volatile bool Hold = true;
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
if (!Hold) return await base.ReadAsync(address, type, ct).ConfigureAwait(false);
entered.TrySetResult();
await release.ConfigureAwait(false);
throw new IOException("socket reset");
}
}
/// <summary>
/// Review minor 8 — a read that started against the PREVIOUS table and fails after a re-initialise must not
/// write its Degraded onto the new generation's health. Positive control: the same failure under the live
/// generation does degrade.
/// </summary>
[Fact]
public async Task A_read_from_the_previous_generation_cannot_degrade_the_new_one()
{
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var clients = new List<HeldReadClient>();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte))],
Probe = new FocasProbeOptions { Enabled = false },
Timeout = TimeSpan.FromSeconds(10),
}, "focas-gen", new FakeFocasClientFactory
{
Customise = () => { var c = new HeldReadClient(entered, release.Task); lock (clients) clients.Add(c); return c; },
});
await drv.InitializeAsync("{}", CancellationToken.None);
var stale = drv.ReadAsync(["raw/R100"], CancellationToken.None);
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
await drv.InitializeAsync("{}", CancellationToken.None); // a new generation, Healthy
release.SetResult();
(await stale.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the old generation's failure must not land on the new one");
// Positive control: a failure under the live generation degrades.
var live = drv.ReadAsync(["raw/R100"], CancellationToken.None);
(await live.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A write the driver refuses (NotSupportedException) is not a comms fault either.</summary>
[Fact]
public async Task Write_refusal_does_not_degrade()
{
var client = new ThrowingWriteClient();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/W", Host, "R100", FocasDataType.Byte, Writable: true))],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-write-refusal", new FakeFocasClientFactory { Customise = () => client });
await drv.InitializeAsync("{}", CancellationToken.None);
var result = await drv.WriteAsync([new WriteRequest("raw/W", (sbyte)1)], CancellationToken.None);
result[0].StatusCode.ShouldBe(FocasStatusMapper.BadNotSupported);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(CancellationToken.None);
}
private sealed class ThrowingWriteClient : FakeFocasClient
{
public override Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken ct) =>
throw new NotSupportedException("bit writes are not supported on this area");
}
}