fd64fed217
I1: Task.WhenAll waits for the slowest device, so concurrency alone still let a device that accepts TCP and then stalls cost (its tag count x call timeout) per batch. After the first timeout, transport failure or comms-class status on a device, its remaining tags in the batch now fail BadCommunicationError immediately, the same as the coalescer's FailMembers, which now also stops after a failed range. FIXED: tags are still served from the cache. A malformed address is BadNodeIdUnknown and does not count as a link failure. Test: a 10-tag stalled device beside a healthy one completes in under 3 call timeouts with the healthy value Good. The P9 test now also asserts per-client read counts and that two reads overlapped across devices. Minors: - 1: a superseded init releases its table only if it was never published (flag). - 2: a fixed-tree refusal credits LastSuccessfulRead; a value-path refusal after a comms fault restores Healthy. - 5: alarm/fixed/refusal health is reset inside the publish lock, after the generation check. - 6: FocasFixedTreeHealthTests wait on served-failure/refusal counters, not wall-clock. - 8: data-path health writes (reads, coalescer, writes, poll errors, refusals) carry the generation they started under and are dropped once a lifecycle call has moved on. Each write is a check-and-set under the lifecycle lock. Also adds a unit case where 20 vendor records with type 0 and empty messages stay vendor. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
213 lines
10 KiB
C#
213 lines
10 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
|
|
|
|
/// <summary>
|
|
/// #740 driver-lane health checklist (drivers.md, #527/#611): a deterministic CNC refusal — the control
|
|
/// answered with a non-OK return code, or with a value the tag's type cannot hold (vacant macro, fractional
|
|
/// macro on an integer tag) — is per-tag Bad with ONE Warning per tag and NO health write, and it still
|
|
/// proves the device answered (LastSuccessfulRead advances). Only a comms-class status degrades the driver.
|
|
/// Before this, every non-Good status latched Degraded every poll, so one vacant macro made a reachable CNC
|
|
/// look sick forever.
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class FocasReadHealthContractTests
|
|
{
|
|
private const string Host = "focas://10.0.0.5:8193";
|
|
|
|
private sealed class WarningCounter : ILogger<FocasDriver>
|
|
{
|
|
private int _warnings;
|
|
public int Warnings => Volatile.Read(ref _warnings);
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
|
Func<TState, Exception?, string> formatter)
|
|
{
|
|
if (logLevel == LogLevel.Warning && formatter(state, exception).Contains("refused", StringComparison.Ordinal))
|
|
Interlocked.Increment(ref _warnings);
|
|
}
|
|
}
|
|
|
|
private static (FocasDriver drv, FakeFocasClient client, WarningCounter log) NewDriver(uint r100Status)
|
|
{
|
|
var client = new FakeFocasClient { Values = { ["R101"] = (sbyte)1 } };
|
|
client.ReadStatuses["R100"] = r100Status;
|
|
var log = new WarningCounter();
|
|
var drv = new FocasDriver(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host)],
|
|
RawTags =
|
|
[
|
|
FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte)),
|
|
FocasRawTags.Entry(new FocasTagDefinition("raw/R101", Host, "R101", FocasDataType.Byte)),
|
|
],
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
|
|
}, "focas-contract", new FakeFocasClientFactory { Customise = () => client }, log);
|
|
return (drv, client, log);
|
|
}
|
|
|
|
/// <summary>A refusal leaves health alone, warns once, and still credits the answer.</summary>
|
|
[Theory]
|
|
[InlineData(FocasStatusMapper.BadOutOfRange)] // EW_NUMBER
|
|
[InlineData(FocasStatusMapper.BadNotSupported)] // EW_FUNC / EW_NOOPT
|
|
[InlineData(FocasStatusMapper.BadNoData)] // vacant macro
|
|
[InlineData(FocasStatusMapper.BadTypeMismatch)] // fractional macro on an integer tag
|
|
[InlineData(FocasStatusMapper.BadDeviceFailure)] // EW_BUSY
|
|
public async Task Cnc_refusal_is_per_tag_bad_with_one_warning_and_no_health_write(uint refusal)
|
|
{
|
|
var (drv, _, log) = NewDriver(refusal);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
var atInit = drv.GetHealth().LastSuccessfulRead;
|
|
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
await Task.Delay(5);
|
|
var snap = (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0];
|
|
snap.StatusCode.ShouldBe(refusal);
|
|
}
|
|
|
|
var health = drv.GetHealth();
|
|
health.State.ShouldBe(DriverState.Healthy, "a CNC refusal is not a comms fault");
|
|
health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(atInit!.Value, "the CNC answered — that is evidence of life");
|
|
log.Warnings.ShouldBe(1, "one Warning per refused tag, not one per poll");
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>A refused tag that recovers and is refused again warns again (the dedup clears on Good).</summary>
|
|
[Fact]
|
|
public async Task Refusal_warning_rearms_after_the_tag_reads_good()
|
|
{
|
|
var (drv, client, log) = NewDriver(FocasStatusMapper.BadOutOfRange);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
client.ReadStatuses.Remove("R100");
|
|
client.Values["R100"] = (sbyte)3;
|
|
(await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
|
|
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
|
|
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
log.Warnings.ShouldBe(2);
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>Positive control: a comms-class status still degrades, preserving LastSuccessfulRead.</summary>
|
|
[Fact]
|
|
public async Task Comms_status_still_degrades_with_last_successful_read_preserved()
|
|
{
|
|
var (drv, _, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
await drv.ReadAsync(["raw/R101"], CancellationToken.None);
|
|
var lastGood = drv.GetHealth().LastSuccessfulRead;
|
|
|
|
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
|
|
var health = drv.GetHealth();
|
|
health.State.ShouldBe(DriverState.Degraded);
|
|
health.LastSuccessfulRead.ShouldBe(lastGood);
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review minor 2 — a device that recovers from a comms fault and then only REFUSES (every tag it serves is
|
|
/// refused) is answering, so it is restored to Healthy instead of staying Degraded forever.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Refusal_after_a_comms_fault_restores_healthy()
|
|
{
|
|
var (drv, client, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "positive control: the comms fault degraded it");
|
|
|
|
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
|
|
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
|
|
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>A read whose CNC answer is held until released, then fails at the transport.</summary>
|
|
private sealed class HeldReadClient(TaskCompletionSource entered, Task release) : FakeFocasClient
|
|
{
|
|
public volatile bool Hold = true;
|
|
|
|
public override async Task<(object? value, uint status)> ReadAsync(
|
|
FocasAddress address, FocasDataType type, CancellationToken ct)
|
|
{
|
|
if (!Hold) return await base.ReadAsync(address, type, ct).ConfigureAwait(false);
|
|
entered.TrySetResult();
|
|
await release.ConfigureAwait(false);
|
|
throw new IOException("socket reset");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review minor 8 — a read that started against the PREVIOUS table and fails after a re-initialise must not
|
|
/// write its Degraded onto the new generation's health. Positive control: the same failure under the live
|
|
/// generation does degrade.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_read_from_the_previous_generation_cannot_degrade_the_new_one()
|
|
{
|
|
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var clients = new List<HeldReadClient>();
|
|
var drv = new FocasDriver(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host)],
|
|
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte))],
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
Timeout = TimeSpan.FromSeconds(10),
|
|
}, "focas-gen", new FakeFocasClientFactory
|
|
{
|
|
Customise = () => { var c = new HeldReadClient(entered, release.Task); lock (clients) clients.Add(c); return c; },
|
|
});
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
|
|
var stale = drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
|
await drv.InitializeAsync("{}", CancellationToken.None); // a new generation, Healthy
|
|
release.SetResult();
|
|
(await stale.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
|
|
|
|
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the old generation's failure must not land on the new one");
|
|
|
|
// Positive control: a failure under the live generation degrades.
|
|
var live = drv.ReadAsync(["raw/R100"], CancellationToken.None);
|
|
(await live.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>A write the driver refuses (NotSupportedException) is not a comms fault either.</summary>
|
|
[Fact]
|
|
public async Task Write_refusal_does_not_degrade()
|
|
{
|
|
var client = new ThrowingWriteClient();
|
|
var drv = new FocasDriver(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host)],
|
|
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/W", Host, "R100", FocasDataType.Byte, Writable: true))],
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
}, "focas-write-refusal", new FakeFocasClientFactory { Customise = () => client });
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
|
|
var result = await drv.WriteAsync([new WriteRequest("raw/W", (sbyte)1)], CancellationToken.None);
|
|
|
|
result[0].StatusCode.ShouldBe(FocasStatusMapper.BadNotSupported);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
|
await drv.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
private sealed class ThrowingWriteClient : FakeFocasClient
|
|
{
|
|
public override Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken ct) =>
|
|
throw new NotSupportedException("bit writes are not supported on this area");
|
|
}
|
|
}
|