using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
///
/// #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a
/// driver serving only FIXED: tags (Identity / Axes / Program / Status — the typical FOCAS shape) read
/// Healthy on /hosts with every node Bad, and the #527 comms-loss watchdog could never fire. The loop
/// now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past
/// the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen.
///
[Trait("Category", "Unit")]
public sealed class FocasFixedTreeHealthTests
{
private const string Host = "focas://10.0.0.5:8193";
private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300);
private static readonly TimeSpan BlipBudget = TimeSpan.FromMilliseconds(1500);
/// A CNC that can be switched off (transport failure) or made to refuse the dynamic read.
private sealed class CncSwitch
{
public volatile bool Down;
public volatile bool Refuse;
/// Transport failures served so far — what the tests wait on instead of wall-clock delays.
public int Failures;
/// Refusals served so far.
public int Refusals;
}
private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient
{
private bool Down => sw.Down;
private bool Refuse => sw.Refuse;
public override Task GetSysInfoAsync(CancellationToken ct)
{
if (!Down) return base.GetSysInfoAsync(ct);
Interlocked.Increment(ref sw.Failures);
throw new IOException("socket closed");
}
public override Task ReadDynamicAsync(int axisIndex, CancellationToken ct)
{
if (Down)
{
Interlocked.Increment(ref sw.Failures);
throw new IOException("socket closed");
}
if (Refuse)
{
Interlocked.Increment(ref sw.Refusals);
throw new FocasWireException("cnc_rddynamic2 refused", rc: 6);
}
return base.ReadDynamicAsync(axisIndex, ct);
}
}
private static FocasDriver FixedOnlyDriver(CncSwitch sw, TimeSpan? budget = null) => new(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))],
Probe = new FocasProbeOptions { Enabled = false },
FixedTree = new FocasFixedTreeOptions
{
PollInterval = TimeSpan.FromMilliseconds(20),
ProgramPollInterval = TimeSpan.Zero,
TimerPollInterval = TimeSpan.Zero,
},
}, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) })
{
FixedTreeStaleBudgetOverride = budget ?? Stale,
};
///
/// Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving
/// LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with
/// LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy.
///
[Fact]
public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns()
{
var sw = new CncSwitch();
// A wide budget (review round 2 m4): two 20 ms ticks always land well inside it, so the "blip does not
// degrade" absence check below runs unconditionally instead of being skipped on a slow agent.
await using var drv = FixedOnlyDriver(sw, BlipBudget);
await drv.InitializeAsync("{}", CancellationToken.None);
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
var first = drv.GetHealth().LastSuccessfulRead!.Value;
// An answered axis pass is a successful read: wait for it to move rather than sleeping.
await WaitUntil(() => drv.GetHealth().LastSuccessfulRead > first);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
sw.Down = true;
var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value;
// Two failed ticks have demonstrably happened (positive evidence for the absence check below) …
await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2);
// … inside the budget (asserted, not assumed), and the driver is still Healthy.
(DateTime.UtcNow - lastAnswer).ShouldBeLessThan(BlipBudget, "the two failed ticks must land inside the budget");
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade");
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded);
var frozen = drv.GetHealth().LastSuccessfulRead;
var failuresAtDegrade = Volatile.Read(ref sw.Failures);
await WaitUntil(() => Volatile.Read(ref sw.Failures) >= failuresAtDegrade + 5);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent");
sw.Down = false;
await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
}
/// A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes.
[Fact]
public async Task Bootstrap_that_never_succeeds_degrades_the_driver()
{
var sw = new CncSwitch { Down = true };
await using var drv = FixedOnlyDriver(sw);
await drv.InitializeAsync("{}", CancellationToken.None);
// The bootstrap retries every 2 s, so the first failure is what has to trip it.
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6));
sw.Down = false;
}
///
/// A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not
/// degraded (else the watchdog would reconnect-loop a production control).
///
[Fact]
public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade()
{
var sw = new CncSwitch();
await using var drv = FixedOnlyDriver(sw);
await drv.InitializeAsync("{}", CancellationToken.None);
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
sw.Refuse = true;
var started = DateTime.UtcNow;
await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 1);
var atFirstRefusal = drv.GetHealth().LastSuccessfulRead!.Value;
// Refused ticks really happened, for longer than the staleness budget …
await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 5 && DateTime.UtcNow - started > Stale * 2);
// … and the driver stayed Healthy, with each refusal crediting the answer (review minor 2).
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(atFirstRefusal,
"a refusal is an answer from the CNC — LastSuccessfulRead advances");
sw.Refuse = false;
}
private static async Task WaitUntil(Func condition, TimeSpan? timeout = null)
{
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5));
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
await Task.Delay(10);
}
}
}