6423ed272f
The fixed-tree loop swallowed every failure at Debug and never touched health, so a driver serving only FIXED: tags (Identity/Axes/Program/Status, the typical FOCAS shape) read Healthy on /hosts with every node Bad and the #527 watchdog could never fire. The fixed tree is now a third health path beside the value and alarm paths, combined in GetHealth (worst state, latest LastSuccessfulRead): - an answered bootstrap / axis pass clears the device's failure and advances LastSuccessfulRead; - a transport failure degrades the driver once the device has been silent past the same staleness budget that turns its values Bad (from its last answer, or from loop start when it never answered), with LastSuccessfulRead frozen; - a CNC refusal (non-OK rc) is not a comms loss: it clears the failure and credits nothing. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
134 lines
5.9 KiB
C#
134 lines
5.9 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
|
|
|
|
/// <summary>
|
|
/// #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a
|
|
/// driver serving only <c>FIXED:</c> tags (Identity / Axes / Program / Status — the typical FOCAS shape) read
|
|
/// Healthy on <c>/hosts</c> with every node Bad, and the #527 comms-loss watchdog could never fire. The loop
|
|
/// now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past
|
|
/// the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen.
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class FocasFixedTreeHealthTests
|
|
{
|
|
private const string Host = "focas://10.0.0.5:8193";
|
|
private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300);
|
|
|
|
/// <summary>A CNC that can be switched off (transport failure) or made to refuse the dynamic read.</summary>
|
|
private sealed class CncSwitch
|
|
{
|
|
public volatile bool Down;
|
|
public volatile bool Refuse;
|
|
}
|
|
|
|
private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient
|
|
{
|
|
private bool Down => sw.Down;
|
|
private bool Refuse => sw.Refuse;
|
|
|
|
public override Task<FocasSysInfo> GetSysInfoAsync(CancellationToken ct) =>
|
|
Down ? throw new IOException("socket closed") : base.GetSysInfoAsync(ct);
|
|
|
|
public override Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken ct)
|
|
{
|
|
if (Down) throw new IOException("socket closed");
|
|
if (Refuse) throw new FocasWireException("cnc_rddynamic2 refused", rc: 6);
|
|
return base.ReadDynamicAsync(axisIndex, ct);
|
|
}
|
|
}
|
|
|
|
private static FocasDriver FixedOnlyDriver(CncSwitch sw) => new(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host)],
|
|
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))],
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
FixedTree = new FocasFixedTreeOptions
|
|
{
|
|
PollInterval = TimeSpan.FromMilliseconds(20),
|
|
ProgramPollInterval = TimeSpan.Zero,
|
|
TimerPollInterval = TimeSpan.Zero,
|
|
},
|
|
}, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) })
|
|
{
|
|
FixedTreeStaleBudgetOverride = Stale,
|
|
};
|
|
|
|
/// <summary>
|
|
/// Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving
|
|
/// LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with
|
|
/// LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns()
|
|
{
|
|
var sw = new CncSwitch();
|
|
await using var drv = FixedOnlyDriver(sw);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
|
|
|
|
var first = drv.GetHealth().LastSuccessfulRead;
|
|
await Task.Delay(100);
|
|
var healthy = drv.GetHealth();
|
|
healthy.State.ShouldBe(DriverState.Healthy);
|
|
healthy.LastSuccessfulRead!.Value.ShouldBeGreaterThan(first!.Value, "an answered axis pass is a successful read");
|
|
|
|
sw.Down = true;
|
|
await Task.Delay(100);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade");
|
|
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded);
|
|
var frozen = drv.GetHealth().LastSuccessfulRead;
|
|
await Task.Delay(150);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
|
|
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent");
|
|
|
|
sw.Down = false;
|
|
await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy);
|
|
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
|
|
}
|
|
|
|
/// <summary>A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes.</summary>
|
|
[Fact]
|
|
public async Task Bootstrap_that_never_succeeds_degrades_the_driver()
|
|
{
|
|
var sw = new CncSwitch { Down = true };
|
|
await using var drv = FixedOnlyDriver(sw);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
// The bootstrap retries every 2 s, so the first failure is what has to trip it.
|
|
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6));
|
|
sw.Down = false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not
|
|
/// degraded (else the watchdog would reconnect-loop a production control).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade()
|
|
{
|
|
var sw = new CncSwitch();
|
|
await using var drv = FixedOnlyDriver(sw);
|
|
await drv.InitializeAsync("{}", CancellationToken.None);
|
|
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
|
|
|
|
sw.Refuse = true;
|
|
await Task.Delay(Stale * 3);
|
|
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
|
sw.Refuse = false;
|
|
}
|
|
|
|
private static async Task WaitUntil(Func<bool> condition, TimeSpan? timeout = null)
|
|
{
|
|
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5));
|
|
while (!condition())
|
|
{
|
|
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
|
|
await Task.Delay(10);
|
|
}
|
|
}
|
|
}
|