using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire; namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; /// /// #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a /// driver serving only FIXED: tags (Identity / Axes / Program / Status — the typical FOCAS shape) read /// Healthy on /hosts with every node Bad, and the #527 comms-loss watchdog could never fire. The loop /// now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past /// the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen. /// [Trait("Category", "Unit")] public sealed class FocasFixedTreeHealthTests { private const string Host = "focas://10.0.0.5:8193"; private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300); private static readonly TimeSpan BlipBudget = TimeSpan.FromMilliseconds(1500); /// A CNC that can be switched off (transport failure) or made to refuse the dynamic read. private sealed class CncSwitch { public volatile bool Down; public volatile bool Refuse; /// Transport failures served so far — what the tests wait on instead of wall-clock delays. public int Failures; /// Refusals served so far. public int Refusals; } private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient { private bool Down => sw.Down; private bool Refuse => sw.Refuse; public override Task GetSysInfoAsync(CancellationToken ct) { if (!Down) return base.GetSysInfoAsync(ct); Interlocked.Increment(ref sw.Failures); throw new IOException("socket closed"); } public override Task ReadDynamicAsync(int axisIndex, CancellationToken ct) { if (Down) { Interlocked.Increment(ref sw.Failures); throw new IOException("socket closed"); } if (Refuse) { Interlocked.Increment(ref sw.Refusals); throw new FocasWireException("cnc_rddynamic2 refused", rc: 6); } return base.ReadDynamicAsync(axisIndex, ct); } } private static FocasDriver FixedOnlyDriver(CncSwitch sw, TimeSpan? budget = null) => new(new FocasDriverOptions { Devices = [new FocasDeviceOptions(Host)], RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))], Probe = new FocasProbeOptions { Enabled = false }, FixedTree = new FocasFixedTreeOptions { PollInterval = TimeSpan.FromMilliseconds(20), ProgramPollInterval = TimeSpan.Zero, TimerPollInterval = TimeSpan.Zero, }, }, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) }) { FixedTreeStaleBudgetOverride = budget ?? Stale, }; /// /// Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving /// LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with /// LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy. /// [Fact] public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns() { var sw = new CncSwitch(); // A wide budget (review round 2 m4): two 20 ms ticks always land well inside it, so the "blip does not // degrade" absence check below runs unconditionally instead of being skipped on a slow agent. await using var drv = FixedOnlyDriver(sw, BlipBudget); await drv.InitializeAsync("{}", CancellationToken.None); await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); var first = drv.GetHealth().LastSuccessfulRead!.Value; // An answered axis pass is a successful read: wait for it to move rather than sleeping. await WaitUntil(() => drv.GetHealth().LastSuccessfulRead > first); drv.GetHealth().State.ShouldBe(DriverState.Healthy); sw.Down = true; var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value; // Two failed ticks have demonstrably happened (positive evidence for the absence check below) … await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2); // … inside the budget (asserted, not assumed), and the driver is still Healthy. (DateTime.UtcNow - lastAnswer).ShouldBeLessThan(BlipBudget, "the two failed ticks must land inside the budget"); drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded); var frozen = drv.GetHealth().LastSuccessfulRead; var failuresAtDegrade = Volatile.Read(ref sw.Failures); await WaitUntil(() => Volatile.Read(ref sw.Failures) >= failuresAtDegrade + 5); drv.GetHealth().State.ShouldBe(DriverState.Degraded); drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent"); sw.Down = false; await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy); drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value); } /// A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes. [Fact] public async Task Bootstrap_that_never_succeeds_degrades_the_driver() { var sw = new CncSwitch { Down = true }; await using var drv = FixedOnlyDriver(sw); await drv.InitializeAsync("{}", CancellationToken.None); // The bootstrap retries every 2 s, so the first failure is what has to trip it. await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6)); sw.Down = false; } /// /// A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not /// degraded (else the watchdog would reconnect-loop a production control). /// [Fact] public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade() { var sw = new CncSwitch(); await using var drv = FixedOnlyDriver(sw); await drv.InitializeAsync("{}", CancellationToken.None); await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); sw.Refuse = true; var started = DateTime.UtcNow; await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 1); var atFirstRefusal = drv.GetHealth().LastSuccessfulRead!.Value; // Refused ticks really happened, for longer than the staleness budget … await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 5 && DateTime.UtcNow - started > Stale * 2); // … and the driver stayed Healthy, with each refusal crediting the answer (review minor 2). drv.GetHealth().State.ShouldBe(DriverState.Healthy); drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(atFirstRefusal, "a refusal is an answer from the CNC — LastSuccessfulRead advances"); sw.Refuse = false; } private static async Task WaitUntil(Func condition, TimeSpan? timeout = null) { var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5)); while (!condition()) { if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached"); await Task.Delay(10); } } }