lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;

namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;

/// <summary>
///     #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a
///     driver serving only <c>FIXED:</c> tags (Identity / Axes / Program / Status — the typical FOCAS shape) read
///     Healthy on <c>/hosts</c> with every node Bad, and the #527 comms-loss watchdog could never fire. The loop
///     now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past
///     the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasFixedTreeHealthTests
{
    private const string Host = "focas://10.0.0.5:8193";
    private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300);
    private static readonly TimeSpan BlipBudget = TimeSpan.FromMilliseconds(1500);

    /// <summary>A CNC that can be switched off (transport failure) or made to refuse the dynamic read.</summary>
    private sealed class CncSwitch
    {
        public volatile bool Down;
        public volatile bool Refuse;
        /// <summary>Transport failures served so far — what the tests wait on instead of wall-clock delays.</summary>
        public int Failures;
        /// <summary>Refusals served so far.</summary>
        public int Refusals;
    }

    private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient
    {
        private bool Down => sw.Down;
        private bool Refuse => sw.Refuse;

        public override Task<FocasSysInfo> GetSysInfoAsync(CancellationToken ct)
        {
            if (!Down) return base.GetSysInfoAsync(ct);
            Interlocked.Increment(ref sw.Failures);
            throw new IOException("socket closed");
        }

        public override Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken ct)
        {
            if (Down)
            {
                Interlocked.Increment(ref sw.Failures);
                throw new IOException("socket closed");
            }
            if (Refuse)
            {
                Interlocked.Increment(ref sw.Refusals);
                throw new FocasWireException("cnc_rddynamic2 refused", rc: 6);
            }
            return base.ReadDynamicAsync(axisIndex, ct);
        }
    }

    private static FocasDriver FixedOnlyDriver(CncSwitch sw, TimeSpan? budget = null) => new(new FocasDriverOptions
    {
        Devices = [new FocasDeviceOptions(Host)],
        RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))],
        Probe = new FocasProbeOptions { Enabled = false },
        FixedTree = new FocasFixedTreeOptions
        {
            PollInterval = TimeSpan.FromMilliseconds(20),
            ProgramPollInterval = TimeSpan.Zero,
            TimerPollInterval = TimeSpan.Zero,
        },
    }, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) })
    {
        FixedTreeStaleBudgetOverride = budget ?? Stale,
    };

    /// <summary>
    ///     Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving
    ///     LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with
    ///     LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy.
    /// </summary>
    [Fact]
    public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns()
    {
        var sw = new CncSwitch();
        // A wide budget (review round 2 m4): two 20 ms ticks always land well inside it, so the "blip does not
        // degrade" absence check below runs unconditionally instead of being skipped on a slow agent.
        await using var drv = FixedOnlyDriver(sw, BlipBudget);
        await drv.InitializeAsync("{}", CancellationToken.None);
        await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);

        var first = drv.GetHealth().LastSuccessfulRead!.Value;
        // An answered axis pass is a successful read: wait for it to move rather than sleeping.
        await WaitUntil(() => drv.GetHealth().LastSuccessfulRead > first);
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);

        sw.Down = true;
        var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value;
        // Two failed ticks have demonstrably happened (positive evidence for the absence check below) …
        await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2);
        // … inside the budget (asserted, not assumed), and the driver is still Healthy.
        (DateTime.UtcNow - lastAnswer).ShouldBeLessThan(BlipBudget, "the two failed ticks must land inside the budget");
        drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade");
        await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded);
        var frozen = drv.GetHealth().LastSuccessfulRead;
        var failuresAtDegrade = Volatile.Read(ref sw.Failures);
        await WaitUntil(() => Volatile.Read(ref sw.Failures) >= failuresAtDegrade + 5);
        drv.GetHealth().State.ShouldBe(DriverState.Degraded);
        drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent");

        sw.Down = false;
        await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy);
        drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
    }

    /// <summary>A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes.</summary>
    [Fact]
    public async Task Bootstrap_that_never_succeeds_degrades_the_driver()
    {
        var sw = new CncSwitch { Down = true };
        await using var drv = FixedOnlyDriver(sw);
        await drv.InitializeAsync("{}", CancellationToken.None);
        // The bootstrap retries every 2 s, so the first failure is what has to trip it.
        await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6));
        sw.Down = false;
    }

    /// <summary>
    ///     A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not
    ///     degraded (else the watchdog would reconnect-loop a production control).
    /// </summary>
    [Fact]
    public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade()
    {
        var sw = new CncSwitch();
        await using var drv = FixedOnlyDriver(sw);
        await drv.InitializeAsync("{}", CancellationToken.None);
        await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);

        sw.Refuse = true;
        var started = DateTime.UtcNow;
        await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 1);
        var atFirstRefusal = drv.GetHealth().LastSuccessfulRead!.Value;
        // Refused ticks really happened, for longer than the staleness budget …
        await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 5 && DateTime.UtcNow - started > Stale * 2);
        // … and the driver stayed Healthy, with each refusal crediting the answer (review minor 2).
        drv.GetHealth().State.ShouldBe(DriverState.Healthy);
        drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(atFirstRefusal,
            "a refusal is an answer from the CNC — LastSuccessfulRead advances");
        sw.Refuse = false;
    }

    private static async Task WaitUntil(Func<bool> condition, TimeSpan? timeout = null)
    {
        var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5));
        while (!condition())
        {
            if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
            await Task.Delay(10);
        }
    }
}