lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectConditionCommsLossTests.cs

using System.Collections.Concurrent;
using System.Net.Http;
using System.Net.Sockets;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;

/// <summary>
///     lmxopcua#852 — the #846 refused-connection fast path reaches native CONDITIONS too: once the refusals are
///     confirmed every told condition gets a quality-only <c>BadNoCommunication</c> (state and ack untouched, no
///     Raise/Clear), once per outage; the first delivery afterwards lifts it with a quality-only Good — even when that
///     delivery does not mention the condition. Ambiguous failures stay with the watchdog.
/// </summary>
/// <remarks>Every absence is followed by an event that MUST arrive, asserted to be the next one.</remarks>
public sealed class MTConnectConditionCommsLossTests
{
    private const string ConditionRawPath = "Plant/MT/dev1/SystemCond";
    private const string ConditionId = "dev1_system_cond";
    private const long Instance = 1655000000L;
    private const uint BadNoCommunication = 0x80310000u;

    private static readonly DateTime ObservedAt = new(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc);
    private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);

    private static CancellationToken Ct => TestContext.Current.CancellationToken;

    private static Exception Refused() =>
        new HttpRequestException("Connection refused (fixture-agent:5000)", new SocketException((int)SocketError.ConnectionRefused));

    private static MTConnectStreamsResult Chunk(long first, params MTConnectObservation[] observations) =>
        new(Instance, first + 1, first, observations);

    private static MTConnectObservation Cond(string state, string? nativeCode = null, string? text = null) =>
        new(ConditionId, state, ObservedAt, Condition: new MTConnectConditionDetail(text, nativeCode, null, null, "SYSTEM"));

    private static async Task<(MTConnectDriver Driver, CannedAgentClient Agent, ConcurrentQueue<AlarmEventArgs> Events)>
        SubscribedAsync()
    {
        var agent = CannedAgentClient.FromFixtures();
        var driver = new MTConnectDriver(
            new MTConnectDriverOptions
            {
                AgentUri = "http://fixture-agent:5000",
                RequestTimeoutMs = 5000,
                CommsLossConfirmMs = 0,
                Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 0, MaxBackoffMs = 200 },
                RawTags = [new RawTagEntry(ConditionRawPath, $$"""{"address":"{{ConditionId}}"}""", false, "dev1")],
            },
            "mt-852",
            _ => agent);
        await driver.InitializeAsync("{}", Ct);

        var events = new ConcurrentQueue<AlarmEventArgs>();
        driver.OnAlarmEvent += (_, e) => events.Enqueue(e);
        await driver.SubscribeAlarmsAsync([ConditionRawPath], Ct);
        await agent.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
        return (driver, agent, events);
    }

    private static async Task WaitUntil(Func<bool> condition, int timeoutMs = 10_000)
    {
        var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
        while (!condition())
        {
            if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
            await Task.Delay(20, Ct);
        }
    }

    private static async Task RecoverAsync(CannedAgentClient agent)
    {
        agent.SampleFailure = null;
        var calls = agent.SampleCallCount;
        await agent.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
    }

    [Fact]
    public async Task A_refused_agent_annotates_an_active_condition_bad_once_and_the_first_delivery_lifts_it()
    {
        var (driver, agent, events) = await SubscribedAsync();
        try
        {
            await agent.PumpAsync(Chunk(108, Cond("Fault", "E17", "overtemp"))).WaitAsync(Watchdog, Ct);
            events.ShouldHaveSingleItem().Kind.ShouldBe(AlarmTransitionKind.Raise);

            agent.SampleFailure = Refused();
            agent.EndStream();
            await WaitUntil(() => events.Count == 2);
            var bad = events.Last();
            bad.IsQualityOnly.ShouldBeTrue("the Active state and the ack must not move — quality only");
            bad.QualityStatusCode.ShouldBe(BadNoCommunication);
            bad.ConditionId.ShouldBe(ConditionRawPath);
            bad.OpcUaSeverity.ShouldBe(800);
            driver.CommsLossMarked.ShouldBeTrue();

            await Task.Delay(400, Ct); // more refusals: annotated ONCE per outage
            events.Count.ShouldBe(2);

            // The Agent is back; an observation-free heartbeat does not mention the condition, and must still lift it.
            await RecoverAsync(agent);
            await agent.PumpAsync(Chunk(109)).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => events.Count == 3);
            var good = events.Last();
            good.IsQualityOnly.ShouldBeTrue("an unchanged Fault is not re-raised (that would un-acknowledge it)");
            good.QualityStatusCode.ShouldBe(0u);

            // Positive control for "nothing else was emitted": the next real transition is the next event.
            await agent.PumpAsync(Chunk(110, Cond("Normal"))).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => events.Count == 4);
            events.Last().Kind.ShouldBe(AlarmTransitionKind.Clear);
        }
        finally
        {
            await driver.ShutdownAsync(CancellationToken.None);
        }
    }

    [Fact]
    public async Task An_inactive_told_condition_is_annotated_too()
    {
        var (driver, agent, events) = await SubscribedAsync();
        try
        {
            // Raise + Clear: the server holds a told Normal statement for the condition.
            await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
            await agent.PumpAsync(Chunk(109, Cond("Normal"))).WaitAsync(Watchdog, Ct);
            events.Count.ShouldBe(2);

            agent.SampleFailure = Refused();
            agent.EndStream();
            await WaitUntil(() => events.Count == 3);
            events.Last().IsQualityOnly.ShouldBeTrue();
            events.Last().QualityStatusCode.ShouldBe(BadNoCommunication);

            await RecoverAsync(agent);
            await agent.PumpAsync(Chunk(110)).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => events.Count == 4);
            events.Last().IsQualityOnly.ShouldBeTrue();
            events.Last().QualityStatusCode.ShouldBe(0u);
        }
        finally
        {
            await driver.ShutdownAsync(CancellationToken.None);
        }
    }

    [Fact]
    public async Task A_condition_already_unavailable_is_not_re_annotated_and_stays_bad_on_recovery()
    {
        var (driver, agent, events) = await SubscribedAsync();
        try
        {
            await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
            await agent.PumpAsync(Chunk(109, Cond("UNAVAILABLE"))).WaitAsync(Watchdog, Ct);
            events.Count.ShouldBe(2);
            events.Last().QualityStatusCode.ShouldBe(BadNoCommunication);

            agent.SampleFailure = Refused();
            agent.EndStream();
            await WaitUntil(() => driver.CommsLossMarked);

            // The Agent answers, but the adapter is still UNAVAILABLE: no Good may be told.
            await RecoverAsync(agent);
            await agent.PumpAsync(Chunk(110)).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => !driver.CommsLossMarked);

            // Positive control: the adapter returns with the same Fault — quality lifted, NOT re-raised — and that
            // is the very next event after the UNAVAILABLE annotation.
            await agent.PumpAsync(Chunk(111, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => events.Count >= 3);
            events.Count.ShouldBe(3);
            events.Last().IsQualityOnly.ShouldBeTrue();
            events.Last().QualityStatusCode.ShouldBe(0u);
        }
        finally
        {
            await driver.ShutdownAsync(CancellationToken.None);
        }
    }

    [Fact]
    public async Task An_ambiguous_failure_annotates_no_condition()
    {
        var (driver, agent, events) = await SubscribedAsync();
        try
        {
            await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
            events.Count.ShouldBe(1);

            agent.SampleFailure = new TimeoutException("the Agent did not answer /sample in time");
            var calls = agent.SampleCallCount;
            agent.EndStream();
            await agent.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
            driver.CommsLossMarked.ShouldBeFalse();

            // Positive control: the next event is the real Clear, not a quality annotation.
            await RecoverAsync(agent);
            await agent.PumpAsync(Chunk(109, Cond("Normal"))).WaitAsync(Watchdog, Ct);
            await WaitUntil(() => events.Count >= 2);
            events.Count.ShouldBe(2);
            events.Last().Kind.ShouldBe(AlarmTransitionKind.Clear);
        }
        finally
        {
            await driver.ShutdownAsync(CancellationToken.None);
        }
    }

    [Fact]
    public void CommsLost_keeps_the_told_statement_and_says_nothing_twice()
    {
        var handle = new MTConnectAlarmSubscriptionHandle(1);
        var warning = new MTConnectConditionState(
            MTConnectConditionLevel.Warning, new MTConnectConditionDetail("coolant", "W1", null, null, "SYSTEM"), ObservedAt, 1);
        var told = new MTConnectToldCondition(warning, QualityBad: false);
        var at = ObservedAt.AddMinutes(5);

        var (evt, after) = MTConnectConditionAlarmMapper.CommsLost(told, ConditionRawPath, ConditionId, handle, at);

        evt.ShouldNotBeNull();
        evt!.IsQualityOnly.ShouldBeTrue();
        evt.QualityStatusCode.ShouldBe(BadNoCommunication);
        evt.SourceTimestampUtc.ShouldBe(at);
        evt.OpcUaSeverity.ShouldBe(500);
        after.ShouldBe(told with { QualityBad = true });

        MTConnectConditionAlarmMapper.CommsLost(after, ConditionRawPath, ConditionId, handle, at).Event.ShouldBeNull();
        MTConnectConditionAlarmMapper.CommsLost(
            new MTConnectToldCondition(null, QualityBad: true), ConditionRawPath, ConditionId, handle, at).Event.ShouldBeNull();
    }
}