mxaccessgw/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/FailoverAlarmConsumerTests.cs

using System;
using System.Collections.Generic;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
using Xunit;

namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess;

/// <summary>
///     Unit tests for <see cref="FailoverAlarmConsumer"/>: prove the
///     auto-failover (consecutive primary COM failures → standby) and
///     auto-failback (consecutive clean probes → primary) state machine,
///     active-child transition forwarding, and active-child delegation of
///     acknowledgments. Fakes stand in for both children so this needs no
///     AVEVA install.
/// </summary>
public sealed class FailoverAlarmConsumerTests
{
    /// <summary>
    ///     Primary fake whose Subscribe/PollOnce throw a COMException while
    ///     <see cref="ThrowOnPoll"/> is set, modeling a wnwrap consumer that
    ///     surfaces COM HRESULT failures. Can also re-raise a transition so
    ///     before-failover forwarding can be exercised.
    /// </summary>
    private sealed class FlakyPrimary : IMxAccessAlarmConsumer
    {
        /// <summary>Raised when the fake forwards a simulated alarm transition via <see cref="Raise"/>.</summary>
        public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;

        public bool ThrowOnPoll = true;

        /// <summary>
        ///     When set, <see cref="PollOnce"/> throws
        ///     <see cref="OutOfMemoryException"/> instead of a
        ///     <see cref="System.Runtime.InteropServices.COMException"/>, to
        ///     exercise the OOM-safe exception filter.
        /// </summary>
        public bool ThrowOutOfMemoryOnPoll;

        public int Polls;

        /// <summary>
        ///     Number of times <see cref="Subscribe"/> has been called.
        ///     Incremented at entry, before any throw, so every attempt is
        ///     counted regardless of whether <see cref="ThrowOnPoll"/> is set.
        /// </summary>
        public int SubscribeCount;

        /// <inheritdoc />
        public void Subscribe(string s)
        {
            SubscribeCount++;
            if (ThrowOnPoll)
            {
                throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
            }
        }

        /// <inheritdoc />
        public void PollOnce()
        {
            Polls++;
            if (ThrowOutOfMemoryOnPoll)
            {
                throw new OutOfMemoryException("simulated allocation failure");
            }

            if (ThrowOnPoll)
            {
                throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
            }
        }

        /// <inheritdoc />
        public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 11;

        /// <inheritdoc />
        public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 11;

        /// <inheritdoc />
        public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms() => Array.Empty<MxAlarmSnapshotRecord>();

        /// <inheritdoc />
        public void Dispose() { }

        /// <summary>Raises <see cref="AlarmTransitionEmitted"/> with the given event, simulating a COM-forwarded transition.</summary>
        /// <param name="e">The transition event to forward.</param>
        public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
    }

    /// <summary>
    ///     Standby fake (subtag stand-in): never throws, records that it was
    ///     armed, and can re-raise a transition.
    /// </summary>
    private sealed class StubStandby : IMxAccessAlarmConsumer
    {
        /// <summary>Raised when the fake forwards a simulated alarm transition via <see cref="Raise"/>.</summary>
        public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;

        public bool Subscribed;

        /// <summary>
        ///     When set, <see cref="SnapshotActiveAlarms"/> throws — modeling a
        ///     priming-snapshot failure during failover.
        /// </summary>
        public bool ThrowOnSnapshot;

        /// <summary>Number of <see cref="SnapshotActiveAlarms"/> calls.</summary>
        public int SnapshotCalls;

        /// <inheritdoc />
        public void Subscribe(string s) => Subscribed = true;

        /// <inheritdoc />
        public void PollOnce() { }

        /// <inheritdoc />
        public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 22;

        /// <inheritdoc />
        public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 22;

        /// <inheritdoc />
        public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms()
        {
            SnapshotCalls++;
            if (ThrowOnSnapshot)
            {
                throw new InvalidOperationException("priming snapshot failed");
            }

            return Array.Empty<MxAlarmSnapshotRecord>();
        }

        /// <inheritdoc />
        public void Dispose() { }

        /// <summary>Raises <see cref="AlarmTransitionEmitted"/> with the given event, simulating a COM-forwarded transition.</summary>
        /// <param name="e">The transition event to forward.</param>
        public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
    }

    private static MxAlarmTransitionEvent SampleTransition() => new MxAlarmTransitionEvent
    {
        Record = new MxAlarmSnapshotRecord { AlarmGuid = Guid.NewGuid() },
        PreviousState = MxAlarmStateKind.Unspecified,
    };

    /// <summary>Proves that the consumer switches to the subtag standby after the primary fails the configured threshold of consecutive times.</summary>
    [Fact]
    public void Primary_FailsThresholdTimes_SwitchesToSubtag()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
        sut.ProviderModeChanged += (_, e) => changes.Add(e);

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // failure 1 (primary), standby armed
        Assert.True(standby.Subscribed);
        Assert.Empty(changes);

        sut.PollOnce(); // failure 2
        Assert.Empty(changes);

        sut.PollOnce(); // failure 3 → switch

        Assert.Single(changes);
        Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
        Assert.Equal(unchecked((int)0x80004005), changes[0].HResult);
    }

    /// <summary>Proves that once failed over, transitions raised by the standby are forwarded through the consumer's <c>AlarmTransitionEmitted</c> event.</summary>
    [Fact]
    public void AfterSwitch_StandbyTransitionsAreForwarded()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        MxAlarmTransitionEvent? forwarded = null;
        sut.AlarmTransitionEmitted += (_, e) => forwarded = e;

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag immediately
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        MxAlarmTransitionEvent transition = SampleTransition();
        standby.Raise(transition);

        Assert.Same(transition, forwarded);
    }

    /// <summary>Proves that once the primary heals, the consumer fails back to it only after the configured number of consecutive clean probes.</summary>
    [Fact]
    public void WhileDegraded_PrimaryHeals_FailsBackAfterStableProbes()
    {
        // threshold=1 so the initial Subscribe failure (PollOnce path) immediately
        // switches to Subtag. stableProbes=2 means two consecutive clean PollOnce
        // calls are needed before failback. ProbeOnce must NOT call Subscribe —
        // WnWrapAlarmConsumer is single-subscribe; re-calling would always throw.
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 2);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
        sut.ProviderModeChanged += (_, e) => changes.Add(e);

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → Subtag (mode change 1)
        Assert.Single(changes);
        Assert.Equal(AlarmProviderMode.Subtag, changes[changes.Count - 1].Mode);

        // Primary heals: PollOnce stops throwing. ProbeOnce should call only
        // PollOnce (not Subscribe) to detect recovery.
        primary.ThrowOnPoll = false;
        int subscribeCountAfterFailover = primary.SubscribeCount;

        sut.ProbeOnce(); // cleanProbes=1 — not yet at stableProbes=2
        Assert.Single(changes);

        sut.ProbeOnce(); // cleanProbes=2 → failback to Alarmmgr (mode change 2)

        Assert.Equal(2, changes.Count);
        Assert.Equal(AlarmProviderMode.Alarmmgr, changes[changes.Count - 1].Mode);
        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
        Assert.Equal(0, changes[changes.Count - 1].HResult);

        // ProbeOnce must not have called Subscribe at all during probing.
        Assert.Equal(subscribeCountAfterFailover, primary.SubscribeCount);
    }

    /// <summary>Proves that before any failover, transitions raised by the primary are forwarded but transitions from the inactive standby are suppressed.</summary>
    [Fact]
    public void BeforeFailover_PrimaryTransitionsAreForwarded()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false }; // healthy, can Raise
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        List<MxAlarmTransitionEvent> forwarded = new List<MxAlarmTransitionEvent>();
        sut.AlarmTransitionEmitted += (_, e) => forwarded.Add(e);

        sut.Subscribe(@"\\HOST\Galaxy!Area");
        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);

        MxAlarmTransitionEvent fromPrimary = SampleTransition();
        primary.Raise(fromPrimary); // active=Primary → forwarded
        Assert.Single(forwarded);
        Assert.Same(fromPrimary, forwarded[0]);

        standby.Raise(SampleTransition()); // standby not active → suppressed
        Assert.Single(forwarded);
    }

    /// <summary>
    ///     Proves that <see cref="FailoverAlarmConsumer.ProbeOnce"/> never calls
    ///     <c>Subscribe</c> on the primary while degraded. The production primary
    ///     (<see cref="WnWrapAlarmConsumer"/>) is single-subscribe; a second
    ///     <c>Subscribe</c> call would always throw and make failback impossible.
    ///     The probe must re-poll the still-subscribed primary via
    ///     <c>PollOnce</c> only.
    /// </summary>
    [Fact]
    public void ProbeOnce_DoesNotCallPrimarySubscribe()
    {
        // threshold=1 → first Subscribe failure immediately switches to Subtag.
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe attempt (throws) → Subtag

        // Capture how many Subscribe calls the initial setup caused (exactly 1:
        // the attempt that threw and triggered failover).
        int subscribeCountAfterSetup = primary.SubscribeCount;
        Assert.Equal(1, subscribeCountAfterSetup);
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        // Let PollOnce succeed so ProbeOnce progresses without throwing.
        primary.ThrowOnPoll = false;

        // Drive several ProbeOnce calls — none should touch Subscribe.
        sut.ProbeOnce();
        sut.ProbeOnce();
        sut.ProbeOnce(); // stableProbes=3 → failback on this call

        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
        Assert.Equal(subscribeCountAfterSetup, primary.SubscribeCount);
    }

    /// <summary>Proves that acknowledgment calls delegate to whichever child (primary or standby) is currently active.</summary>
    [Fact]
    public void Acknowledge_DelegatesToActiveChild()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        sut.Subscribe(@"\\HOST\Galaxy!Area");

        // Active = Primary → primary's sentinel value (11).
        Assert.Equal(11, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
        Assert.Equal(11, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));

        // Force a failover by failing the primary past threshold.
        primary.ThrowOnPoll = true;
        sut.PollOnce(); // threshold=1 → switch to Standby
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        // Active = Standby → standby's sentinel value (22).
        Assert.Equal(22, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
        Assert.Equal(22, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));
    }

    /// <summary>
    ///     Proves that an intermittent failure during failback probing resets the
    ///     clean-probe counter to zero, requiring a fresh unbroken run of
    ///     <see cref="FailoverSettings.StableProbes"/> before failing back.
    /// </summary>
    [Fact]
    public void FailbackProbe_IntermittentFailure_ResetsCleanCount()
    {
        var primary = new FlakyPrimary { ThrowOnPoll = true };
        var standby = new StubStandby();
        using var sut = new FailoverAlarmConsumer(primary, standby, new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3));
        sut.Subscribe(@"\\HOST\Galaxy!Area");           // threshold=1 → switch to Subtag
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        primary.ThrowOnPoll = false;
        sut.ProbeOnce();  // clean 1
        sut.ProbeOnce();  // clean 2
        primary.ThrowOnPoll = true;
        sut.ProbeOnce();  // fails → reset to 0
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        primary.ThrowOnPoll = false;
        sut.ProbeOnce();  // clean 1
        sut.ProbeOnce();  // clean 2
        sut.ProbeOnce();  // clean 3 → failback
        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
    }

    /// <summary>
    ///     When the standby's priming
    ///     <c>SnapshotActiveAlarms</c> throws during failover, the switch must
    ///     still (a) fire <c>ProviderModeChanged</c> so the gateway learns the
    ///     feed went degraded, (b) leave <see cref="FailoverAlarmConsumer.Mode"/>
    ///     in Subtag, and (c) not rethrow out of <c>PollOnce</c> (which on the
    ///     real STA would land in the poll loop's trailing catch and permanently
    ///     stop alarm delivery).
    /// </summary>
    [Fact]
    public void Failover_WhenStandbyPrimingSnapshotThrows_StillRaisesModeChangeAndDoesNotRethrow()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby { ThrowOnSnapshot = true };
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
        sut.ProviderModeChanged += (_, e) => changes.Add(e);

        // threshold=1 → the Subscribe failure triggers the switch, which primes
        // the standby snapshot (throwing). The exception must be contained.
        Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));

        Assert.Null(escaped);
        Assert.Single(changes);
        Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
        Assert.True(standby.SnapshotCalls >= 1); // priming was attempted

        // A subsequent degraded PollOnce (standby.PollOnce + ProbeOnce) must also
        // not rethrow the snapshot failure.
        Exception? pollEscaped = Record.Exception(() => sut.PollOnce());
        Assert.Null(pollEscaped);
    }

    /// <summary>
    ///     When a <c>ProviderModeChanged</c> subscriber's
    ///     handler throws (modeling the AlarmCommandHandler's event-queue enqueue
    ///     overflowing at capacity), the switch must still take effect and the
    ///     exception must not escape the switch path into the poll loop.
    /// </summary>
    [Fact]
    public void Failover_WhenModeChangedHandlerThrows_SwitchStillTakesEffectAndDoesNotRethrow()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        int handlerInvocations = 0;
        sut.ProviderModeChanged += (_, _) =>
        {
            handlerInvocations++;
            throw new InvalidOperationException("subscriber handler blew up");
        };

        Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));

        Assert.Null(escaped);
        Assert.Equal(1, handlerInvocations);          // the event still fired
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // the switch still took effect
    }

    /// <summary>
    ///     With a non-zero
    ///     <see cref="FailoverSettings.ProbeIntervalSeconds"/>, two back-to-back
    ///     <c>ProbeOnce</c> calls must throttle — the second falls inside the
    ///     interval and must NOT re-poll the primary. Two consecutive calls
    ///     reliably fall inside any interval of one second or more, so this needs
    ///     no injected clock.
    /// </summary>
    [Fact]
    public void ProbeOnce_WithNonZeroInterval_ThrottlesSecondProbeWithinInterval()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
        StubStandby standby = new StubStandby();
        // stableProbes high enough that a single clean probe cannot fail back,
        // so Mode stays Subtag and ProbeOnce remains the throttled path.
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 3600, stableProbes: 5);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);

        primary.ThrowOnPoll = false; // primary healthy so a probe would poll cleanly

        sut.ProbeOnce(); // first probe runs: re-polls the primary
        int pollsAfterFirstProbe = primary.Polls;
        Assert.Equal(1, pollsAfterFirstProbe);

        sut.ProbeOnce(); // within the 3600s interval → throttled, must NOT re-poll

        Assert.Equal(pollsAfterFirstProbe, primary.Polls);
        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
    }

    /// <summary>
    ///     <c>RunPrimary</c>'s
    ///     <c>when (ex is not OutOfMemoryException)</c> filter must let an
    ///     <see cref="OutOfMemoryException"/> propagate rather than swallowing it
    ///     and counting it toward the failover threshold. No mode change must
    ///     fire — a fatal allocation failure is not a clean degraded handoff.
    /// </summary>
    [Fact]
    public void RunPrimary_WhenPrimaryThrowsOutOfMemory_PropagatesAndDoesNotFailOver()
    {
        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false, ThrowOutOfMemoryOnPoll = true };
        StubStandby standby = new StubStandby();
        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);

        bool modeChanged = false;
        sut.ProviderModeChanged += (_, _) => modeChanged = true;

        sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe path does not poll; no throw here

        Assert.Throws<OutOfMemoryException>(() => sut.PollOnce());
        Assert.False(modeChanged);
        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
    }

    /// <summary>
    ///     Verifies that <see cref="FailoverSettings"/> clamps out-of-range
    ///     <c>threshold</c> and <c>stableProbes</c> values (and negative
    ///     <c>probeIntervalSeconds</c>) to their safe minimums so a misconfigured
    ///     bind cannot change failover semantics.
    /// </summary>
    /// <param name="threshold">The raw, possibly out-of-range threshold to construct with.</param>
    /// <param name="probeInterval">The raw, possibly negative probe interval (seconds) to construct with.</param>
    /// <param name="stableProbes">The raw, possibly out-of-range stable-probe count to construct with.</param>
    /// <param name="expectedThreshold">The expected clamped <see cref="FailoverSettings.Threshold"/>.</param>
    /// <param name="expectedProbeInterval">The expected clamped <see cref="FailoverSettings.ProbeIntervalSeconds"/>.</param>
    /// <param name="expectedStableProbes">The expected clamped <see cref="FailoverSettings.StableProbes"/>.</param>
    [Theory]
    [InlineData(0, 0, 0, 1, 0, 1)]
    [InlineData(-5, -5, -5, 1, 0, 1)]
    [InlineData(3, 7, 2, 3, 7, 2)]
    public void FailoverSettings_ClampsSubMinimumValues(
        int threshold,
        int probeInterval,
        int stableProbes,
        int expectedThreshold,
        int expectedProbeInterval,
        int expectedStableProbes)
    {
        FailoverSettings settings = new FailoverSettings(threshold, probeInterval, stableProbes);

        Assert.Equal(expectedThreshold, settings.Threshold);
        Assert.Equal(expectedProbeInterval, settings.ProbeIntervalSeconds);
        Assert.Equal(expectedStableProbes, settings.StableProbes);
    }
}