using System; using System.Collections.Generic; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.MxAccess; using Xunit; namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess; /// /// Unit tests for : prove the /// auto-failover (consecutive primary COM failures → standby) and /// auto-failback (consecutive clean probes → primary) state machine, /// active-child transition forwarding, and active-child delegation of /// acknowledgments. Fakes stand in for both children so this needs no /// AVEVA install. /// public sealed class FailoverAlarmConsumerTests { /// /// Primary fake whose Subscribe/PollOnce throw a COMException while /// is set, modeling a wnwrap consumer that /// surfaces COM HRESULT failures. Can also re-raise a transition so /// before-failover forwarding can be exercised. /// private sealed class FlakyPrimary : IMxAccessAlarmConsumer { /// Raised when the fake forwards a simulated alarm transition via . public event EventHandler? AlarmTransitionEmitted; public bool ThrowOnPoll = true; /// /// When set, throws /// instead of a /// , to /// exercise the OOM-safe exception filter. /// public bool ThrowOutOfMemoryOnPoll; public int Polls; /// /// Number of times has been called. /// Incremented at entry, before any throw, so every attempt is /// counted regardless of whether is set. /// public int SubscribeCount; /// public void Subscribe(string s) { SubscribeCount++; if (ThrowOnPoll) { throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005)); } } /// public void PollOnce() { Polls++; if (ThrowOutOfMemoryOnPoll) { throw new OutOfMemoryException("simulated allocation failure"); } if (ThrowOnPoll) { throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005)); } } /// public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 11; /// public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 11; /// public IReadOnlyList SnapshotActiveAlarms() => Array.Empty(); /// public void Dispose() { } /// Raises with the given event, simulating a COM-forwarded transition. /// The transition event to forward. public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e); } /// /// Standby fake (subtag stand-in): never throws, records that it was /// armed, and can re-raise a transition. /// private sealed class StubStandby : IMxAccessAlarmConsumer { /// Raised when the fake forwards a simulated alarm transition via . public event EventHandler? AlarmTransitionEmitted; public bool Subscribed; /// /// When set, throws — modeling a /// priming-snapshot failure during failover. /// public bool ThrowOnSnapshot; /// Number of calls. public int SnapshotCalls; /// public void Subscribe(string s) => Subscribed = true; /// public void PollOnce() { } /// public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 22; /// public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 22; /// public IReadOnlyList SnapshotActiveAlarms() { SnapshotCalls++; if (ThrowOnSnapshot) { throw new InvalidOperationException("priming snapshot failed"); } return Array.Empty(); } /// public void Dispose() { } /// Raises with the given event, simulating a COM-forwarded transition. /// The transition event to forward. public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e); } private static MxAlarmTransitionEvent SampleTransition() => new MxAlarmTransitionEvent { Record = new MxAlarmSnapshotRecord { AlarmGuid = Guid.NewGuid() }, PreviousState = MxAlarmStateKind.Unspecified, }; /// Proves that the consumer switches to the subtag standby after the primary fails the configured threshold of consecutive times. [Fact] public void Primary_FailsThresholdTimes_SwitchesToSubtag() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); List changes = new List(); sut.ProviderModeChanged += (_, e) => changes.Add(e); sut.Subscribe(@"\\HOST\Galaxy!Area"); // failure 1 (primary), standby armed Assert.True(standby.Subscribed); Assert.Empty(changes); sut.PollOnce(); // failure 2 Assert.Empty(changes); sut.PollOnce(); // failure 3 → switch Assert.Single(changes); Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode); Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); Assert.Equal(unchecked((int)0x80004005), changes[0].HResult); } /// Proves that once failed over, transitions raised by the standby are forwarded through the consumer's AlarmTransitionEmitted event. [Fact] public void AfterSwitch_StandbyTransitionsAreForwarded() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); MxAlarmTransitionEvent? forwarded = null; sut.AlarmTransitionEmitted += (_, e) => forwarded = e; sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag immediately Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); MxAlarmTransitionEvent transition = SampleTransition(); standby.Raise(transition); Assert.Same(transition, forwarded); } /// Proves that once the primary heals, the consumer fails back to it only after the configured number of consecutive clean probes. [Fact] public void WhileDegraded_PrimaryHeals_FailsBackAfterStableProbes() { // threshold=1 so the initial Subscribe failure (PollOnce path) immediately // switches to Subtag. stableProbes=2 means two consecutive clean PollOnce // calls are needed before failback. ProbeOnce must NOT call Subscribe — // WnWrapAlarmConsumer is single-subscribe; re-calling would always throw. FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 2); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); List changes = new List(); sut.ProviderModeChanged += (_, e) => changes.Add(e); sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → Subtag (mode change 1) Assert.Single(changes); Assert.Equal(AlarmProviderMode.Subtag, changes[changes.Count - 1].Mode); // Primary heals: PollOnce stops throwing. ProbeOnce should call only // PollOnce (not Subscribe) to detect recovery. primary.ThrowOnPoll = false; int subscribeCountAfterFailover = primary.SubscribeCount; sut.ProbeOnce(); // cleanProbes=1 — not yet at stableProbes=2 Assert.Single(changes); sut.ProbeOnce(); // cleanProbes=2 → failback to Alarmmgr (mode change 2) Assert.Equal(2, changes.Count); Assert.Equal(AlarmProviderMode.Alarmmgr, changes[changes.Count - 1].Mode); Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode); Assert.Equal(0, changes[changes.Count - 1].HResult); // ProbeOnce must not have called Subscribe at all during probing. Assert.Equal(subscribeCountAfterFailover, primary.SubscribeCount); } /// Proves that before any failover, transitions raised by the primary are forwarded but transitions from the inactive standby are suppressed. [Fact] public void BeforeFailover_PrimaryTransitionsAreForwarded() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false }; // healthy, can Raise StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); List forwarded = new List(); sut.AlarmTransitionEmitted += (_, e) => forwarded.Add(e); sut.Subscribe(@"\\HOST\Galaxy!Area"); Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode); MxAlarmTransitionEvent fromPrimary = SampleTransition(); primary.Raise(fromPrimary); // active=Primary → forwarded Assert.Single(forwarded); Assert.Same(fromPrimary, forwarded[0]); standby.Raise(SampleTransition()); // standby not active → suppressed Assert.Single(forwarded); } /// /// Proves that never calls /// Subscribe on the primary while degraded. The production primary /// () is single-subscribe; a second /// Subscribe call would always throw and make failback impossible. /// The probe must re-poll the still-subscribed primary via /// PollOnce only. /// [Fact] public void ProbeOnce_DoesNotCallPrimarySubscribe() { // threshold=1 → first Subscribe failure immediately switches to Subtag. FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe attempt (throws) → Subtag // Capture how many Subscribe calls the initial setup caused (exactly 1: // the attempt that threw and triggered failover). int subscribeCountAfterSetup = primary.SubscribeCount; Assert.Equal(1, subscribeCountAfterSetup); Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // Let PollOnce succeed so ProbeOnce progresses without throwing. primary.ThrowOnPoll = false; // Drive several ProbeOnce calls — none should touch Subscribe. sut.ProbeOnce(); sut.ProbeOnce(); sut.ProbeOnce(); // stableProbes=3 → failback on this call Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode); Assert.Equal(subscribeCountAfterSetup, primary.SubscribeCount); } /// Proves that acknowledgment calls delegate to whichever child (primary or standby) is currently active. [Fact] public void Acknowledge_DelegatesToActiveChild() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); sut.Subscribe(@"\\HOST\Galaxy!Area"); // Active = Primary → primary's sentinel value (11). Assert.Equal(11, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full")); Assert.Equal(11, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full")); // Force a failover by failing the primary past threshold. primary.ThrowOnPoll = true; sut.PollOnce(); // threshold=1 → switch to Standby Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // Active = Standby → standby's sentinel value (22). Assert.Equal(22, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full")); Assert.Equal(22, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full")); } /// /// Proves that an intermittent failure during failback probing resets the /// clean-probe counter to zero, requiring a fresh unbroken run of /// before failing back. /// [Fact] public void FailbackProbe_IntermittentFailure_ResetsCleanCount() { var primary = new FlakyPrimary { ThrowOnPoll = true }; var standby = new StubStandby(); using var sut = new FailoverAlarmConsumer(primary, standby, new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3)); sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); primary.ThrowOnPoll = false; sut.ProbeOnce(); // clean 1 sut.ProbeOnce(); // clean 2 primary.ThrowOnPoll = true; sut.ProbeOnce(); // fails → reset to 0 Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); primary.ThrowOnPoll = false; sut.ProbeOnce(); // clean 1 sut.ProbeOnce(); // clean 2 sut.ProbeOnce(); // clean 3 → failback Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode); } /// /// When the standby's priming /// SnapshotActiveAlarms throws during failover, the switch must /// still (a) fire ProviderModeChanged so the gateway learns the /// feed went degraded, (b) leave /// in Subtag, and (c) not rethrow out of PollOnce (which on the /// real STA would land in the poll loop's trailing catch and permanently /// stop alarm delivery). /// [Fact] public void Failover_WhenStandbyPrimingSnapshotThrows_StillRaisesModeChangeAndDoesNotRethrow() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby { ThrowOnSnapshot = true }; FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); List changes = new List(); sut.ProviderModeChanged += (_, e) => changes.Add(e); // threshold=1 → the Subscribe failure triggers the switch, which primes // the standby snapshot (throwing). The exception must be contained. Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area")); Assert.Null(escaped); Assert.Single(changes); Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode); Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); Assert.True(standby.SnapshotCalls >= 1); // priming was attempted // A subsequent degraded PollOnce (standby.PollOnce + ProbeOnce) must also // not rethrow the snapshot failure. Exception? pollEscaped = Record.Exception(() => sut.PollOnce()); Assert.Null(pollEscaped); } /// /// When a ProviderModeChanged subscriber's /// handler throws (modeling the AlarmCommandHandler's event-queue enqueue /// overflowing at capacity), the switch must still take effect and the /// exception must not escape the switch path into the poll loop. /// [Fact] public void Failover_WhenModeChangedHandlerThrows_SwitchStillTakesEffectAndDoesNotRethrow() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); int handlerInvocations = 0; sut.ProviderModeChanged += (_, _) => { handlerInvocations++; throw new InvalidOperationException("subscriber handler blew up"); }; Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area")); Assert.Null(escaped); Assert.Equal(1, handlerInvocations); // the event still fired Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // the switch still took effect } /// /// With a non-zero /// , two back-to-back /// ProbeOnce calls must throttle — the second falls inside the /// interval and must NOT re-poll the primary. Two consecutive calls /// reliably fall inside any interval of one second or more, so this needs /// no injected clock. /// [Fact] public void ProbeOnce_WithNonZeroInterval_ThrottlesSecondProbeWithinInterval() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true }; StubStandby standby = new StubStandby(); // stableProbes high enough that a single clean probe cannot fail back, // so Mode stays Subtag and ProbeOnce remains the throttled path. FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 3600, stableProbes: 5); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); primary.ThrowOnPoll = false; // primary healthy so a probe would poll cleanly sut.ProbeOnce(); // first probe runs: re-polls the primary int pollsAfterFirstProbe = primary.Polls; Assert.Equal(1, pollsAfterFirstProbe); sut.ProbeOnce(); // within the 3600s interval → throttled, must NOT re-poll Assert.Equal(pollsAfterFirstProbe, primary.Polls); Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); } /// /// RunPrimary's /// when (ex is not OutOfMemoryException) filter must let an /// propagate rather than swallowing it /// and counting it toward the failover threshold. No mode change must /// fire — a fatal allocation failure is not a clean degraded handoff. /// [Fact] public void RunPrimary_WhenPrimaryThrowsOutOfMemory_PropagatesAndDoesNotFailOver() { FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false, ThrowOutOfMemoryOnPoll = true }; StubStandby standby = new StubStandby(); FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1); using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings); bool modeChanged = false; sut.ProviderModeChanged += (_, _) => modeChanged = true; sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe path does not poll; no throw here Assert.Throws(() => sut.PollOnce()); Assert.False(modeChanged); Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode); } /// /// Verifies that clamps out-of-range /// threshold and stableProbes values (and negative /// probeIntervalSeconds) to their safe minimums so a misconfigured /// bind cannot change failover semantics. /// /// The raw, possibly out-of-range threshold to construct with. /// The raw, possibly negative probe interval (seconds) to construct with. /// The raw, possibly out-of-range stable-probe count to construct with. /// The expected clamped . /// The expected clamped . /// The expected clamped . [Theory] [InlineData(0, 0, 0, 1, 0, 1)] [InlineData(-5, -5, -5, 1, 0, 1)] [InlineData(3, 7, 2, 3, 7, 2)] public void FailoverSettings_ClampsSubMinimumValues( int threshold, int probeInterval, int stableProbes, int expectedThreshold, int expectedProbeInterval, int expectedStableProbes) { FailoverSettings settings = new FailoverSettings(threshold, probeInterval, stableProbes); Assert.Equal(expectedThreshold, settings.Threshold); Assert.Equal(expectedProbeInterval, settings.ProbeIntervalSeconds); Assert.Equal(expectedStableProbes, settings.StableProbes); } }