using System;
using System.Collections.Generic;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
using Xunit;
namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess;
///
/// Unit tests for : prove the
/// auto-failover (consecutive primary COM failures → standby) and
/// auto-failback (consecutive clean probes → primary) state machine,
/// active-child transition forwarding, and active-child delegation of
/// acknowledgments. Fakes stand in for both children so this needs no
/// AVEVA install.
///
public sealed class FailoverAlarmConsumerTests
{
///
/// Primary fake whose Subscribe/PollOnce throw a COMException while
/// is set, modeling a wnwrap consumer that
/// surfaces COM HRESULT failures. Can also re-raise a transition so
/// before-failover forwarding can be exercised.
///
private sealed class FlakyPrimary : IMxAccessAlarmConsumer
{
/// Raised when the fake forwards a simulated alarm transition via .
public event EventHandler? AlarmTransitionEmitted;
public bool ThrowOnPoll = true;
///
/// When set, throws
/// instead of a
/// , to
/// exercise the OOM-safe exception filter.
///
public bool ThrowOutOfMemoryOnPoll;
public int Polls;
///
/// Number of times has been called.
/// Incremented at entry, before any throw, so every attempt is
/// counted regardless of whether is set.
///
public int SubscribeCount;
///
public void Subscribe(string s)
{
SubscribeCount++;
if (ThrowOnPoll)
{
throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
}
}
///
public void PollOnce()
{
Polls++;
if (ThrowOutOfMemoryOnPoll)
{
throw new OutOfMemoryException("simulated allocation failure");
}
if (ThrowOnPoll)
{
throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
}
}
///
public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 11;
///
public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 11;
///
public IReadOnlyList SnapshotActiveAlarms() => Array.Empty();
///
public void Dispose() { }
/// Raises with the given event, simulating a COM-forwarded transition.
/// The transition event to forward.
public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
}
///
/// Standby fake (subtag stand-in): never throws, records that it was
/// armed, and can re-raise a transition.
///
private sealed class StubStandby : IMxAccessAlarmConsumer
{
/// Raised when the fake forwards a simulated alarm transition via .
public event EventHandler? AlarmTransitionEmitted;
public bool Subscribed;
///
/// When set, throws — modeling a
/// priming-snapshot failure during failover.
///
public bool ThrowOnSnapshot;
/// Number of calls.
public int SnapshotCalls;
///
public void Subscribe(string s) => Subscribed = true;
///
public void PollOnce() { }
///
public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 22;
///
public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 22;
///
public IReadOnlyList SnapshotActiveAlarms()
{
SnapshotCalls++;
if (ThrowOnSnapshot)
{
throw new InvalidOperationException("priming snapshot failed");
}
return Array.Empty();
}
///
public void Dispose() { }
/// Raises with the given event, simulating a COM-forwarded transition.
/// The transition event to forward.
public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
}
private static MxAlarmTransitionEvent SampleTransition() => new MxAlarmTransitionEvent
{
Record = new MxAlarmSnapshotRecord { AlarmGuid = Guid.NewGuid() },
PreviousState = MxAlarmStateKind.Unspecified,
};
/// Proves that the consumer switches to the subtag standby after the primary fails the configured threshold of consecutive times.
[Fact]
public void Primary_FailsThresholdTimes_SwitchesToSubtag()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
List changes = new List();
sut.ProviderModeChanged += (_, e) => changes.Add(e);
sut.Subscribe(@"\\HOST\Galaxy!Area"); // failure 1 (primary), standby armed
Assert.True(standby.Subscribed);
Assert.Empty(changes);
sut.PollOnce(); // failure 2
Assert.Empty(changes);
sut.PollOnce(); // failure 3 → switch
Assert.Single(changes);
Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
Assert.Equal(unchecked((int)0x80004005), changes[0].HResult);
}
/// Proves that once failed over, transitions raised by the standby are forwarded through the consumer's AlarmTransitionEmitted event.
[Fact]
public void AfterSwitch_StandbyTransitionsAreForwarded()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
MxAlarmTransitionEvent? forwarded = null;
sut.AlarmTransitionEmitted += (_, e) => forwarded = e;
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag immediately
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
MxAlarmTransitionEvent transition = SampleTransition();
standby.Raise(transition);
Assert.Same(transition, forwarded);
}
/// Proves that once the primary heals, the consumer fails back to it only after the configured number of consecutive clean probes.
[Fact]
public void WhileDegraded_PrimaryHeals_FailsBackAfterStableProbes()
{
// threshold=1 so the initial Subscribe failure (PollOnce path) immediately
// switches to Subtag. stableProbes=2 means two consecutive clean PollOnce
// calls are needed before failback. ProbeOnce must NOT call Subscribe —
// WnWrapAlarmConsumer is single-subscribe; re-calling would always throw.
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 2);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
List changes = new List();
sut.ProviderModeChanged += (_, e) => changes.Add(e);
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → Subtag (mode change 1)
Assert.Single(changes);
Assert.Equal(AlarmProviderMode.Subtag, changes[changes.Count - 1].Mode);
// Primary heals: PollOnce stops throwing. ProbeOnce should call only
// PollOnce (not Subscribe) to detect recovery.
primary.ThrowOnPoll = false;
int subscribeCountAfterFailover = primary.SubscribeCount;
sut.ProbeOnce(); // cleanProbes=1 — not yet at stableProbes=2
Assert.Single(changes);
sut.ProbeOnce(); // cleanProbes=2 → failback to Alarmmgr (mode change 2)
Assert.Equal(2, changes.Count);
Assert.Equal(AlarmProviderMode.Alarmmgr, changes[changes.Count - 1].Mode);
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
Assert.Equal(0, changes[changes.Count - 1].HResult);
// ProbeOnce must not have called Subscribe at all during probing.
Assert.Equal(subscribeCountAfterFailover, primary.SubscribeCount);
}
/// Proves that before any failover, transitions raised by the primary are forwarded but transitions from the inactive standby are suppressed.
[Fact]
public void BeforeFailover_PrimaryTransitionsAreForwarded()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false }; // healthy, can Raise
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
List forwarded = new List();
sut.AlarmTransitionEmitted += (_, e) => forwarded.Add(e);
sut.Subscribe(@"\\HOST\Galaxy!Area");
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
MxAlarmTransitionEvent fromPrimary = SampleTransition();
primary.Raise(fromPrimary); // active=Primary → forwarded
Assert.Single(forwarded);
Assert.Same(fromPrimary, forwarded[0]);
standby.Raise(SampleTransition()); // standby not active → suppressed
Assert.Single(forwarded);
}
///
/// Proves that never calls
/// Subscribe on the primary while degraded. The production primary
/// () is single-subscribe; a second
/// Subscribe call would always throw and make failback impossible.
/// The probe must re-poll the still-subscribed primary via
/// PollOnce only.
///
[Fact]
public void ProbeOnce_DoesNotCallPrimarySubscribe()
{
// threshold=1 → first Subscribe failure immediately switches to Subtag.
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe attempt (throws) → Subtag
// Capture how many Subscribe calls the initial setup caused (exactly 1:
// the attempt that threw and triggered failover).
int subscribeCountAfterSetup = primary.SubscribeCount;
Assert.Equal(1, subscribeCountAfterSetup);
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
// Let PollOnce succeed so ProbeOnce progresses without throwing.
primary.ThrowOnPoll = false;
// Drive several ProbeOnce calls — none should touch Subscribe.
sut.ProbeOnce();
sut.ProbeOnce();
sut.ProbeOnce(); // stableProbes=3 → failback on this call
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
Assert.Equal(subscribeCountAfterSetup, primary.SubscribeCount);
}
/// Proves that acknowledgment calls delegate to whichever child (primary or standby) is currently active.
[Fact]
public void Acknowledge_DelegatesToActiveChild()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
sut.Subscribe(@"\\HOST\Galaxy!Area");
// Active = Primary → primary's sentinel value (11).
Assert.Equal(11, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
Assert.Equal(11, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));
// Force a failover by failing the primary past threshold.
primary.ThrowOnPoll = true;
sut.PollOnce(); // threshold=1 → switch to Standby
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
// Active = Standby → standby's sentinel value (22).
Assert.Equal(22, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
Assert.Equal(22, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));
}
///
/// Proves that an intermittent failure during failback probing resets the
/// clean-probe counter to zero, requiring a fresh unbroken run of
/// before failing back.
///
[Fact]
public void FailbackProbe_IntermittentFailure_ResetsCleanCount()
{
var primary = new FlakyPrimary { ThrowOnPoll = true };
var standby = new StubStandby();
using var sut = new FailoverAlarmConsumer(primary, standby, new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3));
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
primary.ThrowOnPoll = false;
sut.ProbeOnce(); // clean 1
sut.ProbeOnce(); // clean 2
primary.ThrowOnPoll = true;
sut.ProbeOnce(); // fails → reset to 0
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
primary.ThrowOnPoll = false;
sut.ProbeOnce(); // clean 1
sut.ProbeOnce(); // clean 2
sut.ProbeOnce(); // clean 3 → failback
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
}
///
/// When the standby's priming
/// SnapshotActiveAlarms throws during failover, the switch must
/// still (a) fire ProviderModeChanged so the gateway learns the
/// feed went degraded, (b) leave
/// in Subtag, and (c) not rethrow out of PollOnce (which on the
/// real STA would land in the poll loop's trailing catch and permanently
/// stop alarm delivery).
///
[Fact]
public void Failover_WhenStandbyPrimingSnapshotThrows_StillRaisesModeChangeAndDoesNotRethrow()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby { ThrowOnSnapshot = true };
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
List changes = new List();
sut.ProviderModeChanged += (_, e) => changes.Add(e);
// threshold=1 → the Subscribe failure triggers the switch, which primes
// the standby snapshot (throwing). The exception must be contained.
Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));
Assert.Null(escaped);
Assert.Single(changes);
Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
Assert.True(standby.SnapshotCalls >= 1); // priming was attempted
// A subsequent degraded PollOnce (standby.PollOnce + ProbeOnce) must also
// not rethrow the snapshot failure.
Exception? pollEscaped = Record.Exception(() => sut.PollOnce());
Assert.Null(pollEscaped);
}
///
/// When a ProviderModeChanged subscriber's
/// handler throws (modeling the AlarmCommandHandler's event-queue enqueue
/// overflowing at capacity), the switch must still take effect and the
/// exception must not escape the switch path into the poll loop.
///
[Fact]
public void Failover_WhenModeChangedHandlerThrows_SwitchStillTakesEffectAndDoesNotRethrow()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
int handlerInvocations = 0;
sut.ProviderModeChanged += (_, _) =>
{
handlerInvocations++;
throw new InvalidOperationException("subscriber handler blew up");
};
Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));
Assert.Null(escaped);
Assert.Equal(1, handlerInvocations); // the event still fired
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // the switch still took effect
}
///
/// With a non-zero
/// , two back-to-back
/// ProbeOnce calls must throttle — the second falls inside the
/// interval and must NOT re-poll the primary. Two consecutive calls
/// reliably fall inside any interval of one second or more, so this needs
/// no injected clock.
///
[Fact]
public void ProbeOnce_WithNonZeroInterval_ThrottlesSecondProbeWithinInterval()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
StubStandby standby = new StubStandby();
// stableProbes high enough that a single clean probe cannot fail back,
// so Mode stays Subtag and ProbeOnce remains the throttled path.
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 3600, stableProbes: 5);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
primary.ThrowOnPoll = false; // primary healthy so a probe would poll cleanly
sut.ProbeOnce(); // first probe runs: re-polls the primary
int pollsAfterFirstProbe = primary.Polls;
Assert.Equal(1, pollsAfterFirstProbe);
sut.ProbeOnce(); // within the 3600s interval → throttled, must NOT re-poll
Assert.Equal(pollsAfterFirstProbe, primary.Polls);
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
}
///
/// RunPrimary's
/// when (ex is not OutOfMemoryException) filter must let an
/// propagate rather than swallowing it
/// and counting it toward the failover threshold. No mode change must
/// fire — a fatal allocation failure is not a clean degraded handoff.
///
[Fact]
public void RunPrimary_WhenPrimaryThrowsOutOfMemory_PropagatesAndDoesNotFailOver()
{
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false, ThrowOutOfMemoryOnPoll = true };
StubStandby standby = new StubStandby();
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
bool modeChanged = false;
sut.ProviderModeChanged += (_, _) => modeChanged = true;
sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe path does not poll; no throw here
Assert.Throws(() => sut.PollOnce());
Assert.False(modeChanged);
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
}
///
/// Verifies that clamps out-of-range
/// threshold and stableProbes values (and negative
/// probeIntervalSeconds) to their safe minimums so a misconfigured
/// bind cannot change failover semantics.
///
/// The raw, possibly out-of-range threshold to construct with.
/// The raw, possibly negative probe interval (seconds) to construct with.
/// The raw, possibly out-of-range stable-probe count to construct with.
/// The expected clamped .
/// The expected clamped .
/// The expected clamped .
[Theory]
[InlineData(0, 0, 0, 1, 0, 1)]
[InlineData(-5, -5, -5, 1, 0, 1)]
[InlineData(3, 7, 2, 3, 7, 2)]
public void FailoverSettings_ClampsSubMinimumValues(
int threshold,
int probeInterval,
int stableProbes,
int expectedThreshold,
int expectedProbeInterval,
int expectedStableProbes)
{
FailoverSettings settings = new FailoverSettings(threshold, probeInterval, stableProbes);
Assert.Equal(expectedThreshold, settings.Threshold);
Assert.Equal(expectedProbeInterval, settings.ProbeIntervalSeconds);
Assert.Equal(expectedStableProbes, settings.StableProbes);
}
}