Joseph Doherty
295 lines
12 KiB
C#
295 lines
12 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
|
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
|
|
using Xunit;
|
|
|
|
namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess;
|
|
|
|
/// <summary>
|
|
/// Unit tests for <see cref="FailoverAlarmConsumer"/>: prove the
|
|
/// auto-failover (consecutive primary COM failures → standby) and
|
|
/// auto-failback (consecutive clean probes → primary) state machine,
|
|
/// active-child transition forwarding, and active-child delegation of
|
|
/// acknowledgments. Fakes stand in for both children so this needs no
|
|
/// AVEVA install.
|
|
/// </summary>
|
|
public sealed class FailoverAlarmConsumerTests
|
|
{
|
|
/// <summary>
|
|
/// Primary fake whose Subscribe/PollOnce throw a COMException while
|
|
/// <see cref="ThrowOnPoll"/> is set, modeling a wnwrap consumer that
|
|
/// surfaces COM HRESULT failures. Can also re-raise a transition so
|
|
/// before-failover forwarding can be exercised.
|
|
/// </summary>
|
|
private sealed class FlakyPrimary : IMxAccessAlarmConsumer
|
|
{
|
|
public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;
|
|
|
|
public bool ThrowOnPoll = true;
|
|
public int Polls;
|
|
|
|
/// <summary>
|
|
/// Number of times <see cref="Subscribe"/> has been called.
|
|
/// Incremented at entry, before any throw, so every attempt is
|
|
/// counted regardless of whether <see cref="ThrowOnPoll"/> is set.
|
|
/// </summary>
|
|
public int SubscribeCount;
|
|
|
|
public void Subscribe(string s)
|
|
{
|
|
SubscribeCount++;
|
|
if (ThrowOnPoll)
|
|
{
|
|
throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
|
|
}
|
|
}
|
|
|
|
public void PollOnce()
|
|
{
|
|
Polls++;
|
|
if (ThrowOnPoll)
|
|
{
|
|
throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
|
|
}
|
|
}
|
|
|
|
public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 11;
|
|
|
|
public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 11;
|
|
|
|
public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms() => Array.Empty<MxAlarmSnapshotRecord>();
|
|
|
|
public void Dispose() { }
|
|
|
|
public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Standby fake (subtag stand-in): never throws, records that it was
|
|
/// armed, and can re-raise a transition.
|
|
/// </summary>
|
|
private sealed class StubStandby : IMxAccessAlarmConsumer
|
|
{
|
|
public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;
|
|
|
|
public bool Subscribed;
|
|
|
|
public void Subscribe(string s) => Subscribed = true;
|
|
|
|
public void PollOnce() { }
|
|
|
|
public int AcknowledgeByGuid(Guid g, string c, string a, string b, string d, string e) => 22;
|
|
|
|
public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 22;
|
|
|
|
public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms() => Array.Empty<MxAlarmSnapshotRecord>();
|
|
|
|
public void Dispose() { }
|
|
|
|
public void Raise(MxAlarmTransitionEvent e) => AlarmTransitionEmitted?.Invoke(this, e);
|
|
}
|
|
|
|
private static MxAlarmTransitionEvent SampleTransition() => new MxAlarmTransitionEvent
|
|
{
|
|
Record = new MxAlarmSnapshotRecord { AlarmGuid = Guid.NewGuid() },
|
|
PreviousState = MxAlarmStateKind.Unspecified,
|
|
};
|
|
|
|
[Fact]
|
|
public void Primary_FailsThresholdTimes_SwitchesToSubtag()
|
|
{
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
|
|
sut.ProviderModeChanged += (_, e) => changes.Add(e);
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area"); // failure 1 (primary), standby armed
|
|
Assert.True(standby.Subscribed);
|
|
Assert.Empty(changes);
|
|
|
|
sut.PollOnce(); // failure 2
|
|
Assert.Empty(changes);
|
|
|
|
sut.PollOnce(); // failure 3 → switch
|
|
|
|
Assert.Single(changes);
|
|
Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
Assert.Equal(unchecked((int)0x80004005), changes[0].HResult);
|
|
}
|
|
|
|
[Fact]
|
|
public void AfterSwitch_StandbyTransitionsAreForwarded()
|
|
{
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
MxAlarmTransitionEvent? forwarded = null;
|
|
sut.AlarmTransitionEmitted += (_, e) => forwarded = e;
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag immediately
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
|
|
MxAlarmTransitionEvent transition = SampleTransition();
|
|
standby.Raise(transition);
|
|
|
|
Assert.Same(transition, forwarded);
|
|
}
|
|
|
|
[Fact]
|
|
public void WhileDegraded_PrimaryHeals_FailsBackAfterStableProbes()
|
|
{
|
|
// threshold=1 so the initial Subscribe failure (PollOnce path) immediately
|
|
// switches to Subtag. stableProbes=2 means two consecutive clean PollOnce
|
|
// calls are needed before failback. ProbeOnce must NOT call Subscribe —
|
|
// WnWrapAlarmConsumer is single-subscribe; re-calling would always throw.
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 2);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
|
|
sut.ProviderModeChanged += (_, e) => changes.Add(e);
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → Subtag (mode change 1)
|
|
Assert.Single(changes);
|
|
Assert.Equal(AlarmProviderMode.Subtag, changes[changes.Count - 1].Mode);
|
|
|
|
// Primary heals: PollOnce stops throwing. ProbeOnce should call only
|
|
// PollOnce (not Subscribe) to detect recovery.
|
|
primary.ThrowOnPoll = false;
|
|
int subscribeCountAfterFailover = primary.SubscribeCount;
|
|
|
|
sut.ProbeOnce(); // cleanProbes=1 — not yet at stableProbes=2
|
|
Assert.Single(changes);
|
|
|
|
sut.ProbeOnce(); // cleanProbes=2 → failback to Alarmmgr (mode change 2)
|
|
|
|
Assert.Equal(2, changes.Count);
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, changes[changes.Count - 1].Mode);
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
|
|
Assert.Equal(0, changes[changes.Count - 1].HResult);
|
|
|
|
// ProbeOnce must not have called Subscribe at all during probing.
|
|
Assert.Equal(subscribeCountAfterFailover, primary.SubscribeCount);
|
|
}
|
|
|
|
[Fact]
|
|
public void BeforeFailover_PrimaryTransitionsAreForwarded()
|
|
{
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false }; // healthy, can Raise
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 3, probeIntervalSeconds: 0, stableProbes: 1);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
List<MxAlarmTransitionEvent> forwarded = new List<MxAlarmTransitionEvent>();
|
|
sut.AlarmTransitionEmitted += (_, e) => forwarded.Add(e);
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area");
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
|
|
|
|
MxAlarmTransitionEvent fromPrimary = SampleTransition();
|
|
primary.Raise(fromPrimary); // active=Primary → forwarded
|
|
Assert.Single(forwarded);
|
|
Assert.Same(fromPrimary, forwarded[0]);
|
|
|
|
standby.Raise(SampleTransition()); // standby not active → suppressed
|
|
Assert.Single(forwarded);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Proves that <see cref="FailoverAlarmConsumer.ProbeOnce"/> never calls
|
|
/// <c>Subscribe</c> on the primary while degraded. The production primary
|
|
/// (<see cref="WnWrapAlarmConsumer"/>) is single-subscribe; a second
|
|
/// <c>Subscribe</c> call would always throw and make failback impossible.
|
|
/// The probe must re-poll the still-subscribed primary via
|
|
/// <c>PollOnce</c> only.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ProbeOnce_DoesNotCallPrimarySubscribe()
|
|
{
|
|
// threshold=1 → first Subscribe failure immediately switches to Subtag.
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe attempt #1 (throws) → Subtag
|
|
|
|
// Capture how many Subscribe calls the initial setup caused (exactly 1:
|
|
// the attempt that threw and triggered failover).
|
|
int subscribeCountAfterSetup = primary.SubscribeCount;
|
|
Assert.Equal(1, subscribeCountAfterSetup);
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
|
|
// Let PollOnce succeed so ProbeOnce progresses without throwing.
|
|
primary.ThrowOnPoll = false;
|
|
|
|
// Drive several ProbeOnce calls — none should touch Subscribe.
|
|
sut.ProbeOnce();
|
|
sut.ProbeOnce();
|
|
sut.ProbeOnce(); // stableProbes=3 → failback on this call
|
|
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
|
|
Assert.Equal(subscribeCountAfterSetup, primary.SubscribeCount);
|
|
}
|
|
|
|
[Fact]
|
|
public void Acknowledge_DelegatesToActiveChild()
|
|
{
|
|
FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false };
|
|
StubStandby standby = new StubStandby();
|
|
FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
|
|
using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
|
|
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area");
|
|
|
|
// Active = Primary → primary's sentinel value (11).
|
|
Assert.Equal(11, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
|
|
Assert.Equal(11, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));
|
|
|
|
// Force a failover by failing the primary past threshold.
|
|
primary.ThrowOnPoll = true;
|
|
sut.PollOnce(); // threshold=1 → switch to Standby
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
|
|
// Active = Standby → standby's sentinel value (22).
|
|
Assert.Equal(22, sut.AcknowledgeByGuid(Guid.NewGuid(), "c", "n", "node", "dom", "full"));
|
|
Assert.Equal(22, sut.AcknowledgeByName("a", "p", "g", "c", "n", "node", "dom", "full"));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Proves that an intermittent failure during failback probing resets the
|
|
/// clean-probe counter to zero, requiring a fresh unbroken run of
|
|
/// <see cref="FailoverSettings.StableProbes"/> before failing back.
|
|
/// </summary>
|
|
[Fact]
|
|
public void FailbackProbe_IntermittentFailure_ResetsCleanCount()
|
|
{
|
|
var primary = new FlakyPrimary { ThrowOnPoll = true };
|
|
var standby = new StubStandby();
|
|
using var sut = new FailoverAlarmConsumer(primary, standby, new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 3));
|
|
sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
|
|
primary.ThrowOnPoll = false;
|
|
sut.ProbeOnce(); // clean 1
|
|
sut.ProbeOnce(); // clean 2
|
|
primary.ThrowOnPoll = true;
|
|
sut.ProbeOnce(); // fails → reset to 0
|
|
Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
|
|
|
|
primary.ThrowOnPoll = false;
|
|
sut.ProbeOnce(); // clean 1
|
|
sut.ProbeOnce(); // clean 2
|
|
sut.ProbeOnce(); // clean 3 → failback
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
|
|
}
|
|
}
|