Verify automatic alarm failover through the gateway

This commit is contained in:
Joseph Doherty
2026-11-21 10:13:38 -05:00
parent 9293c4c913
commit 8045afe174
4 changed files with 147 additions and 13 deletions
@@ -743,7 +743,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary> /// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
/// <returns>A task that represents the live test.</returns> /// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact] [RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm() public Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
{
return VerifyNativeAlarmAsync();
}
/// <summary>Verifies automatic failover and failback on one live alarm feed.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact(requireAlarmFaults: true)]
public async Task GatewayGrpc_WithRustWorker_StreamsAcrossAutomaticAlarmFailoverAndFailback()
{
string marker = Path.Combine(Path.GetTempPath(), $"mxgw-alarm-fault-{Guid.NewGuid():N}");
try
{
await VerifyNativeAlarmAsync(marker).ConfigureAwait(false);
}
finally
{
File.Delete(marker);
}
}
private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
{ {
const string tag = "TestMachine_001.TestAlarm001"; const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag; const string reference = "Galaxy!TestArea." + tag;
@@ -751,14 +772,37 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty; string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}"; string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath(); string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions IAlarmWatchListResolver? resolver = null;
if (faultMarker is not null)
{
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
{
AlarmFullReference = reference,
SourceObjectReference = "TestMachine_001",
ActiveSubtag = tag + ".InAlarm",
AckedSubtag = tag + ".Acked",
});
}
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output, faultMarker), output, new AlarmsOptions
{ {
Enabled = true, Enabled = true,
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea", SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" }, Fallback = new AlarmFallbackOptions
}); {
Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
ConsecutiveFailureThreshold = 3,
FailbackProbeIntervalSeconds = 1,
FailbackStableProbes = 3,
Discovery = new AlarmDiscoveryOptions
{
UseGalaxyRepository = false,
IncludeAttributes = faultMarker is null ? [] : [tag],
},
},
}, resolver);
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor); await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85)); using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
CancellationToken token = deadline.Token; CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false); await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single()); using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
@@ -792,7 +836,8 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return reply; return reply;
} }
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null) async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
{ {
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false)) while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{ {
@@ -800,14 +845,29 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
&& (ackComment is null || transition.OperatorComment == ackComment)) && (ackComment is null || transition.OperatorComment == ackComment))
{ {
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider); Assert.Equal(provider, transition.SourceProvider);
Assert.False(transition.Degraded); Assert.Equal(provider == AlarmProviderMode.Subtag, transition.Degraded);
return transition; return transition;
} }
} }
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition."); throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
} }
async Task WaitForModeAsync(AlarmProviderMode mode, string reason)
{
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
AlarmProviderStatus? status = feed.ResponseStream.Current.ProviderStatus;
if (status?.Mode == mode && status.Reason == reason)
{
Assert.Equal(mode == AlarmProviderMode.Subtag, status.Degraded);
output.WriteLine($"StreamAlarms provider switched to {mode} ({status.Reason}).");
return;
}
}
throw new InvalidOperationException($"Alarm stream ended before switching to {mode}.");
}
MxCommandRequest? restore = null; MxCommandRequest? restore = null;
try try
{ {
@@ -832,7 +892,6 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
.ConfigureAwait(false)).AuthenticateUser.UserId; .ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId); Assert.NotEqual(0, userId);
MxCommandRequest WriteAlarm(bool active) MxCommandRequest WriteAlarm(bool active)
{ {
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0); MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
@@ -858,6 +917,25 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
restore = WriteAlarm(original.Value.BoolValue); restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
if (faultMarker is not null)
{
File.WriteAllText(faultMarker, string.Empty);
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
File.Delete(faultMarker);
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
return;
}
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{ {
AlarmFullReference = reference, AlarmFullReference = reference,
@@ -924,15 +1002,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return app; return app;
} }
private sealed class SingleAlarmWatchListResolver(AlarmSubtagTarget target) : IAlarmWatchListResolver
{
public Task<AlarmWatchListResolution> ResolveAsync(AlarmsOptions options, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(new AlarmWatchListResolution([target], false, false, null));
}
}
private sealed class RustWorkerLiveFactAttribute : FactAttribute private sealed class RustWorkerLiveFactAttribute : FactAttribute
{ {
public RustWorkerLiveFactAttribute() public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
{ {
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS"; const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn)) if (!IntegrationTestEnvironment.IsEnabled(optIn))
{ {
Skip = $"Set {optIn}=1 to run the Rust worker live test."; Skip = $"Set {optIn}=1 to run the Rust worker live test.";
} }
else if (requireAlarmFaults && !IntegrationTestEnvironment.IsEnabled("MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST"))
{
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
}
} }
} }
@@ -2907,11 +2998,13 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// <param name="processFactory">Factory for creating worker processes.</param> /// <param name="processFactory">Factory for creating worker processes.</param>
/// <param name="output">Test output helper for logging.</param> /// <param name="output">Test output helper for logging.</param>
/// <param name="alarms">Optional configuration for the central alarm monitor.</param> /// <param name="alarms">Optional configuration for the central alarm monitor.</param>
/// <param name="watchListResolver">Optional resolver for the monitor's subtag watch list.</param>
public GatewayServiceFixture( public GatewayServiceFixture(
string workerExecutablePath, string workerExecutablePath,
IWorkerProcessFactory processFactory, IWorkerProcessFactory processFactory,
ITestOutputHelper output, ITestOutputHelper output,
AlarmsOptions? alarms = null) AlarmsOptions? alarms = null,
IAlarmWatchListResolver? watchListResolver = null)
{ {
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms)); IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output))); _loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
@@ -2939,7 +3032,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
_metrics); _metrics);
Monitor = new GatewayAlarmMonitor( Monitor = new GatewayAlarmMonitor(
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options, sessionManager, watchListResolver ?? EmptyAlarmWatchListResolver.Instance, _metrics, options,
_loggerFactory.CreateLogger<GatewayAlarmMonitor>()); _loggerFactory.CreateLogger<GatewayAlarmMonitor>());
Service = new MxAccessGatewayService( Service = new MxAccessGatewayService(
sessionManager, sessionManager,
@@ -3216,13 +3309,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// <summary> /// <summary>
/// Factory that launches worker processes and records their outputs for testing. /// Factory that launches worker processes and records their outputs for testing.
/// </summary> /// </summary>
private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory private sealed class TestWorkerProcessFactory(ITestOutputHelper output, string? alarmFaultMarker = null) : IWorkerProcessFactory
{ {
private readonly ConcurrentBag<TestWorkerProcess> processes = []; private readonly ConcurrentBag<TestWorkerProcess> processes = [];
/// <inheritdoc /> /// <inheritdoc />
public IWorkerProcess Start(ProcessStartInfo startInfo) public IWorkerProcess Start(ProcessStartInfo startInfo)
{ {
if (alarmFaultMarker is not null)
{
startInfo.Environment["MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER"] = alarmFaultMarker;
}
startInfo.RedirectStandardError = true; startInfo.RedirectStandardError = true;
startInfo.RedirectStandardOutput = true; startInfo.RedirectStandardOutput = true;
startInfo.UseShellExecute = false; startInfo.UseShellExecute = false;
+3
View File
@@ -4,6 +4,9 @@ version = "0.1.0"
edition = "2021" edition = "2021"
publish = false publish = false
[features]
alarm-test-faults = []
[dependencies] [dependencies]
prost = "0.13.5" prost = "0.13.5"
prost-types = "0.13.5" prost-types = "0.13.5"
+19
View File
@@ -116,6 +116,25 @@ password. The test accepts credentials through
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~GatewayGrpc_WithRustWorker_' dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~GatewayGrpc_WithRustWorker_'
``` ```
### Verify automatic alarm failover and failback
Build a debug worker with the test fault enabled and run the dedicated live
gateway test on the Windows development host:
```powershell
cargo build --locked --features alarm-test-faults --target i686-pc-windows-msvc --manifest-path src/mxgateway-worker-rust/Cargo.toml
$env:MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS = '1'
$env:MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST = '1'
$env:MXGATEWAY_LIVE_MXACCESS_WORKER_EXE = (Resolve-Path 'src/mxgateway-worker-rust/target/i686-pc-windows-msvc/debug/mxgateway-worker-rust.exe').Path
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~StreamsAcrossAutomaticAlarmFailoverAndFailback'
```
The test temporarily makes the launched worker's Alarmmgr snapshot fetches
return an error. It drives the same development alarm through
`Alarmmgr → Subtag → Alarmmgr` and checks both provider statuses and subsequent
transitions on one open `StreamAlarms` call. The fault injection is compiled only
when both debug assertions and the Cargo feature are enabled.
## COM interface evidence ## COM interface evidence
The native adapter uses the dual `ILMXProxyServer` interface from the installed The native adapter uses the dual `ILMXProxyServer` interface from the installed
@@ -49,6 +49,8 @@ pub struct WnWrap {
v2: ComReference, v2: ComReference,
initialized: bool, initialized: bool,
registered: bool, registered: bool,
#[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
snapshot_fault_marker: Option<std::path::PathBuf>,
} }
impl WnWrap { impl WnWrap {
@@ -66,6 +68,9 @@ impl WnWrap {
v2: ComReference::new(v2)?, v2: ComReference::new(v2)?,
initialized: false, initialized: false,
registered: false, registered: false,
#[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
snapshot_fault_marker: std::env::var_os("MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER")
.map(std::path::PathBuf::from),
}; };
let application = wide("OtOpcUa.ZB.MOM.WW.MxGateway.Worker"); let application = wide("OtOpcUa.ZB.MOM.WW.MxGateway.Worker");
@@ -117,6 +122,15 @@ impl WnWrap {
} }
pub fn snapshot(&self, limit: i32) -> Result<String, HRESULT> { pub fn snapshot(&self, limit: i32) -> Result<String, HRESULT> {
// The live test can fail this worker's polls without stopping the shared Alarmmgr.
#[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
if self
.snapshot_fault_marker
.as_ref()
.is_some_and(|path| path.is_file())
{
return Err(E_FAIL);
}
struct XmlVariant(VARIANT); struct XmlVariant(VARIANT);
impl Drop for XmlVariant { impl Drop for XmlVariant {
fn drop(&mut self) { fn drop(&mut self) {