diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
index 4d029d74..2389df3f 100644
--- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
@@ -743,7 +743,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.
/// A task that represents the live test.
[RustWorkerLiveFact]
- public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
+ public Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
+ {
+ return VerifyNativeAlarmAsync();
+ }
+
+ /// Verifies automatic failover and failback on one live alarm feed.
+ /// A task that represents the live test.
+ [RustWorkerLiveFact(requireAlarmFaults: true)]
+ public async Task GatewayGrpc_WithRustWorker_StreamsAcrossAutomaticAlarmFailoverAndFailback()
+ {
+ string marker = Path.Combine(Path.GetTempPath(), $"mxgw-alarm-fault-{Guid.NewGuid():N}");
+ try
+ {
+ await VerifyNativeAlarmAsync(marker).ConfigureAwait(false);
+ }
+ finally
+ {
+ File.Delete(marker);
+ }
+ }
+
+ private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
@@ -751,14 +772,37 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
- await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
+ IAlarmWatchListResolver? resolver = null;
+ if (faultMarker is not null)
+ {
+ resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
+ {
+ AlarmFullReference = reference,
+ SourceObjectReference = "TestMachine_001",
+ ActiveSubtag = tag + ".InAlarm",
+ AckedSubtag = tag + ".Acked",
+ });
+ }
+
+ await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output, faultMarker), output, new AlarmsOptions
{
Enabled = true,
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
- Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
- });
+ Fallback = new AlarmFallbackOptions
+ {
+ Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
+ ConsecutiveFailureThreshold = 3,
+ FailbackProbeIntervalSeconds = 1,
+ FailbackStableProbes = 3,
+ Discovery = new AlarmDiscoveryOptions
+ {
+ UseGalaxyRepository = false,
+ IncludeAttributes = faultMarker is null ? [] : [tag],
+ },
+ },
+ }, resolver);
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
- using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
+ using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
@@ -792,7 +836,8 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return reply;
}
- async Task WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
+ async Task WaitForTransitionAsync(
+ AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
{
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
@@ -800,14 +845,29 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
&& (ackComment is null || transition.OperatorComment == ackComment))
{
- Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
- Assert.False(transition.Degraded);
+ Assert.Equal(provider, transition.SourceProvider);
+ Assert.Equal(provider == AlarmProviderMode.Subtag, transition.Degraded);
return transition;
}
}
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
}
+ async Task WaitForModeAsync(AlarmProviderMode mode, string reason)
+ {
+ while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
+ {
+ AlarmProviderStatus? status = feed.ResponseStream.Current.ProviderStatus;
+ if (status?.Mode == mode && status.Reason == reason)
+ {
+ Assert.Equal(mode == AlarmProviderMode.Subtag, status.Degraded);
+ output.WriteLine($"StreamAlarms provider switched to {mode} ({status.Reason}).");
+ return;
+ }
+ }
+ throw new InvalidOperationException($"Alarm stream ended before switching to {mode}.");
+ }
+
MxCommandRequest? restore = null;
try
{
@@ -832,7 +892,6 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
-
MxCommandRequest WriteAlarm(bool active)
{
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
@@ -858,6 +917,25 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
+ if (faultMarker is not null)
+ {
+ File.WriteAllText(faultMarker, string.Empty);
+ await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
+ await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
+ await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
+ await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
+ await WaitForTransitionAsync(AlarmTransitionKind.Raise, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
+ await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
+ await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
+
+ File.Delete(faultMarker);
+ await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
+ await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
+ await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
+ output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
+ return;
+ }
+
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference,
@@ -924,15 +1002,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return app;
}
+ private sealed class SingleAlarmWatchListResolver(AlarmSubtagTarget target) : IAlarmWatchListResolver
+ {
+ public Task ResolveAsync(AlarmsOptions options, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return Task.FromResult(new AlarmWatchListResolution([target], false, false, null));
+ }
+ }
+
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
- public RustWorkerLiveFactAttribute()
+ public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
{
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
}
+ else if (requireAlarmFaults && !IntegrationTestEnvironment.IsEnabled("MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST"))
+ {
+ Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
+ }
}
}
@@ -2907,11 +2998,13 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// Factory for creating worker processes.
/// Test output helper for logging.
/// Optional configuration for the central alarm monitor.
+ /// Optional resolver for the monitor's subtag watch list.
public GatewayServiceFixture(
string workerExecutablePath,
IWorkerProcessFactory processFactory,
ITestOutputHelper output,
- AlarmsOptions? alarms = null)
+ AlarmsOptions? alarms = null,
+ IAlarmWatchListResolver? watchListResolver = null)
{
IOptions options = Options.Create(CreateOptions(workerExecutablePath, alarms));
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
@@ -2939,7 +3032,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
_metrics);
Monitor = new GatewayAlarmMonitor(
- sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
+ sessionManager, watchListResolver ?? EmptyAlarmWatchListResolver.Instance, _metrics, options,
_loggerFactory.CreateLogger());
Service = new MxAccessGatewayService(
sessionManager,
@@ -3216,13 +3309,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
///
/// Factory that launches worker processes and records their outputs for testing.
///
- private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory
+ private sealed class TestWorkerProcessFactory(ITestOutputHelper output, string? alarmFaultMarker = null) : IWorkerProcessFactory
{
private readonly ConcurrentBag processes = [];
///
public IWorkerProcess Start(ProcessStartInfo startInfo)
{
+ if (alarmFaultMarker is not null)
+ {
+ startInfo.Environment["MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER"] = alarmFaultMarker;
+ }
+
startInfo.RedirectStandardError = true;
startInfo.RedirectStandardOutput = true;
startInfo.UseShellExecute = false;
diff --git a/src/mxgateway-worker-rust/Cargo.toml b/src/mxgateway-worker-rust/Cargo.toml
index 5e7df6f1..a831ef2d 100644
--- a/src/mxgateway-worker-rust/Cargo.toml
+++ b/src/mxgateway-worker-rust/Cargo.toml
@@ -4,6 +4,9 @@ version = "0.1.0"
edition = "2021"
publish = false
+[features]
+alarm-test-faults = []
+
[dependencies]
prost = "0.13.5"
prost-types = "0.13.5"
diff --git a/src/mxgateway-worker-rust/README.md b/src/mxgateway-worker-rust/README.md
index 7f7e6407..3fe27abe 100644
--- a/src/mxgateway-worker-rust/README.md
+++ b/src/mxgateway-worker-rust/README.md
@@ -116,6 +116,25 @@ password. The test accepts credentials through
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~GatewayGrpc_WithRustWorker_'
```
+### Verify automatic alarm failover and failback
+
+Build a debug worker with the test fault enabled and run the dedicated live
+gateway test on the Windows development host:
+
+```powershell
+cargo build --locked --features alarm-test-faults --target i686-pc-windows-msvc --manifest-path src/mxgateway-worker-rust/Cargo.toml
+$env:MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS = '1'
+$env:MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST = '1'
+$env:MXGATEWAY_LIVE_MXACCESS_WORKER_EXE = (Resolve-Path 'src/mxgateway-worker-rust/target/i686-pc-windows-msvc/debug/mxgateway-worker-rust.exe').Path
+dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~StreamsAcrossAutomaticAlarmFailoverAndFailback'
+```
+
+The test temporarily makes the launched worker's Alarmmgr snapshot fetches
+return an error. It drives the same development alarm through
+`Alarmmgr → Subtag → Alarmmgr` and checks both provider statuses and subsequent
+transitions on one open `StreamAlarms` call. The fault injection is compiled only
+when both debug assertions and the Cargo feature are enabled.
+
## COM interface evidence
The native adapter uses the dual `ILMXProxyServer` interface from the installed
diff --git a/src/mxgateway-worker-rust/src/alarm/wnwrap.rs b/src/mxgateway-worker-rust/src/alarm/wnwrap.rs
index 81137060..5dcc279b 100644
--- a/src/mxgateway-worker-rust/src/alarm/wnwrap.rs
+++ b/src/mxgateway-worker-rust/src/alarm/wnwrap.rs
@@ -49,6 +49,8 @@ pub struct WnWrap {
v2: ComReference,
initialized: bool,
registered: bool,
+ #[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
+ snapshot_fault_marker: Option,
}
impl WnWrap {
@@ -66,6 +68,9 @@ impl WnWrap {
v2: ComReference::new(v2)?,
initialized: false,
registered: false,
+ #[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
+ snapshot_fault_marker: std::env::var_os("MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER")
+ .map(std::path::PathBuf::from),
};
let application = wide("OtOpcUa.ZB.MOM.WW.MxGateway.Worker");
@@ -117,6 +122,15 @@ impl WnWrap {
}
pub fn snapshot(&self, limit: i32) -> Result {
+ // The live test can fail this worker's polls without stopping the shared Alarmmgr.
+ #[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
+ if self
+ .snapshot_fault_marker
+ .as_ref()
+ .is_some_and(|path| path.is_file())
+ {
+ return Err(E_FAIL);
+ }
struct XmlVariant(VARIANT);
impl Drop for XmlVariant {
fn drop(&mut self) {