diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs index 4d029d74..2389df3f 100644 --- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs +++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs @@ -743,7 +743,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) /// Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC. /// A task that represents the live test. [RustWorkerLiveFact] - public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm() + public Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm() + { + return VerifyNativeAlarmAsync(); + } + + /// Verifies automatic failover and failback on one live alarm feed. + /// A task that represents the live test. + [RustWorkerLiveFact(requireAlarmFaults: true)] + public async Task GatewayGrpc_WithRustWorker_StreamsAcrossAutomaticAlarmFailoverAndFailback() + { + string marker = Path.Combine(Path.GetTempPath(), $"mxgw-alarm-fault-{Guid.NewGuid():N}"); + try + { + await VerifyNativeAlarmAsync(marker).ConfigureAwait(false); + } + finally + { + File.Delete(marker); + } + } + + private async Task VerifyNativeAlarmAsync(string? faultMarker = null) { const string tag = "TestMachine_001.TestAlarm001"; const string reference = "Galaxy!TestArea." + tag; @@ -751,14 +772,37 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty; string comment = $"Rust worker smoke {Guid.NewGuid():N}"; string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath(); - await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions + IAlarmWatchListResolver? resolver = null; + if (faultMarker is not null) + { + resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget + { + AlarmFullReference = reference, + SourceObjectReference = "TestMachine_001", + ActiveSubtag = tag + ".InAlarm", + AckedSubtag = tag + ".Acked", + }); + } + + await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output, faultMarker), output, new AlarmsOptions { Enabled = true, SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea", - Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" }, - }); + Fallback = new AlarmFallbackOptions + { + Mode = faultMarker is null ? "ForceAlarmManager" : "Auto", + ConsecutiveFailureThreshold = 3, + FailbackProbeIntervalSeconds = 1, + FailbackStableProbes = 3, + Discovery = new AlarmDiscoveryOptions + { + UseGalaxyRepository = false, + IncludeAttributes = faultMarker is null ? [] : [tag], + }, + }, + }, resolver); await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor); - using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85)); + using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120)); CancellationToken token = deadline.Token; await app.StartAsync(token).ConfigureAwait(false); using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single()); @@ -792,7 +836,8 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) return reply; } - async Task WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null) + async Task WaitForTransitionAsync( + AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr) { while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false)) { @@ -800,14 +845,29 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected && (ackComment is null || transition.OperatorComment == ackComment)) { - Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider); - Assert.False(transition.Degraded); + Assert.Equal(provider, transition.SourceProvider); + Assert.Equal(provider == AlarmProviderMode.Subtag, transition.Degraded); return transition; } } throw new InvalidOperationException($"Alarm stream ended before the {expected} transition."); } + async Task WaitForModeAsync(AlarmProviderMode mode, string reason) + { + while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false)) + { + AlarmProviderStatus? status = feed.ResponseStream.Current.ProviderStatus; + if (status?.Mode == mode && status.Reason == reason) + { + Assert.Equal(mode == AlarmProviderMode.Subtag, status.Degraded); + output.WriteLine($"StreamAlarms provider switched to {mode} ({status.Reason})."); + return; + } + } + throw new InvalidOperationException($"Alarm stream ended before switching to {mode}."); + } + MxCommandRequest? restore = null; try { @@ -832,7 +892,6 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) .ConfigureAwait(false)).AuthenticateUser.UserId; Assert.NotEqual(0, userId); - MxCommandRequest WriteAlarm(bool active) { MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0); @@ -858,6 +917,25 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) restore = WriteAlarm(original.Value.BoolValue); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); + if (faultMarker is not null) + { + File.WriteAllText(faultMarker, string.Empty); + await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false); + await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false); + await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false); + await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false); + await WaitForTransitionAsync(AlarmTransitionKind.Raise, provider: AlarmProviderMode.Subtag).ConfigureAwait(false); + await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false); + await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false); + + File.Delete(faultMarker); + await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false); + await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false); + await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); + output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise."); + return; + } + AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest { AlarmFullReference = reference, @@ -924,15 +1002,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) return app; } + private sealed class SingleAlarmWatchListResolver(AlarmSubtagTarget target) : IAlarmWatchListResolver + { + public Task ResolveAsync(AlarmsOptions options, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(new AlarmWatchListResolution([target], false, false, null)); + } + } + private sealed class RustWorkerLiveFactAttribute : FactAttribute { - public RustWorkerLiveFactAttribute() + public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false) { const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS"; if (!IntegrationTestEnvironment.IsEnabled(optIn)) { Skip = $"Set {optIn}=1 to run the Rust worker live test."; } + else if (requireAlarmFaults && !IntegrationTestEnvironment.IsEnabled("MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST")) + { + Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1."; + } } } @@ -2907,11 +2998,13 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) /// Factory for creating worker processes. /// Test output helper for logging. /// Optional configuration for the central alarm monitor. + /// Optional resolver for the monitor's subtag watch list. public GatewayServiceFixture( string workerExecutablePath, IWorkerProcessFactory processFactory, ITestOutputHelper output, - AlarmsOptions? alarms = null) + AlarmsOptions? alarms = null, + IAlarmWatchListResolver? watchListResolver = null) { IOptions options = Options.Create(CreateOptions(workerExecutablePath, alarms)); _loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output))); @@ -2939,7 +3032,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) _metrics); Monitor = new GatewayAlarmMonitor( - sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options, + sessionManager, watchListResolver ?? EmptyAlarmWatchListResolver.Instance, _metrics, options, _loggerFactory.CreateLogger()); Service = new MxAccessGatewayService( sessionManager, @@ -3216,13 +3309,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) /// /// Factory that launches worker processes and records their outputs for testing. /// - private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory + private sealed class TestWorkerProcessFactory(ITestOutputHelper output, string? alarmFaultMarker = null) : IWorkerProcessFactory { private readonly ConcurrentBag processes = []; /// public IWorkerProcess Start(ProcessStartInfo startInfo) { + if (alarmFaultMarker is not null) + { + startInfo.Environment["MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER"] = alarmFaultMarker; + } + startInfo.RedirectStandardError = true; startInfo.RedirectStandardOutput = true; startInfo.UseShellExecute = false; diff --git a/src/mxgateway-worker-rust/Cargo.toml b/src/mxgateway-worker-rust/Cargo.toml index 5e7df6f1..a831ef2d 100644 --- a/src/mxgateway-worker-rust/Cargo.toml +++ b/src/mxgateway-worker-rust/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" publish = false +[features] +alarm-test-faults = [] + [dependencies] prost = "0.13.5" prost-types = "0.13.5" diff --git a/src/mxgateway-worker-rust/README.md b/src/mxgateway-worker-rust/README.md index 7f7e6407..3fe27abe 100644 --- a/src/mxgateway-worker-rust/README.md +++ b/src/mxgateway-worker-rust/README.md @@ -116,6 +116,25 @@ password. The test accepts credentials through dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~GatewayGrpc_WithRustWorker_' ``` +### Verify automatic alarm failover and failback + +Build a debug worker with the test fault enabled and run the dedicated live +gateway test on the Windows development host: + +```powershell +cargo build --locked --features alarm-test-faults --target i686-pc-windows-msvc --manifest-path src/mxgateway-worker-rust/Cargo.toml +$env:MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS = '1' +$env:MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST = '1' +$env:MXGATEWAY_LIVE_MXACCESS_WORKER_EXE = (Resolve-Path 'src/mxgateway-worker-rust/target/i686-pc-windows-msvc/debug/mxgateway-worker-rust.exe').Path +dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~StreamsAcrossAutomaticAlarmFailoverAndFailback' +``` + +The test temporarily makes the launched worker's Alarmmgr snapshot fetches +return an error. It drives the same development alarm through +`Alarmmgr → Subtag → Alarmmgr` and checks both provider statuses and subsequent +transitions on one open `StreamAlarms` call. The fault injection is compiled only +when both debug assertions and the Cargo feature are enabled. + ## COM interface evidence The native adapter uses the dual `ILMXProxyServer` interface from the installed diff --git a/src/mxgateway-worker-rust/src/alarm/wnwrap.rs b/src/mxgateway-worker-rust/src/alarm/wnwrap.rs index 81137060..5dcc279b 100644 --- a/src/mxgateway-worker-rust/src/alarm/wnwrap.rs +++ b/src/mxgateway-worker-rust/src/alarm/wnwrap.rs @@ -49,6 +49,8 @@ pub struct WnWrap { v2: ComReference, initialized: bool, registered: bool, + #[cfg(all(debug_assertions, feature = "alarm-test-faults"))] + snapshot_fault_marker: Option, } impl WnWrap { @@ -66,6 +68,9 @@ impl WnWrap { v2: ComReference::new(v2)?, initialized: false, registered: false, + #[cfg(all(debug_assertions, feature = "alarm-test-faults"))] + snapshot_fault_marker: std::env::var_os("MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER") + .map(std::path::PathBuf::from), }; let application = wide("OtOpcUa.ZB.MOM.WW.MxGateway.Worker"); @@ -117,6 +122,15 @@ impl WnWrap { } pub fn snapshot(&self, limit: i32) -> Result { + // The live test can fail this worker's polls without stopping the shared Alarmmgr. + #[cfg(all(debug_assertions, feature = "alarm-test-faults"))] + if self + .snapshot_fault_marker + .as_ref() + .is_some_and(|path| path.is_file()) + { + return Err(E_FAIL); + } struct XmlVariant(VARIANT); impl Drop for XmlVariant { fn drop(&mut self) {