diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs index 2389df3f..364e6df2 100644 --- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs +++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs @@ -764,7 +764,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) } } - private async Task VerifyNativeAlarmAsync(string? faultMarker = null) + /// Verifies alarm streaming while an external controller stops and restores Alarmmgr. + /// A task that represents the live test. + [RustWorkerLiveFact(requireExternalAlarmOutage: true)] + public Task GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage() + { + string directory = Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")!; + Assert.True(Directory.Exists(directory), "Create an empty outage coordination directory before launching the test."); + Assert.Empty(Directory.EnumerateFileSystemEntries(directory)); + return VerifyNativeAlarmAsync(outageDirectory: directory); + } + + private async Task VerifyNativeAlarmAsync(string? faultMarker = null, string? outageDirectory = null) { const string tag = "TestMachine_001.TestAlarm001"; const string reference = "Galaxy!TestArea." + tag; @@ -772,8 +783,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty; string comment = $"Rust worker smoke {Guid.NewGuid():N}"; string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath(); + bool verifyFailover = faultMarker is not null || outageDirectory is not null; IAlarmWatchListResolver? resolver = null; - if (faultMarker is not null) + if (verifyFailover) { resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget { @@ -790,19 +802,19 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea", Fallback = new AlarmFallbackOptions { - Mode = faultMarker is null ? "ForceAlarmManager" : "Auto", + Mode = verifyFailover ? "Auto" : "ForceAlarmManager", ConsecutiveFailureThreshold = 3, FailbackProbeIntervalSeconds = 1, FailbackStableProbes = 3, Discovery = new AlarmDiscoveryOptions { UseGalaxyRepository = false, - IncludeAttributes = faultMarker is null ? [] : [tag], + IncludeAttributes = verifyFailover ? [tag] : [], }, }, }, resolver); await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor); - using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120)); + using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(outageDirectory is null ? 120 : 180)); CancellationToken token = deadline.Token; await app.StartAsync(token).ConfigureAwait(false); using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single()); @@ -820,6 +832,15 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) } Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed."); + using (CancellationTokenSource startup = CancellationTokenSource.CreateLinkedTokenSource(token)) + { + startup.CancelAfter(TimeSpan.FromSeconds(45)); + while (fixture.Monitor.State != GatewayAlarmMonitorState.Monitoring) + { + await Task.Delay(100, startup.Token).ConfigureAwait(false); + } + } + OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest { ClientSessionName = "rust-native-alarm-driver", @@ -917,9 +938,10 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) restore = WriteAlarm(original.Value.BoolValue); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); - if (faultMarker is not null) + if (verifyFailover) { - File.WriteAllText(faultMarker, string.Empty); + string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr"); + File.WriteAllText(stopRequest, string.Empty); await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false); await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false); @@ -928,7 +950,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false); - File.Delete(faultMarker); + if (faultMarker is not null) + { + File.Delete(faultMarker); + } + else + { + File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty); + } await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false); await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); @@ -1013,7 +1042,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) private sealed class RustWorkerLiveFactAttribute : FactAttribute { - public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false) + public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false, bool requireExternalAlarmOutage = false) { const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS"; if (!IntegrationTestEnvironment.IsEnabled(optIn)) @@ -1024,6 +1053,11 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) { Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1."; } + else if (requireExternalAlarmOutage + && string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY"))) + { + Skip = "Set MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY and run an external Alarmmgr outage controller."; + } } } diff --git a/src/mxgateway-worker-rust/README.md b/src/mxgateway-worker-rust/README.md index 3fe27abe..1b2b79bf 100644 --- a/src/mxgateway-worker-rust/README.md +++ b/src/mxgateway-worker-rust/README.md @@ -135,6 +135,15 @@ return an error. It drives the same development alarm through transitions on one open `StreamAlarms` call. The fault injection is compiled only when both debug assertions and the Cargo feature are enabled. +For a real outage, build the normal worker and point +`MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY` at a new, empty directory on the host. +Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live +test options above. An external controller stops Alarmmgr when the test creates +`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`. +The controller should also restore Alarmmgr on timeout or failure. The test +uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr +raise after recovery on the original stream. + ## COM interface evidence The native adapter uses the dual `ILMXProxyServer` interface from the installed