diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
index 2389df3f..364e6df2 100644
--- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
@@ -764,7 +764,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
- private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
+ /// Verifies alarm streaming while an external controller stops and restores Alarmmgr.
+ /// A task that represents the live test.
+ [RustWorkerLiveFact(requireExternalAlarmOutage: true)]
+ public Task GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage()
+ {
+ string directory = Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")!;
+ Assert.True(Directory.Exists(directory), "Create an empty outage coordination directory before launching the test.");
+ Assert.Empty(Directory.EnumerateFileSystemEntries(directory));
+ return VerifyNativeAlarmAsync(outageDirectory: directory);
+ }
+
+ private async Task VerifyNativeAlarmAsync(string? faultMarker = null, string? outageDirectory = null)
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
@@ -772,8 +783,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
+ bool verifyFailover = faultMarker is not null || outageDirectory is not null;
IAlarmWatchListResolver? resolver = null;
- if (faultMarker is not null)
+ if (verifyFailover)
{
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
{
@@ -790,19 +802,19 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions
{
- Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
+ Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
ConsecutiveFailureThreshold = 3,
FailbackProbeIntervalSeconds = 1,
FailbackStableProbes = 3,
Discovery = new AlarmDiscoveryOptions
{
UseGalaxyRepository = false,
- IncludeAttributes = faultMarker is null ? [] : [tag],
+ IncludeAttributes = verifyFailover ? [tag] : [],
},
},
}, resolver);
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
- using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
+ using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(outageDirectory is null ? 120 : 180));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
@@ -820,6 +832,15 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
+ using (CancellationTokenSource startup = CancellationTokenSource.CreateLinkedTokenSource(token))
+ {
+ startup.CancelAfter(TimeSpan.FromSeconds(45));
+ while (fixture.Monitor.State != GatewayAlarmMonitorState.Monitoring)
+ {
+ await Task.Delay(100, startup.Token).ConfigureAwait(false);
+ }
+ }
+
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
@@ -917,9 +938,10 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
- if (faultMarker is not null)
+ if (verifyFailover)
{
- File.WriteAllText(faultMarker, string.Empty);
+ string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr");
+ File.WriteAllText(stopRequest, string.Empty);
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
@@ -928,7 +950,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
- File.Delete(faultMarker);
+ if (faultMarker is not null)
+ {
+ File.Delete(faultMarker);
+ }
+ else
+ {
+ File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
+ }
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
@@ -1013,7 +1042,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
- public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
+ public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false, bool requireExternalAlarmOutage = false)
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
@@ -1024,6 +1053,11 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
}
+ else if (requireExternalAlarmOutage
+ && string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")))
+ {
+ Skip = "Set MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY and run an external Alarmmgr outage controller.";
+ }
}
}
diff --git a/src/mxgateway-worker-rust/README.md b/src/mxgateway-worker-rust/README.md
index 3fe27abe..1b2b79bf 100644
--- a/src/mxgateway-worker-rust/README.md
+++ b/src/mxgateway-worker-rust/README.md
@@ -135,6 +135,15 @@ return an error. It drives the same development alarm through
transitions on one open `StreamAlarms` call. The fault injection is compiled only
when both debug assertions and the Cargo feature are enabled.
+For a real outage, build the normal worker and point
+`MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY` at a new, empty directory on the host.
+Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
+test options above. An external controller stops Alarmmgr when the test creates
+`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
+The controller should also restore Alarmmgr on timeout or failure. The test
+uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
+raise after recovery on the original stream.
+
## COM interface evidence
The native adapter uses the dual `ILMXProxyServer` interface from the installed