Add a live test for a real Alarmmgr outage

This commit is contained in:
Joseph Doherty
2026-11-21 10:28:54 -05:00
parent 8045afe174
commit 04655b78fa
2 changed files with 52 additions and 9 deletions
@@ -764,7 +764,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
/// <summary>Verifies alarm streaming while an external controller stops and restores Alarmmgr.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact(requireExternalAlarmOutage: true)]
public Task GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage()
{
string directory = Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")!;
Assert.True(Directory.Exists(directory), "Create an empty outage coordination directory before launching the test.");
Assert.Empty(Directory.EnumerateFileSystemEntries(directory));
return VerifyNativeAlarmAsync(outageDirectory: directory);
}
private async Task VerifyNativeAlarmAsync(string? faultMarker = null, string? outageDirectory = null)
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
@@ -772,8 +783,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
bool verifyFailover = faultMarker is not null || outageDirectory is not null;
IAlarmWatchListResolver? resolver = null;
if (faultMarker is not null)
if (verifyFailover)
{
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
{
@@ -790,19 +802,19 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions
{
Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
ConsecutiveFailureThreshold = 3,
FailbackProbeIntervalSeconds = 1,
FailbackStableProbes = 3,
Discovery = new AlarmDiscoveryOptions
{
UseGalaxyRepository = false,
IncludeAttributes = faultMarker is null ? [] : [tag],
IncludeAttributes = verifyFailover ? [tag] : [],
},
},
}, resolver);
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(outageDirectory is null ? 120 : 180));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
@@ -820,6 +832,15 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
using (CancellationTokenSource startup = CancellationTokenSource.CreateLinkedTokenSource(token))
{
startup.CancelAfter(TimeSpan.FromSeconds(45));
while (fixture.Monitor.State != GatewayAlarmMonitorState.Monitoring)
{
await Task.Delay(100, startup.Token).ConfigureAwait(false);
}
}
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
@@ -917,9 +938,10 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
if (faultMarker is not null)
if (verifyFailover)
{
File.WriteAllText(faultMarker, string.Empty);
string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr");
File.WriteAllText(stopRequest, string.Empty);
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
@@ -928,7 +950,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
File.Delete(faultMarker);
if (faultMarker is not null)
{
File.Delete(faultMarker);
}
else
{
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
}
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
@@ -1013,7 +1042,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false, bool requireExternalAlarmOutage = false)
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
@@ -1024,6 +1053,11 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
}
else if (requireExternalAlarmOutage
&& string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")))
{
Skip = "Set MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY and run an external Alarmmgr outage controller.";
}
}
}
+9
View File
@@ -135,6 +135,15 @@ return an error. It drives the same development alarm through
transitions on one open `StreamAlarms` call. The fault injection is compiled only
when both debug assertions and the Cargo feature are enabled.
For a real outage, build the normal worker and point
`MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY` at a new, empty directory on the host.
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
test options above. An external controller stops Alarmmgr when the test creates
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
The controller should also restore Alarmmgr on timeout or failure. The test
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
raise after recovery on the original stream.
## COM interface evidence
The native adapter uses the dual `ILMXProxyServer` interface from the installed