Add a live test for a real Alarmmgr outage
This commit is contained in:
@@ -764,7 +764,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
|
/// <summary>Verifies alarm streaming while an external controller stops and restores Alarmmgr.</summary>
|
||||||
|
/// <returns>A task that represents the live test.</returns>
|
||||||
|
[RustWorkerLiveFact(requireExternalAlarmOutage: true)]
|
||||||
|
public Task GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage()
|
||||||
|
{
|
||||||
|
string directory = Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")!;
|
||||||
|
Assert.True(Directory.Exists(directory), "Create an empty outage coordination directory before launching the test.");
|
||||||
|
Assert.Empty(Directory.EnumerateFileSystemEntries(directory));
|
||||||
|
return VerifyNativeAlarmAsync(outageDirectory: directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task VerifyNativeAlarmAsync(string? faultMarker = null, string? outageDirectory = null)
|
||||||
{
|
{
|
||||||
const string tag = "TestMachine_001.TestAlarm001";
|
const string tag = "TestMachine_001.TestAlarm001";
|
||||||
const string reference = "Galaxy!TestArea." + tag;
|
const string reference = "Galaxy!TestArea." + tag;
|
||||||
@@ -772,8 +783,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
|
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
|
||||||
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
|
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
|
||||||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||||||
|
bool verifyFailover = faultMarker is not null || outageDirectory is not null;
|
||||||
IAlarmWatchListResolver? resolver = null;
|
IAlarmWatchListResolver? resolver = null;
|
||||||
if (faultMarker is not null)
|
if (verifyFailover)
|
||||||
{
|
{
|
||||||
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
|
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
|
||||||
{
|
{
|
||||||
@@ -790,19 +802,19 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
|
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
|
||||||
Fallback = new AlarmFallbackOptions
|
Fallback = new AlarmFallbackOptions
|
||||||
{
|
{
|
||||||
Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
|
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
|
||||||
ConsecutiveFailureThreshold = 3,
|
ConsecutiveFailureThreshold = 3,
|
||||||
FailbackProbeIntervalSeconds = 1,
|
FailbackProbeIntervalSeconds = 1,
|
||||||
FailbackStableProbes = 3,
|
FailbackStableProbes = 3,
|
||||||
Discovery = new AlarmDiscoveryOptions
|
Discovery = new AlarmDiscoveryOptions
|
||||||
{
|
{
|
||||||
UseGalaxyRepository = false,
|
UseGalaxyRepository = false,
|
||||||
IncludeAttributes = faultMarker is null ? [] : [tag],
|
IncludeAttributes = verifyFailover ? [tag] : [],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}, resolver);
|
}, resolver);
|
||||||
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
|
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
|
||||||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
|
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(outageDirectory is null ? 120 : 180));
|
||||||
CancellationToken token = deadline.Token;
|
CancellationToken token = deadline.Token;
|
||||||
await app.StartAsync(token).ConfigureAwait(false);
|
await app.StartAsync(token).ConfigureAwait(false);
|
||||||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||||||
@@ -820,6 +832,15 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
}
|
}
|
||||||
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
|
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
|
||||||
|
|
||||||
|
using (CancellationTokenSource startup = CancellationTokenSource.CreateLinkedTokenSource(token))
|
||||||
|
{
|
||||||
|
startup.CancelAfter(TimeSpan.FromSeconds(45));
|
||||||
|
while (fixture.Monitor.State != GatewayAlarmMonitorState.Monitoring)
|
||||||
|
{
|
||||||
|
await Task.Delay(100, startup.Token).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||||
{
|
{
|
||||||
ClientSessionName = "rust-native-alarm-driver",
|
ClientSessionName = "rust-native-alarm-driver",
|
||||||
@@ -917,9 +938,10 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
restore = WriteAlarm(original.Value.BoolValue);
|
restore = WriteAlarm(original.Value.BoolValue);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||||||
|
|
||||||
if (faultMarker is not null)
|
if (verifyFailover)
|
||||||
{
|
{
|
||||||
File.WriteAllText(faultMarker, string.Empty);
|
string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr");
|
||||||
|
File.WriteAllText(stopRequest, string.Empty);
|
||||||
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
|
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
|
||||||
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
||||||
@@ -928,7 +950,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
||||||
|
|
||||||
File.Delete(faultMarker);
|
if (faultMarker is not null)
|
||||||
|
{
|
||||||
|
File.Delete(faultMarker);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
|
||||||
|
}
|
||||||
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
||||||
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||||||
@@ -1013,7 +1042,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
|
|
||||||
private sealed class RustWorkerLiveFactAttribute : FactAttribute
|
private sealed class RustWorkerLiveFactAttribute : FactAttribute
|
||||||
{
|
{
|
||||||
public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
|
public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false, bool requireExternalAlarmOutage = false)
|
||||||
{
|
{
|
||||||
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
|
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
|
||||||
if (!IntegrationTestEnvironment.IsEnabled(optIn))
|
if (!IntegrationTestEnvironment.IsEnabled(optIn))
|
||||||
@@ -1024,6 +1053,11 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
{
|
{
|
||||||
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
|
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
|
||||||
}
|
}
|
||||||
|
else if (requireExternalAlarmOutage
|
||||||
|
&& string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY")))
|
||||||
|
{
|
||||||
|
Skip = "Set MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY and run an external Alarmmgr outage controller.";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -135,6 +135,15 @@ return an error. It drives the same development alarm through
|
|||||||
transitions on one open `StreamAlarms` call. The fault injection is compiled only
|
transitions on one open `StreamAlarms` call. The fault injection is compiled only
|
||||||
when both debug assertions and the Cargo feature are enabled.
|
when both debug assertions and the Cargo feature are enabled.
|
||||||
|
|
||||||
|
For a real outage, build the normal worker and point
|
||||||
|
`MXGATEWAY_TEST_ALARMMGR_OUTAGE_DIRECTORY` at a new, empty directory on the host.
|
||||||
|
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
|
||||||
|
test options above. An external controller stops Alarmmgr when the test creates
|
||||||
|
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
|
||||||
|
The controller should also restore Alarmmgr on timeout or failure. The test
|
||||||
|
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
|
||||||
|
raise after recovery on the original stream.
|
||||||
|
|
||||||
## COM interface evidence
|
## COM interface evidence
|
||||||
|
|
||||||
The native adapter uses the dual `ILMXProxyServer` interface from the installed
|
The native adapter uses the dual `ILMXProxyServer` interface from the installed
|
||||||
|
|||||||
Reference in New Issue
Block a user