|
|
|
@@ -743,7 +743,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
|
|
|
|
|
/// <returns>A task that represents the live test.</returns>
|
|
|
|
|
[RustWorkerLiveFact]
|
|
|
|
|
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
|
|
|
|
|
public Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
|
|
|
|
|
{
|
|
|
|
|
return VerifyNativeAlarmAsync();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies automatic failover and failback on one live alarm feed.</summary>
|
|
|
|
|
/// <returns>A task that represents the live test.</returns>
|
|
|
|
|
[RustWorkerLiveFact(requireAlarmFaults: true)]
|
|
|
|
|
public async Task GatewayGrpc_WithRustWorker_StreamsAcrossAutomaticAlarmFailoverAndFailback()
|
|
|
|
|
{
|
|
|
|
|
string marker = Path.Combine(Path.GetTempPath(), $"mxgw-alarm-fault-{Guid.NewGuid():N}");
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
await VerifyNativeAlarmAsync(marker).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
|
File.Delete(marker);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task VerifyNativeAlarmAsync(string? faultMarker = null)
|
|
|
|
|
{
|
|
|
|
|
const string tag = "TestMachine_001.TestAlarm001";
|
|
|
|
|
const string reference = "Galaxy!TestArea." + tag;
|
|
|
|
@@ -751,14 +772,37 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
|
|
|
|
|
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
|
|
|
|
|
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
|
|
|
|
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
|
|
|
|
|
IAlarmWatchListResolver? resolver = null;
|
|
|
|
|
if (faultMarker is not null)
|
|
|
|
|
{
|
|
|
|
|
resolver = new SingleAlarmWatchListResolver(new AlarmSubtagTarget
|
|
|
|
|
{
|
|
|
|
|
AlarmFullReference = reference,
|
|
|
|
|
SourceObjectReference = "TestMachine_001",
|
|
|
|
|
ActiveSubtag = tag + ".InAlarm",
|
|
|
|
|
AckedSubtag = tag + ".Acked",
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output, faultMarker), output, new AlarmsOptions
|
|
|
|
|
{
|
|
|
|
|
Enabled = true,
|
|
|
|
|
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
|
|
|
|
|
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
|
|
|
|
|
});
|
|
|
|
|
Fallback = new AlarmFallbackOptions
|
|
|
|
|
{
|
|
|
|
|
Mode = faultMarker is null ? "ForceAlarmManager" : "Auto",
|
|
|
|
|
ConsecutiveFailureThreshold = 3,
|
|
|
|
|
FailbackProbeIntervalSeconds = 1,
|
|
|
|
|
FailbackStableProbes = 3,
|
|
|
|
|
Discovery = new AlarmDiscoveryOptions
|
|
|
|
|
{
|
|
|
|
|
UseGalaxyRepository = false,
|
|
|
|
|
IncludeAttributes = faultMarker is null ? [] : [tag],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}, resolver);
|
|
|
|
|
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
|
|
|
|
|
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
|
|
|
|
|
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(120));
|
|
|
|
|
CancellationToken token = deadline.Token;
|
|
|
|
|
await app.StartAsync(token).ConfigureAwait(false);
|
|
|
|
|
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
|
|
|
@@ -792,7 +836,8 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
return reply;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
|
|
|
|
|
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
|
|
|
|
|
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
|
|
|
|
|
{
|
|
|
|
|
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
|
|
|
|
{
|
|
|
|
@@ -800,14 +845,29 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
|
|
|
|
|
&& (ackComment is null || transition.OperatorComment == ackComment))
|
|
|
|
|
{
|
|
|
|
|
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
|
|
|
|
|
Assert.False(transition.Degraded);
|
|
|
|
|
Assert.Equal(provider, transition.SourceProvider);
|
|
|
|
|
Assert.Equal(provider == AlarmProviderMode.Subtag, transition.Degraded);
|
|
|
|
|
return transition;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async Task WaitForModeAsync(AlarmProviderMode mode, string reason)
|
|
|
|
|
{
|
|
|
|
|
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
|
|
|
|
{
|
|
|
|
|
AlarmProviderStatus? status = feed.ResponseStream.Current.ProviderStatus;
|
|
|
|
|
if (status?.Mode == mode && status.Reason == reason)
|
|
|
|
|
{
|
|
|
|
|
Assert.Equal(mode == AlarmProviderMode.Subtag, status.Degraded);
|
|
|
|
|
output.WriteLine($"StreamAlarms provider switched to {mode} ({status.Reason}).");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
throw new InvalidOperationException($"Alarm stream ended before switching to {mode}.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MxCommandRequest? restore = null;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
@@ -832,7 +892,6 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
|
|
|
|
Assert.NotEqual(0, userId);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MxCommandRequest WriteAlarm(bool active)
|
|
|
|
|
{
|
|
|
|
|
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
|
|
|
|
@@ -858,6 +917,25 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
restore = WriteAlarm(original.Value.BoolValue);
|
|
|
|
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
|
|
|
|
|
|
|
|
|
if (faultMarker is not null)
|
|
|
|
|
{
|
|
|
|
|
File.WriteAllText(faultMarker, string.Empty);
|
|
|
|
|
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
|
|
|
|
|
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
|
|
|
|
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
|
|
|
|
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
|
|
|
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
|
|
|
|
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
|
|
|
|
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
|
|
|
|
|
|
|
|
|
File.Delete(faultMarker);
|
|
|
|
|
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
|
|
|
|
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
|
|
|
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
|
|
|
|
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
|
|
|
|
|
{
|
|
|
|
|
AlarmFullReference = reference,
|
|
|
|
@@ -924,15 +1002,28 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
return app;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private sealed class SingleAlarmWatchListResolver(AlarmSubtagTarget target) : IAlarmWatchListResolver
|
|
|
|
|
{
|
|
|
|
|
public Task<AlarmWatchListResolution> ResolveAsync(AlarmsOptions options, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
|
return Task.FromResult(new AlarmWatchListResolution([target], false, false, null));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private sealed class RustWorkerLiveFactAttribute : FactAttribute
|
|
|
|
|
{
|
|
|
|
|
public RustWorkerLiveFactAttribute()
|
|
|
|
|
public RustWorkerLiveFactAttribute(bool requireAlarmFaults = false)
|
|
|
|
|
{
|
|
|
|
|
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
|
|
|
|
|
if (!IntegrationTestEnvironment.IsEnabled(optIn))
|
|
|
|
|
{
|
|
|
|
|
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
|
|
|
|
|
}
|
|
|
|
|
else if (requireAlarmFaults && !IntegrationTestEnvironment.IsEnabled("MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST"))
|
|
|
|
|
{
|
|
|
|
|
Skip = "Build the debug worker with alarm-test-faults and set MXGATEWAY_RUN_RUST_ALARM_FAILOVER_TEST=1.";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -2907,11 +2998,13 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
/// <param name="processFactory">Factory for creating worker processes.</param>
|
|
|
|
|
/// <param name="output">Test output helper for logging.</param>
|
|
|
|
|
/// <param name="alarms">Optional configuration for the central alarm monitor.</param>
|
|
|
|
|
/// <param name="watchListResolver">Optional resolver for the monitor's subtag watch list.</param>
|
|
|
|
|
public GatewayServiceFixture(
|
|
|
|
|
string workerExecutablePath,
|
|
|
|
|
IWorkerProcessFactory processFactory,
|
|
|
|
|
ITestOutputHelper output,
|
|
|
|
|
AlarmsOptions? alarms = null)
|
|
|
|
|
AlarmsOptions? alarms = null,
|
|
|
|
|
IAlarmWatchListResolver? watchListResolver = null)
|
|
|
|
|
{
|
|
|
|
|
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
|
|
|
|
|
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
|
|
|
|
@@ -2939,7 +3032,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
_metrics);
|
|
|
|
|
|
|
|
|
|
Monitor = new GatewayAlarmMonitor(
|
|
|
|
|
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
|
|
|
|
|
sessionManager, watchListResolver ?? EmptyAlarmWatchListResolver.Instance, _metrics, options,
|
|
|
|
|
_loggerFactory.CreateLogger<GatewayAlarmMonitor>());
|
|
|
|
|
Service = new MxAccessGatewayService(
|
|
|
|
|
sessionManager,
|
|
|
|
@@ -3216,13 +3309,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Factory that launches worker processes and records their outputs for testing.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory
|
|
|
|
|
private sealed class TestWorkerProcessFactory(ITestOutputHelper output, string? alarmFaultMarker = null) : IWorkerProcessFactory
|
|
|
|
|
{
|
|
|
|
|
private readonly ConcurrentBag<TestWorkerProcess> processes = [];
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public IWorkerProcess Start(ProcessStartInfo startInfo)
|
|
|
|
|
{
|
|
|
|
|
if (alarmFaultMarker is not null)
|
|
|
|
|
{
|
|
|
|
|
startInfo.Environment["MXGATEWAY_TEST_ALARM_SNAPSHOT_FAULT_MARKER"] = alarmFaultMarker;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
startInfo.RedirectStandardError = true;
|
|
|
|
|
startInfo.RedirectStandardOutput = true;
|
|
|
|
|
startInfo.UseShellExecute = false;
|
|
|
|
|