Run alarm providers natively in the Rust worker
This commit is contained in:
@@ -10,10 +10,12 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
@@ -597,7 +599,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Verifies managed alarm provider callbacks forwarded by the Rust worker.</summary>
|
||||
/// <summary>Verifies the Rust worker’s native subtag alarm provider and standby replacement.</summary>
|
||||
/// <returns>A task that represents the live test.</returns>
|
||||
[RustWorkerLiveFact]
|
||||
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
|
||||
@@ -738,13 +740,185 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
}
|
||||
}
|
||||
|
||||
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service)
|
||||
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
|
||||
/// <returns>A task that represents the live test.</returns>
|
||||
[RustWorkerLiveFact]
|
||||
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
|
||||
{
|
||||
const string tag = "TestMachine_001.TestAlarm001";
|
||||
const string reference = "Galaxy!TestArea." + tag;
|
||||
string user = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER") ?? "Administrator";
|
||||
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
|
||||
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
|
||||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
|
||||
{
|
||||
Enabled = true,
|
||||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
|
||||
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
|
||||
});
|
||||
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
|
||||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
|
||||
CancellationToken token = deadline.Token;
|
||||
await app.StartAsync(token).ConfigureAwait(false);
|
||||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||||
|
||||
using AsyncServerStreamingCall<AlarmFeedMessage> feed = client.StreamAlarms(
|
||||
new StreamAlarmsRequest { AlarmFilterPrefix = reference }, cancellationToken: token);
|
||||
bool snapshotComplete = false;
|
||||
bool alarmmgrReady = false;
|
||||
while ((!snapshotComplete || !alarmmgrReady) && await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||||
{
|
||||
AlarmFeedMessage frame = feed.ResponseStream.Current;
|
||||
snapshotComplete |= frame.SnapshotComplete;
|
||||
alarmmgrReady |= frame.ProviderStatus?.Mode == AlarmProviderMode.Alarmmgr;
|
||||
}
|
||||
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
|
||||
|
||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "rust-native-alarm-driver",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||
string sessionId = opened.SessionId;
|
||||
|
||||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
MxCommandReply reply = await client.InvokeAsync(request, deadline: DateTime.UtcNow.AddSeconds(15),
|
||||
cancellationToken: cancellationToken).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
|
||||
$"{reply.Kind} returned {reply.ProtocolStatus.Code}: {reply.DiagnosticMessage}");
|
||||
return reply;
|
||||
}
|
||||
|
||||
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
|
||||
{
|
||||
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||||
{
|
||||
OnAlarmTransitionEvent? transition = feed.ResponseStream.Current.Transition;
|
||||
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
|
||||
&& (ackComment is null || transition.OperatorComment == ackComment))
|
||||
{
|
||||
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
|
||||
Assert.False(transition.Degraded);
|
||||
return transition;
|
||||
}
|
||||
}
|
||||
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
|
||||
}
|
||||
|
||||
MxCommandRequest? restore = null;
|
||||
try
|
||||
{
|
||||
int server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
|
||||
MxCommandReply read = await InvokeAsync(new MxCommandRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.ReadBulk,
|
||||
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 5000, TagAddresses = { tag } },
|
||||
},
|
||||
}, token).ConfigureAwait(false);
|
||||
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
|
||||
Assert.True(original.WasSuccessful, original.ErrorMessage);
|
||||
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
|
||||
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
||||
add.Command.AddItem.ItemDefinition = tag;
|
||||
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
||||
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
||||
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
||||
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
||||
Assert.NotEqual(0, userId);
|
||||
|
||||
|
||||
MxCommandRequest WriteAlarm(bool active)
|
||||
{
|
||||
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
|
||||
request.Command.WriteSecured.Value = new MxValue { DataType = MxDataType.Boolean, BoolValue = active };
|
||||
return request;
|
||||
}
|
||||
|
||||
async Task WriteConfirmedAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
MxCommandReply reply = await InvokeAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
Assert.NotEmpty(reply.Statuses);
|
||||
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||||
}
|
||||
|
||||
// The dev rig’s test UDA is normally false. Clear an existing test episode before raising a fresh one.
|
||||
if (original.Value.BoolValue)
|
||||
{
|
||||
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
||||
restore = WriteAlarm(true);
|
||||
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
|
||||
}
|
||||
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
||||
restore = WriteAlarm(original.Value.BoolValue);
|
||||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||||
|
||||
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
|
||||
{
|
||||
AlarmFullReference = reference,
|
||||
OperatorUser = $"mxgw-smoke-{Guid.NewGuid():N}",
|
||||
Comment = "This acknowledgement must be rejected",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.MxaccessFailure, rejected.ProtocolStatus.Code);
|
||||
Assert.True(rejected.HasHresult && rejected.Hresult != 0);
|
||||
|
||||
AcknowledgeAlarmReply ack = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
|
||||
{
|
||||
AlarmFullReference = reference, OperatorUser = user, OperatorPassword = password, Comment = comment,
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.True(ack.ProtocolStatus.Code == ProtocolStatusCode.Ok, ack.DiagnosticMessage);
|
||||
Assert.True(ack.HasHresult);
|
||||
Assert.Equal(0, ack.Hresult);
|
||||
Assert.DoesNotContain("unconfirmed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("not observed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
|
||||
OnAlarmTransitionEvent acknowledged = await WaitForTransitionAsync(AlarmTransitionKind.Acknowledge, comment)
|
||||
.ConfigureAwait(false);
|
||||
Assert.Equal(user, acknowledged.OperatorUser, ignoreCase: true);
|
||||
output.WriteLine($"Alarmmgr streamed Raise and Acknowledge with operator {acknowledged.OperatorUser} and the requested comment.");
|
||||
|
||||
if (!original.Value.BoolValue)
|
||||
{
|
||||
await WriteConfirmedAsync(restore, token).ConfigureAwait(false);
|
||||
restore = null;
|
||||
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
|
||||
output.WriteLine("The gateway streamed Clear after the test UDA was restored.");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
if (restore is not null)
|
||||
{
|
||||
MxCommandReply restored = await InvokeAsync(restore, CancellationToken.None).ConfigureAwait(false);
|
||||
Assert.NotEmpty(restored.Statuses);
|
||||
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
|
||||
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service, GatewayAlarmMonitor? monitor = null)
|
||||
{
|
||||
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
|
||||
builder.WebHost.ConfigureKestrel(options =>
|
||||
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
|
||||
builder.Services.AddGrpc();
|
||||
builder.Services.AddSingleton(service);
|
||||
if (monitor is not null)
|
||||
{
|
||||
builder.Services.AddSingleton<IHostedService>(monitor);
|
||||
}
|
||||
WebApplication app = builder.Build();
|
||||
app.MapGrpcService<MxAccessGatewayService>();
|
||||
return app;
|
||||
@@ -2732,12 +2906,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
/// <param name="workerExecutablePath">Path to the worker process executable.</param>
|
||||
/// <param name="processFactory">Factory for creating worker processes.</param>
|
||||
/// <param name="output">Test output helper for logging.</param>
|
||||
/// <param name="alarms">Optional configuration for the central alarm monitor.</param>
|
||||
public GatewayServiceFixture(
|
||||
string workerExecutablePath,
|
||||
IWorkerProcessFactory processFactory,
|
||||
ITestOutputHelper output)
|
||||
ITestOutputHelper output,
|
||||
AlarmsOptions? alarms = null)
|
||||
{
|
||||
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath));
|
||||
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
|
||||
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
|
||||
WorkerProcessLauncher launcher = new(
|
||||
options,
|
||||
@@ -2762,6 +2938,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
options,
|
||||
_metrics);
|
||||
|
||||
Monitor = new GatewayAlarmMonitor(
|
||||
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
|
||||
_loggerFactory.CreateLogger<GatewayAlarmMonitor>());
|
||||
Service = new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
new GatewayRequestIdentityAccessor(),
|
||||
@@ -2771,12 +2950,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
eventStreamService,
|
||||
_metrics,
|
||||
_loggerFactory.CreateLogger<MxAccessGatewayService>(),
|
||||
new ZB.MOM.WW.MxGateway.Server.Alarms.GatewayAlarmMonitor(
|
||||
sessionManager,
|
||||
EmptyAlarmWatchListResolver.Instance,
|
||||
_metrics,
|
||||
options,
|
||||
_loggerFactory.CreateLogger<ZB.MOM.WW.MxGateway.Server.Alarms.GatewayAlarmMonitor>()));
|
||||
Monitor);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2784,6 +2958,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
/// </summary>
|
||||
public MxAccessGatewayService Service { get; }
|
||||
|
||||
/// <summary>The central alarm monitor for tests of the public alarm RPCs.</summary>
|
||||
public GatewayAlarmMonitor Monitor { get; }
|
||||
|
||||
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
|
||||
public SessionManager Manager { get; }
|
||||
|
||||
@@ -2807,19 +2984,22 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await Monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
foreach (GatewaySession session in _registry.Snapshot())
|
||||
{
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Monitor.Dispose();
|
||||
_loggerFactory.Dispose();
|
||||
_metrics.Dispose();
|
||||
}
|
||||
|
||||
private static GatewayOptions CreateOptions(string workerExecutablePath)
|
||||
private static GatewayOptions CreateOptions(string workerExecutablePath, AlarmsOptions? alarms)
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
Alarms = alarms ?? new AlarmsOptions(),
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
ExecutablePath = workerExecutablePath,
|
||||
@@ -2833,7 +3013,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
Sessions = new SessionOptions
|
||||
{
|
||||
DefaultCommandTimeoutSeconds = 15,
|
||||
MaxSessions = 1,
|
||||
MaxSessions = alarms?.Enabled == true ? 2 : 1,
|
||||
},
|
||||
Events = new EventOptions
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user