Run alarm providers natively in the Rust worker

This commit is contained in:
Joseph Doherty
2026-11-21 09:53:37 -05:00
parent dbc5ea0c25
commit 9293c4c913
13 changed files with 1638 additions and 430 deletions
@@ -10,10 +10,12 @@ using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
@@ -597,7 +599,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
/// <summary>Verifies managed alarm provider callbacks forwarded by the Rust worker.</summary>
/// <summary>Verifies the Rust worker’s native subtag alarm provider and standby replacement.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
@@ -738,13 +740,185 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service)
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
string user = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER") ?? "Administrator";
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
{
Enabled = true,
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
});
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
using AsyncServerStreamingCall<AlarmFeedMessage> feed = client.StreamAlarms(
new StreamAlarmsRequest { AlarmFilterPrefix = reference }, cancellationToken: token);
bool snapshotComplete = false;
bool alarmmgrReady = false;
while ((!snapshotComplete || !alarmmgrReady) && await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
AlarmFeedMessage frame = feed.ResponseStream.Current;
snapshotComplete |= frame.SnapshotComplete;
alarmmgrReady |= frame.ProviderStatus?.Mode == AlarmProviderMode.Alarmmgr;
}
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await client.InvokeAsync(request, deadline: DateTime.UtcNow.AddSeconds(15),
cancellationToken: cancellationToken).ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
$"{reply.Kind} returned {reply.ProtocolStatus.Code}: {reply.DiagnosticMessage}");
return reply;
}
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
{
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
OnAlarmTransitionEvent? transition = feed.ResponseStream.Current.Transition;
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
&& (ackComment is null || transition.OperatorComment == ackComment))
{
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
Assert.False(transition.Degraded);
return transition;
}
}
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
}
MxCommandRequest? restore = null;
try
{
int server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply read = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
Command = new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 5000, TagAddresses = { tag } },
},
}, token).ConfigureAwait(false);
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
Assert.True(original.WasSuccessful, original.ErrorMessage);
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
MxCommandRequest WriteAlarm(bool active)
{
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
request.Command.WriteSecured.Value = new MxValue { DataType = MxDataType.Boolean, BoolValue = active };
return request;
}
async Task WriteConfirmedAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await InvokeAsync(request, cancellationToken).ConfigureAwait(false);
Assert.NotEmpty(reply.Statuses);
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
// The dev rig’s test UDA is normally false. Clear an existing test episode before raising a fresh one.
if (original.Value.BoolValue)
{
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
restore = WriteAlarm(true);
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
}
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference,
OperatorUser = $"mxgw-smoke-{Guid.NewGuid():N}",
Comment = "This acknowledgement must be rejected",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, rejected.ProtocolStatus.Code);
Assert.True(rejected.HasHresult && rejected.Hresult != 0);
AcknowledgeAlarmReply ack = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference, OperatorUser = user, OperatorPassword = password, Comment = comment,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.True(ack.ProtocolStatus.Code == ProtocolStatusCode.Ok, ack.DiagnosticMessage);
Assert.True(ack.HasHresult);
Assert.Equal(0, ack.Hresult);
Assert.DoesNotContain("unconfirmed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("not observed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
OnAlarmTransitionEvent acknowledged = await WaitForTransitionAsync(AlarmTransitionKind.Acknowledge, comment)
.ConfigureAwait(false);
Assert.Equal(user, acknowledged.OperatorUser, ignoreCase: true);
output.WriteLine($"Alarmmgr streamed Raise and Acknowledge with operator {acknowledged.OperatorUser} and the requested comment.");
if (!original.Value.BoolValue)
{
await WriteConfirmedAsync(restore, token).ConfigureAwait(false);
restore = null;
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
output.WriteLine("The gateway streamed Clear after the test UDA was restored.");
}
}
finally
{
try
{
if (restore is not null)
{
MxCommandReply restored = await InvokeAsync(restore, CancellationToken.None).ConfigureAwait(false);
Assert.NotEmpty(restored.Statuses);
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
}
finally
{
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
}
}
}
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service, GatewayAlarmMonitor? monitor = null)
{
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
builder.WebHost.ConfigureKestrel(options =>
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
builder.Services.AddGrpc();
builder.Services.AddSingleton(service);
if (monitor is not null)
{
builder.Services.AddSingleton<IHostedService>(monitor);
}
WebApplication app = builder.Build();
app.MapGrpcService<MxAccessGatewayService>();
return app;
@@ -2732,12 +2906,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// <param name="workerExecutablePath">Path to the worker process executable.</param>
/// <param name="processFactory">Factory for creating worker processes.</param>
/// <param name="output">Test output helper for logging.</param>
/// <param name="alarms">Optional configuration for the central alarm monitor.</param>
public GatewayServiceFixture(
string workerExecutablePath,
IWorkerProcessFactory processFactory,
ITestOutputHelper output)
ITestOutputHelper output,
AlarmsOptions? alarms = null)
{
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath));
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
WorkerProcessLauncher launcher = new(
options,
@@ -2762,6 +2938,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
options,
_metrics);
Monitor = new GatewayAlarmMonitor(
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
_loggerFactory.CreateLogger<GatewayAlarmMonitor>());
Service = new MxAccessGatewayService(
sessionManager,
new GatewayRequestIdentityAccessor(),
@@ -2771,12 +2950,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
eventStreamService,
_metrics,
_loggerFactory.CreateLogger<MxAccessGatewayService>(),
new ZB.MOM.WW.MxGateway.Server.Alarms.GatewayAlarmMonitor(
sessionManager,
EmptyAlarmWatchListResolver.Instance,
_metrics,
options,
_loggerFactory.CreateLogger<ZB.MOM.WW.MxGateway.Server.Alarms.GatewayAlarmMonitor>()));
Monitor);
}
/// <summary>
@@ -2784,6 +2958,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// </summary>
public MxAccessGatewayService Service { get; }
/// <summary>The central alarm monitor for tests of the public alarm RPCs.</summary>
public GatewayAlarmMonitor Monitor { get; }
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
public SessionManager Manager { get; }
@@ -2807,19 +2984,22 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
await Monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
foreach (GatewaySession session in _registry.Snapshot())
{
await session.DisposeAsync().ConfigureAwait(false);
}
Monitor.Dispose();
_loggerFactory.Dispose();
_metrics.Dispose();
}
private static GatewayOptions CreateOptions(string workerExecutablePath)
private static GatewayOptions CreateOptions(string workerExecutablePath, AlarmsOptions? alarms)
{
return new GatewayOptions
{
Alarms = alarms ?? new AlarmsOptions(),
Worker = new WorkerOptions
{
ExecutablePath = workerExecutablePath,
@@ -2833,7 +3013,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
Sessions = new SessionOptions
{
DefaultCommandTimeoutSeconds = 15,
MaxSessions = 1,
MaxSessions = alarms?.Enabled == true ? 2 : 1,
},
Events = new EventOptions
{