Stream live MXAccess advice events from the Rust worker

This commit is contained in:
Joseph Doherty
2026-11-21 08:56:01 -05:00
parent 1a44b5ac31
commit 00a597b08b
9 changed files with 1560 additions and 46 deletions
@@ -1,9 +1,15 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Text;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
@@ -17,6 +23,7 @@ using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.IntegrationTests.TestSupport;
using Xunit.Abstractions;
using SessionOptions = ZB.MOM.WW.MxGateway.Server.Configuration.SessionOptions;
namespace ZB.MOM.WW.MxGateway.IntegrationTests;
@@ -115,6 +122,163 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
/// <summary>Verifies Rust worker callbacks over the gateway's live HTTP/2 gRPC endpoint.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsChangingIntegersAndUnadvises()
{
const string itemDefinition = "TestMachine_001.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerPath, processFactory, output);
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
builder.WebHost.ConfigureKestrel(options =>
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
builder.Services.AddGrpc();
builder.Services.AddSingleton(fixture.Service);
await using WebApplication app = builder.Build();
app.MapGrpcService<MxAccessGatewayService>();
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "rust-worker-live-events" },
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
try
{
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request)
{
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
.ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok, reply.ToString());
return reply;
}
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-worker-info",
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
}).ConfigureAwait(false);
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
int serverHandle = registration.Register.ServerHandle;
MxCommandRequest addItem = CreateAddItemRequest(sessionId, serverHandle);
addItem.Command.AddItem.ItemDefinition = itemDefinition;
MxCommandReply added = await InvokeAsync(addItem).ConfigureAwait(false);
int itemHandle = added.AddItem.ItemHandle;
using (AsyncServerStreamingCall<MxEvent> stream = client.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
cancellationToken: token))
{
Assert.True(await stream.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(stream.ResponseStream.Current.StreamAttached);
await InvokeAsync(CreateAdviseRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
MxEvent? first = null;
bool observedChange = false;
ulong lastSequence = 0;
while (await stream.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.True(change.WorkerSequence > lastSequence);
lastSequence = change.WorkerSequence;
if (change.Family != MxEventFamily.OnDataChange ||
change.Value?.KindCase != MxValue.KindOneofCase.Int32Value ||
(change.Quality & 0xc0) != 0xc0)
{
continue;
}
Assert.Equal(sessionId, change.SessionId);
Assert.Equal(serverHandle, change.ServerHandle);
Assert.Equal(itemHandle, change.ItemHandle);
Assert.NotNull(change.SourceTimestamp);
Assert.NotNull(change.WorkerTimestamp);
Assert.Contains(change.Statuses, status => status.Category == MxStatusCategory.Ok);
if (first is not null && change.Value.Int32Value != first.Value.Int32Value)
{
output.WriteLine(
$"{itemDefinition} delivered through gateway: {first.Value.Int32Value} -> {change.Value.Int32Value}, " +
$"worker sequences {first.WorkerSequence} -> {change.WorkerSequence}, quality {change.Quality}.");
observedChange = true;
break;
}
first ??= change;
}
Assert.True(observedChange, "The gateway did not deliver distinct TestChangingInt values.");
await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-unadvise",
Command = new MxCommand
{
Kind = MxCommandKind.UnAdvise,
UnAdvise = new UnAdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
},
}).ConfigureAwait(false);
// A callback queued before UnAdvise may still cross the pipe.
// Observe the same stream and reject updates generated after acknowledgement.
DateTime unadvisedAt = DateTime.UtcNow;
using CancellationTokenSource quiet = CancellationTokenSource.CreateLinkedTokenSource(token);
quiet.CancelAfter(TimeSpan.FromSeconds(2));
try
{
while (await stream.ResponseStream.MoveNext(quiet.Token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.False(change.Family == MxEventFamily.OnDataChange &&
change.ServerHandle == serverHandle && change.ItemHandle == itemHandle &&
change.WorkerTimestamp.ToDateTime() >= unadvisedAt,
"The worker sent a new data change after UnAdvise completed.");
}
Assert.Fail("The event stream ended before the quiet period elapsed.");
}
catch (RpcException exception) when (exception.StatusCode == StatusCode.Cancelled && quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
catch (OperationCanceledException) when (quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
}
}
finally
{
CloseSessionReply closed = await client.CloseSessionAsync(
new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
}
}
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
public RustWorkerLiveFactAttribute()
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
{
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
}
}
}
/// <summary>
/// Verifies that a Write command round-trips through live MXAccess against an advised item
/// and that the worker emits a matching <see cref="MxEventFamily.OnWriteComplete"/> event