using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Text;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.IntegrationTests.TestSupport;
using Xunit.Abstractions;
using SessionOptions = ZB.MOM.WW.MxGateway.Server.Configuration.SessionOptions;
namespace ZB.MOM.WW.MxGateway.IntegrationTests;
[Collection(LiveResourcesCollection.Name)]
[Trait("Category", "LiveMxAccess")]
public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{
private static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(15);
private static readonly TimeSpan StreamShutdownTimeout = TimeSpan.FromSeconds(10);
///
/// Verifies that a gateway session can register, add item, advise, and stream events from live MXAccess.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_RegistersAdvisesStreamsDataAndCloses()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-smoke",
ClientCorrelationId = "live-open",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
output.WriteLine($"OpenSession status={openReply.ProtocolStatus.Code} session={sessionId} worker_pid={openReply.WorkerProcessId}");
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
Assert.True(registerReply.Register.ServerHandle > 0);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// A live MXAccess provider can deliver an initial registration-state
// or bad-quality bootstrap event before the OnDataChange the worker
// is contracted to emit. Match on the family rather than trusting
// whatever event arrives first so a genuine ordering defect cannot
// pass spuriously or leave a later wrong event unverified.
MxEvent dataChange = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(dataChange);
Assert.Equal(MxEventFamily.OnDataChange, dataChange.Family);
Assert.Equal(sessionId, dataChange.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, dataChange.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, dataChange.ItemHandle);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
/// Verifies Rust worker callbacks over the gateway's live HTTP/2 gRPC endpoint.
/// A task that represents the live test.
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsChangingIntegersAndUnadvises()
{
const string itemDefinition = "TestMachine_001.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerPath, processFactory, output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "rust-worker-live-events" },
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
try
{
async Task InvokeAsync(MxCommandRequest request)
{
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
.ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok, reply.ToString());
return reply;
}
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-worker-info",
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
}).ConfigureAwait(false);
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
int serverHandle = registration.Register.ServerHandle;
MxCommandRequest addItem = CreateAddItemRequest(sessionId, serverHandle);
addItem.Command.AddItem.ItemDefinition = itemDefinition;
MxCommandReply added = await InvokeAsync(addItem).ConfigureAwait(false);
int itemHandle = added.AddItem.ItemHandle;
using (AsyncServerStreamingCall stream = client.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
cancellationToken: token))
{
Assert.True(await stream.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(stream.ResponseStream.Current.StreamAttached);
await InvokeAsync(CreateAdviseRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
MxEvent? first = null;
bool observedChange = false;
ulong lastSequence = 0;
while (await stream.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.True(change.WorkerSequence > lastSequence);
lastSequence = change.WorkerSequence;
if (change.Family != MxEventFamily.OnDataChange ||
change.Value?.KindCase != MxValue.KindOneofCase.Int32Value ||
(change.Quality & 0xc0) != 0xc0)
{
continue;
}
Assert.Equal(sessionId, change.SessionId);
Assert.Equal(serverHandle, change.ServerHandle);
Assert.Equal(itemHandle, change.ItemHandle);
Assert.NotNull(change.SourceTimestamp);
Assert.NotNull(change.WorkerTimestamp);
Assert.Contains(change.Statuses, status => status.Category == MxStatusCategory.Ok);
if (first is not null && change.Value.Int32Value != first.Value.Int32Value)
{
output.WriteLine(
$"{itemDefinition} delivered through gateway: {first.Value.Int32Value} -> {change.Value.Int32Value}, " +
$"worker sequences {first.WorkerSequence} -> {change.WorkerSequence}, quality {change.Quality}.");
observedChange = true;
break;
}
first ??= change;
}
Assert.True(observedChange, "The gateway did not deliver distinct TestChangingInt values.");
await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-unadvise",
Command = new MxCommand
{
Kind = MxCommandKind.UnAdvise,
UnAdvise = new UnAdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
},
}).ConfigureAwait(false);
// A callback queued before UnAdvise may still cross the pipe.
// Observe the same stream and reject updates generated after acknowledgement.
DateTime unadvisedAt = DateTime.UtcNow;
using CancellationTokenSource quiet = CancellationTokenSource.CreateLinkedTokenSource(token);
quiet.CancelAfter(TimeSpan.FromSeconds(2));
try
{
while (await stream.ResponseStream.MoveNext(quiet.Token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.False(change.Family == MxEventFamily.OnDataChange &&
change.ServerHandle == serverHandle && change.ItemHandle == itemHandle &&
change.WorkerTimestamp.ToDateTime() >= unadvisedAt,
"The worker sent a new data change after UnAdvise completed.");
}
Assert.Fail("The event stream ended before the quiet period elapsed.");
}
catch (RpcException exception) when (exception.StatusCode == StatusCode.Cancelled && quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
catch (OperationCanceledException) when (quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
}
}
finally
{
CloseSessionReply closed = await client.CloseSessionAsync(
new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
}
}
/// Verifies committed Rust worker writes and their callbacks through the gateway.
/// A task that represents the live test.
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_WritesAndReceivesOnWriteComplete()
{
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "rust-worker-live-write" },
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
MxCommandRequest? restore = null;
try
{
async Task InvokeAsync(MxCommandRequest request)
{
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
.ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
$"{request.Command.Kind}: {reply.ProtocolStatus.Code} HRESULT={reply.Hresult} {reply.DiagnosticMessage}");
return reply;
}
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-write-worker-info",
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
}).ConfigureAwait(false);
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
int serverHandle = registration.Register.ServerHandle;
MxCommandReply added = await InvokeAsync(CreateAddItemRequest(sessionId, serverHandle)).ConfigureAwait(false);
int itemHandle = added.AddItem.ItemHandle;
using AsyncServerStreamingCall events = client.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
cancellationToken: token);
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
await InvokeAsync(CreateAdviseSupervisoryRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
bool IsTarget(MxEvent update) => update.SessionId == sessionId &&
update.ServerHandle == serverHandle && update.ItemHandle == itemHandle;
bool IsGoodInt(MxEvent update) => IsTarget(update) && update.Family == MxEventFamily.OnDataChange &&
update.Value?.KindCase == MxValue.KindOneofCase.Int32Value && (update.Quality & 0xc0) == 0xc0;
MxEvent? initial = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
if (IsGoodInt(events.ResponseStream.Current))
{
initial = events.ResponseStream.Current;
break;
}
}
Assert.NotNull(initial);
int original = initial.Value.Int32Value;
int written = original == 1 ? 2 : 1;
async Task CheckWriteAsync(MxCommandRequest request, int expected, DateTime? expectedSource = null)
{
MxCommandReply reply = await InvokeAsync(request).ConfigureAwait(false);
Assert.Equal(request.Command.Kind, reply.Kind);
Assert.Equal(0, reply.Hresult);
if (request.Command.Kind == MxCommandKind.WriteBulk)
{
Assert.Collection(reply.WriteBulk.Results,
unsupported =>
{
Assert.Equal(itemHandle, unsupported.ItemHandle);
Assert.False(unsupported.WasSuccessful);
Assert.NotEmpty(unsupported.ErrorMessage);
},
accepted =>
{
Assert.Equal(itemHandle, accepted.ItemHandle);
Assert.True(accepted.WasSuccessful, accepted.ErrorMessage);
});
}
else
{
Assert.NotEmpty(reply.Statuses);
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
MxEvent? completion = null;
MxEvent? changed = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent update = events.ResponseStream.Current;
if (IsTarget(update) && update.Family == MxEventFamily.OnWriteComplete)
{
completion = update;
}
if (IsGoodInt(update) && update.Value.Int32Value == expected)
{
changed = update;
}
if (completion is not null && changed is not null)
{
break;
}
}
Assert.NotNull(completion);
Assert.NotNull(changed);
Assert.NotEmpty(completion.Statuses);
Assert.All(completion.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
if (request.Command.Kind != MxCommandKind.WriteBulk)
{
Assert.Equal(reply.Statuses.ToArray(), completion.Statuses.ToArray());
}
if (expectedSource is DateTime timestamp)
{
Assert.Equal(timestamp, changed.SourceTimestamp?.ToDateTime());
}
output.WriteLine(
$"{request.Command.Kind} committed {IntegrationTestEnvironment.LiveMxAccessItem}={expected}; " +
$"OnWriteComplete sequence={completion.WorkerSequence}, readback sequence={changed.WorkerSequence}.");
}
restore = CreateWriteRequest(sessionId, serverHandle, itemHandle, initial.Value);
MxCommandRequest write = CreateWriteRequest(sessionId, serverHandle, itemHandle,
new MxValue { DataType = MxDataType.Integer, Int32Value = written });
write.ClientCorrelationId = "rust-write-round-trip";
await CheckWriteAsync(write, written).ConfigureAwait(false);
int bulkValue = original == 3 ? 4 : 3;
var bulk = new WriteBulkCommand { ServerHandle = serverHandle };
bulk.Entries.Add(new WriteBulkEntry
{
ItemHandle = itemHandle,
Value = new MxValue { RawValue = Google.Protobuf.ByteString.CopyFrom(new byte[] { 0 }) },
});
bulk.Entries.Add(new WriteBulkEntry
{
ItemHandle = itemHandle,
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = bulkValue },
});
await CheckWriteAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-write-bulk-round-trip",
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = bulk },
}, bulkValue).ConfigureAwait(false);
DateTime timestamp = DateTime.UtcNow.AddSeconds(-30);
timestamp = new DateTime(timestamp.Ticks - timestamp.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc);
MxCommandRequest write2 = CreateWrite2Request(sessionId, serverHandle, itemHandle, timestamp);
write2.Command.Write2.Value = initial.Value;
await CheckWriteAsync(write2, original, timestamp).ConfigureAwait(false);
restore = null;
// The test item is unprotected. Exercise both secured native entry
// points with its current value and surface provider refusal as HRESULT.
foreach (MxCommand command in new MxCommand[]
{
new() { Kind = MxCommandKind.WriteSecured, WriteSecured = new WriteSecuredCommand
{
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
} },
new() { Kind = MxCommandKind.WriteSecured2, WriteSecured2 = new WriteSecured2Command
{
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
TimestampValue = write2.Command.Write2.TimestampValue,
} },
})
{
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
{
SessionId = sessionId, ClientCorrelationId = $"rust-{command.Kind}", Command = command,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(command.Kind, reply.Kind);
Assert.True(reply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"{command.Kind}: {reply.ProtocolStatus.Code} {reply.DiagnosticMessage}");
if (reply.ProtocolStatus.Code == ProtocolStatusCode.MxaccessFailure)
{
Assert.True(reply.HasHresult && reply.Hresult < 0);
}
output.WriteLine($"{command.Kind} returned {reply.ProtocolStatus.Code}, HRESULT=0x{reply.Hresult:X8}.");
}
}
finally
{
try
{
if (restore is not null)
{
MxCommandReply restored = await client.InvokeAsync(restore, deadline: DateTime.UtcNow.AddSeconds(10))
.ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, restored.ProtocolStatus.Code);
Assert.NotEmpty(restored.Statuses);
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
}
finally
{
CloseSessionReply closed = await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
}
}
}
/// Exercises bulk operations and both snapshot read paths over gRPC.
/// A task that represents the live test.
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_BulkItemsAndReads()
{
const string first = "TestMachine_001.TestChangingInt";
const string second = "TestMachine_002.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(60));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-bulk-items",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task InvokeAsync(MxCommand command)
{
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
{
SessionId = id, ClientCorrelationId = Guid.NewGuid().ToString("N"), Command = command,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
return reply;
}
int server = (await InvokeAsync(CreateRegisterRequest(id).Command).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply extended = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItem2,
AddItem2 = new AddItem2Command { ServerHandle = server, ItemDefinition = "TestChangingInt", ItemContext = "TestMachine_003" },
}).ConfigureAwait(false);
Assert.True(extended.AddItem2.ItemHandle > 0);
MxCommandReply subscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeBulk,
SubscribeBulk = new SubscribeBulkCommand { ServerHandle = server, TagAddresses = { first } },
}).ConfigureAwait(false);
SubscribeResult subscription = Assert.Single(subscribed.SubscribeBulk.Results);
Assert.True(subscription.WasSuccessful, subscription.ErrorMessage);
// Observe the permanent advice before checking the worker's value cache.
using (AsyncServerStreamingCall events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true },
cancellationToken: token))
{
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = events.ResponseStream.Current;
if (change.Family == MxEventFamily.OnDataChange && change.ItemHandle == subscription.ItemHandle)
{
break;
}
}
}
MxCommandReply read = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 3000, TagAddresses = { first, second, second } },
}).ConfigureAwait(false);
Assert.Collection(read.ReadBulk.Results,
cached => { Assert.True(cached.WasSuccessful, cached.ErrorMessage); Assert.True(cached.WasCached); },
snapshot => { Assert.True(snapshot.WasSuccessful, snapshot.ErrorMessage); Assert.False(snapshot.WasCached); },
duplicate =>
{
Assert.True(duplicate.WasSuccessful, duplicate.ErrorMessage);
Assert.False(duplicate.WasCached);
Assert.Equal(read.ReadBulk.Results[1].ItemHandle, duplicate.ItemHandle);
});
MxCommandReply added = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItemBulk,
AddItemBulk = new AddItemBulkCommand { ServerHandle = server, TagAddresses = { second } },
}).ConfigureAwait(false);
SubscribeResult item = Assert.Single(added.AddItemBulk.Results);
Assert.True(item.WasSuccessful, item.ErrorMessage);
MxCommandReply advised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AdviseItemBulk,
AdviseItemBulk = new AdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(advised.AdviseItemBulk.Results).WasSuccessful);
MxCommandReply unadvised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnAdviseItemBulk,
UnAdviseItemBulk = new UnAdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unadvised.UnAdviseItemBulk.Results).WasSuccessful);
MxCommandReply removed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.RemoveItemBulk,
RemoveItemBulk = new RemoveItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle, extended.AddItem2.ItemHandle } },
}).ConfigureAwait(false);
Assert.All(removed.RemoveItemBulk.Results, result => Assert.True(result.WasSuccessful, result.ErrorMessage));
MxCommandReply unsubscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeBulk,
UnsubscribeBulk = new UnsubscribeBulkCommand { ServerHandle = server, ItemHandles = { subscription.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unsubscribed.UnsubscribeBulk.Results).WasSuccessful);
output.WriteLine("Bulk add, advise, cached and snapshot reads, unadvise, remove, subscribe, and unsubscribe passed.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
/// Verifies the Rust worker’s native subtag alarm provider and standby replacement.
/// A task that represents the live test.
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
{
const string reference = "Galaxy!DEV.TestMachine_001.RustWorkerSmoke";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(75));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-alarms",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task AlarmAsync(MxCommand command)
{
WorkerCommandReply envelope = await fixture.Manager.InvokeAsync(id,
new WorkerCommand { Command = command }, token).ConfigureAwait(false);
Assert.Equal(command.Kind, envelope.Reply.Kind);
return envelope.Reply;
}
using AsyncServerStreamingCall events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true }, cancellationToken: token);
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
MxCommandReply subscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
ForcedMode = AlarmProviderMode.Subtag,
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(subscribed.ProtocolStatus.Code == ProtocolStatusCode.Ok, subscribed.DiagnosticMessage);
MxEvent? transition = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
if (events.ResponseStream.Current.OnAlarmTransition?.AlarmFullReference == reference)
{
transition = events.ResponseStream.Current;
break;
}
}
Assert.NotNull(transition);
Assert.Equal(id, transition.SessionId);
Assert.True(transition.WorkerSequence > 0);
Assert.True(transition.OnAlarmTransition.Degraded);
output.WriteLine($"Forwarded {transition.OnAlarmTransition.TransitionKind} at worker sequence {transition.WorkerSequence}.");
MxCommandReply snapshot = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.QueryActiveAlarms,
QueryActiveAlarmsCommand = new QueryActiveAlarmsCommand { AlarmFilterPrefix = reference },
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, snapshot.ProtocolStatus.Code);
Assert.Contains(snapshot.QueryActiveAlarms.Snapshots, alarm => alarm.AlarmFullReference == reference);
MxCommandReply unknownAck = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarm,
AcknowledgeAlarmCommand = new AcknowledgeAlarmCommand { AlarmGuid = Guid.NewGuid().ToString() },
}).ConfigureAwait(false);
Assert.NotNull(unknownAck.AcknowledgeAlarm);
Assert.NotEqual(0, unknownAck.AcknowledgeAlarm.NativeStatus);
MxCommandReply unknownName = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarmByName,
AcknowledgeAlarmByNameCommand = new AcknowledgeAlarmByNameCommand
{
AlarmName = "RustWorkerMissingAlarm", ProviderName = "Galaxy", GroupName = "DEV",
},
}).ConfigureAwait(false);
Assert.NotNull(unknownName.AcknowledgeAlarm);
Assert.NotEqual(0, unknownName.AcknowledgeAlarm.NativeStatus);
MxCommandReply update = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, update.ProtocolStatus.Code);
MxCommandReply unsubscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeAlarms,
UnsubscribeAlarms = new UnsubscribeAlarmsCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, unsubscribed.ProtocolStatus.Code);
MxCommandReply fallback = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
FallbackEnabled = true,
Failover = new AlarmFailoverConfig
{
ConsecutiveFailureThreshold = 1, FailbackProbeIntervalSeconds = 30, FailbackStableProbes = 3,
},
},
}).ConfigureAwait(false);
Assert.True(fallback.ProtocolStatus.Code == ProtocolStatusCode.Ok, fallback.DiagnosticMessage);
MxCommandReply replacement = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
{
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(replacement.ProtocolStatus.Code == ProtocolStatusCode.Ok, replacement.DiagnosticMessage);
output.WriteLine("Updated the live alarm provider's fallback watch list.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
/// Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.
/// A task that represents the live test.
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
string user = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER") ?? "Administrator";
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
{
Enabled = true,
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
});
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
using AsyncServerStreamingCall feed = client.StreamAlarms(
new StreamAlarmsRequest { AlarmFilterPrefix = reference }, cancellationToken: token);
bool snapshotComplete = false;
bool alarmmgrReady = false;
while ((!snapshotComplete || !alarmmgrReady) && await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
AlarmFeedMessage frame = feed.ResponseStream.Current;
snapshotComplete |= frame.SnapshotComplete;
alarmmgrReady |= frame.ProviderStatus?.Mode == AlarmProviderMode.Alarmmgr;
}
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
async Task InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await client.InvokeAsync(request, deadline: DateTime.UtcNow.AddSeconds(15),
cancellationToken: cancellationToken).ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
$"{reply.Kind} returned {reply.ProtocolStatus.Code}: {reply.DiagnosticMessage}");
return reply;
}
async Task WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
{
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
OnAlarmTransitionEvent? transition = feed.ResponseStream.Current.Transition;
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
&& (ackComment is null || transition.OperatorComment == ackComment))
{
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
Assert.False(transition.Degraded);
return transition;
}
}
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
}
MxCommandRequest? restore = null;
try
{
int server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply read = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
Command = new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 5000, TagAddresses = { tag } },
},
}, token).ConfigureAwait(false);
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
Assert.True(original.WasSuccessful, original.ErrorMessage);
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
MxCommandRequest WriteAlarm(bool active)
{
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
request.Command.WriteSecured.Value = new MxValue { DataType = MxDataType.Boolean, BoolValue = active };
return request;
}
async Task WriteConfirmedAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await InvokeAsync(request, cancellationToken).ConfigureAwait(false);
Assert.NotEmpty(reply.Statuses);
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
// The dev rig’s test UDA is normally false. Clear an existing test episode before raising a fresh one.
if (original.Value.BoolValue)
{
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
restore = WriteAlarm(true);
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
}
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference,
OperatorUser = $"mxgw-smoke-{Guid.NewGuid():N}",
Comment = "This acknowledgement must be rejected",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, rejected.ProtocolStatus.Code);
Assert.True(rejected.HasHresult && rejected.Hresult != 0);
AcknowledgeAlarmReply ack = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference, OperatorUser = user, OperatorPassword = password, Comment = comment,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.True(ack.ProtocolStatus.Code == ProtocolStatusCode.Ok, ack.DiagnosticMessage);
Assert.True(ack.HasHresult);
Assert.Equal(0, ack.Hresult);
Assert.DoesNotContain("unconfirmed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("not observed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
OnAlarmTransitionEvent acknowledged = await WaitForTransitionAsync(AlarmTransitionKind.Acknowledge, comment)
.ConfigureAwait(false);
Assert.Equal(user, acknowledged.OperatorUser, ignoreCase: true);
output.WriteLine($"Alarmmgr streamed Raise and Acknowledge with operator {acknowledged.OperatorUser} and the requested comment.");
if (!original.Value.BoolValue)
{
await WriteConfirmedAsync(restore, token).ConfigureAwait(false);
restore = null;
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
output.WriteLine("The gateway streamed Clear after the test UDA was restored.");
}
}
finally
{
try
{
if (restore is not null)
{
MxCommandReply restored = await InvokeAsync(restore, CancellationToken.None).ConfigureAwait(false);
Assert.NotEmpty(restored.Statuses);
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
}
finally
{
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
}
}
}
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service, GatewayAlarmMonitor? monitor = null)
{
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
builder.WebHost.ConfigureKestrel(options =>
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
builder.Services.AddGrpc();
builder.Services.AddSingleton(service);
if (monitor is not null)
{
builder.Services.AddSingleton(monitor);
}
WebApplication app = builder.Build();
app.MapGrpcService();
return app;
}
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
public RustWorkerLiveFactAttribute()
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
{
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
}
}
}
///
/// Verifies that a Write command round-trips through live MXAccess against an advised item
/// and that the worker emits a matching event
/// — the proof of round-trip the cross-language client e2e runner relies on.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_WritesValueToAdvisedItem()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write",
ClientCorrelationId = "live-open-write",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
MxCommandReply writeReply = await fixture.Service.Invoke(
CreateWriteRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write", writeReply);
// Happy-path Write: the worker COM call succeeded so HResultConverter
// produces ProtocolStatusCode.Ok. An MXAccess rejection (a write to a
// bad item, a secured-item failure) would surface as
// ProtocolStatusCode.MxaccessFailure with a non-zero hresult — never
// as an RpcException / transport fault, because the command still
// completed its round-trip to the worker and back.
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Write, writeReply.Kind);
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
// after the underlying provider acknowledges the write — that is
// the event the cross-language client e2e runner asserts on. We
// scan the recorded stream (so an interleaving OnDataChange does
// not preempt the match) for an OnWriteComplete carrying the same
// server/item handles the Write command targeted.
MxEvent writeComplete = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnWriteComplete
&& candidate.ServerHandle == registerReply.Register.ServerHandle
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writeComplete);
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
Assert.Equal(sessionId, writeComplete.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
// The stream task must not be in a faulted state. ShutDownAsync's
// broad catch would otherwise swallow the fault and silently let
// this Write-parity coverage pass against a broken event pipeline.
Assert.False(
streamTask.IsFaulted,
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
}
finally
{
// Cancel the stream call before draining so StreamEvents observes
// cancellation rather than blocking on the channel. Any unhandled
// stream-task fault is rethrown from ShutDownAsync into the test.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
}
}
///
/// Verifies that a Write2 command (the B8 extended write carrying a timestamp
/// value as its second context parameter) round-trips through live MXAccess against an
/// advised item and raises a matching event.
/// A regression that short-circuits Write2 to InvalidRequest (or drops the
/// timestamp) fails here, which no other live test catches (IntegrationTests-033, #212).
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_Write2ValueWithTimestampToAdvisedItem()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
MxValue? originalValue = null;
bool restored = false;
int serverHandle = 0;
int itemHandle = 0;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write2",
ClientCorrelationId = "live-open-write2",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
serverHandle = registerReply.Register.ServerHandle;
itemHandle = addItemReply.AddItem.ItemHandle;
// Supervisory, not plain, advise: MXAccess refuses a user-id-0 write on a
// plainly advised item with SecurityError/1008. Proven on the DEV gateway
// (wonder-app-vd03, 2026-11-12, #212): on MESReceiver_025.MoveInBatchID, Write2
// applied the value after AdviseSupervisory and got SecurityError/1008 after Advise.
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseSupervisoryRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AdviseSupervisory", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// The write below can now really apply, so remember the value to restore.
MxEvent initialValue = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(initialValue);
originalValue = initialValue.Value;
// A whole-second instant 30 s in the past, so a stored source timestamp
// can be compared exactly and cannot be confused with the write time.
DateTime sentTimestampUtc = DateTime.UtcNow.AddSeconds(-30);
sentTimestampUtc = new DateTime(
sentTimestampUtc.Ticks - (sentTimestampUtc.Ticks % TimeSpan.TicksPerSecond),
DateTimeKind.Utc);
output.WriteLine(
$"Write2 timestamp sent={sentTimestampUtc:O} box zone={TimeZoneInfo.Local.Id} offset={TimeZoneInfo.Local.GetUtcOffset(sentTimestampUtc)}");
MxCommandReply writeReply = await fixture.Service.Invoke(
CreateWrite2Request(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle,
sentTimestampUtc),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write2", writeReply);
// Ok means the worker reached the COM Write2 call and COM accepted it.
// InvalidRequest here means the executor rejected the payload
// (missing value/timestamp) before reaching MXAccess. Ok alone does
// not mean the provider applied the value; the completion status
// below carries that. (The 2026-11-12 windev run used a plain Advise
// and got SecurityError/1008 on the completion; see #212.)
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Write2, writeReply.Kind);
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
// after the underlying provider acknowledges the write — that is
// the event the cross-language client e2e runner asserts on. We
// scan the recorded stream (so an interleaving OnDataChange does
// not preempt the match) for an OnWriteComplete carrying the same
// server/item handles the Write command targeted.
MxEvent writeComplete = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnWriteComplete
&& candidate.ServerHandle == registerReply.Register.ServerHandle
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writeComplete);
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
Assert.Equal(sessionId, writeComplete.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
foreach (MxStatusProxy status in writeComplete.Statuses)
{
output.WriteLine($"Write2 completion category={status.Category} detail={status.Detail}");
}
// #228: MXAccess reads the Write2 time argument as box-local time, so
// the worker must convert the UTC instant before the COM call. When the
// provider applied the write, the source timestamp it reports must be
// the instant sent. On a UTC box this cannot fail either way, so the
// zone is logged above. A refused write (no data change) is logged,
// not asserted, as before.
bool applied = writeComplete.Statuses.Count > 0
&& writeComplete.Statuses.All(status => status.Category == MxStatusCategory.Ok);
if (applied)
{
MxEvent writtenValue = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle
&& candidate.WorkerSequence > initialValue.WorkerSequence,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writtenValue);
DateTime? storedTimestampUtc = writtenValue.SourceTimestamp?.ToDateTime();
output.WriteLine($"Write2 timestamp stored={storedTimestampUtc:O}");
Assert.Equal(sentTimestampUtc, storedTimestampUtc);
}
else
{
output.WriteLine("Write2 timestamp not checked: the completion was not Ok, so nothing was applied.");
}
// Restore the original value with a plain Write on the same supervisory advise.
MxCommandReply restoreReply = await fixture.Service.Invoke(
CreateWriteRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle,
originalValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write (restore)", restoreReply);
Assert.Equal(ProtocolStatusCode.Ok, restoreReply.ProtocolStatus.Code);
restored = true;
// The stream task must not be in a faulted state. ShutDownAsync's
// broad catch would otherwise swallow the fault and silently let
// this Write2-parity coverage pass against a broken event pipeline.
Assert.False(
streamTask.IsFaulted,
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
}
finally
{
if (!restored && originalValue is not null && sessionId is not null && itemHandle != 0)
{
// Best effort: do not leave the Write2 test value behind after a failed
// assertion. Logged, not thrown, so the original failure is not masked.
try
{
MxCommandReply restoreReply = await fixture.Service.Invoke(
CreateWriteRequest(sessionId, serverHandle, itemHandle, originalValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write (restore after failure)", restoreReply);
}
catch (Exception ex)
{
output.WriteLine($"Restore after failure threw {ex.GetType().Name}: {ex.Message}");
}
}
// Cancel the stream call before draining so StreamEvents observes
// cancellation rather than blocking on the channel. Any unhandled
// stream-task fault is rethrown from ShutDownAsync into the test.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
}
}
///
/// Verifies that an AddItem against an invalid server handle surfaces the MXAccess failure
/// without faulting the gateway transport, exercising the invalid-handle parity path.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_InvalidHandleCommand_SurfacesFailureWithoutTransportFault()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-invalid-handle",
ClientCorrelationId = "live-open-invalid",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
// Deliberately skip Register: server handle 0x7FFFFFFF was never
// issued by MXAccess. The worker must invoke COM and relay the
// invalid-handle failure rather than the gateway short-circuiting.
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle: int.MaxValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem(invalid-handle)", addItemReply);
// MXAccess parity: an invalid handle is an MXAccess-level failure.
// The command still completed its worker round-trip, so the gateway
// must reply with ProtocolStatusCode.MxaccessFailure and a non-zero
// hresult carrying the COM failure (per HResultConverter) — never a
// gRPC transport fault. The assertion below just checks the status
// is not Ok; the failure detail lives in hresult / the status proxies.
Assert.NotEqual(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(
addItemReply.AddItem is null || addItemReply.AddItem.ItemHandle <= 0,
"Invalid-handle AddItem must not yield a usable item handle.");
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
///
/// Verifies the MXAccess teardown chain: Unadvise then RemoveItem then Unregister
/// each return , and the worker stops emitting
/// OnDataChange events for the un-advised item. Exercises the lifecycle-ordering
/// parity CLAUDE.md singles out as a "do not synthesize" rule.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_UnadviseRemoveItemUnregister_TeardownOrderingParity()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-teardown",
ClientCorrelationId = "live-open-teardown",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// Wait for an OnDataChange to prove the subscription is live before tearing it down.
MxEvent firstDataChange = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ServerHandle == serverHandle
&& candidate.ItemHandle == itemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(firstDataChange);
// 1) UnAdvise — must reply Ok; the worker must stop emitting OnDataChange
// for this (server, item) pair after this returns.
MxCommandReply unadviseReply = await fixture.Service.Invoke(
CreateUnAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("UnAdvise", unadviseReply);
Assert.Equal(ProtocolStatusCode.Ok, unadviseReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.UnAdvise, unadviseReply.Kind);
// 2) RemoveItem — must reply Ok against the same handles.
MxCommandReply removeItemReply = await fixture.Service.Invoke(
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("RemoveItem", removeItemReply);
Assert.Equal(ProtocolStatusCode.Ok, removeItemReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.RemoveItem, removeItemReply.Kind);
// 3) Unregister — closes the client session inside the worker.
MxCommandReply unregisterReply = await fixture.Service.Invoke(
CreateUnregisterRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Unregister", unregisterReply);
Assert.Equal(ProtocolStatusCode.Ok, unregisterReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Unregister, unregisterReply.Kind);
// Parity rule: after UnAdvise returns Ok the worker must stop emitting
// OnDataChange for this (server, item) pair. Events the provider already
// published before that ack are in-flight and not a regression — the rule
// only constrains events generated AFTER the teardown returned. So the
// "before" baseline is taken *after* a first settle window drains those
// in-flight events, not before UnAdvise was issued (which races against
// the round-trip + STA dispatch + pipe send window).
//
// RecordingServerStreamWriter.Messages returns a snapshot copy under its
// own lock, so iterating after each settle window is safe without external
// sync.
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
int dataChangeCountAfterFirstSettle = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == itemHandle);
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
int dataChangeCountAfterSecondSettle = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == itemHandle);
output.WriteLine(
$"DataChange count after first settle={dataChangeCountAfterFirstSettle} after second settle={dataChangeCountAfterSecondSettle}");
Assert.Equal(dataChangeCountAfterFirstSettle, dataChangeCountAfterSecondSettle);
// A RemoveItem against the just-freed item handle must not silently succeed —
// the worker has to relay MXAccess's invalid-handle response. Closing the
// session is enough for parity, but we sanity-check that re-using the freed
// pair does not accidentally appear Ok.
MxCommandReply secondRemoveItemReply = await fixture.Service.Invoke(
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("RemoveItem(stale)", secondRemoveItemReply);
Assert.NotEqual(ProtocolStatusCode.Ok, secondRemoveItemReply.ProtocolStatus.Code);
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
///
/// Verifies the MXAccess WriteSecured path: AuthenticateUser resolves a
/// user id, then WriteSecured against the advised item completes its round-trip
/// to the worker and back. CLAUDE.md singles out WriteSecured ordering as a
/// parity surface the gateway must not "fix" — the test asserts the reply kind and
/// protocol status, not a fabricated outcome.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_WriteSecured_AuthenticatedRoundTripParity()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
// CLAUDE.md's credential-redaction rule covers every log
// surface the test sees, not just the reply's DiagnosticMessage. Wire a buffering
// wrapper around output and route the worker stdout/stderr echo and the gateway
// ILogger sink through it so the post-run assertion covers the accumulated test
// output. A regression that logged the request body, the WorkerCommandRequest
// envelope, or printed the credential from inside the worker is caught here
// even if the bare DiagnosticMessage check still passes.
RecordingTestOutputHelper recordedOutput = new(output);
TestWorkerProcessFactory processFactory = new(recordedOutput);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
// Stream events so a regression that emitted an OperationComplete or
// OnWriteComplete with wrong handles would still be observable via the test
// output (we don't assert a specific event here — the docs note successful
// writes raise only OnWriteComplete, but WriteSecured against an unprotected
// item commonly fails with 0x80004021 in this provider, which raises no event).
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write-secured",
ClientCorrelationId = "live-open-write-secured",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// AuthenticateUser resolves an ArchestrA user id for the WriteSecured call.
// Credentials are env-overridable so the test honors the gateway's "do not
// log secrets" rule and works against either MXAccess's own user store or
// the LmxOpcUa-baseline GLAuth-bridged ArchestrA identity (admin/password —
// see glauth.md; the shared GLAuth's dev-user password is "password", not
// the old per-box "admin123").
MxCommandReply authReply = await fixture.Service.Invoke(
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
new TestServerCallContext()).ConfigureAwait(false);
// AuthenticateUser is allowed to fail — MXAccess parity means a rejected
// credential is a legitimate reply, not a bug, so this must never be an
// Assert. But a silent fall-through to "operator only" (currentUserId 0)
// quietly demotes the rest of this test from an authenticated round-trip
// to a bare failure-reply-shape check, which a stale rig credential (e.g.
// one that has expired) would otherwise hide indefinitely in nightly runs. Log the
// outcome loudly — never the password (verifyUser only) — so that
// degradation is visible in the test output even though it never fails
// the test by itself.
bool authenticateUserSucceeded = authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok
&& authReply.AuthenticateUser is not null
&& authReply.AuthenticateUser.UserId != 0;
recordedOutput.WriteLine(
authenticateUserSucceeded
? $"AuthenticateUser SUCCEEDED for user={verifyUser} user_id={authReply.AuthenticateUser!.UserId}"
: $"AuthenticateUser FAILED (credential rejected or not recognized by the provider) "
+ $"for user={verifyUser} status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} "
+ $"diagnostic={authReply.DiagnosticMessage} -- WriteSecured below will run as an "
+ $"unauthenticated (currentUserId=0) failure-reply-shape check only, not an "
+ $"authenticated round-trip. If this is unexpected, verify "
+ $"MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER/_PASSWORD against the rig's ArchestrA "
+ $"identity (see glauth.md).");
int currentUserId = authenticateUserSucceeded ? authReply.AuthenticateUser!.UserId : 0;
MxCommandReply writeSecuredReply = await fixture.Service.Invoke(
CreateWriteSecuredRequest(
sessionId,
serverHandle,
itemHandle,
currentUserId,
verifierUserId: 0),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "WriteSecured", writeSecuredReply);
// Parity: the command itself completed its round-trip — the reply kind is
// WriteSecured and the gateway protocol status is set. The MXAccess outcome
// (Ok for an unprotected provider, MxaccessFailure with hresult 0x80004021
// when the item is not WriteSecured-eligible) lives in protocol_status +
// hresult, never as a transport fault. The diagnostic message must never
// contain the credential.
Assert.Equal(MxCommandKind.WriteSecured, writeSecuredReply.Kind);
Assert.True(
writeSecuredReply.ProtocolStatus.Code is ProtocolStatusCode.Ok
or ProtocolStatusCode.MxaccessFailure,
$"Unexpected WriteSecured protocol status {writeSecuredReply.ProtocolStatus.Code}.");
Assert.DoesNotContain(verifyPassword, writeSecuredReply.DiagnosticMessage ?? string.Empty, StringComparison.Ordinal);
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
// CLAUDE.md credential contract: passwords and WriteSecured payloads must never
// reach logs. The buffered output covers the gateway ILogger sink, worker
// stdout/stderr, and every direct WriteLine the test body issued. A regression
// that dumped the request envelope, the AuthenticateUserCommand body, or any
// command-level WriteSecured payload would land here and trip this assertion.
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
}
///
/// B8 live verification of the COM commands the B-bundle added against a fake
/// IMxAccessServer: AuthenticateUser, ArchestrAUserToId, Suspend,
/// and Activate. The contract being proven is that each command round-trips
/// to the worker and back carrying a real MXAccess outcome (Ok / an MxStatusProxy /
/// a non-zero HResult) and is NOT short-circuited to INVALID_REQUEST the way an
/// unimplemented command would be. MXAccess-level rejections (a wrong item class for
/// Suspend/Activate commonly returns 0x80070057) are parity, not test failures — we
/// assert the reply kind plus a non-INVALID_REQUEST protocol status, and log the
/// HResult for the record.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_NewComCommands_RoundTripWithRealReplies()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
// Credential-redaction: AuthenticateUser carries a password. Route every test
// log surface through the buffering helper so the post-run assertion proves the
// password never reached the gateway logger, worker stdout/stderr, or any
// WriteLine the test body issued (same pattern as the WriteSecured parity test).
RecordingTestOutputHelper recordedOutput = new(output);
TestWorkerProcessFactory processFactory = new(recordedOutput);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
string? sessionId = null;
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-new-com-commands",
ClientCorrelationId = "live-open-new-com",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
// AuthenticateUser — the B-bundle command under live verification. Before the
// B-bundle this command was unimplemented and the worker short-circuited it to
// INVALID_REQUEST. It must now produce a real reply (Ok with a user id when the
// provider accepts the credential, or a real MXAccess HResult when it does not).
MxCommandReply authReply = await fixture.Service.Invoke(
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
new TestServerCallContext()).ConfigureAwait(false);
recordedOutput.WriteLine(
$"AuthenticateUser status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} user_id={authReply.AuthenticateUser?.UserId}");
Assert.Equal(MxCommandKind.AuthenticateUser, authReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, authReply.ProtocolStatus.Code);
Assert.True(
authReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"AuthenticateUser must surface a real MXAccess outcome, got {authReply.ProtocolStatus.Code}.");
int authenticatedUserId =
authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok && authReply.AuthenticateUser is not null
? authReply.AuthenticateUser.UserId
: 0;
if (authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
{
// On the dev rig AuthenticateUser("Administrator","") resolves to user id 1.
// Don't pin the exact value (provider/user-store dependent) — just prove a
// success carried a usable, non-zero ArchestrA user id through the reply.
Assert.NotEqual(0, authenticatedUserId);
}
// ArchestrAUserToId — resolves an ArchestrA user GUID to an integer user id.
// We feed an empty/placeholder GUID: the value is provider-dependent, so the
// assertion is the parity one (real reply, never INVALID_REQUEST). A non-zero
// HResult here is the expected MXAccess rejection of an unknown GUID.
MxCommandReply userToIdReply = await fixture.Service.Invoke(
CreateArchestrAUserToIdRequest(sessionId, serverHandle, userIdGuid: string.Empty),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "ArchestrAUserToId", userToIdReply);
recordedOutput.WriteLine($"ArchestrAUserToId user_id={userToIdReply.ArchestraUserToId?.UserId}");
Assert.Equal(MxCommandKind.ArchestraUserToId, userToIdReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, userToIdReply.ProtocolStatus.Code);
Assert.True(
userToIdReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"ArchestrAUserToId must surface a real MXAccess outcome, got {userToIdReply.ProtocolStatus.Code}.");
if (userToIdReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
{
// On the dev rig ArchestrAUserToId with a valid GUID resolves to user_id=1.
// Don't pin the exact id (provider-dependent) — just prove the Ok path carried
// a usable non-zero ArchestrA user id through the reply payload.
Assert.NotNull(userToIdReply.ArchestraUserToId);
Assert.NotEqual(0, userToIdReply.ArchestraUserToId.UserId);
}
// Suspend / Activate against the added-but-not-advised item (no Advise was issued
// between AddItem and this call). The dev-rig TestInt item class may not be
// suspendable (MXAccess returns 0x80070057 / E_INVALIDARG for a wrong item class
// — see B8 notes). That is MXAccess parity: assert the reply kind and a
// non-INVALID_REQUEST status, surface the HResult and MxStatusProxy for the
// record, and do NOT treat a provider-side rejection as a test failure.
MxCommandReply suspendReply = await fixture.Service.Invoke(
CreateSuspendRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Suspend", suspendReply);
recordedOutput.WriteLine(
$"Suspend status_proxy success={suspendReply.Suspend?.Status?.Success} hresult=0x{(uint)suspendReply.Hresult:X8}");
Assert.Equal(MxCommandKind.Suspend, suspendReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, suspendReply.ProtocolStatus.Code);
Assert.True(
suspendReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"Suspend must surface a real MXAccess outcome, got {suspendReply.ProtocolStatus.Code}.");
MxCommandReply activateReply = await fixture.Service.Invoke(
CreateActivateRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Activate", activateReply);
recordedOutput.WriteLine(
$"Activate status_proxy success={activateReply.Activate?.Status?.Success} hresult=0x{(uint)activateReply.Hresult:X8}");
Assert.Equal(MxCommandKind.Activate, activateReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, activateReply.ProtocolStatus.Code);
Assert.True(
activateReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"Activate must surface a real MXAccess outcome, got {activateReply.ProtocolStatus.Code}.");
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
// Credential contract: the AuthenticateUser password must never reach any log
// surface (gateway logger, worker stdout/stderr, or test WriteLine).
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
}
///
/// B8 §3.2 buffered-data path: adds a BUFFERED item (AddBufferedItem), sets the
/// buffered update interval (SetBufferedUpdateInterval), advises it, then attempts
/// to observe an event carrying multiple
/// samples so the worker's multi-sample conversion (VariantConverter →
/// OnBufferedDataChangeEvent quality/timestamp arrays) can be validated live.
///
/// The AddBufferedItem + SetBufferedUpdateInterval round-trips are asserted unconditionally
/// (they are the B-bundle commands under verification). The buffered EVENT capture is
/// best-effort: if the rig's object logic does not drive a buffered batch within the live
/// event timeout (the same environmental limitation seen with the externally-undrivable
/// alarm rig), the test records the buffered conversion as an unverified residual rather
/// than failing — the command path is proven, the live multi-sample conversion is not.
/// When a batch IS captured, the converted value and quality/timestamp arrays are asserted
/// to be non-empty and internally consistent (no crash, no dropped payload).
///
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_BufferedItem_AddsSetsIntervalAndAttemptsCapture()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
// AddBufferedItem takes (item_definition, item_context) like AddItem2. The dev rig
// exposes TestChildObject.TestInt; the buffered form is item="TestInt",
// context="TestChildObject" (per B8 notes). Split the configured live item so a
// custom MXGATEWAY_LIVE_MXACCESS_ITEM override still works.
(string bufferedItem, string bufferedContext) = SplitLiveItemForBuffered(IntegrationTestEnvironment.LiveMxAccessItem);
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-buffered",
ClientCorrelationId = "live-open-buffered",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
// SetBufferedUpdateInterval first so the buffered cadence is established before
// the item is added/advised. MXAccess rounds to 100ms units and rejects < 1.
MxCommandReply intervalReply = await fixture.Service.Invoke(
CreateSetBufferedUpdateIntervalRequest(sessionId, serverHandle, updateIntervalMilliseconds: 1000),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("SetBufferedUpdateInterval", intervalReply);
Assert.Equal(MxCommandKind.SetBufferedUpdateInterval, intervalReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, intervalReply.ProtocolStatus.Code);
Assert.Equal(ProtocolStatusCode.Ok, intervalReply.ProtocolStatus.Code);
// AddBufferedItem — must return a real item handle (the dev rig yields handle 1).
MxCommandReply addBufferedReply = await fixture.Service.Invoke(
CreateAddBufferedItemRequest(sessionId, serverHandle, bufferedItem, bufferedContext),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddBufferedItem", addBufferedReply);
Assert.Equal(MxCommandKind.AddBufferedItem, addBufferedReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, addBufferedReply.ProtocolStatus.Code);
Assert.Equal(ProtocolStatusCode.Ok, addBufferedReply.ProtocolStatus.Code);
Assert.NotNull(addBufferedReply.AddBufferedItem);
int bufferedItemHandle = addBufferedReply.AddBufferedItem.ItemHandle;
Assert.True(bufferedItemHandle > 0, "AddBufferedItem must yield a usable item handle.");
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, bufferedItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise(buffered)", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// Best-effort capture of a SAMPLE-BEARING buffered batch.
//
// Live observation (B8, re-read 2026-11-13): immediately after Advise the provider
// delivers one OnBufferedDataChange with data_type=NoData and zero quality and
// timestamp samples. Its status row is ConfigurationError: detail 6
// (InvalidReference) for the (item, context) form used here, and detail 1002
// (InvalidPropertyId) for a full reference to an attribute without a buffer. It is
// a resolution failure, not a bootstrap. The DEV Galaxy has no buffered attribute,
// so no sample-bearing batch can arrive (docs/LiveCaptures-2026-11-13.md §5). The
// converter handling it without crashing is still worth keeping. We match only a
// batch that carries samples, so the test validates a multi-sample conversion if
// the item is pointed at a buffered attribute.
MxEvent? bufferedBatch = null;
try
{
bufferedBatch = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnBufferedDataChange
&& candidate.ServerHandle == serverHandle
&& candidate.ItemHandle == bufferedItemHandle
&& candidate.OnBufferedDataChange is not null
&& (CountArrayElements(candidate.OnBufferedDataChange.QualityValues) > 0
|| CountArrayElements(candidate.OnBufferedDataChange.TimestampValues) > 0),
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
}
catch (TimeoutException ex)
{
output.WriteLine($"B8: sample-bearing batch predicate timed out: {ex.Message}");
bufferedBatch = null;
}
// Whether or not a sample-bearing batch arrived, record what buffered events the
// rig DID deliver (typically just the empty NoData bootstrap) for the record.
int bootstrapBufferedEvents = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnBufferedDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == bufferedItemHandle);
if (bufferedBatch is null)
{
// RESIDUAL (documented): the command path (AddBufferedItem +
// SetBufferedUpdateInterval + Advise) is proven and the buffered EVENT plumbing
// is live (the NoData resolution-failure event arrives and converts without
// crashing), but the rig has no buffered attribute to deliver a sample-bearing
// batch (docs/LiveCaptures-2026-11-13.md §5).
// The §3.2 OnBufferedDataChange MULTI-SAMPLE conversion therefore remains
// unverified live. This is environmental, not a defect — let the test pass.
output.WriteLine(
"B8 RESIDUAL: AddBufferedItem/SetBufferedUpdateInterval/Advise round-tripped and "
+ $"{bootstrapBufferedEvents} OnBufferedDataChange event(s) arrived (empty NoData "
+ "bootstrap, converted without crash/drop), but no sample-bearing buffered batch "
+ $"was observed within {IntegrationTestEnvironment.LiveMxAccessEventTimeout}. Live "
+ "§3.2 multi-sample conversion remains unverified (rig object logic may not drive "
+ "buffered samples on demand).");
// The residual claim ("empty NoData bootstrap arrives and converts without crashing")
// is only meaningful if at least one OnBufferedDataChange event arrived. Assert that
// the buffered subscription registered and the worker's event plumbing fired at all.
Assert.True(
bootstrapBufferedEvents > 0,
"No OnBufferedDataChange event arrived at all after Advise; the buffered subscription may not have registered.");
return;
}
// A SAMPLE-BEARING buffered batch was captured — validate the §3.2 conversion.
LogEvent(bufferedBatch);
OnBufferedDataChangeEvent body = bufferedBatch.OnBufferedDataChange;
Assert.NotNull(body);
int qualityCount = CountArrayElements(body.QualityValues);
int timestampCount = CountArrayElements(body.TimestampValues);
output.WriteLine(
$"B8 CAPTURED buffered batch: data_type={body.DataType} raw_data_type={body.RawDataType} "
+ $"quality_samples={qualityCount} timestamp_samples={timestampCount} "
+ $"value_kind={bufferedBatch.Value?.KindCase}");
// The predicate guaranteed at least one sample; the converted aggregate value
// must also exist (no crash, no dropped payload).
Assert.True(
qualityCount > 0 || timestampCount > 0,
"Sample-bearing OnBufferedDataChange lost its samples after the predicate matched.");
Assert.NotNull(bufferedBatch.Value);
// When MXAccess delivers parallel quality + timestamp arrays the converted
// arrays must agree in length; a mismatch is a real conversion defect (a sample
// was dropped on one side).
if (qualityCount > 0 && timestampCount > 0)
{
Assert.Equal(qualityCount, timestampCount);
}
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
///
/// Verifies that killing the worker process marks the session
/// with a clean fault classification — the gateway
/// must observe the abnormal exit, transition the session, and surface a non-empty
/// fault description rather than hanging or crashing.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_AbnormalWorkerExit_MarksSessionFaulted()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-abnormal-exit",
ClientCorrelationId = "live-open-abnormal",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
// Kill the worker process directly. WorkerClient's read loop hits an
// end-of-stream on the named pipe and routes through SetFaulted; the
// session manager then marks the session Faulted. We avoid CloseSession
// so the transition is driven by the abnormal exit, not a graceful path.
processFactory.KillAllAndDetach();
DateTimeOffset waitDeadline = DateTimeOffset.UtcNow + StreamShutdownTimeout;
SessionState observedState = SessionState.Unspecified;
string? observedFault = null;
while (DateTimeOffset.UtcNow < waitDeadline)
{
if (fixture.TryGetSession(sessionId, out GatewaySession? session))
{
observedState = session.State;
observedFault = session.FinalFault;
if (observedState == SessionState.Faulted)
{
break;
}
}
await Task.Delay(TimeSpan.FromMilliseconds(50)).ConfigureAwait(false);
}
output.WriteLine($"AbnormalExit observed_state={observedState} fault={observedFault}");
Assert.Equal(SessionState.Faulted, observedState);
Assert.False(string.IsNullOrWhiteSpace(observedFault), "Faulted session must carry a non-empty fault description.");
// The fault classification must come from a known worker-client error code so
// operators get an actionable cause string rather than an opaque exception
// trace. We accept the classifications WorkerClient actually drives on an
// abnormal exit (kill-the-process path): the read loop hits EndOfStream and
// calls SetFaulted with WorkerClientErrorCode.PipeDisconnected and the
// message "Worker pipe disconnected." (see WorkerClient.cs:378-381). The
// earlier broad list (including "worker") matched every WorkerClient fault
// message (they all begin with "Worker"); tighten to the pipe/disconnect/
// end-of-stream classifications that match THIS path, so a regression that
// routed an unrelated fault here would surface as a test failure rather
// than silently passing. "heartbeat" is dropped
// because HeartbeatGraceSeconds (15s) exceeds the StreamShutdownTimeout
// (10s) poll window, so a heartbeat-expired transition can never be
// observed inside this test.
Assert.True(
observedFault!.Contains("pipe disconnected", StringComparison.OrdinalIgnoreCase)
|| observedFault.Contains("end of stream", StringComparison.OrdinalIgnoreCase),
$"Fault description '{observedFault}' did not match a known abnormal-exit classification "
+ "(expected 'pipe disconnected' or 'end of stream' from WorkerClient's EndOfStream path).");
// Also assert the StreamEvents call observed the fault
// — the chain that puts the session into Faulted goes through ReadEventsAsync
// propagating a WorkerClientException into EventStreamService, which calls
// session.MarkFaulted. The gateway then maps the WorkerClientException to an
// RpcException at the public boundary (MxAccessGatewayService.MapException →
// MapWorkerClientException). Polling session.State alone would silently pass
// if a future refactor moved MarkFaulted off the stream-consumption path —
// assert the streamTask itself terminated with a fault so the test couples
// to the actual fault-propagation path. Compare to the inverse assertion in
// the Write parity test (line 217: Assert.False(streamTask.IsFaulted, ...)).
try
{
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (Exception streamException)
{
output.WriteLine($"StreamEvents task terminated with: {streamException.GetType().Name}: {streamException.Message}");
}
Assert.True(
streamTask.IsCompleted,
"StreamEvents task did not complete within the shutdown timeout after the worker was killed.");
Assert.True(
streamTask.IsFaulted,
"StreamEvents task must fault on abnormal worker exit, not complete cleanly — "
+ "the fault-propagation path from WorkerClient.SetFaulted through ReadEventsAsync is the contract.");
}
finally
{
streamCancellation.Cancel();
// sessionId is intentionally null here — the session is already faulted and a
// CloseSession round-trip would just log a cleanup failure. We still wait for
// the worker process exit so the next test starts with a clean state.
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
}
}
///
/// TST-05: the real worker answers the four non-terminating control commands off-STA in
/// WorkerPipeSession.HandleControlCommandAsync. The default suite only ever sees the
/// FakeWorkerHarness canned replies, so this drives Ping, GetSessionState,
/// GetWorkerInfo, and DrainEvents through the gateway against a live worker and
/// asserts each reply carries real worker state rather than an INVALID_REQUEST.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ControlCommands_ReturnRealWorkerState()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-control",
ClientCorrelationId = "live-open-control",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
MxCommandReply pingReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand { Message = "live-ping-echo" },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Ping", pingReply);
Assert.Equal(ProtocolStatusCode.Ok, pingReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Ping, pingReply.Kind);
MxCommandReply stateReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-session-state", new MxCommand
{
Kind = MxCommandKind.GetSessionState,
GetSessionState = new GetSessionStateCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetSessionState", stateReply);
Assert.Equal(ProtocolStatusCode.Ok, stateReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetSessionState, stateReply.Kind);
Assert.NotNull(stateReply.SessionState);
Assert.Equal(SessionState.Ready, stateReply.SessionState.State);
MxCommandReply infoReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-worker-info", new MxCommand
{
Kind = MxCommandKind.GetWorkerInfo,
GetWorkerInfo = new GetWorkerInfoCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetWorkerInfo", infoReply);
Assert.Equal(ProtocolStatusCode.Ok, infoReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetWorkerInfo, infoReply.Kind);
Assert.NotNull(infoReply.WorkerInfo);
output.WriteLine(
$"GetWorkerInfo pid={infoReply.WorkerInfo.WorkerProcessId} version={infoReply.WorkerInfo.WorkerVersion} "
+ $"progid={infoReply.WorkerInfo.MxaccessProgid} clsid={infoReply.WorkerInfo.MxaccessClsid}");
Assert.Equal(openReply.WorkerProcessId, infoReply.WorkerInfo.WorkerProcessId);
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.WorkerVersion));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessProgid));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessClsid));
// No StreamEvents subscriber is attached, so whatever the worker queued since start
// is still drainable; the reply must carry the DrainEvents payload (possibly empty).
MxCommandReply drainReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-drain-events", new MxCommand
{
Kind = MxCommandKind.DrainEvents,
DrainEvents = new DrainEventsCommand { MaxEvents = 16 },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("DrainEvents", drainReply);
Assert.Equal(ProtocolStatusCode.Ok, drainReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.DrainEvents, drainReply.Kind);
Assert.Equal(MxCommandReply.PayloadOneofCase.DrainEvents, drainReply.PayloadCase);
Assert.True(drainReply.DrainEvents.Events.Count <= 16);
output.WriteLine($"DrainEvents drained={drainReply.DrainEvents.Events.Count}");
// A control command must not disturb the session: it is still Ready afterwards.
Assert.True(fixture.TryGetSession(sessionId, out GatewaySession? session));
Assert.Equal(SessionState.Ready, session!.State);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
///
/// TST-05: ShutdownWorker writes its OK reply before the worker shuts down, so the
/// reply reaches the caller (a reply written after the pipe closed would surface as a
/// pipe-disconnected failure instead), the worker process then exits on its own, and a later
/// command on the session does not succeed. Uses its own fixture and worker because it
/// terminates the worker.
///
/// A task that represents the asynchronous operation.
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ShutdownWorker_RepliesOkThenWorkerExits()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-shutdown-worker",
ClientCorrelationId = "live-open-shutdown-worker",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
// The gateway observes a worker's pipe closing through the event-stream read loop
// (the same path the abnormal-exit fact relies on); without a subscriber the session
// would sit in Ready until the heartbeat grace expired.
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply shutdownReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-shutdown-worker", new MxCommand
{
Kind = MxCommandKind.ShutdownWorker,
ShutdownWorker = new ShutdownWorkerCommand
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(5)),
},
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("ShutdownWorker", shutdownReply);
Assert.Equal(ProtocolStatusCode.Ok, shutdownReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.ShutdownWorker, shutdownReply.Kind);
// The worker exits on its own — no CloseSession, no kill. WaitForProcessesAsync
// throws on timeout, which fails the test.
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
// The session can no longer serve commands once its worker is gone: either the
// gateway rejects the call outright or the reply is not Ok.
MxCommandReply? afterReply = null;
Exception? afterException = null;
try
{
afterReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping-after-shutdown", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("PingAfterShutdown", afterReply);
}
catch (Exception exception)
{
afterException = exception;
output.WriteLine($"PingAfterShutdown rejected: {exception.GetType().Name}: {exception.Message}");
}
Assert.True(
afterException is not null || afterReply!.ProtocolStatus.Code != ProtocolStatusCode.Ok,
"A command sent after ShutdownWorker must not succeed against the exited worker.");
// Logged, not asserted: the gateway's WorkerClient moves to Closed on the shutdown
// ack, but GatewaySession.State is not driven from that transition today, so the
// public session state still reads Ready here (observed live on windev). That is a
// gateway-side gap, not a worker one — this fact pins the worker half of the
// contract: OK reply first, then a clean exit.
string observedState = fixture.TryGetSession(sessionId, out GatewaySession? session)
? session.State.ToString()
: "";
output.WriteLine($"ShutdownWorker observed_session_state={observedState}");
}
finally
{
// The worker is gone; a CloseSession round-trip would only log a cleanup failure.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
}
}
private static MxCommandRequest CreateControlRequest(
string sessionId,
string correlationId,
MxCommand command)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = correlationId,
Command = command,
};
}
///
/// Closes the session and drains the event stream / worker processes without letting a
/// cleanup timeout mask the original failure from the test body.
///
///
/// When , a faulted is rethrown so the
/// test fails on a silent stream-task exception (the Write parity test relies on this so
/// stream-side defects in event delivery are visible). When , all
/// cleanup exceptions are logged and swallowed so a real test-body assertion failure is not
/// masked by a shutdown timeout.
///
private async Task ShutDownAsync(
GatewayServiceFixture fixture,
TestWorkerProcessFactory processFactory,
string? sessionId,
Task? streamTask,
bool propagateStreamFaults = false)
{
Exception? streamFault = null;
try
{
if (!string.IsNullOrWhiteSpace(sessionId))
{
await CloseSessionAsync(fixture, sessionId).ConfigureAwait(false);
}
}
catch (Exception ex)
{
output.WriteLine($"Cleanup error during session close: {ex}");
}
if (streamTask is not null)
{
try
{
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
// A linked CancellationToken on the streaming TestServerCallContext is the
// intended way to stop StreamEvents promptly — treat the resulting
// OperationCanceledException as a clean shutdown, not a fault.
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Message}");
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
{
// MxAccessGatewayService.MapException intentionally converts the
// server-side OperationCanceledException into RpcException(Cancelled)
// so real gRPC clients see the standard cancellation status. Treat
// that as the same clean-shutdown signal as a raw OCE.
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Status.Detail}");
}
catch (Exception ex)
{
// Cleanup runs in a finally block. By default a faulted StreamEvents task is
// logged and swallowed so a test-body assertion failure is not masked. When
// the caller opts into propagateStreamFaults (the Write parity test), we
// rethrow the fault after the worker-process wait so a silent stream-side
// defect actually fails the test.
output.WriteLine($"Event stream task faulted during shutdown: {ex}");
if (propagateStreamFaults)
{
streamFault = ex;
}
}
}
try
{
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (Exception ex)
{
output.WriteLine($"Cleanup error while waiting for worker processes to exit: {ex}");
}
if (streamFault is not null)
{
throw streamFault;
}
}
private static MxCommandRequest CreateRegisterRequest(string sessionId)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-register",
Command = new MxCommand
{
Kind = MxCommandKind.Register,
Register = new RegisterCommand
{
ClientName = IntegrationTestEnvironment.LiveMxAccessClientName,
},
},
};
}
private static MxCommandRequest CreateAddItemRequest(
string sessionId,
int serverHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-add-item",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand
{
ServerHandle = serverHandle,
ItemDefinition = IntegrationTestEnvironment.LiveMxAccessItem,
},
},
};
}
private static MxCommandRequest CreateAdviseRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-advise",
Command = new MxCommand
{
Kind = MxCommandKind.Advise,
Advise = new AdviseCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateAdviseSupervisoryRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-advise-supervisory",
Command = new MxCommand
{
Kind = MxCommandKind.AdviseSupervisory,
AdviseSupervisory = new AdviseSupervisoryCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateWriteRequest(
string sessionId,
int serverHandle,
int itemHandle,
MxValue value)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write-restore",
Command = new MxCommand
{
Kind = MxCommandKind.Write,
Write = new WriteCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = value,
},
},
};
}
private static MxCommandRequest CreateWriteRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write",
Command = new MxCommand
{
Kind = MxCommandKind.Write,
Write = new WriteCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 1,
},
},
},
};
}
private static MxCommandRequest CreateWrite2Request(
string sessionId,
int serverHandle,
int itemHandle,
DateTime timestampUtc)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write2",
Command = new MxCommand
{
Kind = MxCommandKind.Write2,
Write2 = new Write2Command
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 2,
},
TimestampValue = new MxValue
{
DataType = MxDataType.Time,
VariantType = "VT_DATE",
TimestampValue = Timestamp.FromDateTime(timestampUtc),
},
},
},
};
}
private static MxCommandRequest CreateUnAdviseRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-unadvise",
Command = new MxCommand
{
Kind = MxCommandKind.UnAdvise,
UnAdvise = new UnAdviseCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateRemoveItemRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-remove-item",
Command = new MxCommand
{
Kind = MxCommandKind.RemoveItem,
RemoveItem = new RemoveItemCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateUnregisterRequest(
string sessionId,
int serverHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-unregister",
Command = new MxCommand
{
Kind = MxCommandKind.Unregister,
Unregister = new UnregisterCommand
{
ServerHandle = serverHandle,
},
},
};
}
private static MxCommandRequest CreateAuthenticateUserRequest(
string sessionId,
int serverHandle,
string verifyUser,
string verifyPassword)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-authenticate-user",
Command = new MxCommand
{
Kind = MxCommandKind.AuthenticateUser,
AuthenticateUser = new AuthenticateUserCommand
{
ServerHandle = serverHandle,
VerifyUser = verifyUser,
VerifyUserPassword = verifyPassword,
},
},
};
}
private static MxCommandRequest CreateArchestrAUserToIdRequest(
string sessionId,
int serverHandle,
string userIdGuid)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-archestra-user-to-id",
Command = new MxCommand
{
Kind = MxCommandKind.ArchestraUserToId,
ArchestraUserToId = new ArchestrAUserToIdCommand
{
ServerHandle = serverHandle,
UserIdGuid = userIdGuid,
},
},
};
}
private static MxCommandRequest CreateAddBufferedItemRequest(
string sessionId,
int serverHandle,
string itemDefinition,
string itemContext)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-add-buffered-item",
Command = new MxCommand
{
Kind = MxCommandKind.AddBufferedItem,
AddBufferedItem = new AddBufferedItemCommand
{
ServerHandle = serverHandle,
ItemDefinition = itemDefinition,
ItemContext = itemContext,
},
},
};
}
private static MxCommandRequest CreateSetBufferedUpdateIntervalRequest(
string sessionId,
int serverHandle,
int updateIntervalMilliseconds)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-set-buffered-update-interval",
Command = new MxCommand
{
Kind = MxCommandKind.SetBufferedUpdateInterval,
SetBufferedUpdateInterval = new SetBufferedUpdateIntervalCommand
{
ServerHandle = serverHandle,
UpdateIntervalMilliseconds = updateIntervalMilliseconds,
},
},
};
}
private static MxCommandRequest CreateSuspendRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-suspend",
Command = new MxCommand
{
Kind = MxCommandKind.Suspend,
Suspend = new SuspendCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateActivateRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-activate",
Command = new MxCommand
{
Kind = MxCommandKind.Activate,
Activate = new ActivateCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateWriteSecuredRequest(
string sessionId,
int serverHandle,
int itemHandle,
int currentUserId,
int verifierUserId)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write-secured",
Command = new MxCommand
{
Kind = MxCommandKind.WriteSecured,
WriteSecured = new WriteSecuredCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
CurrentUserId = currentUserId,
VerifierUserId = verifierUserId,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 2,
},
},
},
};
}
private static (string VerifyUser, string VerifyPassword) ResolveLiveMxAccessSecuredCredentials()
{
string verifyUser = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER")
?? "admin";
string verifyPassword = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD")
?? "password";
return (verifyUser, verifyPassword);
}
///
/// Splits a dotted MXAccess reference (e.g. "TestChildObject.TestInt") into the
/// (item_definition, item_context) pair AddBufferedItem expects — attribute name and
/// owning object. An undotted reference is passed through with an empty context.
///
private static (string Item, string Context) SplitLiveItemForBuffered(string liveItem)
{
int lastDot = liveItem.LastIndexOf('.');
if (lastDot < 0 || lastDot >= liveItem.Length - 1)
{
return (liveItem, string.Empty);
}
string context = liveItem[..lastDot];
string item = liveItem[(lastDot + 1)..];
return (item, context);
}
///
/// Counts the elements in a converted buffered across whichever
/// typed-array oneof case the VariantConverter populated, so the buffered-capture
/// assertions are independent of the rig item's element type.
///
private static int CountArrayElements(MxArray? array)
{
if (array is null)
{
return 0;
}
return array.ValuesCase switch
{
MxArray.ValuesOneofCase.BoolValues => array.BoolValues.Values.Count,
MxArray.ValuesOneofCase.Int32Values => array.Int32Values.Values.Count,
MxArray.ValuesOneofCase.Int64Values => array.Int64Values.Values.Count,
MxArray.ValuesOneofCase.FloatValues => array.FloatValues.Values.Count,
MxArray.ValuesOneofCase.DoubleValues => array.DoubleValues.Values.Count,
MxArray.ValuesOneofCase.StringValues => array.StringValues.Values.Count,
MxArray.ValuesOneofCase.TimestampValues => array.TimestampValues.Values.Count,
MxArray.ValuesOneofCase.RawValues => array.RawValues.Values.Count,
_ => 0,
};
}
private static int CountMatchingEvents(
RecordingServerStreamWriter writer,
Func predicate)
{
int count = 0;
foreach (MxEvent message in writer.Messages)
{
if (predicate(message))
{
count++;
}
}
return count;
}
private async Task CloseSessionAsync(
GatewayServiceFixture fixture,
string sessionId)
{
CloseSessionReply closeReply = await fixture.Service.CloseSession(
new CloseSessionRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-close",
},
new TestServerCallContext()).ConfigureAwait(false);
output.WriteLine($"CloseSession status={closeReply.ProtocolStatus.Code} final_state={closeReply.FinalState}");
}
private void LogReply(
string method,
MxCommandReply reply)
{
LogReplyTo(output, method, reply);
}
private static void LogReplyTo(
ITestOutputHelper sink,
string method,
MxCommandReply reply)
{
sink.WriteLine(
$"{method} status={reply.ProtocolStatus.Code} hresult={reply.Hresult} diagnostic={reply.DiagnosticMessage}");
foreach (MxStatusProxy status in reply.Statuses)
{
sink.WriteLine(
$"{method} mxstatus success={status.Success} category={status.Category} detail={status.Detail} text={status.DiagnosticText}");
}
}
private void LogEvent(MxEvent dataChange)
{
output.WriteLine(
$"Event family={dataChange.Family} worker_sequence={dataChange.WorkerSequence} server_handle={dataChange.ServerHandle} item_handle={dataChange.ItemHandle} quality={dataChange.Quality}");
output.WriteLine(
$"Event value_type={dataChange.Value?.DataType} raw_status={dataChange.RawStatus}");
}
///
/// Test fixture that assembles the gateway service with a worker process factory for live MXAccess testing.
///
private sealed class GatewayServiceFixture : IAsyncDisposable
{
private readonly GatewayMetrics _metrics = new();
private readonly SessionRegistry _registry = new();
private readonly ILoggerFactory _loggerFactory;
///
/// Initializes the fixture with worker executable path, factory, and test output helper.
///
/// Path to the worker process executable.
/// Factory for creating worker processes.
/// Test output helper for logging.
/// Optional configuration for the central alarm monitor.
public GatewayServiceFixture(
string workerExecutablePath,
IWorkerProcessFactory processFactory,
ITestOutputHelper output,
AlarmsOptions? alarms = null)
{
IOptions options = Options.Create(CreateOptions(workerExecutablePath, alarms));
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
WorkerProcessLauncher launcher = new(
options,
processFactory,
new WorkerProcessStartedProbe(),
_metrics);
SessionWorkerClientFactory workerClientFactory = new(
launcher,
options,
_metrics,
_loggerFactory);
SessionManager sessionManager = new(
_registry,
workerClientFactory,
options,
_metrics,
logger: _loggerFactory.CreateLogger());
Manager = sessionManager;
MxAccessGrpcMapper mapper = new();
EventStreamService eventStreamService = new(
sessionManager,
options,
_metrics);
Monitor = new GatewayAlarmMonitor(
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
_loggerFactory.CreateLogger());
Service = new MxAccessGatewayService(
sessionManager,
new GatewayRequestIdentityAccessor(),
new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(),
mapper,
eventStreamService,
_metrics,
_loggerFactory.CreateLogger(),
Monitor);
}
///
/// The assembled gateway service instance.
///
public MxAccessGatewayService Service { get; }
/// The central alarm monitor for tests of the public alarm RPCs.
public GatewayAlarmMonitor Monitor { get; }
/// The manager used by the gateway's internal alarm monitor.
public SessionManager Manager { get; }
///
/// Looks up a session by id directly against the in-process registry. The abnormal
/// worker-exit test needs to observe the session's State / FinalFault as the gateway
/// transitions it to Faulted, which the public gRPC API only exposes indirectly via
/// CloseSession's reply (and not before a graceful close completes).
///
/// The session identifier.
/// The session if found; otherwise null.
/// if a session with the given id was found; otherwise .
public bool TryGetSession(string sessionId, [MaybeNullWhen(false)] out GatewaySession session)
{
return _registry.TryGet(sessionId, out session);
}
///
/// Disposes the fixture resources and closes all sessions.
///
/// A task that represents the asynchronous operation.
public async ValueTask DisposeAsync()
{
await Monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
foreach (GatewaySession session in _registry.Snapshot())
{
await session.DisposeAsync().ConfigureAwait(false);
}
Monitor.Dispose();
_loggerFactory.Dispose();
_metrics.Dispose();
}
private static GatewayOptions CreateOptions(string workerExecutablePath, AlarmsOptions? alarms)
{
return new GatewayOptions
{
Alarms = alarms ?? new AlarmsOptions(),
Worker = new WorkerOptions
{
ExecutablePath = workerExecutablePath,
StartupTimeoutSeconds = 30,
ShutdownTimeoutSeconds = 15,
HeartbeatCheckIntervalSeconds = 5,
HeartbeatGraceSeconds = 15,
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
RequiredArchitecture = WorkerArchitecture.X86,
},
Sessions = new SessionOptions
{
DefaultCommandTimeoutSeconds = 15,
MaxSessions = alarms?.Enabled == true ? 2 : 1,
},
Events = new EventOptions
{
QueueCapacity = 32,
},
};
}
}
///
/// Gathers messages written to a server stream for test inspection.
///
private sealed class RecordingServerStreamWriter : IServerStreamWriter, IDisposable
{
private readonly object syncRoot = new();
private readonly List messages = [];
private readonly SemaphoreSlim messageArrived = new(0);
///
/// All messages that have been written to the stream.
///
public IReadOnlyList Messages
{
get
{
lock (syncRoot)
{
return messages.ToArray();
}
}
}
///
/// Inherited write options.
///
public WriteOptions? WriteOptions { get; set; }
///
/// Records the message unless the token is already cancelled.
///
/// The message to write.
/// Token that cancels the write.
/// A task that represents the asynchronous operation.
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
// The interface default throws NotSupportedException for a cancelable token; the
// gateway passes the call's token on every streamed write.
cancellationToken.ThrowIfCancellationRequested();
return WriteAsync(message);
}
///
/// Records the message and signals any pending waiter.
///
/// The message to write.
/// A task that represents the asynchronous operation.
public Task WriteAsync(T message)
{
lock (syncRoot)
{
messages.Add(message);
}
messageArrived.Release();
return Task.CompletedTask;
}
///
/// Waits for the first recorded message that satisfies ,
/// up to the specified timeout. Earlier non-matching messages (for example a
/// registration-state bootstrap event) are skipped rather than treated as the result.
///
/// Filter the awaited message must satisfy.
/// The maximum total time to wait.
///
/// Token observed alongside the timeout so a per-test cancellation (for example the
/// gRPC call context's token) aborts the wait promptly instead of hanging until the
/// timeout elapses.
///
/// The first message that satisfies the predicate.
public async Task WaitForMessageAsync(
Func predicate,
TimeSpan timeout,
CancellationToken cancellationToken = default)
{
using CancellationTokenSource timeoutCancellation = new(timeout);
using CancellationTokenSource linkedCancellation =
CancellationTokenSource.CreateLinkedTokenSource(timeoutCancellation.Token, cancellationToken);
int scanned = 0;
while (true)
{
T[] snapshot;
lock (syncRoot)
{
snapshot = messages.ToArray();
}
for (; scanned < snapshot.Length; scanned++)
{
if (predicate(snapshot[scanned]))
{
return snapshot[scanned];
}
}
try
{
await messageArrived.WaitAsync(linkedCancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (timeoutCancellation.IsCancellationRequested)
{
throw new TimeoutException(
$"No stream message satisfied the predicate within {timeout}. Recorded {scanned} message(s).");
}
}
}
///
/// Releases the wait handle backing messageArrived. The writer owns an
/// field so it must be disposable itself; the leak
/// is otherwise bounded only by how many opt-in live tests run.
///
public void Dispose()
{
messageArrived.Dispose();
}
}
///
/// Minimal stub for invoking the gRPC service
/// in-process. It is a hand-written fake with no verification behavior — it
/// only supplies the context values the service reads during a call.
///
private sealed class TestServerCallContext(CancellationToken cancellationToken = default) : ServerCallContext
{
private readonly Metadata requestHeaders = [];
private readonly Metadata responseTrailers = [];
private readonly Dictionary