Files
mxaccessgw/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs
T
2026-11-21 09:53:37 -05:00

3516 lines
169 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Text;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.IntegrationTests.TestSupport;
using Xunit.Abstractions;
using SessionOptions = ZB.MOM.WW.MxGateway.Server.Configuration.SessionOptions;
namespace ZB.MOM.WW.MxGateway.IntegrationTests;
[Collection(LiveResourcesCollection.Name)]
[Trait("Category", "LiveMxAccess")]
public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{
private static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(15);
private static readonly TimeSpan StreamShutdownTimeout = TimeSpan.FromSeconds(10);
/// <summary>
/// Verifies that a gateway session can register, add item, advise, and stream events from live MXAccess.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_RegistersAdvisesStreamsDataAndCloses()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-smoke",
ClientCorrelationId = "live-open",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
output.WriteLine($"OpenSession status={openReply.ProtocolStatus.Code} session={sessionId} worker_pid={openReply.WorkerProcessId}");
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
Assert.True(registerReply.Register.ServerHandle > 0);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// A live MXAccess provider can deliver an initial registration-state
// or bad-quality bootstrap event before the OnDataChange the worker
// is contracted to emit. Match on the family rather than trusting
// whatever event arrives first so a genuine ordering defect cannot
// pass spuriously or leave a later wrong event unverified.
MxEvent dataChange = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(dataChange);
Assert.Equal(MxEventFamily.OnDataChange, dataChange.Family);
Assert.Equal(sessionId, dataChange.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, dataChange.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, dataChange.ItemHandle);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
/// <summary>Verifies Rust worker callbacks over the gateway's live HTTP/2 gRPC endpoint.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsChangingIntegersAndUnadvises()
{
const string itemDefinition = "TestMachine_001.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerPath, processFactory, output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "rust-worker-live-events" },
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
try
{
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request)
{
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
.ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok, reply.ToString());
return reply;
}
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-worker-info",
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
}).ConfigureAwait(false);
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
int serverHandle = registration.Register.ServerHandle;
MxCommandRequest addItem = CreateAddItemRequest(sessionId, serverHandle);
addItem.Command.AddItem.ItemDefinition = itemDefinition;
MxCommandReply added = await InvokeAsync(addItem).ConfigureAwait(false);
int itemHandle = added.AddItem.ItemHandle;
using (AsyncServerStreamingCall<MxEvent> stream = client.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
cancellationToken: token))
{
Assert.True(await stream.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(stream.ResponseStream.Current.StreamAttached);
await InvokeAsync(CreateAdviseRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
MxEvent? first = null;
bool observedChange = false;
ulong lastSequence = 0;
while (await stream.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.True(change.WorkerSequence > lastSequence);
lastSequence = change.WorkerSequence;
if (change.Family != MxEventFamily.OnDataChange ||
change.Value?.KindCase != MxValue.KindOneofCase.Int32Value ||
(change.Quality & 0xc0) != 0xc0)
{
continue;
}
Assert.Equal(sessionId, change.SessionId);
Assert.Equal(serverHandle, change.ServerHandle);
Assert.Equal(itemHandle, change.ItemHandle);
Assert.NotNull(change.SourceTimestamp);
Assert.NotNull(change.WorkerTimestamp);
Assert.Contains(change.Statuses, status => status.Category == MxStatusCategory.Ok);
if (first is not null && change.Value.Int32Value != first.Value.Int32Value)
{
output.WriteLine(
$"{itemDefinition} delivered through gateway: {first.Value.Int32Value} -> {change.Value.Int32Value}, " +
$"worker sequences {first.WorkerSequence} -> {change.WorkerSequence}, quality {change.Quality}.");
observedChange = true;
break;
}
first ??= change;
}
Assert.True(observedChange, "The gateway did not deliver distinct TestChangingInt values.");
await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-unadvise",
Command = new MxCommand
{
Kind = MxCommandKind.UnAdvise,
UnAdvise = new UnAdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
},
}).ConfigureAwait(false);
// A callback queued before UnAdvise may still cross the pipe.
// Observe the same stream and reject updates generated after acknowledgement.
DateTime unadvisedAt = DateTime.UtcNow;
using CancellationTokenSource quiet = CancellationTokenSource.CreateLinkedTokenSource(token);
quiet.CancelAfter(TimeSpan.FromSeconds(2));
try
{
while (await stream.ResponseStream.MoveNext(quiet.Token).ConfigureAwait(false))
{
MxEvent change = stream.ResponseStream.Current;
Assert.False(change.Family == MxEventFamily.OnDataChange &&
change.ServerHandle == serverHandle && change.ItemHandle == itemHandle &&
change.WorkerTimestamp.ToDateTime() >= unadvisedAt,
"The worker sent a new data change after UnAdvise completed.");
}
Assert.Fail("The event stream ended before the quiet period elapsed.");
}
catch (RpcException exception) when (exception.StatusCode == StatusCode.Cancelled && quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
catch (OperationCanceledException) when (quiet.IsCancellationRequested)
{
Assert.False(token.IsCancellationRequested);
}
}
}
finally
{
CloseSessionReply closed = await client.CloseSessionAsync(
new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
}
}
/// <summary>Verifies committed Rust worker writes and their callbacks through the gateway.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_WritesAndReceivesOnWriteComplete()
{
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "rust-worker-live-write" },
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
MxCommandRequest? restore = null;
try
{
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request)
{
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
.ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
$"{request.Command.Kind}: {reply.ProtocolStatus.Code} HRESULT={reply.Hresult} {reply.DiagnosticMessage}");
return reply;
}
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-write-worker-info",
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
}).ConfigureAwait(false);
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
int serverHandle = registration.Register.ServerHandle;
MxCommandReply added = await InvokeAsync(CreateAddItemRequest(sessionId, serverHandle)).ConfigureAwait(false);
int itemHandle = added.AddItem.ItemHandle;
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
cancellationToken: token);
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
await InvokeAsync(CreateAdviseSupervisoryRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
bool IsTarget(MxEvent update) => update.SessionId == sessionId &&
update.ServerHandle == serverHandle && update.ItemHandle == itemHandle;
bool IsGoodInt(MxEvent update) => IsTarget(update) && update.Family == MxEventFamily.OnDataChange &&
update.Value?.KindCase == MxValue.KindOneofCase.Int32Value && (update.Quality & 0xc0) == 0xc0;
MxEvent? initial = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
if (IsGoodInt(events.ResponseStream.Current))
{
initial = events.ResponseStream.Current;
break;
}
}
Assert.NotNull(initial);
int original = initial.Value.Int32Value;
int written = original == 1 ? 2 : 1;
async Task CheckWriteAsync(MxCommandRequest request, int expected, DateTime? expectedSource = null)
{
MxCommandReply reply = await InvokeAsync(request).ConfigureAwait(false);
Assert.Equal(request.Command.Kind, reply.Kind);
Assert.Equal(0, reply.Hresult);
if (request.Command.Kind == MxCommandKind.WriteBulk)
{
Assert.Collection(reply.WriteBulk.Results,
unsupported =>
{
Assert.Equal(itemHandle, unsupported.ItemHandle);
Assert.False(unsupported.WasSuccessful);
Assert.NotEmpty(unsupported.ErrorMessage);
},
accepted =>
{
Assert.Equal(itemHandle, accepted.ItemHandle);
Assert.True(accepted.WasSuccessful, accepted.ErrorMessage);
});
}
else
{
Assert.NotEmpty(reply.Statuses);
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
MxEvent? completion = null;
MxEvent? changed = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent update = events.ResponseStream.Current;
if (IsTarget(update) && update.Family == MxEventFamily.OnWriteComplete)
{
completion = update;
}
if (IsGoodInt(update) && update.Value.Int32Value == expected)
{
changed = update;
}
if (completion is not null && changed is not null)
{
break;
}
}
Assert.NotNull(completion);
Assert.NotNull(changed);
Assert.NotEmpty(completion.Statuses);
Assert.All(completion.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
if (request.Command.Kind != MxCommandKind.WriteBulk)
{
Assert.Equal(reply.Statuses.ToArray(), completion.Statuses.ToArray());
}
if (expectedSource is DateTime timestamp)
{
Assert.Equal(timestamp, changed.SourceTimestamp?.ToDateTime());
}
output.WriteLine(
$"{request.Command.Kind} committed {IntegrationTestEnvironment.LiveMxAccessItem}={expected}; " +
$"OnWriteComplete sequence={completion.WorkerSequence}, readback sequence={changed.WorkerSequence}.");
}
restore = CreateWriteRequest(sessionId, serverHandle, itemHandle, initial.Value);
MxCommandRequest write = CreateWriteRequest(sessionId, serverHandle, itemHandle,
new MxValue { DataType = MxDataType.Integer, Int32Value = written });
write.ClientCorrelationId = "rust-write-round-trip";
await CheckWriteAsync(write, written).ConfigureAwait(false);
int bulkValue = original == 3 ? 4 : 3;
var bulk = new WriteBulkCommand { ServerHandle = serverHandle };
bulk.Entries.Add(new WriteBulkEntry
{
ItemHandle = itemHandle,
Value = new MxValue { RawValue = Google.Protobuf.ByteString.CopyFrom(new byte[] { 0 }) },
});
bulk.Entries.Add(new WriteBulkEntry
{
ItemHandle = itemHandle,
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = bulkValue },
});
await CheckWriteAsync(new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "rust-write-bulk-round-trip",
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = bulk },
}, bulkValue).ConfigureAwait(false);
DateTime timestamp = DateTime.UtcNow.AddSeconds(-30);
timestamp = new DateTime(timestamp.Ticks - timestamp.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc);
MxCommandRequest write2 = CreateWrite2Request(sessionId, serverHandle, itemHandle, timestamp);
write2.Command.Write2.Value = initial.Value;
await CheckWriteAsync(write2, original, timestamp).ConfigureAwait(false);
restore = null;
// The test item is unprotected. Exercise both secured native entry
// points with its current value and surface provider refusal as HRESULT.
foreach (MxCommand command in new MxCommand[]
{
new() { Kind = MxCommandKind.WriteSecured, WriteSecured = new WriteSecuredCommand
{
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
} },
new() { Kind = MxCommandKind.WriteSecured2, WriteSecured2 = new WriteSecured2Command
{
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
TimestampValue = write2.Command.Write2.TimestampValue,
} },
})
{
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
{
SessionId = sessionId, ClientCorrelationId = $"rust-{command.Kind}", Command = command,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(command.Kind, reply.Kind);
Assert.True(reply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"{command.Kind}: {reply.ProtocolStatus.Code} {reply.DiagnosticMessage}");
if (reply.ProtocolStatus.Code == ProtocolStatusCode.MxaccessFailure)
{
Assert.True(reply.HasHresult && reply.Hresult < 0);
}
output.WriteLine($"{command.Kind} returned {reply.ProtocolStatus.Code}, HRESULT=0x{reply.Hresult:X8}.");
}
}
finally
{
try
{
if (restore is not null)
{
MxCommandReply restored = await client.InvokeAsync(restore, deadline: DateTime.UtcNow.AddSeconds(10))
.ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, restored.ProtocolStatus.Code);
Assert.NotEmpty(restored.Statuses);
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
}
finally
{
CloseSessionReply closed = await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
}
}
}
/// <summary>Exercises bulk operations and both snapshot read paths over gRPC.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_BulkItemsAndReads()
{
const string first = "TestMachine_001.TestChangingInt";
const string second = "TestMachine_002.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(60));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-bulk-items",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task<MxCommandReply> InvokeAsync(MxCommand command)
{
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
{
SessionId = id, ClientCorrelationId = Guid.NewGuid().ToString("N"), Command = command,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
return reply;
}
int server = (await InvokeAsync(CreateRegisterRequest(id).Command).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply extended = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItem2,
AddItem2 = new AddItem2Command { ServerHandle = server, ItemDefinition = "TestChangingInt", ItemContext = "TestMachine_003" },
}).ConfigureAwait(false);
Assert.True(extended.AddItem2.ItemHandle > 0);
MxCommandReply subscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeBulk,
SubscribeBulk = new SubscribeBulkCommand { ServerHandle = server, TagAddresses = { first } },
}).ConfigureAwait(false);
SubscribeResult subscription = Assert.Single(subscribed.SubscribeBulk.Results);
Assert.True(subscription.WasSuccessful, subscription.ErrorMessage);
// Observe the permanent advice before checking the worker's value cache.
using (AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true },
cancellationToken: token))
{
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = events.ResponseStream.Current;
if (change.Family == MxEventFamily.OnDataChange && change.ItemHandle == subscription.ItemHandle)
{
break;
}
}
}
MxCommandReply read = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 3000, TagAddresses = { first, second, second } },
}).ConfigureAwait(false);
Assert.Collection(read.ReadBulk.Results,
cached => { Assert.True(cached.WasSuccessful, cached.ErrorMessage); Assert.True(cached.WasCached); },
snapshot => { Assert.True(snapshot.WasSuccessful, snapshot.ErrorMessage); Assert.False(snapshot.WasCached); },
duplicate =>
{
Assert.True(duplicate.WasSuccessful, duplicate.ErrorMessage);
Assert.False(duplicate.WasCached);
Assert.Equal(read.ReadBulk.Results[1].ItemHandle, duplicate.ItemHandle);
});
MxCommandReply added = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItemBulk,
AddItemBulk = new AddItemBulkCommand { ServerHandle = server, TagAddresses = { second } },
}).ConfigureAwait(false);
SubscribeResult item = Assert.Single(added.AddItemBulk.Results);
Assert.True(item.WasSuccessful, item.ErrorMessage);
MxCommandReply advised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AdviseItemBulk,
AdviseItemBulk = new AdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(advised.AdviseItemBulk.Results).WasSuccessful);
MxCommandReply unadvised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnAdviseItemBulk,
UnAdviseItemBulk = new UnAdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unadvised.UnAdviseItemBulk.Results).WasSuccessful);
MxCommandReply removed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.RemoveItemBulk,
RemoveItemBulk = new RemoveItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle, extended.AddItem2.ItemHandle } },
}).ConfigureAwait(false);
Assert.All(removed.RemoveItemBulk.Results, result => Assert.True(result.WasSuccessful, result.ErrorMessage));
MxCommandReply unsubscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeBulk,
UnsubscribeBulk = new UnsubscribeBulkCommand { ServerHandle = server, ItemHandles = { subscription.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unsubscribed.UnsubscribeBulk.Results).WasSuccessful);
output.WriteLine("Bulk add, advise, cached and snapshot reads, unadvise, remove, subscribe, and unsubscribe passed.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
/// <summary>Verifies the Rust worker’s native subtag alarm provider and standby replacement.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
{
const string reference = "Galaxy!DEV.TestMachine_001.RustWorkerSmoke";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(75));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-alarms",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task<MxCommandReply> AlarmAsync(MxCommand command)
{
WorkerCommandReply envelope = await fixture.Manager.InvokeAsync(id,
new WorkerCommand { Command = command }, token).ConfigureAwait(false);
Assert.Equal(command.Kind, envelope.Reply.Kind);
return envelope.Reply;
}
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true }, cancellationToken: token);
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
MxCommandReply subscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
ForcedMode = AlarmProviderMode.Subtag,
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(subscribed.ProtocolStatus.Code == ProtocolStatusCode.Ok, subscribed.DiagnosticMessage);
MxEvent? transition = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
if (events.ResponseStream.Current.OnAlarmTransition?.AlarmFullReference == reference)
{
transition = events.ResponseStream.Current;
break;
}
}
Assert.NotNull(transition);
Assert.Equal(id, transition.SessionId);
Assert.True(transition.WorkerSequence > 0);
Assert.True(transition.OnAlarmTransition.Degraded);
output.WriteLine($"Forwarded {transition.OnAlarmTransition.TransitionKind} at worker sequence {transition.WorkerSequence}.");
MxCommandReply snapshot = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.QueryActiveAlarms,
QueryActiveAlarmsCommand = new QueryActiveAlarmsCommand { AlarmFilterPrefix = reference },
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, snapshot.ProtocolStatus.Code);
Assert.Contains(snapshot.QueryActiveAlarms.Snapshots, alarm => alarm.AlarmFullReference == reference);
MxCommandReply unknownAck = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarm,
AcknowledgeAlarmCommand = new AcknowledgeAlarmCommand { AlarmGuid = Guid.NewGuid().ToString() },
}).ConfigureAwait(false);
Assert.NotNull(unknownAck.AcknowledgeAlarm);
Assert.NotEqual(0, unknownAck.AcknowledgeAlarm.NativeStatus);
MxCommandReply unknownName = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarmByName,
AcknowledgeAlarmByNameCommand = new AcknowledgeAlarmByNameCommand
{
AlarmName = "RustWorkerMissingAlarm", ProviderName = "Galaxy", GroupName = "DEV",
},
}).ConfigureAwait(false);
Assert.NotNull(unknownName.AcknowledgeAlarm);
Assert.NotEqual(0, unknownName.AcknowledgeAlarm.NativeStatus);
MxCommandReply update = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, update.ProtocolStatus.Code);
MxCommandReply unsubscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeAlarms,
UnsubscribeAlarms = new UnsubscribeAlarmsCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, unsubscribed.ProtocolStatus.Code);
MxCommandReply fallback = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
FallbackEnabled = true,
Failover = new AlarmFailoverConfig
{
ConsecutiveFailureThreshold = 1, FailbackProbeIntervalSeconds = 30, FailbackStableProbes = 3,
},
},
}).ConfigureAwait(false);
Assert.True(fallback.ProtocolStatus.Code == ProtocolStatusCode.Ok, fallback.DiagnosticMessage);
MxCommandReply replacement = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
{
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(replacement.ProtocolStatus.Code == ProtocolStatusCode.Ok, replacement.DiagnosticMessage);
output.WriteLine("Updated the live alarm provider's fallback watch list.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
{
const string tag = "TestMachine_001.TestAlarm001";
const string reference = "Galaxy!TestArea." + tag;
string user = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER") ?? "Administrator";
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
{
Enabled = true,
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
});
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
using AsyncServerStreamingCall<AlarmFeedMessage> feed = client.StreamAlarms(
new StreamAlarmsRequest { AlarmFilterPrefix = reference }, cancellationToken: token);
bool snapshotComplete = false;
bool alarmmgrReady = false;
while ((!snapshotComplete || !alarmmgrReady) && await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
AlarmFeedMessage frame = feed.ResponseStream.Current;
snapshotComplete |= frame.SnapshotComplete;
alarmmgrReady |= frame.ProviderStatus?.Mode == AlarmProviderMode.Alarmmgr;
}
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await client.InvokeAsync(request, deadline: DateTime.UtcNow.AddSeconds(15),
cancellationToken: cancellationToken).ResponseAsync.ConfigureAwait(false);
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
$"{reply.Kind} returned {reply.ProtocolStatus.Code}: {reply.DiagnosticMessage}");
return reply;
}
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
{
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
OnAlarmTransitionEvent? transition = feed.ResponseStream.Current.Transition;
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
&& (ackComment is null || transition.OperatorComment == ackComment))
{
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
Assert.False(transition.Degraded);
return transition;
}
}
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
}
MxCommandRequest? restore = null;
try
{
int server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply read = await InvokeAsync(new MxCommandRequest
{
SessionId = sessionId,
Command = new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 5000, TagAddresses = { tag } },
},
}, token).ConfigureAwait(false);
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
Assert.True(original.WasSuccessful, original.ErrorMessage);
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
MxCommandRequest WriteAlarm(bool active)
{
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
request.Command.WriteSecured.Value = new MxValue { DataType = MxDataType.Boolean, BoolValue = active };
return request;
}
async Task WriteConfirmedAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
MxCommandReply reply = await InvokeAsync(request, cancellationToken).ConfigureAwait(false);
Assert.NotEmpty(reply.Statuses);
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
// The dev rig’s test UDA is normally false. Clear an existing test episode before raising a fresh one.
if (original.Value.BoolValue)
{
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
restore = WriteAlarm(true);
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
}
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
restore = WriteAlarm(original.Value.BoolValue);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference,
OperatorUser = $"mxgw-smoke-{Guid.NewGuid():N}",
Comment = "This acknowledgement must be rejected",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, rejected.ProtocolStatus.Code);
Assert.True(rejected.HasHresult && rejected.Hresult != 0);
AcknowledgeAlarmReply ack = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
{
AlarmFullReference = reference, OperatorUser = user, OperatorPassword = password, Comment = comment,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.True(ack.ProtocolStatus.Code == ProtocolStatusCode.Ok, ack.DiagnosticMessage);
Assert.True(ack.HasHresult);
Assert.Equal(0, ack.Hresult);
Assert.DoesNotContain("unconfirmed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("not observed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
OnAlarmTransitionEvent acknowledged = await WaitForTransitionAsync(AlarmTransitionKind.Acknowledge, comment)
.ConfigureAwait(false);
Assert.Equal(user, acknowledged.OperatorUser, ignoreCase: true);
output.WriteLine($"Alarmmgr streamed Raise and Acknowledge with operator {acknowledged.OperatorUser} and the requested comment.");
if (!original.Value.BoolValue)
{
await WriteConfirmedAsync(restore, token).ConfigureAwait(false);
restore = null;
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
output.WriteLine("The gateway streamed Clear after the test UDA was restored.");
}
}
finally
{
try
{
if (restore is not null)
{
MxCommandReply restored = await InvokeAsync(restore, CancellationToken.None).ConfigureAwait(false);
Assert.NotEmpty(restored.Statuses);
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
}
}
finally
{
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
}
}
}
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service, GatewayAlarmMonitor? monitor = null)
{
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
builder.WebHost.ConfigureKestrel(options =>
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
builder.Services.AddGrpc();
builder.Services.AddSingleton(service);
if (monitor is not null)
{
builder.Services.AddSingleton<IHostedService>(monitor);
}
WebApplication app = builder.Build();
app.MapGrpcService<MxAccessGatewayService>();
return app;
}
private sealed class RustWorkerLiveFactAttribute : FactAttribute
{
public RustWorkerLiveFactAttribute()
{
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
if (!IntegrationTestEnvironment.IsEnabled(optIn))
{
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
}
}
}
/// <summary>
/// Verifies that a Write command round-trips through live MXAccess against an advised item
/// and that the worker emits a matching <see cref="MxEventFamily.OnWriteComplete"/> event
/// — the proof of round-trip the cross-language client e2e runner relies on.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_WritesValueToAdvisedItem()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write",
ClientCorrelationId = "live-open-write",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
MxCommandReply writeReply = await fixture.Service.Invoke(
CreateWriteRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write", writeReply);
// Happy-path Write: the worker COM call succeeded so HResultConverter
// produces ProtocolStatusCode.Ok. An MXAccess rejection (a write to a
// bad item, a secured-item failure) would surface as
// ProtocolStatusCode.MxaccessFailure with a non-zero hresult — never
// as an RpcException / transport fault, because the command still
// completed its round-trip to the worker and back.
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Write, writeReply.Kind);
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
// after the underlying provider acknowledges the write — that is
// the event the cross-language client e2e runner asserts on. We
// scan the recorded stream (so an interleaving OnDataChange does
// not preempt the match) for an OnWriteComplete carrying the same
// server/item handles the Write command targeted.
MxEvent writeComplete = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnWriteComplete
&& candidate.ServerHandle == registerReply.Register.ServerHandle
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writeComplete);
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
Assert.Equal(sessionId, writeComplete.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
// The stream task must not be in a faulted state. ShutDownAsync's
// broad catch would otherwise swallow the fault and silently let
// this Write-parity coverage pass against a broken event pipeline.
Assert.False(
streamTask.IsFaulted,
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
}
finally
{
// Cancel the stream call before draining so StreamEvents observes
// cancellation rather than blocking on the channel. Any unhandled
// stream-task fault is rethrown from ShutDownAsync into the test.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies that a <c>Write2</c> command (the B8 extended write carrying a timestamp
/// value as its second context parameter) round-trips through live MXAccess against an
/// advised item and raises a matching <see cref="MxEventFamily.OnWriteComplete"/> event.
/// A regression that short-circuits <c>Write2</c> to <c>InvalidRequest</c> (or drops the
/// timestamp) fails here, which no other live test catches (IntegrationTests-033, #212).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_Write2ValueWithTimestampToAdvisedItem()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
MxValue? originalValue = null;
bool restored = false;
int serverHandle = 0;
int itemHandle = 0;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write2",
ClientCorrelationId = "live-open-write2",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(addItemReply.AddItem.ItemHandle > 0);
serverHandle = registerReply.Register.ServerHandle;
itemHandle = addItemReply.AddItem.ItemHandle;
// Supervisory, not plain, advise: MXAccess refuses a user-id-0 write on a
// plainly advised item with SecurityError/1008. Proven on the DEV gateway
// (wonder-app-vd03, 2026-11-12, #212): on MESReceiver_025.MoveInBatchID, Write2
// applied the value after AdviseSupervisory and got SecurityError/1008 after Advise.
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseSupervisoryRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AdviseSupervisory", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// The write below can now really apply, so remember the value to restore.
MxEvent initialValue = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(initialValue);
originalValue = initialValue.Value;
// A whole-second instant 30 s in the past, so a stored source timestamp
// can be compared exactly and cannot be confused with the write time.
DateTime sentTimestampUtc = DateTime.UtcNow.AddSeconds(-30);
sentTimestampUtc = new DateTime(
sentTimestampUtc.Ticks - (sentTimestampUtc.Ticks % TimeSpan.TicksPerSecond),
DateTimeKind.Utc);
output.WriteLine(
$"Write2 timestamp sent={sentTimestampUtc:O} box zone={TimeZoneInfo.Local.Id} offset={TimeZoneInfo.Local.GetUtcOffset(sentTimestampUtc)}");
MxCommandReply writeReply = await fixture.Service.Invoke(
CreateWrite2Request(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle,
sentTimestampUtc),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write2", writeReply);
// Ok means the worker reached the COM Write2 call and COM accepted it.
// InvalidRequest here means the executor rejected the payload
// (missing value/timestamp) before reaching MXAccess. Ok alone does
// not mean the provider applied the value; the completion status
// below carries that. (The 2026-11-12 windev run used a plain Advise
// and got SecurityError/1008 on the completion; see #212.)
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Write2, writeReply.Kind);
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
// after the underlying provider acknowledges the write — that is
// the event the cross-language client e2e runner asserts on. We
// scan the recorded stream (so an interleaving OnDataChange does
// not preempt the match) for an OnWriteComplete carrying the same
// server/item handles the Write command targeted.
MxEvent writeComplete = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnWriteComplete
&& candidate.ServerHandle == registerReply.Register.ServerHandle
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writeComplete);
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
Assert.Equal(sessionId, writeComplete.SessionId);
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
foreach (MxStatusProxy status in writeComplete.Statuses)
{
output.WriteLine($"Write2 completion category={status.Category} detail={status.Detail}");
}
// #228: MXAccess reads the Write2 time argument as box-local time, so
// the worker must convert the UTC instant before the COM call. When the
// provider applied the write, the source timestamp it reports must be
// the instant sent. On a UTC box this cannot fail either way, so the
// zone is logged above. A refused write (no data change) is logged,
// not asserted, as before.
bool applied = writeComplete.Statuses.Count > 0
&& writeComplete.Statuses.All(status => status.Category == MxStatusCategory.Ok);
if (applied)
{
MxEvent writtenValue = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle
&& candidate.WorkerSequence > initialValue.WorkerSequence,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(writtenValue);
DateTime? storedTimestampUtc = writtenValue.SourceTimestamp?.ToDateTime();
output.WriteLine($"Write2 timestamp stored={storedTimestampUtc:O}");
Assert.Equal(sentTimestampUtc, storedTimestampUtc);
}
else
{
output.WriteLine("Write2 timestamp not checked: the completion was not Ok, so nothing was applied.");
}
// Restore the original value with a plain Write on the same supervisory advise.
MxCommandReply restoreReply = await fixture.Service.Invoke(
CreateWriteRequest(
sessionId,
registerReply.Register.ServerHandle,
addItemReply.AddItem.ItemHandle,
originalValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write (restore)", restoreReply);
Assert.Equal(ProtocolStatusCode.Ok, restoreReply.ProtocolStatus.Code);
restored = true;
// The stream task must not be in a faulted state. ShutDownAsync's
// broad catch would otherwise swallow the fault and silently let
// this Write2-parity coverage pass against a broken event pipeline.
Assert.False(
streamTask.IsFaulted,
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
}
finally
{
if (!restored && originalValue is not null && sessionId is not null && itemHandle != 0)
{
// Best effort: do not leave the Write2 test value behind after a failed
// assertion. Logged, not thrown, so the original failure is not masked.
try
{
MxCommandReply restoreReply = await fixture.Service.Invoke(
CreateWriteRequest(sessionId, serverHandle, itemHandle, originalValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Write (restore after failure)", restoreReply);
}
catch (Exception ex)
{
output.WriteLine($"Restore after failure threw {ex.GetType().Name}: {ex.Message}");
}
}
// Cancel the stream call before draining so StreamEvents observes
// cancellation rather than blocking on the channel. Any unhandled
// stream-task fault is rethrown from ShutDownAsync into the test.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies that an AddItem against an invalid server handle surfaces the MXAccess failure
/// without faulting the gateway transport, exercising the invalid-handle parity path.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_InvalidHandleCommand_SurfacesFailureWithoutTransportFault()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-invalid-handle",
ClientCorrelationId = "live-open-invalid",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
// Deliberately skip Register: server handle 0x7FFFFFFF was never
// issued by MXAccess. The worker must invoke COM and relay the
// invalid-handle failure rather than the gateway short-circuiting.
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle: int.MaxValue),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem(invalid-handle)", addItemReply);
// MXAccess parity: an invalid handle is an MXAccess-level failure.
// The command still completed its worker round-trip, so the gateway
// must reply with ProtocolStatusCode.MxaccessFailure and a non-zero
// hresult carrying the COM failure (per HResultConverter) — never a
// gRPC transport fault. The assertion below just checks the status
// is not Ok; the failure detail lives in hresult / the status proxies.
Assert.NotEqual(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
Assert.True(
addItemReply.AddItem is null || addItemReply.AddItem.ItemHandle <= 0,
"Invalid-handle AddItem must not yield a usable item handle.");
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies the MXAccess teardown chain: Unadvise then RemoveItem then Unregister
/// each return <see cref="ProtocolStatusCode.Ok"/>, and the worker stops emitting
/// OnDataChange events for the un-advised item. Exercises the lifecycle-ordering
/// parity CLAUDE.md singles out as a "do not synthesize" rule.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_UnadviseRemoveItemUnregister_TeardownOrderingParity()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-teardown",
ClientCorrelationId = "live-open-teardown",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// Wait for an OnDataChange to prove the subscription is live before tearing it down.
MxEvent firstDataChange = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnDataChange
&& candidate.ServerHandle == serverHandle
&& candidate.ItemHandle == itemHandle,
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
LogEvent(firstDataChange);
// 1) UnAdvise — must reply Ok; the worker must stop emitting OnDataChange
// for this (server, item) pair after this returns.
MxCommandReply unadviseReply = await fixture.Service.Invoke(
CreateUnAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("UnAdvise", unadviseReply);
Assert.Equal(ProtocolStatusCode.Ok, unadviseReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.UnAdvise, unadviseReply.Kind);
// 2) RemoveItem — must reply Ok against the same handles.
MxCommandReply removeItemReply = await fixture.Service.Invoke(
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("RemoveItem", removeItemReply);
Assert.Equal(ProtocolStatusCode.Ok, removeItemReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.RemoveItem, removeItemReply.Kind);
// 3) Unregister — closes the client session inside the worker.
MxCommandReply unregisterReply = await fixture.Service.Invoke(
CreateUnregisterRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Unregister", unregisterReply);
Assert.Equal(ProtocolStatusCode.Ok, unregisterReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Unregister, unregisterReply.Kind);
// Parity rule: after UnAdvise returns Ok the worker must stop emitting
// OnDataChange for this (server, item) pair. Events the provider already
// published before that ack are in-flight and not a regression — the rule
// only constrains events generated AFTER the teardown returned. So the
// "before" baseline is taken *after* a first settle window drains those
// in-flight events, not before UnAdvise was issued (which races against
// the round-trip + STA dispatch + pipe send window).
//
// RecordingServerStreamWriter.Messages returns a snapshot copy under its
// own lock, so iterating after each settle window is safe without external
// sync.
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
int dataChangeCountAfterFirstSettle = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == itemHandle);
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
int dataChangeCountAfterSecondSettle = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == itemHandle);
output.WriteLine(
$"DataChange count after first settle={dataChangeCountAfterFirstSettle} after second settle={dataChangeCountAfterSecondSettle}");
Assert.Equal(dataChangeCountAfterFirstSettle, dataChangeCountAfterSecondSettle);
// A RemoveItem against the just-freed item handle must not silently succeed —
// the worker has to relay MXAccess's invalid-handle response. Closing the
// session is enough for parity, but we sanity-check that re-using the freed
// pair does not accidentally appear Ok.
MxCommandReply secondRemoveItemReply = await fixture.Service.Invoke(
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("RemoveItem(stale)", secondRemoveItemReply);
Assert.NotEqual(ProtocolStatusCode.Ok, secondRemoveItemReply.ProtocolStatus.Code);
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies the MXAccess <c>WriteSecured</c> path: <c>AuthenticateUser</c> resolves a
/// user id, then <c>WriteSecured</c> against the advised item completes its round-trip
/// to the worker and back. CLAUDE.md singles out <c>WriteSecured</c> ordering as a
/// parity surface the gateway must not "fix" — the test asserts the reply kind and
/// protocol status, not a fabricated outcome.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_WriteSecured_AuthenticatedRoundTripParity()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
// CLAUDE.md's credential-redaction rule covers every log
// surface the test sees, not just the reply's DiagnosticMessage. Wire a buffering
// wrapper around output and route the worker stdout/stderr echo and the gateway
// ILogger sink through it so the post-run assertion covers the accumulated test
// output. A regression that logged the request body, the WorkerCommandRequest
// envelope, or printed the credential from inside the worker is caught here
// even if the bare DiagnosticMessage check still passes.
RecordingTestOutputHelper recordedOutput = new(output);
TestWorkerProcessFactory processFactory = new(recordedOutput);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
// Stream events so a regression that emitted an OperationComplete or
// OnWriteComplete with wrong handles would still be observable via the test
// output (we don't assert a specific event here — the docs note successful
// writes raise only OnWriteComplete, but WriteSecured against an unprotected
// item commonly fails with 0x80004021 in this provider, which raises no event).
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-write-secured",
ClientCorrelationId = "live-open-write-secured",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Advise", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// AuthenticateUser resolves an ArchestrA user id for the WriteSecured call.
// Credentials are env-overridable so the test honors the gateway's "do not
// log secrets" rule and works against either MXAccess's own user store or
// the LmxOpcUa-baseline GLAuth-bridged ArchestrA identity (admin/password —
// see glauth.md; the shared GLAuth's dev-user password is "password", not
// the old per-box "admin123").
MxCommandReply authReply = await fixture.Service.Invoke(
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
new TestServerCallContext()).ConfigureAwait(false);
// AuthenticateUser is allowed to fail — MXAccess parity means a rejected
// credential is a legitimate reply, not a bug, so this must never be an
// Assert. But a silent fall-through to "operator only" (currentUserId 0)
// quietly demotes the rest of this test from an authenticated round-trip
// to a bare failure-reply-shape check, which a stale rig credential (e.g.
// one that has expired) would otherwise hide indefinitely in nightly runs. Log the
// outcome loudly — never the password (verifyUser only) — so that
// degradation is visible in the test output even though it never fails
// the test by itself.
bool authenticateUserSucceeded = authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok
&& authReply.AuthenticateUser is not null
&& authReply.AuthenticateUser.UserId != 0;
recordedOutput.WriteLine(
authenticateUserSucceeded
? $"AuthenticateUser SUCCEEDED for user={verifyUser} user_id={authReply.AuthenticateUser!.UserId}"
: $"AuthenticateUser FAILED (credential rejected or not recognized by the provider) "
+ $"for user={verifyUser} status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} "
+ $"diagnostic={authReply.DiagnosticMessage} -- WriteSecured below will run as an "
+ $"unauthenticated (currentUserId=0) failure-reply-shape check only, not an "
+ $"authenticated round-trip. If this is unexpected, verify "
+ $"MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER/_PASSWORD against the rig's ArchestrA "
+ $"identity (see glauth.md).");
int currentUserId = authenticateUserSucceeded ? authReply.AuthenticateUser!.UserId : 0;
MxCommandReply writeSecuredReply = await fixture.Service.Invoke(
CreateWriteSecuredRequest(
sessionId,
serverHandle,
itemHandle,
currentUserId,
verifierUserId: 0),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "WriteSecured", writeSecuredReply);
// Parity: the command itself completed its round-trip — the reply kind is
// WriteSecured and the gateway protocol status is set. The MXAccess outcome
// (Ok for an unprotected provider, MxaccessFailure with hresult 0x80004021
// when the item is not WriteSecured-eligible) lives in protocol_status +
// hresult, never as a transport fault. The diagnostic message must never
// contain the credential.
Assert.Equal(MxCommandKind.WriteSecured, writeSecuredReply.Kind);
Assert.True(
writeSecuredReply.ProtocolStatus.Code is ProtocolStatusCode.Ok
or ProtocolStatusCode.MxaccessFailure,
$"Unexpected WriteSecured protocol status {writeSecuredReply.ProtocolStatus.Code}.");
Assert.DoesNotContain(verifyPassword, writeSecuredReply.DiagnosticMessage ?? string.Empty, StringComparison.Ordinal);
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
// CLAUDE.md credential contract: passwords and WriteSecured payloads must never
// reach logs. The buffered output covers the gateway ILogger sink, worker
// stdout/stderr, and every direct WriteLine the test body issued. A regression
// that dumped the request envelope, the AuthenticateUserCommand body, or any
// command-level WriteSecured payload would land here and trip this assertion.
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
}
/// <summary>
/// B8 live verification of the COM commands the B-bundle added against a fake
/// IMxAccessServer: <c>AuthenticateUser</c>, <c>ArchestrAUserToId</c>, <c>Suspend</c>,
/// and <c>Activate</c>. The contract being proven is that each command round-trips
/// to the worker and back carrying a real MXAccess outcome (Ok / an MxStatusProxy /
/// a non-zero HResult) and is NOT short-circuited to <c>INVALID_REQUEST</c> the way an
/// unimplemented command would be. MXAccess-level rejections (a wrong item class for
/// Suspend/Activate commonly returns 0x80070057) are parity, not test failures — we
/// assert the reply kind plus a non-INVALID_REQUEST protocol status, and log the
/// HResult for the record.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_NewComCommands_RoundTripWithRealReplies()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
// Credential-redaction: AuthenticateUser carries a password. Route every test
// log surface through the buffering helper so the post-run assertion proves the
// password never reached the gateway logger, worker stdout/stderr, or any
// WriteLine the test body issued (same pattern as the WriteSecured parity test).
RecordingTestOutputHelper recordedOutput = new(output);
TestWorkerProcessFactory processFactory = new(recordedOutput);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
string? sessionId = null;
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-new-com-commands",
ClientCorrelationId = "live-open-new-com",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, serverHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "AddItem", addItemReply);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
int itemHandle = addItemReply.AddItem.ItemHandle;
// AuthenticateUser — the B-bundle command under live verification. Before the
// B-bundle this command was unimplemented and the worker short-circuited it to
// INVALID_REQUEST. It must now produce a real reply (Ok with a user id when the
// provider accepts the credential, or a real MXAccess HResult when it does not).
MxCommandReply authReply = await fixture.Service.Invoke(
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
new TestServerCallContext()).ConfigureAwait(false);
recordedOutput.WriteLine(
$"AuthenticateUser status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} user_id={authReply.AuthenticateUser?.UserId}");
Assert.Equal(MxCommandKind.AuthenticateUser, authReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, authReply.ProtocolStatus.Code);
Assert.True(
authReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"AuthenticateUser must surface a real MXAccess outcome, got {authReply.ProtocolStatus.Code}.");
int authenticatedUserId =
authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok && authReply.AuthenticateUser is not null
? authReply.AuthenticateUser.UserId
: 0;
if (authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
{
// On the dev rig AuthenticateUser("Administrator","") resolves to user id 1.
// Don't pin the exact value (provider/user-store dependent) — just prove a
// success carried a usable, non-zero ArchestrA user id through the reply.
Assert.NotEqual(0, authenticatedUserId);
}
// ArchestrAUserToId — resolves an ArchestrA user GUID to an integer user id.
// We feed an empty/placeholder GUID: the value is provider-dependent, so the
// assertion is the parity one (real reply, never INVALID_REQUEST). A non-zero
// HResult here is the expected MXAccess rejection of an unknown GUID.
MxCommandReply userToIdReply = await fixture.Service.Invoke(
CreateArchestrAUserToIdRequest(sessionId, serverHandle, userIdGuid: string.Empty),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "ArchestrAUserToId", userToIdReply);
recordedOutput.WriteLine($"ArchestrAUserToId user_id={userToIdReply.ArchestraUserToId?.UserId}");
Assert.Equal(MxCommandKind.ArchestraUserToId, userToIdReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, userToIdReply.ProtocolStatus.Code);
Assert.True(
userToIdReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"ArchestrAUserToId must surface a real MXAccess outcome, got {userToIdReply.ProtocolStatus.Code}.");
if (userToIdReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
{
// On the dev rig ArchestrAUserToId with a valid GUID resolves to user_id=1.
// Don't pin the exact id (provider-dependent) — just prove the Ok path carried
// a usable non-zero ArchestrA user id through the reply payload.
Assert.NotNull(userToIdReply.ArchestraUserToId);
Assert.NotEqual(0, userToIdReply.ArchestraUserToId.UserId);
}
// Suspend / Activate against the added-but-not-advised item (no Advise was issued
// between AddItem and this call). The dev-rig TestInt item class may not be
// suspendable (MXAccess returns 0x80070057 / E_INVALIDARG for a wrong item class
// — see B8 notes). That is MXAccess parity: assert the reply kind and a
// non-INVALID_REQUEST status, surface the HResult and MxStatusProxy for the
// record, and do NOT treat a provider-side rejection as a test failure.
MxCommandReply suspendReply = await fixture.Service.Invoke(
CreateSuspendRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Suspend", suspendReply);
recordedOutput.WriteLine(
$"Suspend status_proxy success={suspendReply.Suspend?.Status?.Success} hresult=0x{(uint)suspendReply.Hresult:X8}");
Assert.Equal(MxCommandKind.Suspend, suspendReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, suspendReply.ProtocolStatus.Code);
Assert.True(
suspendReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"Suspend must surface a real MXAccess outcome, got {suspendReply.ProtocolStatus.Code}.");
MxCommandReply activateReply = await fixture.Service.Invoke(
CreateActivateRequest(sessionId, serverHandle, itemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReplyTo(recordedOutput, "Activate", activateReply);
recordedOutput.WriteLine(
$"Activate status_proxy success={activateReply.Activate?.Status?.Success} hresult=0x{(uint)activateReply.Hresult:X8}");
Assert.Equal(MxCommandKind.Activate, activateReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, activateReply.ProtocolStatus.Code);
Assert.True(
activateReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
$"Activate must surface a real MXAccess outcome, got {activateReply.ProtocolStatus.Code}.");
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
// Credential contract: the AuthenticateUser password must never reach any log
// surface (gateway logger, worker stdout/stderr, or test WriteLine).
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
}
/// <summary>
/// B8 §3.2 buffered-data path: adds a BUFFERED item (<c>AddBufferedItem</c>), sets the
/// buffered update interval (<c>SetBufferedUpdateInterval</c>), advises it, then attempts
/// to observe an <see cref="MxEventFamily.OnBufferedDataChange"/> event carrying multiple
/// samples so the worker's multi-sample conversion (VariantConverter →
/// OnBufferedDataChangeEvent quality/timestamp arrays) can be validated live.
/// <para>
/// The AddBufferedItem + SetBufferedUpdateInterval round-trips are asserted unconditionally
/// (they are the B-bundle commands under verification). The buffered EVENT capture is
/// best-effort: if the rig's object logic does not drive a buffered batch within the live
/// event timeout (the same environmental limitation seen with the externally-undrivable
/// alarm rig), the test records the buffered conversion as an unverified residual rather
/// than failing — the command path is proven, the live multi-sample conversion is not.
/// When a batch IS captured, the converted value and quality/timestamp arrays are asserted
/// to be non-empty and internally consistent (no crash, no dropped payload).
/// </para>
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_BufferedItem_AddsSetsIntervalAndAttemptsCapture()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
// AddBufferedItem takes (item_definition, item_context) like AddItem2. The dev rig
// exposes TestChildObject.TestInt; the buffered form is item="TestInt",
// context="TestChildObject" (per B8 notes). Split the configured live item so a
// custom MXGATEWAY_LIVE_MXACCESS_ITEM override still works.
(string bufferedItem, string bufferedContext) = SplitLiveItemForBuffered(IntegrationTestEnvironment.LiveMxAccessItem);
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-buffered",
ClientCorrelationId = "live-open-buffered",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Register", registerReply);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
int serverHandle = registerReply.Register.ServerHandle;
// SetBufferedUpdateInterval first so the buffered cadence is established before
// the item is added/advised. MXAccess rounds to 100ms units and rejects < 1.
MxCommandReply intervalReply = await fixture.Service.Invoke(
CreateSetBufferedUpdateIntervalRequest(sessionId, serverHandle, updateIntervalMilliseconds: 1000),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("SetBufferedUpdateInterval", intervalReply);
Assert.Equal(MxCommandKind.SetBufferedUpdateInterval, intervalReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, intervalReply.ProtocolStatus.Code);
Assert.Equal(ProtocolStatusCode.Ok, intervalReply.ProtocolStatus.Code);
// AddBufferedItem — must return a real item handle (the dev rig yields handle 1).
MxCommandReply addBufferedReply = await fixture.Service.Invoke(
CreateAddBufferedItemRequest(sessionId, serverHandle, bufferedItem, bufferedContext),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("AddBufferedItem", addBufferedReply);
Assert.Equal(MxCommandKind.AddBufferedItem, addBufferedReply.Kind);
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, addBufferedReply.ProtocolStatus.Code);
Assert.Equal(ProtocolStatusCode.Ok, addBufferedReply.ProtocolStatus.Code);
Assert.NotNull(addBufferedReply.AddBufferedItem);
int bufferedItemHandle = addBufferedReply.AddBufferedItem.ItemHandle;
Assert.True(bufferedItemHandle > 0, "AddBufferedItem must yield a usable item handle.");
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, serverHandle, bufferedItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Advise(buffered)", adviseReply);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
// Best-effort capture of a SAMPLE-BEARING buffered batch.
//
// Live observation (B8, re-read 2026-11-13): immediately after Advise the provider
// delivers one OnBufferedDataChange with data_type=NoData and zero quality and
// timestamp samples. Its status row is ConfigurationError: detail 6
// (InvalidReference) for the (item, context) form used here, and detail 1002
// (InvalidPropertyId) for a full reference to an attribute without a buffer. It is
// a resolution failure, not a bootstrap. The DEV Galaxy has no buffered attribute,
// so no sample-bearing batch can arrive (docs/LiveCaptures-2026-11-13.md §5). The
// converter handling it without crashing is still worth keeping. We match only a
// batch that carries samples, so the test validates a multi-sample conversion if
// the item is pointed at a buffered attribute.
MxEvent? bufferedBatch = null;
try
{
bufferedBatch = await eventWriter
.WaitForMessageAsync(
candidate => candidate.Family == MxEventFamily.OnBufferedDataChange
&& candidate.ServerHandle == serverHandle
&& candidate.ItemHandle == bufferedItemHandle
&& candidate.OnBufferedDataChange is not null
&& (CountArrayElements(candidate.OnBufferedDataChange.QualityValues) > 0
|| CountArrayElements(candidate.OnBufferedDataChange.TimestampValues) > 0),
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
streamCancellation.Token)
.ConfigureAwait(false);
}
catch (TimeoutException ex)
{
output.WriteLine($"B8: sample-bearing batch predicate timed out: {ex.Message}");
bufferedBatch = null;
}
// Whether or not a sample-bearing batch arrived, record what buffered events the
// rig DID deliver (typically just the empty NoData bootstrap) for the record.
int bootstrapBufferedEvents = CountMatchingEvents(
eventWriter,
e => e.Family == MxEventFamily.OnBufferedDataChange
&& e.ServerHandle == serverHandle
&& e.ItemHandle == bufferedItemHandle);
if (bufferedBatch is null)
{
// RESIDUAL (documented): the command path (AddBufferedItem +
// SetBufferedUpdateInterval + Advise) is proven and the buffered EVENT plumbing
// is live (the NoData resolution-failure event arrives and converts without
// crashing), but the rig has no buffered attribute to deliver a sample-bearing
// batch (docs/LiveCaptures-2026-11-13.md §5).
// The §3.2 OnBufferedDataChange MULTI-SAMPLE conversion therefore remains
// unverified live. This is environmental, not a defect — let the test pass.
output.WriteLine(
"B8 RESIDUAL: AddBufferedItem/SetBufferedUpdateInterval/Advise round-tripped and "
+ $"{bootstrapBufferedEvents} OnBufferedDataChange event(s) arrived (empty NoData "
+ "bootstrap, converted without crash/drop), but no sample-bearing buffered batch "
+ $"was observed within {IntegrationTestEnvironment.LiveMxAccessEventTimeout}. Live "
+ "§3.2 multi-sample conversion remains unverified (rig object logic may not drive "
+ "buffered samples on demand).");
// The residual claim ("empty NoData bootstrap arrives and converts without crashing")
// is only meaningful if at least one OnBufferedDataChange event arrived. Assert that
// the buffered subscription registered and the worker's event plumbing fired at all.
Assert.True(
bootstrapBufferedEvents > 0,
"No OnBufferedDataChange event arrived at all after Advise; the buffered subscription may not have registered.");
return;
}
// A SAMPLE-BEARING buffered batch was captured — validate the §3.2 conversion.
LogEvent(bufferedBatch);
OnBufferedDataChangeEvent body = bufferedBatch.OnBufferedDataChange;
Assert.NotNull(body);
int qualityCount = CountArrayElements(body.QualityValues);
int timestampCount = CountArrayElements(body.TimestampValues);
output.WriteLine(
$"B8 CAPTURED buffered batch: data_type={body.DataType} raw_data_type={body.RawDataType} "
+ $"quality_samples={qualityCount} timestamp_samples={timestampCount} "
+ $"value_kind={bufferedBatch.Value?.KindCase}");
// The predicate guaranteed at least one sample; the converted aggregate value
// must also exist (no crash, no dropped payload).
Assert.True(
qualityCount > 0 || timestampCount > 0,
"Sample-bearing OnBufferedDataChange lost its samples after the predicate matched.");
Assert.NotNull(bufferedBatch.Value);
// When MXAccess delivers parallel quality + timestamp arrays the converted
// arrays must agree in length; a mismatch is a real conversion defect (a sample
// was dropped on one side).
if (qualityCount > 0 && timestampCount > 0)
{
Assert.Equal(qualityCount, timestampCount);
}
}
finally
{
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
}
}
/// <summary>
/// Verifies that killing the worker process marks the session
/// <see cref="SessionState.Faulted"/> with a clean fault classification — the gateway
/// must observe the abnormal exit, transition the session, and surface a non-empty
/// fault description rather than hanging or crashing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_AbnormalWorkerExit_MarksSessionFaulted()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-abnormal-exit",
ClientCorrelationId = "live-open-abnormal",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
// Kill the worker process directly. WorkerClient's read loop hits an
// end-of-stream on the named pipe and routes through SetFaulted; the
// session manager then marks the session Faulted. We avoid CloseSession
// so the transition is driven by the abnormal exit, not a graceful path.
processFactory.KillAllAndDetach();
DateTimeOffset waitDeadline = DateTimeOffset.UtcNow + StreamShutdownTimeout;
SessionState observedState = SessionState.Unspecified;
string? observedFault = null;
while (DateTimeOffset.UtcNow < waitDeadline)
{
if (fixture.TryGetSession(sessionId, out GatewaySession? session))
{
observedState = session.State;
observedFault = session.FinalFault;
if (observedState == SessionState.Faulted)
{
break;
}
}
await Task.Delay(TimeSpan.FromMilliseconds(50)).ConfigureAwait(false);
}
output.WriteLine($"AbnormalExit observed_state={observedState} fault={observedFault}");
Assert.Equal(SessionState.Faulted, observedState);
Assert.False(string.IsNullOrWhiteSpace(observedFault), "Faulted session must carry a non-empty fault description.");
// The fault classification must come from a known worker-client error code so
// operators get an actionable cause string rather than an opaque exception
// trace. We accept the classifications WorkerClient actually drives on an
// abnormal exit (kill-the-process path): the read loop hits EndOfStream and
// calls SetFaulted with WorkerClientErrorCode.PipeDisconnected and the
// message "Worker pipe disconnected." (see WorkerClient.cs:378-381). The
// earlier broad list (including "worker") matched every WorkerClient fault
// message (they all begin with "Worker"); tighten to the pipe/disconnect/
// end-of-stream classifications that match THIS path, so a regression that
// routed an unrelated fault here would surface as a test failure rather
// than silently passing. "heartbeat" is dropped
// because HeartbeatGraceSeconds (15s) exceeds the StreamShutdownTimeout
// (10s) poll window, so a heartbeat-expired transition can never be
// observed inside this test.
Assert.True(
observedFault!.Contains("pipe disconnected", StringComparison.OrdinalIgnoreCase)
|| observedFault.Contains("end of stream", StringComparison.OrdinalIgnoreCase),
$"Fault description '{observedFault}' did not match a known abnormal-exit classification "
+ "(expected 'pipe disconnected' or 'end of stream' from WorkerClient's EndOfStream path).");
// Also assert the StreamEvents call observed the fault
// — the chain that puts the session into Faulted goes through ReadEventsAsync
// propagating a WorkerClientException into EventStreamService, which calls
// session.MarkFaulted. The gateway then maps the WorkerClientException to an
// RpcException at the public boundary (MxAccessGatewayService.MapException →
// MapWorkerClientException). Polling session.State alone would silently pass
// if a future refactor moved MarkFaulted off the stream-consumption path —
// assert the streamTask itself terminated with a fault so the test couples
// to the actual fault-propagation path. Compare to the inverse assertion in
// the Write parity test (line 217: Assert.False(streamTask.IsFaulted, ...)).
try
{
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (Exception streamException)
{
output.WriteLine($"StreamEvents task terminated with: {streamException.GetType().Name}: {streamException.Message}");
}
Assert.True(
streamTask.IsCompleted,
"StreamEvents task did not complete within the shutdown timeout after the worker was killed.");
Assert.True(
streamTask.IsFaulted,
"StreamEvents task must fault on abnormal worker exit, not complete cleanly — "
+ "the fault-propagation path from WorkerClient.SetFaulted through ReadEventsAsync is the contract.");
}
finally
{
streamCancellation.Cancel();
// sessionId is intentionally null here — the session is already faulted and a
// CloseSession round-trip would just log a cleanup failure. We still wait for
// the worker process exit so the next test starts with a clean state.
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
}
}
/// <summary>
/// TST-05: the real worker answers the four non-terminating control commands off-STA in
/// <c>WorkerPipeSession.HandleControlCommandAsync</c>. The default suite only ever sees the
/// FakeWorkerHarness canned replies, so this drives <c>Ping</c>, <c>GetSessionState</c>,
/// <c>GetWorkerInfo</c>, and <c>DrainEvents</c> through the gateway against a live worker and
/// asserts each reply carries real worker state rather than an <c>INVALID_REQUEST</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ControlCommands_ReturnRealWorkerState()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-control",
ClientCorrelationId = "live-open-control",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
MxCommandReply pingReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand { Message = "live-ping-echo" },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Ping", pingReply);
Assert.Equal(ProtocolStatusCode.Ok, pingReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Ping, pingReply.Kind);
MxCommandReply stateReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-session-state", new MxCommand
{
Kind = MxCommandKind.GetSessionState,
GetSessionState = new GetSessionStateCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetSessionState", stateReply);
Assert.Equal(ProtocolStatusCode.Ok, stateReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetSessionState, stateReply.Kind);
Assert.NotNull(stateReply.SessionState);
Assert.Equal(SessionState.Ready, stateReply.SessionState.State);
MxCommandReply infoReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-worker-info", new MxCommand
{
Kind = MxCommandKind.GetWorkerInfo,
GetWorkerInfo = new GetWorkerInfoCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetWorkerInfo", infoReply);
Assert.Equal(ProtocolStatusCode.Ok, infoReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetWorkerInfo, infoReply.Kind);
Assert.NotNull(infoReply.WorkerInfo);
output.WriteLine(
$"GetWorkerInfo pid={infoReply.WorkerInfo.WorkerProcessId} version={infoReply.WorkerInfo.WorkerVersion} "
+ $"progid={infoReply.WorkerInfo.MxaccessProgid} clsid={infoReply.WorkerInfo.MxaccessClsid}");
Assert.Equal(openReply.WorkerProcessId, infoReply.WorkerInfo.WorkerProcessId);
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.WorkerVersion));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessProgid));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessClsid));
// No StreamEvents subscriber is attached, so whatever the worker queued since start
// is still drainable; the reply must carry the DrainEvents payload (possibly empty).
MxCommandReply drainReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-drain-events", new MxCommand
{
Kind = MxCommandKind.DrainEvents,
DrainEvents = new DrainEventsCommand { MaxEvents = 16 },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("DrainEvents", drainReply);
Assert.Equal(ProtocolStatusCode.Ok, drainReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.DrainEvents, drainReply.Kind);
Assert.Equal(MxCommandReply.PayloadOneofCase.DrainEvents, drainReply.PayloadCase);
Assert.True(drainReply.DrainEvents.Events.Count <= 16);
output.WriteLine($"DrainEvents drained={drainReply.DrainEvents.Events.Count}");
// A control command must not disturb the session: it is still Ready afterwards.
Assert.True(fixture.TryGetSession(sessionId, out GatewaySession? session));
Assert.Equal(SessionState.Ready, session!.State);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
/// <summary>
/// TST-05: <c>ShutdownWorker</c> writes its OK reply before the worker shuts down, so the
/// reply reaches the caller (a reply written after the pipe closed would surface as a
/// pipe-disconnected failure instead), the worker process then exits on its own, and a later
/// command on the session does not succeed. Uses its own fixture and worker because it
/// terminates the worker.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ShutdownWorker_RepliesOkThenWorkerExits()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-shutdown-worker",
ClientCorrelationId = "live-open-shutdown-worker",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
// The gateway observes a worker's pipe closing through the event-stream read loop
// (the same path the abnormal-exit fact relies on); without a subscriber the session
// would sit in Ready until the heartbeat grace expired.
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply shutdownReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-shutdown-worker", new MxCommand
{
Kind = MxCommandKind.ShutdownWorker,
ShutdownWorker = new ShutdownWorkerCommand
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(5)),
},
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("ShutdownWorker", shutdownReply);
Assert.Equal(ProtocolStatusCode.Ok, shutdownReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.ShutdownWorker, shutdownReply.Kind);
// The worker exits on its own — no CloseSession, no kill. WaitForProcessesAsync
// throws on timeout, which fails the test.
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
// The session can no longer serve commands once its worker is gone: either the
// gateway rejects the call outright or the reply is not Ok.
MxCommandReply? afterReply = null;
Exception? afterException = null;
try
{
afterReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping-after-shutdown", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("PingAfterShutdown", afterReply);
}
catch (Exception exception)
{
afterException = exception;
output.WriteLine($"PingAfterShutdown rejected: {exception.GetType().Name}: {exception.Message}");
}
Assert.True(
afterException is not null || afterReply!.ProtocolStatus.Code != ProtocolStatusCode.Ok,
"A command sent after ShutdownWorker must not succeed against the exited worker.");
// Logged, not asserted: the gateway's WorkerClient moves to Closed on the shutdown
// ack, but GatewaySession.State is not driven from that transition today, so the
// public session state still reads Ready here (observed live on windev). That is a
// gateway-side gap, not a worker one — this fact pins the worker half of the
// contract: OK reply first, then a clean exit.
string observedState = fixture.TryGetSession(sessionId, out GatewaySession? session)
? session.State.ToString()
: "<removed>";
output.WriteLine($"ShutdownWorker observed_session_state={observedState}");
}
finally
{
// The worker is gone; a CloseSession round-trip would only log a cleanup failure.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
}
}
private static MxCommandRequest CreateControlRequest(
string sessionId,
string correlationId,
MxCommand command)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = correlationId,
Command = command,
};
}
/// <summary>
/// Closes the session and drains the event stream / worker processes without letting a
/// cleanup timeout mask the original failure from the test body.
/// </summary>
/// <param name="propagateStreamFaults">
/// When <see langword="true"/>, a faulted <paramref name="streamTask"/> is rethrown so the
/// test fails on a silent stream-task exception (the Write parity test relies on this so
/// stream-side defects in event delivery are visible). When <see langword="false"/>, all
/// cleanup exceptions are logged and swallowed so a real test-body assertion failure is not
/// masked by a shutdown timeout.
/// </param>
private async Task ShutDownAsync(
GatewayServiceFixture fixture,
TestWorkerProcessFactory processFactory,
string? sessionId,
Task? streamTask,
bool propagateStreamFaults = false)
{
Exception? streamFault = null;
try
{
if (!string.IsNullOrWhiteSpace(sessionId))
{
await CloseSessionAsync(fixture, sessionId).ConfigureAwait(false);
}
}
catch (Exception ex)
{
output.WriteLine($"Cleanup error during session close: {ex}");
}
if (streamTask is not null)
{
try
{
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
// A linked CancellationToken on the streaming TestServerCallContext is the
// intended way to stop StreamEvents promptly — treat the resulting
// OperationCanceledException as a clean shutdown, not a fault.
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Message}");
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
{
// MxAccessGatewayService.MapException intentionally converts the
// server-side OperationCanceledException into RpcException(Cancelled)
// so real gRPC clients see the standard cancellation status. Treat
// that as the same clean-shutdown signal as a raw OCE.
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Status.Detail}");
}
catch (Exception ex)
{
// Cleanup runs in a finally block. By default a faulted StreamEvents task is
// logged and swallowed so a test-body assertion failure is not masked. When
// the caller opts into propagateStreamFaults (the Write parity test), we
// rethrow the fault after the worker-process wait so a silent stream-side
// defect actually fails the test.
output.WriteLine($"Event stream task faulted during shutdown: {ex}");
if (propagateStreamFaults)
{
streamFault = ex;
}
}
}
try
{
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
}
catch (Exception ex)
{
output.WriteLine($"Cleanup error while waiting for worker processes to exit: {ex}");
}
if (streamFault is not null)
{
throw streamFault;
}
}
private static MxCommandRequest CreateRegisterRequest(string sessionId)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-register",
Command = new MxCommand
{
Kind = MxCommandKind.Register,
Register = new RegisterCommand
{
ClientName = IntegrationTestEnvironment.LiveMxAccessClientName,
},
},
};
}
private static MxCommandRequest CreateAddItemRequest(
string sessionId,
int serverHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-add-item",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand
{
ServerHandle = serverHandle,
ItemDefinition = IntegrationTestEnvironment.LiveMxAccessItem,
},
},
};
}
private static MxCommandRequest CreateAdviseRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-advise",
Command = new MxCommand
{
Kind = MxCommandKind.Advise,
Advise = new AdviseCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateAdviseSupervisoryRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-advise-supervisory",
Command = new MxCommand
{
Kind = MxCommandKind.AdviseSupervisory,
AdviseSupervisory = new AdviseSupervisoryCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateWriteRequest(
string sessionId,
int serverHandle,
int itemHandle,
MxValue value)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write-restore",
Command = new MxCommand
{
Kind = MxCommandKind.Write,
Write = new WriteCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = value,
},
},
};
}
private static MxCommandRequest CreateWriteRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write",
Command = new MxCommand
{
Kind = MxCommandKind.Write,
Write = new WriteCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 1,
},
},
},
};
}
private static MxCommandRequest CreateWrite2Request(
string sessionId,
int serverHandle,
int itemHandle,
DateTime timestampUtc)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write2",
Command = new MxCommand
{
Kind = MxCommandKind.Write2,
Write2 = new Write2Command
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 2,
},
TimestampValue = new MxValue
{
DataType = MxDataType.Time,
VariantType = "VT_DATE",
TimestampValue = Timestamp.FromDateTime(timestampUtc),
},
},
},
};
}
private static MxCommandRequest CreateUnAdviseRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-unadvise",
Command = new MxCommand
{
Kind = MxCommandKind.UnAdvise,
UnAdvise = new UnAdviseCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateRemoveItemRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-remove-item",
Command = new MxCommand
{
Kind = MxCommandKind.RemoveItem,
RemoveItem = new RemoveItemCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateUnregisterRequest(
string sessionId,
int serverHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-unregister",
Command = new MxCommand
{
Kind = MxCommandKind.Unregister,
Unregister = new UnregisterCommand
{
ServerHandle = serverHandle,
},
},
};
}
private static MxCommandRequest CreateAuthenticateUserRequest(
string sessionId,
int serverHandle,
string verifyUser,
string verifyPassword)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-authenticate-user",
Command = new MxCommand
{
Kind = MxCommandKind.AuthenticateUser,
AuthenticateUser = new AuthenticateUserCommand
{
ServerHandle = serverHandle,
VerifyUser = verifyUser,
VerifyUserPassword = verifyPassword,
},
},
};
}
private static MxCommandRequest CreateArchestrAUserToIdRequest(
string sessionId,
int serverHandle,
string userIdGuid)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-archestra-user-to-id",
Command = new MxCommand
{
Kind = MxCommandKind.ArchestraUserToId,
ArchestraUserToId = new ArchestrAUserToIdCommand
{
ServerHandle = serverHandle,
UserIdGuid = userIdGuid,
},
},
};
}
private static MxCommandRequest CreateAddBufferedItemRequest(
string sessionId,
int serverHandle,
string itemDefinition,
string itemContext)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-add-buffered-item",
Command = new MxCommand
{
Kind = MxCommandKind.AddBufferedItem,
AddBufferedItem = new AddBufferedItemCommand
{
ServerHandle = serverHandle,
ItemDefinition = itemDefinition,
ItemContext = itemContext,
},
},
};
}
private static MxCommandRequest CreateSetBufferedUpdateIntervalRequest(
string sessionId,
int serverHandle,
int updateIntervalMilliseconds)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-set-buffered-update-interval",
Command = new MxCommand
{
Kind = MxCommandKind.SetBufferedUpdateInterval,
SetBufferedUpdateInterval = new SetBufferedUpdateIntervalCommand
{
ServerHandle = serverHandle,
UpdateIntervalMilliseconds = updateIntervalMilliseconds,
},
},
};
}
private static MxCommandRequest CreateSuspendRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-suspend",
Command = new MxCommand
{
Kind = MxCommandKind.Suspend,
Suspend = new SuspendCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateActivateRequest(
string sessionId,
int serverHandle,
int itemHandle)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-activate",
Command = new MxCommand
{
Kind = MxCommandKind.Activate,
Activate = new ActivateCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
},
},
};
}
private static MxCommandRequest CreateWriteSecuredRequest(
string sessionId,
int serverHandle,
int itemHandle,
int currentUserId,
int verifierUserId)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-write-secured",
Command = new MxCommand
{
Kind = MxCommandKind.WriteSecured,
WriteSecured = new WriteSecuredCommand
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
CurrentUserId = currentUserId,
VerifierUserId = verifierUserId,
Value = new MxValue
{
DataType = MxDataType.Integer,
Int32Value = 2,
},
},
},
};
}
private static (string VerifyUser, string VerifyPassword) ResolveLiveMxAccessSecuredCredentials()
{
string verifyUser = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER")
?? "admin";
string verifyPassword = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD")
?? "password";
return (verifyUser, verifyPassword);
}
/// <summary>
/// Splits a dotted MXAccess reference (e.g. "TestChildObject.TestInt") into the
/// (item_definition, item_context) pair AddBufferedItem expects — attribute name and
/// owning object. An undotted reference is passed through with an empty context.
/// </summary>
private static (string Item, string Context) SplitLiveItemForBuffered(string liveItem)
{
int lastDot = liveItem.LastIndexOf('.');
if (lastDot < 0 || lastDot >= liveItem.Length - 1)
{
return (liveItem, string.Empty);
}
string context = liveItem[..lastDot];
string item = liveItem[(lastDot + 1)..];
return (item, context);
}
/// <summary>
/// Counts the elements in a converted buffered <see cref="MxArray"/> across whichever
/// typed-array oneof case the VariantConverter populated, so the buffered-capture
/// assertions are independent of the rig item's element type.
/// </summary>
private static int CountArrayElements(MxArray? array)
{
if (array is null)
{
return 0;
}
return array.ValuesCase switch
{
MxArray.ValuesOneofCase.BoolValues => array.BoolValues.Values.Count,
MxArray.ValuesOneofCase.Int32Values => array.Int32Values.Values.Count,
MxArray.ValuesOneofCase.Int64Values => array.Int64Values.Values.Count,
MxArray.ValuesOneofCase.FloatValues => array.FloatValues.Values.Count,
MxArray.ValuesOneofCase.DoubleValues => array.DoubleValues.Values.Count,
MxArray.ValuesOneofCase.StringValues => array.StringValues.Values.Count,
MxArray.ValuesOneofCase.TimestampValues => array.TimestampValues.Values.Count,
MxArray.ValuesOneofCase.RawValues => array.RawValues.Values.Count,
_ => 0,
};
}
private static int CountMatchingEvents(
RecordingServerStreamWriter<MxEvent> writer,
Func<MxEvent, bool> predicate)
{
int count = 0;
foreach (MxEvent message in writer.Messages)
{
if (predicate(message))
{
count++;
}
}
return count;
}
private async Task CloseSessionAsync(
GatewayServiceFixture fixture,
string sessionId)
{
CloseSessionReply closeReply = await fixture.Service.CloseSession(
new CloseSessionRequest
{
SessionId = sessionId,
ClientCorrelationId = "live-close",
},
new TestServerCallContext()).ConfigureAwait(false);
output.WriteLine($"CloseSession status={closeReply.ProtocolStatus.Code} final_state={closeReply.FinalState}");
}
private void LogReply(
string method,
MxCommandReply reply)
{
LogReplyTo(output, method, reply);
}
private static void LogReplyTo(
ITestOutputHelper sink,
string method,
MxCommandReply reply)
{
sink.WriteLine(
$"{method} status={reply.ProtocolStatus.Code} hresult={reply.Hresult} diagnostic={reply.DiagnosticMessage}");
foreach (MxStatusProxy status in reply.Statuses)
{
sink.WriteLine(
$"{method} mxstatus success={status.Success} category={status.Category} detail={status.Detail} text={status.DiagnosticText}");
}
}
private void LogEvent(MxEvent dataChange)
{
output.WriteLine(
$"Event family={dataChange.Family} worker_sequence={dataChange.WorkerSequence} server_handle={dataChange.ServerHandle} item_handle={dataChange.ItemHandle} quality={dataChange.Quality}");
output.WriteLine(
$"Event value_type={dataChange.Value?.DataType} raw_status={dataChange.RawStatus}");
}
/// <summary>
/// Test fixture that assembles the gateway service with a worker process factory for live MXAccess testing.
/// </summary>
private sealed class GatewayServiceFixture : IAsyncDisposable
{
private readonly GatewayMetrics _metrics = new();
private readonly SessionRegistry _registry = new();
private readonly ILoggerFactory _loggerFactory;
/// <summary>
/// Initializes the fixture with worker executable path, factory, and test output helper.
/// </summary>
/// <param name="workerExecutablePath">Path to the worker process executable.</param>
/// <param name="processFactory">Factory for creating worker processes.</param>
/// <param name="output">Test output helper for logging.</param>
/// <param name="alarms">Optional configuration for the central alarm monitor.</param>
public GatewayServiceFixture(
string workerExecutablePath,
IWorkerProcessFactory processFactory,
ITestOutputHelper output,
AlarmsOptions? alarms = null)
{
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
WorkerProcessLauncher launcher = new(
options,
processFactory,
new WorkerProcessStartedProbe(),
_metrics);
SessionWorkerClientFactory workerClientFactory = new(
launcher,
options,
_metrics,
_loggerFactory);
SessionManager sessionManager = new(
_registry,
workerClientFactory,
options,
_metrics,
logger: _loggerFactory.CreateLogger<SessionManager>());
Manager = sessionManager;
MxAccessGrpcMapper mapper = new();
EventStreamService eventStreamService = new(
sessionManager,
options,
_metrics);
Monitor = new GatewayAlarmMonitor(
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
_loggerFactory.CreateLogger<GatewayAlarmMonitor>());
Service = new MxAccessGatewayService(
sessionManager,
new GatewayRequestIdentityAccessor(),
new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(),
mapper,
eventStreamService,
_metrics,
_loggerFactory.CreateLogger<MxAccessGatewayService>(),
Monitor);
}
/// <summary>
/// The assembled gateway service instance.
/// </summary>
public MxAccessGatewayService Service { get; }
/// <summary>The central alarm monitor for tests of the public alarm RPCs.</summary>
public GatewayAlarmMonitor Monitor { get; }
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
public SessionManager Manager { get; }
/// <summary>
/// Looks up a session by id directly against the in-process registry. The abnormal
/// worker-exit test needs to observe the session's State / FinalFault as the gateway
/// transitions it to Faulted, which the public gRPC API only exposes indirectly via
/// CloseSession's reply (and not before a graceful close completes).
/// </summary>
/// <param name="sessionId">The session identifier.</param>
/// <param name="session">The session if found; otherwise null.</param>
/// <returns><see langword="true"/> if a session with the given id was found; otherwise <see langword="false"/>.</returns>
public bool TryGetSession(string sessionId, [MaybeNullWhen(false)] out GatewaySession session)
{
return _registry.TryGet(sessionId, out session);
}
/// <summary>
/// Disposes the fixture resources and closes all sessions.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
await Monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
foreach (GatewaySession session in _registry.Snapshot())
{
await session.DisposeAsync().ConfigureAwait(false);
}
Monitor.Dispose();
_loggerFactory.Dispose();
_metrics.Dispose();
}
private static GatewayOptions CreateOptions(string workerExecutablePath, AlarmsOptions? alarms)
{
return new GatewayOptions
{
Alarms = alarms ?? new AlarmsOptions(),
Worker = new WorkerOptions
{
ExecutablePath = workerExecutablePath,
StartupTimeoutSeconds = 30,
ShutdownTimeoutSeconds = 15,
HeartbeatCheckIntervalSeconds = 5,
HeartbeatGraceSeconds = 15,
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
RequiredArchitecture = WorkerArchitecture.X86,
},
Sessions = new SessionOptions
{
DefaultCommandTimeoutSeconds = 15,
MaxSessions = alarms?.Enabled == true ? 2 : 1,
},
Events = new EventOptions
{
QueueCapacity = 32,
},
};
}
}
/// <summary>
/// Gathers messages written to a server stream for test inspection.
/// </summary>
private sealed class RecordingServerStreamWriter<T> : IServerStreamWriter<T>, IDisposable
{
private readonly object syncRoot = new();
private readonly List<T> messages = [];
private readonly SemaphoreSlim messageArrived = new(0);
/// <summary>
/// All messages that have been written to the stream.
/// </summary>
public IReadOnlyList<T> Messages
{
get
{
lock (syncRoot)
{
return messages.ToArray();
}
}
}
/// <summary>
/// Inherited write options.
/// </summary>
public WriteOptions? WriteOptions { get; set; }
/// <summary>
/// Records the message unless the token is already cancelled.
/// </summary>
/// <param name="message">The message to write.</param>
/// <param name="cancellationToken">Token that cancels the write.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
// The interface default throws NotSupportedException for a cancelable token; the
// gateway passes the call's token on every streamed write.
cancellationToken.ThrowIfCancellationRequested();
return WriteAsync(message);
}
/// <summary>
/// Records the message and signals any pending waiter.
/// </summary>
/// <param name="message">The message to write.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task WriteAsync(T message)
{
lock (syncRoot)
{
messages.Add(message);
}
messageArrived.Release();
return Task.CompletedTask;
}
/// <summary>
/// Waits for the first recorded message that satisfies <paramref name="predicate"/>,
/// up to the specified timeout. Earlier non-matching messages (for example a
/// registration-state bootstrap event) are skipped rather than treated as the result.
/// </summary>
/// <param name="predicate">Filter the awaited message must satisfy.</param>
/// <param name="timeout">The maximum total time to wait.</param>
/// <param name="cancellationToken">
/// Token observed alongside the timeout so a per-test cancellation (for example the
/// gRPC call context's token) aborts the wait promptly instead of hanging until the
/// timeout elapses.
/// </param>
/// <returns>The first message that satisfies the predicate.</returns>
public async Task<T> WaitForMessageAsync(
Func<T, bool> predicate,
TimeSpan timeout,
CancellationToken cancellationToken = default)
{
using CancellationTokenSource timeoutCancellation = new(timeout);
using CancellationTokenSource linkedCancellation =
CancellationTokenSource.CreateLinkedTokenSource(timeoutCancellation.Token, cancellationToken);
int scanned = 0;
while (true)
{
T[] snapshot;
lock (syncRoot)
{
snapshot = messages.ToArray();
}
for (; scanned < snapshot.Length; scanned++)
{
if (predicate(snapshot[scanned]))
{
return snapshot[scanned];
}
}
try
{
await messageArrived.WaitAsync(linkedCancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (timeoutCancellation.IsCancellationRequested)
{
throw new TimeoutException(
$"No stream message satisfied the predicate within {timeout}. Recorded {scanned} message(s).");
}
}
}
/// <summary>
/// Releases the wait handle backing <c>messageArrived</c>. The writer owns an
/// <see cref="IDisposable"/> field so it must be disposable itself; the leak
/// is otherwise bounded only by how many opt-in live tests run.
/// </summary>
public void Dispose()
{
messageArrived.Dispose();
}
}
/// <summary>
/// Minimal <see cref="ServerCallContext"/> stub for invoking the gRPC service
/// in-process. It is a hand-written fake with no verification behavior — it
/// only supplies the context values the service reads during a call.
/// </summary>
private sealed class TestServerCallContext(CancellationToken cancellationToken = default) : ServerCallContext
{
private readonly Metadata requestHeaders = [];
private readonly Metadata responseTrailers = [];
private readonly Dictionary<object, object> userState = [];
private Status status;
private WriteOptions? writeOptions;
/// <inheritdoc />
protected override string MethodCore => "/mxaccess_gateway.v1.MxAccessGateway/Test";
/// <inheritdoc />
protected override string HostCore => "localhost";
/// <inheritdoc />
protected override string PeerCore => "ipv4:127.0.0.1:5000";
/// <inheritdoc />
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
/// <inheritdoc />
protected override Metadata RequestHeadersCore => requestHeaders;
/// <inheritdoc />
protected override CancellationToken CancellationTokenCore => cancellationToken;
/// <inheritdoc />
protected override Metadata ResponseTrailersCore => responseTrailers;
/// <inheritdoc />
protected override Status StatusCore
{
get => status;
set => status = value;
}
/// <inheritdoc />
protected override WriteOptions? WriteOptionsCore
{
get => writeOptions;
set => writeOptions = value;
}
/// <inheritdoc />
protected override AuthContext AuthContextCore { get; } = new(
string.Empty,
new Dictionary<string, List<AuthProperty>>(StringComparer.Ordinal));
/// <inheritdoc />
protected override IDictionary<object, object> UserStateCore => userState;
/// <inheritdoc />
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
{
return Task.CompletedTask;
}
/// <inheritdoc />
protected override ContextPropagationToken CreatePropagationTokenCore(
ContextPropagationOptions? options)
{
throw new NotSupportedException();
}
}
/// <summary>
/// Factory that launches worker processes and records their outputs for testing.
/// </summary>
private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory
{
private readonly ConcurrentBag<TestWorkerProcess> processes = [];
/// <inheritdoc />
public IWorkerProcess Start(ProcessStartInfo startInfo)
{
startInfo.RedirectStandardError = true;
startInfo.RedirectStandardOutput = true;
startInfo.UseShellExecute = false;
Process process = new()
{
StartInfo = startInfo,
EnableRaisingEvents = true,
};
process.OutputDataReceived += (_, args) => WriteWorkerOutput("stdout", args.Data);
process.ErrorDataReceived += (_, args) => WriteWorkerOutput("stderr", args.Data);
if (!process.Start())
{
process.Dispose();
throw new InvalidOperationException("Worker process failed to start.");
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
TestWorkerProcess workerProcess = new(process);
processes.Add(workerProcess);
output.WriteLine($"WorkerProcess started pid={workerProcess.Id} path={startInfo.FileName}");
return workerProcess;
}
/// <summary>
/// Waits for every recorded worker process to exit, up to the specified timeout per process.
/// </summary>
/// <param name="timeout">The maximum time to wait for each process.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task WaitForProcessesAsync(TimeSpan timeout)
{
foreach (TestWorkerProcess process in processes)
{
if (process.HasExited)
{
output.WriteLine($"WorkerProcess exited pid={process.Id} exit_code={process.ExitCode}");
continue;
}
using CancellationTokenSource timeoutCancellation = new(timeout);
await process.WaitForExitAsync(timeoutCancellation.Token).ConfigureAwait(false);
output.WriteLine($"WorkerProcess exited pid={process.Id} exit_code={process.ExitCode}");
}
}
/// <summary>
/// Kills every recorded worker process tree so the abnormal-exit test can simulate a
/// crashed worker without going through the graceful shutdown handshake. Failures to
/// kill an already-dead process are tolerated.
/// </summary>
public void KillAllAndDetach()
{
foreach (TestWorkerProcess process in processes)
{
if (process.HasExited)
{
continue;
}
try
{
process.Kill(entireProcessTree: true);
output.WriteLine($"WorkerProcess killed pid={process.Id} (abnormal-exit simulation)");
}
catch (InvalidOperationException ex)
{
output.WriteLine($"WorkerProcess kill skipped pid={process.Id}: {ex.Message}");
}
}
}
private void WriteWorkerOutput(
string streamName,
string? line)
{
if (!string.IsNullOrWhiteSpace(line))
{
output.WriteLine($"worker_{streamName}: {line}");
}
}
}
/// <summary>
/// Adapter wrapping a System.Diagnostics.Process as IWorkerProcess for testing.
/// </summary>
private sealed class TestWorkerProcess(Process process) : IWorkerProcess
{
/// <inheritdoc />
public int Id => process.Id;
/// <inheritdoc />
public bool HasExited => process.HasExited;
/// <inheritdoc />
public int? ExitCode => process.HasExited ? process.ExitCode : null;
/// <inheritdoc />
public async ValueTask WaitForExitAsync(CancellationToken cancellationToken)
{
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public void Kill(bool entireProcessTree)
{
process.Kill(entireProcessTree);
}
/// <summary>Disposes the wrapped process.</summary>
public void Dispose()
{
process.Dispose();
}
}
/// <summary>
/// Logger provider that writes all output to the test output helper.
/// </summary>
private sealed class TestOutputLoggerProvider(ITestOutputHelper output) : ILoggerProvider
{
/// <summary>Creates a logger that writes to the test output helper for the given category.</summary>
/// <param name="categoryName">Category name for the logger.</param>
/// <returns>The created logger.</returns>
public ILogger CreateLogger(string categoryName)
{
return new TestOutputLogger(output, categoryName);
}
/// <summary>No resources to release; provided to satisfy <see cref="ILoggerProvider"/>.</summary>
public void Dispose()
{
}
}
/// <summary>
/// Logger that writes messages to the test output helper.
/// </summary>
private sealed class TestOutputLogger(
ITestOutputHelper output,
string categoryName) : ILogger
{
/// <summary>Not supported; this test logger does not track scopes.</summary>
/// <typeparam name="TState">The type of the state to begin the scope for.</typeparam>
/// <param name="state">The identifier for the scope.</param>
/// <returns>Always <see langword="null"/>.</returns>
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull
{
return null;
}
/// <summary>Determines whether the given log level is enabled.</summary>
/// <param name="logLevel">The log level to check.</param>
/// <returns><see langword="true"/> if <paramref name="logLevel"/> is at least <see cref="LogLevel.Information"/>; otherwise <see langword="false"/>.</returns>
public bool IsEnabled(LogLevel logLevel)
{
return logLevel >= LogLevel.Information;
}
/// <summary>Writes the formatted log message and any exception to the test output helper.</summary>
/// <typeparam name="TState">The type of the object to be logged.</typeparam>
/// <param name="logLevel">The severity of the log entry.</param>
/// <param name="eventId">The event id associated with the log entry.</param>
/// <param name="state">The entry to be logged, which can be an object or a message string.</param>
/// <param name="exception">The exception related to this entry, if any.</param>
/// <param name="formatter">Function that creates the log message from <paramref name="state"/> and <paramref name="exception"/>.</param>
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
if (!IsEnabled(logLevel))
{
return;
}
output.WriteLine($"{logLevel} {categoryName}: {formatter(state, exception)}");
if (exception is not null)
{
output.WriteLine(exception.ToString());
}
}
}
/// <summary>
/// Buffering wrapper around an <see cref="ITestOutputHelper"/> that mirrors every line
/// written through it into a <see cref="StringBuilder"/> the test owns. The WriteSecured
/// parity test uses this to make CLAUDE.md's "passwords and
/// <c>WriteSecured</c> payloads must never reach logs" rule a property of the entire
/// test output stream — gateway <see cref="ILogger"/> entries (echoed via
/// <see cref="TestOutputLoggerProvider"/>), worker stdout/stderr (echoed via
/// <see cref="TestWorkerProcessFactory.WriteWorkerOutput"/>), and direct
/// <c>output.WriteLine</c> calls all land in the same buffer, so a future maintenance
/// change that prints a credential through any of those channels is caught by the
/// assertion rather than slipping past the existing <c>DiagnosticMessage</c> check.
/// </summary>
private sealed class RecordingTestOutputHelper(ITestOutputHelper inner) : ITestOutputHelper
{
private readonly StringBuilder buffer = new();
private readonly object syncRoot = new();
/// <summary>Gets the accumulated output buffer contents.</summary>
public string Captured
{
get
{
lock (syncRoot)
{
return buffer.ToString();
}
}
}
/// <summary>Writes a line of text to the buffer and inner helper.</summary>
/// <param name="message">The message to write.</param>
public void WriteLine(string message)
{
lock (syncRoot)
{
buffer.AppendLine(message);
}
inner.WriteLine(message);
}
/// <summary>Writes a formatted line of text to the buffer and inner helper.</summary>
/// <param name="format">The message format string.</param>
/// <param name="args">The format arguments.</param>
public void WriteLine(string format, params object[] args)
{
string formatted = string.Format(System.Globalization.CultureInfo.InvariantCulture, format, args);
lock (syncRoot)
{
buffer.AppendLine(formatted);
}
inner.WriteLine(format, args);
}
}
private sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
{
/// <inheritdoc />
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task RecordDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
string target,
ConstraintFailure failure,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
}
}