3516 lines
169 KiB
C#
3516 lines
169 KiB
C#
using System.Collections.Concurrent;
|
||
using System.Diagnostics;
|
||
using System.Diagnostics.CodeAnalysis;
|
||
using System.Net;
|
||
using System.Text;
|
||
using Google.Protobuf.WellKnownTypes;
|
||
using Grpc.Core;
|
||
using Grpc.Net.Client;
|
||
using Microsoft.AspNetCore.Builder;
|
||
using Microsoft.AspNetCore.Hosting;
|
||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Options;
|
||
using ZB.MOM.WW.MxGateway.Contracts;
|
||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||
using ZB.MOM.WW.MxGateway.IntegrationTests.TestSupport;
|
||
using Xunit.Abstractions;
|
||
using SessionOptions = ZB.MOM.WW.MxGateway.Server.Configuration.SessionOptions;
|
||
|
||
namespace ZB.MOM.WW.MxGateway.IntegrationTests;
|
||
|
||
[Collection(LiveResourcesCollection.Name)]
|
||
[Trait("Category", "LiveMxAccess")]
|
||
public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||
{
|
||
private static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(15);
|
||
private static readonly TimeSpan StreamShutdownTimeout = TimeSpan.FromSeconds(10);
|
||
|
||
/// <summary>
|
||
/// Verifies that a gateway session can register, add item, advise, and stream events from live MXAccess.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_RegistersAdvisesStreamsDataAndCloses()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-smoke",
|
||
ClientCorrelationId = "live-open",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
output.WriteLine($"OpenSession status={openReply.ProtocolStatus.Code} session={sessionId} worker_pid={openReply.WorkerProcessId}");
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
Assert.True(openReply.WorkerProcessId > 0);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
Assert.True(registerReply.Register.ServerHandle > 0);
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
Assert.True(addItemReply.AddItem.ItemHandle > 0);
|
||
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseRequest(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Advise", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
// A live MXAccess provider can deliver an initial registration-state
|
||
// or bad-quality bootstrap event before the OnDataChange the worker
|
||
// is contracted to emit. Match on the family rather than trusting
|
||
// whatever event arrives first so a genuine ordering defect cannot
|
||
// pass spuriously or leave a later wrong event unverified.
|
||
MxEvent dataChange = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnDataChange,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(dataChange);
|
||
|
||
Assert.Equal(MxEventFamily.OnDataChange, dataChange.Family);
|
||
Assert.Equal(sessionId, dataChange.SessionId);
|
||
Assert.Equal(registerReply.Register.ServerHandle, dataChange.ServerHandle);
|
||
Assert.Equal(addItemReply.AddItem.ItemHandle, dataChange.ItemHandle);
|
||
}
|
||
finally
|
||
{
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>Verifies Rust worker callbacks over the gateway's live HTTP/2 gRPC endpoint.</summary>
|
||
/// <returns>A task that represents the live test.</returns>
|
||
[RustWorkerLiveFact]
|
||
public async Task GatewayGrpc_WithRustWorker_StreamsChangingIntegersAndUnadvises()
|
||
{
|
||
const string itemDefinition = "TestMachine_001.TestChangingInt";
|
||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerPath, processFactory, output);
|
||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
|
||
CancellationToken token = deadline.Token;
|
||
await app.StartAsync(token).ConfigureAwait(false);
|
||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||
OpenSessionReply opened = await client.OpenSessionAsync(
|
||
new OpenSessionRequest { ClientSessionName = "rust-worker-live-events" },
|
||
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||
string sessionId = opened.SessionId;
|
||
|
||
try
|
||
{
|
||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request)
|
||
{
|
||
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
|
||
.ResponseAsync.ConfigureAwait(false);
|
||
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok, reply.ToString());
|
||
return reply;
|
||
}
|
||
|
||
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "rust-worker-info",
|
||
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
|
||
}).ConfigureAwait(false);
|
||
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
|
||
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
|
||
|
||
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
|
||
int serverHandle = registration.Register.ServerHandle;
|
||
MxCommandRequest addItem = CreateAddItemRequest(sessionId, serverHandle);
|
||
addItem.Command.AddItem.ItemDefinition = itemDefinition;
|
||
MxCommandReply added = await InvokeAsync(addItem).ConfigureAwait(false);
|
||
int itemHandle = added.AddItem.ItemHandle;
|
||
|
||
using (AsyncServerStreamingCall<MxEvent> stream = client.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
|
||
cancellationToken: token))
|
||
{
|
||
Assert.True(await stream.ResponseStream.MoveNext(token).ConfigureAwait(false));
|
||
Assert.NotNull(stream.ResponseStream.Current.StreamAttached);
|
||
await InvokeAsync(CreateAdviseRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
|
||
|
||
MxEvent? first = null;
|
||
bool observedChange = false;
|
||
ulong lastSequence = 0;
|
||
while (await stream.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
MxEvent change = stream.ResponseStream.Current;
|
||
Assert.True(change.WorkerSequence > lastSequence);
|
||
lastSequence = change.WorkerSequence;
|
||
if (change.Family != MxEventFamily.OnDataChange ||
|
||
change.Value?.KindCase != MxValue.KindOneofCase.Int32Value ||
|
||
(change.Quality & 0xc0) != 0xc0)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
Assert.Equal(sessionId, change.SessionId);
|
||
Assert.Equal(serverHandle, change.ServerHandle);
|
||
Assert.Equal(itemHandle, change.ItemHandle);
|
||
Assert.NotNull(change.SourceTimestamp);
|
||
Assert.NotNull(change.WorkerTimestamp);
|
||
Assert.Contains(change.Statuses, status => status.Category == MxStatusCategory.Ok);
|
||
if (first is not null && change.Value.Int32Value != first.Value.Int32Value)
|
||
{
|
||
output.WriteLine(
|
||
$"{itemDefinition} delivered through gateway: {first.Value.Int32Value} -> {change.Value.Int32Value}, " +
|
||
$"worker sequences {first.WorkerSequence} -> {change.WorkerSequence}, quality {change.Quality}.");
|
||
observedChange = true;
|
||
break;
|
||
}
|
||
|
||
first ??= change;
|
||
}
|
||
|
||
Assert.True(observedChange, "The gateway did not deliver distinct TestChangingInt values.");
|
||
|
||
await InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "rust-unadvise",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UnAdvise,
|
||
UnAdvise = new UnAdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
|
||
},
|
||
}).ConfigureAwait(false);
|
||
|
||
// A callback queued before UnAdvise may still cross the pipe.
|
||
// Observe the same stream and reject updates generated after acknowledgement.
|
||
DateTime unadvisedAt = DateTime.UtcNow;
|
||
using CancellationTokenSource quiet = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||
quiet.CancelAfter(TimeSpan.FromSeconds(2));
|
||
try
|
||
{
|
||
while (await stream.ResponseStream.MoveNext(quiet.Token).ConfigureAwait(false))
|
||
{
|
||
MxEvent change = stream.ResponseStream.Current;
|
||
Assert.False(change.Family == MxEventFamily.OnDataChange &&
|
||
change.ServerHandle == serverHandle && change.ItemHandle == itemHandle &&
|
||
change.WorkerTimestamp.ToDateTime() >= unadvisedAt,
|
||
"The worker sent a new data change after UnAdvise completed.");
|
||
}
|
||
|
||
Assert.Fail("The event stream ended before the quiet period elapsed.");
|
||
}
|
||
catch (RpcException exception) when (exception.StatusCode == StatusCode.Cancelled && quiet.IsCancellationRequested)
|
||
{
|
||
Assert.False(token.IsCancellationRequested);
|
||
}
|
||
catch (OperationCanceledException) when (quiet.IsCancellationRequested)
|
||
{
|
||
Assert.False(token.IsCancellationRequested);
|
||
}
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
CloseSessionReply closed = await client.CloseSessionAsync(
|
||
new CloseSessionRequest { SessionId = sessionId },
|
||
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
|
||
}
|
||
}
|
||
|
||
/// <summary>Verifies committed Rust worker writes and their callbacks through the gateway.</summary>
|
||
/// <returns>A task that represents the live test.</returns>
|
||
[RustWorkerLiveFact]
|
||
public async Task GatewayGrpc_WithRustWorker_WritesAndReceivesOnWriteComplete()
|
||
{
|
||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
|
||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(90));
|
||
CancellationToken token = deadline.Token;
|
||
await app.StartAsync(token).ConfigureAwait(false);
|
||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||
OpenSessionReply opened = await client.OpenSessionAsync(
|
||
new OpenSessionRequest { ClientSessionName = "rust-worker-live-write" },
|
||
cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||
string sessionId = opened.SessionId;
|
||
MxCommandRequest? restore = null;
|
||
|
||
try
|
||
{
|
||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request)
|
||
{
|
||
MxCommandReply reply = await client.InvokeAsync(request, cancellationToken: token)
|
||
.ResponseAsync.ConfigureAwait(false);
|
||
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
|
||
$"{request.Command.Kind}: {reply.ProtocolStatus.Code} HRESULT={reply.Hresult} {reply.DiagnosticMessage}");
|
||
return reply;
|
||
}
|
||
|
||
MxCommandReply worker = await InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "rust-write-worker-info",
|
||
Command = new MxCommand { Kind = MxCommandKind.GetWorkerInfo, GetWorkerInfo = new GetWorkerInfoCommand() },
|
||
}).ConfigureAwait(false);
|
||
Assert.EndsWith("-rust", worker.WorkerInfo.WorkerVersion, StringComparison.Ordinal);
|
||
Assert.Equal(opened.WorkerProcessId, worker.WorkerInfo.WorkerProcessId);
|
||
|
||
MxCommandReply registration = await InvokeAsync(CreateRegisterRequest(sessionId)).ConfigureAwait(false);
|
||
int serverHandle = registration.Register.ServerHandle;
|
||
MxCommandReply added = await InvokeAsync(CreateAddItemRequest(sessionId, serverHandle)).ConfigureAwait(false);
|
||
int itemHandle = added.AddItem.ItemHandle;
|
||
|
||
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId, EmitStreamAttached = true },
|
||
cancellationToken: token);
|
||
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
|
||
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
|
||
await InvokeAsync(CreateAdviseSupervisoryRequest(sessionId, serverHandle, itemHandle)).ConfigureAwait(false);
|
||
|
||
bool IsTarget(MxEvent update) => update.SessionId == sessionId &&
|
||
update.ServerHandle == serverHandle && update.ItemHandle == itemHandle;
|
||
bool IsGoodInt(MxEvent update) => IsTarget(update) && update.Family == MxEventFamily.OnDataChange &&
|
||
update.Value?.KindCase == MxValue.KindOneofCase.Int32Value && (update.Quality & 0xc0) == 0xc0;
|
||
|
||
MxEvent? initial = null;
|
||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
if (IsGoodInt(events.ResponseStream.Current))
|
||
{
|
||
initial = events.ResponseStream.Current;
|
||
break;
|
||
}
|
||
}
|
||
|
||
Assert.NotNull(initial);
|
||
int original = initial.Value.Int32Value;
|
||
int written = original == 1 ? 2 : 1;
|
||
|
||
async Task CheckWriteAsync(MxCommandRequest request, int expected, DateTime? expectedSource = null)
|
||
{
|
||
MxCommandReply reply = await InvokeAsync(request).ConfigureAwait(false);
|
||
Assert.Equal(request.Command.Kind, reply.Kind);
|
||
Assert.Equal(0, reply.Hresult);
|
||
if (request.Command.Kind == MxCommandKind.WriteBulk)
|
||
{
|
||
Assert.Collection(reply.WriteBulk.Results,
|
||
unsupported =>
|
||
{
|
||
Assert.Equal(itemHandle, unsupported.ItemHandle);
|
||
Assert.False(unsupported.WasSuccessful);
|
||
Assert.NotEmpty(unsupported.ErrorMessage);
|
||
},
|
||
accepted =>
|
||
{
|
||
Assert.Equal(itemHandle, accepted.ItemHandle);
|
||
Assert.True(accepted.WasSuccessful, accepted.ErrorMessage);
|
||
});
|
||
}
|
||
else
|
||
{
|
||
Assert.NotEmpty(reply.Statuses);
|
||
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||
}
|
||
MxEvent? completion = null;
|
||
MxEvent? changed = null;
|
||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
MxEvent update = events.ResponseStream.Current;
|
||
if (IsTarget(update) && update.Family == MxEventFamily.OnWriteComplete)
|
||
{
|
||
completion = update;
|
||
}
|
||
if (IsGoodInt(update) && update.Value.Int32Value == expected)
|
||
{
|
||
changed = update;
|
||
}
|
||
if (completion is not null && changed is not null)
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
|
||
Assert.NotNull(completion);
|
||
Assert.NotNull(changed);
|
||
Assert.NotEmpty(completion.Statuses);
|
||
Assert.All(completion.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||
if (request.Command.Kind != MxCommandKind.WriteBulk)
|
||
{
|
||
Assert.Equal(reply.Statuses.ToArray(), completion.Statuses.ToArray());
|
||
}
|
||
if (expectedSource is DateTime timestamp)
|
||
{
|
||
Assert.Equal(timestamp, changed.SourceTimestamp?.ToDateTime());
|
||
}
|
||
output.WriteLine(
|
||
$"{request.Command.Kind} committed {IntegrationTestEnvironment.LiveMxAccessItem}={expected}; " +
|
||
$"OnWriteComplete sequence={completion.WorkerSequence}, readback sequence={changed.WorkerSequence}.");
|
||
}
|
||
|
||
restore = CreateWriteRequest(sessionId, serverHandle, itemHandle, initial.Value);
|
||
MxCommandRequest write = CreateWriteRequest(sessionId, serverHandle, itemHandle,
|
||
new MxValue { DataType = MxDataType.Integer, Int32Value = written });
|
||
write.ClientCorrelationId = "rust-write-round-trip";
|
||
await CheckWriteAsync(write, written).ConfigureAwait(false);
|
||
|
||
int bulkValue = original == 3 ? 4 : 3;
|
||
var bulk = new WriteBulkCommand { ServerHandle = serverHandle };
|
||
bulk.Entries.Add(new WriteBulkEntry
|
||
{
|
||
ItemHandle = itemHandle,
|
||
Value = new MxValue { RawValue = Google.Protobuf.ByteString.CopyFrom(new byte[] { 0 }) },
|
||
});
|
||
bulk.Entries.Add(new WriteBulkEntry
|
||
{
|
||
ItemHandle = itemHandle,
|
||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = bulkValue },
|
||
});
|
||
await CheckWriteAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "rust-write-bulk-round-trip",
|
||
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = bulk },
|
||
}, bulkValue).ConfigureAwait(false);
|
||
|
||
DateTime timestamp = DateTime.UtcNow.AddSeconds(-30);
|
||
timestamp = new DateTime(timestamp.Ticks - timestamp.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc);
|
||
MxCommandRequest write2 = CreateWrite2Request(sessionId, serverHandle, itemHandle, timestamp);
|
||
write2.Command.Write2.Value = initial.Value;
|
||
await CheckWriteAsync(write2, original, timestamp).ConfigureAwait(false);
|
||
restore = null;
|
||
|
||
// The test item is unprotected. Exercise both secured native entry
|
||
// points with its current value and surface provider refusal as HRESULT.
|
||
foreach (MxCommand command in new MxCommand[]
|
||
{
|
||
new() { Kind = MxCommandKind.WriteSecured, WriteSecured = new WriteSecuredCommand
|
||
{
|
||
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
|
||
} },
|
||
new() { Kind = MxCommandKind.WriteSecured2, WriteSecured2 = new WriteSecured2Command
|
||
{
|
||
ServerHandle = serverHandle, ItemHandle = itemHandle, Value = initial.Value,
|
||
TimestampValue = write2.Command.Write2.TimestampValue,
|
||
} },
|
||
})
|
||
{
|
||
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId, ClientCorrelationId = $"rust-{command.Kind}", Command = command,
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(command.Kind, reply.Kind);
|
||
Assert.True(reply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
|
||
$"{command.Kind}: {reply.ProtocolStatus.Code} {reply.DiagnosticMessage}");
|
||
if (reply.ProtocolStatus.Code == ProtocolStatusCode.MxaccessFailure)
|
||
{
|
||
Assert.True(reply.HasHresult && reply.Hresult < 0);
|
||
}
|
||
output.WriteLine($"{command.Kind} returned {reply.ProtocolStatus.Code}, HRESULT=0x{reply.Hresult:X8}.");
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
try
|
||
{
|
||
if (restore is not null)
|
||
{
|
||
MxCommandReply restored = await client.InvokeAsync(restore, deadline: DateTime.UtcNow.AddSeconds(10))
|
||
.ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, restored.ProtocolStatus.Code);
|
||
Assert.NotEmpty(restored.Statuses);
|
||
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
CloseSessionReply closed = await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
|
||
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, closed.ProtocolStatus.Code);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Exercises bulk operations and both snapshot read paths over gRPC.</summary>
|
||
/// <returns>A task that represents the live test.</returns>
|
||
[RustWorkerLiveFact]
|
||
public async Task GatewayGrpc_WithRustWorker_BulkItemsAndReads()
|
||
{
|
||
const string first = "TestMachine_001.TestChangingInt";
|
||
const string second = "TestMachine_002.TestChangingInt";
|
||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
|
||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(60));
|
||
CancellationToken token = deadline.Token;
|
||
await app.StartAsync(token).ConfigureAwait(false);
|
||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "rust-worker-bulk-items",
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||
string id = opened.SessionId;
|
||
|
||
try
|
||
{
|
||
async Task<MxCommandReply> InvokeAsync(MxCommand command)
|
||
{
|
||
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = id, ClientCorrelationId = Guid.NewGuid().ToString("N"), Command = command,
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||
return reply;
|
||
}
|
||
|
||
int server = (await InvokeAsync(CreateRegisterRequest(id).Command).ConfigureAwait(false)).Register.ServerHandle;
|
||
MxCommandReply extended = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AddItem2,
|
||
AddItem2 = new AddItem2Command { ServerHandle = server, ItemDefinition = "TestChangingInt", ItemContext = "TestMachine_003" },
|
||
}).ConfigureAwait(false);
|
||
Assert.True(extended.AddItem2.ItemHandle > 0);
|
||
|
||
MxCommandReply subscribed = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.SubscribeBulk,
|
||
SubscribeBulk = new SubscribeBulkCommand { ServerHandle = server, TagAddresses = { first } },
|
||
}).ConfigureAwait(false);
|
||
SubscribeResult subscription = Assert.Single(subscribed.SubscribeBulk.Results);
|
||
Assert.True(subscription.WasSuccessful, subscription.ErrorMessage);
|
||
|
||
// Observe the permanent advice before checking the worker's value cache.
|
||
using (AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
|
||
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true },
|
||
cancellationToken: token))
|
||
{
|
||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
MxEvent change = events.ResponseStream.Current;
|
||
if (change.Family == MxEventFamily.OnDataChange && change.ItemHandle == subscription.ItemHandle)
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
MxCommandReply read = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.ReadBulk,
|
||
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 3000, TagAddresses = { first, second, second } },
|
||
}).ConfigureAwait(false);
|
||
Assert.Collection(read.ReadBulk.Results,
|
||
cached => { Assert.True(cached.WasSuccessful, cached.ErrorMessage); Assert.True(cached.WasCached); },
|
||
snapshot => { Assert.True(snapshot.WasSuccessful, snapshot.ErrorMessage); Assert.False(snapshot.WasCached); },
|
||
duplicate =>
|
||
{
|
||
Assert.True(duplicate.WasSuccessful, duplicate.ErrorMessage);
|
||
Assert.False(duplicate.WasCached);
|
||
Assert.Equal(read.ReadBulk.Results[1].ItemHandle, duplicate.ItemHandle);
|
||
});
|
||
|
||
MxCommandReply added = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AddItemBulk,
|
||
AddItemBulk = new AddItemBulkCommand { ServerHandle = server, TagAddresses = { second } },
|
||
}).ConfigureAwait(false);
|
||
SubscribeResult item = Assert.Single(added.AddItemBulk.Results);
|
||
Assert.True(item.WasSuccessful, item.ErrorMessage);
|
||
MxCommandReply advised = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AdviseItemBulk,
|
||
AdviseItemBulk = new AdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
|
||
}).ConfigureAwait(false);
|
||
Assert.True(Assert.Single(advised.AdviseItemBulk.Results).WasSuccessful);
|
||
MxCommandReply unadvised = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UnAdviseItemBulk,
|
||
UnAdviseItemBulk = new UnAdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
|
||
}).ConfigureAwait(false);
|
||
Assert.True(Assert.Single(unadvised.UnAdviseItemBulk.Results).WasSuccessful);
|
||
MxCommandReply removed = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.RemoveItemBulk,
|
||
RemoveItemBulk = new RemoveItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle, extended.AddItem2.ItemHandle } },
|
||
}).ConfigureAwait(false);
|
||
Assert.All(removed.RemoveItemBulk.Results, result => Assert.True(result.WasSuccessful, result.ErrorMessage));
|
||
MxCommandReply unsubscribed = await InvokeAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UnsubscribeBulk,
|
||
UnsubscribeBulk = new UnsubscribeBulkCommand { ServerHandle = server, ItemHandles = { subscription.ItemHandle } },
|
||
}).ConfigureAwait(false);
|
||
Assert.True(Assert.Single(unsubscribed.UnsubscribeBulk.Results).WasSuccessful);
|
||
output.WriteLine("Bulk add, advise, cached and snapshot reads, unadvise, remove, subscribe, and unsubscribe passed.");
|
||
}
|
||
finally
|
||
{
|
||
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
|
||
{
|
||
SessionId = id,
|
||
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
|
||
}
|
||
}
|
||
|
||
/// <summary>Verifies the Rust worker’s native subtag alarm provider and standby replacement.</summary>
|
||
/// <returns>A task that represents the live test.</returns>
|
||
[RustWorkerLiveFact]
|
||
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
|
||
{
|
||
const string reference = "Galaxy!DEV.TestMachine_001.RustWorkerSmoke";
|
||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
|
||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(75));
|
||
CancellationToken token = deadline.Token;
|
||
await app.StartAsync(token).ConfigureAwait(false);
|
||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "rust-worker-alarms",
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||
string id = opened.SessionId;
|
||
|
||
try
|
||
{
|
||
async Task<MxCommandReply> AlarmAsync(MxCommand command)
|
||
{
|
||
WorkerCommandReply envelope = await fixture.Manager.InvokeAsync(id,
|
||
new WorkerCommand { Command = command }, token).ConfigureAwait(false);
|
||
Assert.Equal(command.Kind, envelope.Reply.Kind);
|
||
return envelope.Reply;
|
||
}
|
||
|
||
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
|
||
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true }, cancellationToken: token);
|
||
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
|
||
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
|
||
MxCommandReply subscribed = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.SubscribeAlarms,
|
||
SubscribeAlarms = new SubscribeAlarmsCommand
|
||
{
|
||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
|
||
ForcedMode = AlarmProviderMode.Subtag,
|
||
WatchList = { new AlarmSubtagTarget
|
||
{
|
||
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
|
||
ActiveSubtag = "TestMachine_001.TestChangingInt",
|
||
} },
|
||
},
|
||
}).ConfigureAwait(false);
|
||
Assert.True(subscribed.ProtocolStatus.Code == ProtocolStatusCode.Ok, subscribed.DiagnosticMessage);
|
||
|
||
MxEvent? transition = null;
|
||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
if (events.ResponseStream.Current.OnAlarmTransition?.AlarmFullReference == reference)
|
||
{
|
||
transition = events.ResponseStream.Current;
|
||
break;
|
||
}
|
||
}
|
||
Assert.NotNull(transition);
|
||
Assert.Equal(id, transition.SessionId);
|
||
Assert.True(transition.WorkerSequence > 0);
|
||
Assert.True(transition.OnAlarmTransition.Degraded);
|
||
output.WriteLine($"Forwarded {transition.OnAlarmTransition.TransitionKind} at worker sequence {transition.WorkerSequence}.");
|
||
|
||
MxCommandReply snapshot = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.QueryActiveAlarms,
|
||
QueryActiveAlarmsCommand = new QueryActiveAlarmsCommand { AlarmFilterPrefix = reference },
|
||
}).ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, snapshot.ProtocolStatus.Code);
|
||
Assert.Contains(snapshot.QueryActiveAlarms.Snapshots, alarm => alarm.AlarmFullReference == reference);
|
||
MxCommandReply unknownAck = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AcknowledgeAlarm,
|
||
AcknowledgeAlarmCommand = new AcknowledgeAlarmCommand { AlarmGuid = Guid.NewGuid().ToString() },
|
||
}).ConfigureAwait(false);
|
||
Assert.NotNull(unknownAck.AcknowledgeAlarm);
|
||
Assert.NotEqual(0, unknownAck.AcknowledgeAlarm.NativeStatus);
|
||
MxCommandReply unknownName = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AcknowledgeAlarmByName,
|
||
AcknowledgeAlarmByNameCommand = new AcknowledgeAlarmByNameCommand
|
||
{
|
||
AlarmName = "RustWorkerMissingAlarm", ProviderName = "Galaxy", GroupName = "DEV",
|
||
},
|
||
}).ConfigureAwait(false);
|
||
Assert.NotNull(unknownName.AcknowledgeAlarm);
|
||
Assert.NotEqual(0, unknownName.AcknowledgeAlarm.NativeStatus);
|
||
MxCommandReply update = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UpdateAlarmWatchList,
|
||
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand(),
|
||
}).ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.MxaccessFailure, update.ProtocolStatus.Code);
|
||
MxCommandReply unsubscribed = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UnsubscribeAlarms,
|
||
UnsubscribeAlarms = new UnsubscribeAlarmsCommand(),
|
||
}).ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, unsubscribed.ProtocolStatus.Code);
|
||
|
||
MxCommandReply fallback = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.SubscribeAlarms,
|
||
SubscribeAlarms = new SubscribeAlarmsCommand
|
||
{
|
||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
|
||
FallbackEnabled = true,
|
||
Failover = new AlarmFailoverConfig
|
||
{
|
||
ConsecutiveFailureThreshold = 1, FailbackProbeIntervalSeconds = 30, FailbackStableProbes = 3,
|
||
},
|
||
},
|
||
}).ConfigureAwait(false);
|
||
Assert.True(fallback.ProtocolStatus.Code == ProtocolStatusCode.Ok, fallback.DiagnosticMessage);
|
||
MxCommandReply replacement = await AlarmAsync(new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UpdateAlarmWatchList,
|
||
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
|
||
{
|
||
WatchList = { new AlarmSubtagTarget
|
||
{
|
||
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
|
||
ActiveSubtag = "TestMachine_001.TestChangingInt",
|
||
} },
|
||
},
|
||
}).ConfigureAwait(false);
|
||
Assert.True(replacement.ProtocolStatus.Code == ProtocolStatusCode.Ok, replacement.DiagnosticMessage);
|
||
output.WriteLine("Updated the live alarm provider's fallback watch list.");
|
||
}
|
||
finally
|
||
{
|
||
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
|
||
{
|
||
SessionId = id,
|
||
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
|
||
}
|
||
}
|
||
|
||
/// <summary>Raises a development Galaxy alarm and acknowledges it as an authenticated operator through gRPC.</summary>
|
||
/// <returns>A task that represents the live test.</returns>
|
||
[RustWorkerLiveFact]
|
||
public async Task GatewayGrpc_WithRustWorker_StreamsAndAcknowledgesNativeAlarm()
|
||
{
|
||
const string tag = "TestMachine_001.TestAlarm001";
|
||
const string reference = "Galaxy!TestArea." + tag;
|
||
string user = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER") ?? "Administrator";
|
||
string password = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD") ?? string.Empty;
|
||
string comment = $"Rust worker smoke {Guid.NewGuid():N}";
|
||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output, new AlarmsOptions
|
||
{
|
||
Enabled = true,
|
||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!TestArea",
|
||
Fallback = new AlarmFallbackOptions { Mode = "Alarmmgr" },
|
||
});
|
||
await using WebApplication app = BuildLoopbackGateway(fixture.Service, fixture.Monitor);
|
||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(85));
|
||
CancellationToken token = deadline.Token;
|
||
await app.StartAsync(token).ConfigureAwait(false);
|
||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||
|
||
using AsyncServerStreamingCall<AlarmFeedMessage> feed = client.StreamAlarms(
|
||
new StreamAlarmsRequest { AlarmFilterPrefix = reference }, cancellationToken: token);
|
||
bool snapshotComplete = false;
|
||
bool alarmmgrReady = false;
|
||
while ((!snapshotComplete || !alarmmgrReady) && await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
AlarmFeedMessage frame = feed.ResponseStream.Current;
|
||
snapshotComplete |= frame.SnapshotComplete;
|
||
alarmmgrReady |= frame.ProviderStatus?.Mode == AlarmProviderMode.Alarmmgr;
|
||
}
|
||
Assert.True(snapshotComplete && alarmmgrReady, "The gateway did not open an Alarmmgr feed.");
|
||
|
||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "rust-native-alarm-driver",
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||
string sessionId = opened.SessionId;
|
||
|
||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||
{
|
||
MxCommandReply reply = await client.InvokeAsync(request, deadline: DateTime.UtcNow.AddSeconds(15),
|
||
cancellationToken: cancellationToken).ResponseAsync.ConfigureAwait(false);
|
||
Assert.True(reply.ProtocolStatus.Code == ProtocolStatusCode.Ok,
|
||
$"{reply.Kind} returned {reply.ProtocolStatus.Code}: {reply.DiagnosticMessage}");
|
||
return reply;
|
||
}
|
||
|
||
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(AlarmTransitionKind expected, string? ackComment = null)
|
||
{
|
||
while (await feed.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||
{
|
||
OnAlarmTransitionEvent? transition = feed.ResponseStream.Current.Transition;
|
||
if (transition?.AlarmFullReference == reference && transition.TransitionKind == expected
|
||
&& (ackComment is null || transition.OperatorComment == ackComment))
|
||
{
|
||
Assert.Equal(AlarmProviderMode.Alarmmgr, transition.SourceProvider);
|
||
Assert.False(transition.Degraded);
|
||
return transition;
|
||
}
|
||
}
|
||
throw new InvalidOperationException($"Alarm stream ended before the {expected} transition.");
|
||
}
|
||
|
||
MxCommandRequest? restore = null;
|
||
try
|
||
{
|
||
int server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
|
||
MxCommandReply read = await InvokeAsync(new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.ReadBulk,
|
||
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 5000, TagAddresses = { tag } },
|
||
},
|
||
}, token).ConfigureAwait(false);
|
||
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
|
||
Assert.True(original.WasSuccessful, original.ErrorMessage);
|
||
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
|
||
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
||
add.Command.AddItem.ItemDefinition = tag;
|
||
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
||
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
||
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
||
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
||
Assert.NotEqual(0, userId);
|
||
|
||
|
||
MxCommandRequest WriteAlarm(bool active)
|
||
{
|
||
MxCommandRequest request = CreateWriteSecuredRequest(sessionId, server, item, userId, verifierUserId: 0);
|
||
request.Command.WriteSecured.Value = new MxValue { DataType = MxDataType.Boolean, BoolValue = active };
|
||
return request;
|
||
}
|
||
|
||
async Task WriteConfirmedAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||
{
|
||
MxCommandReply reply = await InvokeAsync(request, cancellationToken).ConfigureAwait(false);
|
||
Assert.NotEmpty(reply.Statuses);
|
||
Assert.All(reply.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||
}
|
||
|
||
// The dev rig’s test UDA is normally false. Clear an existing test episode before raising a fresh one.
|
||
if (original.Value.BoolValue)
|
||
{
|
||
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
||
restore = WriteAlarm(true);
|
||
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
|
||
}
|
||
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
||
restore = WriteAlarm(original.Value.BoolValue);
|
||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||
|
||
AcknowledgeAlarmReply rejected = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
|
||
{
|
||
AlarmFullReference = reference,
|
||
OperatorUser = $"mxgw-smoke-{Guid.NewGuid():N}",
|
||
Comment = "This acknowledgement must be rejected",
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.Equal(ProtocolStatusCode.MxaccessFailure, rejected.ProtocolStatus.Code);
|
||
Assert.True(rejected.HasHresult && rejected.Hresult != 0);
|
||
|
||
AcknowledgeAlarmReply ack = await client.AcknowledgeAlarmAsync(new AcknowledgeAlarmRequest
|
||
{
|
||
AlarmFullReference = reference, OperatorUser = user, OperatorPassword = password, Comment = comment,
|
||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||
Assert.True(ack.ProtocolStatus.Code == ProtocolStatusCode.Ok, ack.DiagnosticMessage);
|
||
Assert.True(ack.HasHresult);
|
||
Assert.Equal(0, ack.Hresult);
|
||
Assert.DoesNotContain("unconfirmed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
|
||
Assert.DoesNotContain("not observed", ack.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
|
||
OnAlarmTransitionEvent acknowledged = await WaitForTransitionAsync(AlarmTransitionKind.Acknowledge, comment)
|
||
.ConfigureAwait(false);
|
||
Assert.Equal(user, acknowledged.OperatorUser, ignoreCase: true);
|
||
output.WriteLine($"Alarmmgr streamed Raise and Acknowledge with operator {acknowledged.OperatorUser} and the requested comment.");
|
||
|
||
if (!original.Value.BoolValue)
|
||
{
|
||
await WriteConfirmedAsync(restore, token).ConfigureAwait(false);
|
||
restore = null;
|
||
await WaitForTransitionAsync(AlarmTransitionKind.Clear).ConfigureAwait(false);
|
||
output.WriteLine("The gateway streamed Clear after the test UDA was restored.");
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
try
|
||
{
|
||
if (restore is not null)
|
||
{
|
||
MxCommandReply restored = await InvokeAsync(restore, CancellationToken.None).ConfigureAwait(false);
|
||
Assert.NotEmpty(restored.Statuses);
|
||
Assert.All(restored.Statuses, status => Assert.Equal(MxStatusCategory.Ok, status.Category));
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
|
||
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service, GatewayAlarmMonitor? monitor = null)
|
||
{
|
||
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
|
||
builder.WebHost.ConfigureKestrel(options =>
|
||
options.Listen(IPAddress.Loopback, 0, listener => listener.Protocols = HttpProtocols.Http2));
|
||
builder.Services.AddGrpc();
|
||
builder.Services.AddSingleton(service);
|
||
if (monitor is not null)
|
||
{
|
||
builder.Services.AddSingleton<IHostedService>(monitor);
|
||
}
|
||
WebApplication app = builder.Build();
|
||
app.MapGrpcService<MxAccessGatewayService>();
|
||
return app;
|
||
}
|
||
|
||
private sealed class RustWorkerLiveFactAttribute : FactAttribute
|
||
{
|
||
public RustWorkerLiveFactAttribute()
|
||
{
|
||
const string optIn = "MXGATEWAY_RUN_RUST_WORKER_LIVE_TESTS";
|
||
if (!IntegrationTestEnvironment.IsEnabled(optIn))
|
||
{
|
||
Skip = $"Set {optIn}=1 to run the Rust worker live test.";
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a Write command round-trips through live MXAccess against an advised item
|
||
/// and that the worker emits a matching <see cref="MxEventFamily.OnWriteComplete"/> event
|
||
/// — the proof of round-trip the cross-language client e2e runner relies on.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_WritesValueToAdvisedItem()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-write",
|
||
ClientCorrelationId = "live-open-write",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
Assert.True(addItemReply.AddItem.ItemHandle > 0);
|
||
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseRequest(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Advise", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
MxCommandReply writeReply = await fixture.Service.Invoke(
|
||
CreateWriteRequest(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Write", writeReply);
|
||
|
||
// Happy-path Write: the worker COM call succeeded so HResultConverter
|
||
// produces ProtocolStatusCode.Ok. An MXAccess rejection (a write to a
|
||
// bad item, a secured-item failure) would surface as
|
||
// ProtocolStatusCode.MxaccessFailure with a non-zero hresult — never
|
||
// as an RpcException / transport fault, because the command still
|
||
// completed its round-trip to the worker and back.
|
||
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.Write, writeReply.Kind);
|
||
|
||
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
|
||
// after the underlying provider acknowledges the write — that is
|
||
// the event the cross-language client e2e runner asserts on. We
|
||
// scan the recorded stream (so an interleaving OnDataChange does
|
||
// not preempt the match) for an OnWriteComplete carrying the same
|
||
// server/item handles the Write command targeted.
|
||
MxEvent writeComplete = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnWriteComplete
|
||
&& candidate.ServerHandle == registerReply.Register.ServerHandle
|
||
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(writeComplete);
|
||
|
||
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
|
||
Assert.Equal(sessionId, writeComplete.SessionId);
|
||
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
|
||
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
|
||
|
||
// The stream task must not be in a faulted state. ShutDownAsync's
|
||
// broad catch would otherwise swallow the fault and silently let
|
||
// this Write-parity coverage pass against a broken event pipeline.
|
||
Assert.False(
|
||
streamTask.IsFaulted,
|
||
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
|
||
}
|
||
finally
|
||
{
|
||
// Cancel the stream call before draining so StreamEvents observes
|
||
// cancellation rather than blocking on the channel. Any unhandled
|
||
// stream-task fault is rethrown from ShutDownAsync into the test.
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a <c>Write2</c> command (the B8 extended write carrying a timestamp
|
||
/// value as its second context parameter) round-trips through live MXAccess against an
|
||
/// advised item and raises a matching <see cref="MxEventFamily.OnWriteComplete"/> event.
|
||
/// A regression that short-circuits <c>Write2</c> to <c>InvalidRequest</c> (or drops the
|
||
/// timestamp) fails here, which no other live test catches (IntegrationTests-033, #212).
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_Write2ValueWithTimestampToAdvisedItem()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
MxValue? originalValue = null;
|
||
bool restored = false;
|
||
int serverHandle = 0;
|
||
int itemHandle = 0;
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-write2",
|
||
ClientCorrelationId = "live-open-write2",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
Assert.True(addItemReply.AddItem.ItemHandle > 0);
|
||
serverHandle = registerReply.Register.ServerHandle;
|
||
itemHandle = addItemReply.AddItem.ItemHandle;
|
||
|
||
// Supervisory, not plain, advise: MXAccess refuses a user-id-0 write on a
|
||
// plainly advised item with SecurityError/1008. Proven on the DEV gateway
|
||
// (wonder-app-vd03, 2026-11-12, #212): on MESReceiver_025.MoveInBatchID, Write2
|
||
// applied the value after AdviseSupervisory and got SecurityError/1008 after Advise.
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseSupervisoryRequest(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AdviseSupervisory", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
// The write below can now really apply, so remember the value to restore.
|
||
MxEvent initialValue = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnDataChange
|
||
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(initialValue);
|
||
originalValue = initialValue.Value;
|
||
|
||
// A whole-second instant 30 s in the past, so a stored source timestamp
|
||
// can be compared exactly and cannot be confused with the write time.
|
||
DateTime sentTimestampUtc = DateTime.UtcNow.AddSeconds(-30);
|
||
sentTimestampUtc = new DateTime(
|
||
sentTimestampUtc.Ticks - (sentTimestampUtc.Ticks % TimeSpan.TicksPerSecond),
|
||
DateTimeKind.Utc);
|
||
output.WriteLine(
|
||
$"Write2 timestamp sent={sentTimestampUtc:O} box zone={TimeZoneInfo.Local.Id} offset={TimeZoneInfo.Local.GetUtcOffset(sentTimestampUtc)}");
|
||
|
||
MxCommandReply writeReply = await fixture.Service.Invoke(
|
||
CreateWrite2Request(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle,
|
||
sentTimestampUtc),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Write2", writeReply);
|
||
|
||
// Ok means the worker reached the COM Write2 call and COM accepted it.
|
||
// InvalidRequest here means the executor rejected the payload
|
||
// (missing value/timestamp) before reaching MXAccess. Ok alone does
|
||
// not mean the provider applied the value; the completion status
|
||
// below carries that. (The 2026-11-12 windev run used a plain Advise
|
||
// and got SecurityError/1008 on the completion; see #212.)
|
||
Assert.Equal(ProtocolStatusCode.Ok, writeReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.Write2, writeReply.Kind);
|
||
|
||
// Proof of round-trip: MXAccess fires OnWriteComplete (event id 2)
|
||
// after the underlying provider acknowledges the write — that is
|
||
// the event the cross-language client e2e runner asserts on. We
|
||
// scan the recorded stream (so an interleaving OnDataChange does
|
||
// not preempt the match) for an OnWriteComplete carrying the same
|
||
// server/item handles the Write command targeted.
|
||
MxEvent writeComplete = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnWriteComplete
|
||
&& candidate.ServerHandle == registerReply.Register.ServerHandle
|
||
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(writeComplete);
|
||
|
||
Assert.Equal(MxEventFamily.OnWriteComplete, writeComplete.Family);
|
||
Assert.Equal(sessionId, writeComplete.SessionId);
|
||
Assert.Equal(registerReply.Register.ServerHandle, writeComplete.ServerHandle);
|
||
Assert.Equal(addItemReply.AddItem.ItemHandle, writeComplete.ItemHandle);
|
||
|
||
foreach (MxStatusProxy status in writeComplete.Statuses)
|
||
{
|
||
output.WriteLine($"Write2 completion category={status.Category} detail={status.Detail}");
|
||
}
|
||
|
||
// #228: MXAccess reads the Write2 time argument as box-local time, so
|
||
// the worker must convert the UTC instant before the COM call. When the
|
||
// provider applied the write, the source timestamp it reports must be
|
||
// the instant sent. On a UTC box this cannot fail either way, so the
|
||
// zone is logged above. A refused write (no data change) is logged,
|
||
// not asserted, as before.
|
||
bool applied = writeComplete.Statuses.Count > 0
|
||
&& writeComplete.Statuses.All(status => status.Category == MxStatusCategory.Ok);
|
||
if (applied)
|
||
{
|
||
MxEvent writtenValue = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnDataChange
|
||
&& candidate.ItemHandle == addItemReply.AddItem.ItemHandle
|
||
&& candidate.WorkerSequence > initialValue.WorkerSequence,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(writtenValue);
|
||
DateTime? storedTimestampUtc = writtenValue.SourceTimestamp?.ToDateTime();
|
||
output.WriteLine($"Write2 timestamp stored={storedTimestampUtc:O}");
|
||
Assert.Equal(sentTimestampUtc, storedTimestampUtc);
|
||
}
|
||
else
|
||
{
|
||
output.WriteLine("Write2 timestamp not checked: the completion was not Ok, so nothing was applied.");
|
||
}
|
||
|
||
// Restore the original value with a plain Write on the same supervisory advise.
|
||
MxCommandReply restoreReply = await fixture.Service.Invoke(
|
||
CreateWriteRequest(
|
||
sessionId,
|
||
registerReply.Register.ServerHandle,
|
||
addItemReply.AddItem.ItemHandle,
|
||
originalValue),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Write (restore)", restoreReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, restoreReply.ProtocolStatus.Code);
|
||
restored = true;
|
||
|
||
// The stream task must not be in a faulted state. ShutDownAsync's
|
||
// broad catch would otherwise swallow the fault and silently let
|
||
// this Write2-parity coverage pass against a broken event pipeline.
|
||
Assert.False(
|
||
streamTask.IsFaulted,
|
||
streamTask.Exception?.ToString() ?? "Event stream task faulted without an exception.");
|
||
}
|
||
finally
|
||
{
|
||
if (!restored && originalValue is not null && sessionId is not null && itemHandle != 0)
|
||
{
|
||
// Best effort: do not leave the Write2 test value behind after a failed
|
||
// assertion. Logged, not thrown, so the original failure is not masked.
|
||
try
|
||
{
|
||
MxCommandReply restoreReply = await fixture.Service.Invoke(
|
||
CreateWriteRequest(sessionId, serverHandle, itemHandle, originalValue),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Write (restore after failure)", restoreReply);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
output.WriteLine($"Restore after failure threw {ex.GetType().Name}: {ex.Message}");
|
||
}
|
||
}
|
||
|
||
// Cancel the stream call before draining so StreamEvents observes
|
||
// cancellation rather than blocking on the channel. Any unhandled
|
||
// stream-task fault is rethrown from ShutDownAsync into the test.
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask, propagateStreamFaults: true).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that an AddItem against an invalid server handle surfaces the MXAccess failure
|
||
/// without faulting the gateway transport, exercising the invalid-handle parity path.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_InvalidHandleCommand_SurfacesFailureWithoutTransportFault()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
|
||
string? sessionId = null;
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-invalid-handle",
|
||
ClientCorrelationId = "live-open-invalid",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
// Deliberately skip Register: server handle 0x7FFFFFFF was never
|
||
// issued by MXAccess. The worker must invoke COM and relay the
|
||
// invalid-handle failure rather than the gateway short-circuiting.
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, serverHandle: int.MaxValue),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddItem(invalid-handle)", addItemReply);
|
||
|
||
// MXAccess parity: an invalid handle is an MXAccess-level failure.
|
||
// The command still completed its worker round-trip, so the gateway
|
||
// must reply with ProtocolStatusCode.MxaccessFailure and a non-zero
|
||
// hresult carrying the COM failure (per HResultConverter) — never a
|
||
// gRPC transport fault. The assertion below just checks the status
|
||
// is not Ok; the failure detail lives in hresult / the status proxies.
|
||
Assert.NotEqual(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
Assert.True(
|
||
addItemReply.AddItem is null || addItemReply.AddItem.ItemHandle <= 0,
|
||
"Invalid-handle AddItem must not yield a usable item handle.");
|
||
}
|
||
finally
|
||
{
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies the MXAccess teardown chain: Unadvise then RemoveItem then Unregister
|
||
/// each return <see cref="ProtocolStatusCode.Ok"/>, and the worker stops emitting
|
||
/// OnDataChange events for the un-advised item. Exercises the lifecycle-ordering
|
||
/// parity CLAUDE.md singles out as a "do not synthesize" rule.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_UnadviseRemoveItemUnregister_TeardownOrderingParity()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-teardown",
|
||
ClientCorrelationId = "live-open-teardown",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
|
||
int serverHandle = registerReply.Register.ServerHandle;
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, serverHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
int itemHandle = addItemReply.AddItem.ItemHandle;
|
||
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Advise", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
// Wait for an OnDataChange to prove the subscription is live before tearing it down.
|
||
MxEvent firstDataChange = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnDataChange
|
||
&& candidate.ServerHandle == serverHandle
|
||
&& candidate.ItemHandle == itemHandle,
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
LogEvent(firstDataChange);
|
||
|
||
// 1) UnAdvise — must reply Ok; the worker must stop emitting OnDataChange
|
||
// for this (server, item) pair after this returns.
|
||
MxCommandReply unadviseReply = await fixture.Service.Invoke(
|
||
CreateUnAdviseRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("UnAdvise", unadviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, unadviseReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.UnAdvise, unadviseReply.Kind);
|
||
|
||
// 2) RemoveItem — must reply Ok against the same handles.
|
||
MxCommandReply removeItemReply = await fixture.Service.Invoke(
|
||
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("RemoveItem", removeItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, removeItemReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.RemoveItem, removeItemReply.Kind);
|
||
|
||
// 3) Unregister — closes the client session inside the worker.
|
||
MxCommandReply unregisterReply = await fixture.Service.Invoke(
|
||
CreateUnregisterRequest(sessionId, serverHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Unregister", unregisterReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, unregisterReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.Unregister, unregisterReply.Kind);
|
||
|
||
// Parity rule: after UnAdvise returns Ok the worker must stop emitting
|
||
// OnDataChange for this (server, item) pair. Events the provider already
|
||
// published before that ack are in-flight and not a regression — the rule
|
||
// only constrains events generated AFTER the teardown returned. So the
|
||
// "before" baseline is taken *after* a first settle window drains those
|
||
// in-flight events, not before UnAdvise was issued (which races against
|
||
// the round-trip + STA dispatch + pipe send window).
|
||
//
|
||
// RecordingServerStreamWriter.Messages returns a snapshot copy under its
|
||
// own lock, so iterating after each settle window is safe without external
|
||
// sync.
|
||
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
|
||
int dataChangeCountAfterFirstSettle = CountMatchingEvents(
|
||
eventWriter,
|
||
e => e.Family == MxEventFamily.OnDataChange
|
||
&& e.ServerHandle == serverHandle
|
||
&& e.ItemHandle == itemHandle);
|
||
|
||
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
|
||
int dataChangeCountAfterSecondSettle = CountMatchingEvents(
|
||
eventWriter,
|
||
e => e.Family == MxEventFamily.OnDataChange
|
||
&& e.ServerHandle == serverHandle
|
||
&& e.ItemHandle == itemHandle);
|
||
output.WriteLine(
|
||
$"DataChange count after first settle={dataChangeCountAfterFirstSettle} after second settle={dataChangeCountAfterSecondSettle}");
|
||
Assert.Equal(dataChangeCountAfterFirstSettle, dataChangeCountAfterSecondSettle);
|
||
|
||
// A RemoveItem against the just-freed item handle must not silently succeed —
|
||
// the worker has to relay MXAccess's invalid-handle response. Closing the
|
||
// session is enough for parity, but we sanity-check that re-using the freed
|
||
// pair does not accidentally appear Ok.
|
||
MxCommandReply secondRemoveItemReply = await fixture.Service.Invoke(
|
||
CreateRemoveItemRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("RemoveItem(stale)", secondRemoveItemReply);
|
||
Assert.NotEqual(ProtocolStatusCode.Ok, secondRemoveItemReply.ProtocolStatus.Code);
|
||
}
|
||
finally
|
||
{
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies the MXAccess <c>WriteSecured</c> path: <c>AuthenticateUser</c> resolves a
|
||
/// user id, then <c>WriteSecured</c> against the advised item completes its round-trip
|
||
/// to the worker and back. CLAUDE.md singles out <c>WriteSecured</c> ordering as a
|
||
/// parity surface the gateway must not "fix" — the test asserts the reply kind and
|
||
/// protocol status, not a fabricated outcome.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_WriteSecured_AuthenticatedRoundTripParity()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
// CLAUDE.md's credential-redaction rule covers every log
|
||
// surface the test sees, not just the reply's DiagnosticMessage. Wire a buffering
|
||
// wrapper around output and route the worker stdout/stderr echo and the gateway
|
||
// ILogger sink through it so the post-run assertion covers the accumulated test
|
||
// output. A regression that logged the request body, the WorkerCommandRequest
|
||
// envelope, or printed the credential from inside the worker is caught here
|
||
// even if the bare DiagnosticMessage check still passes.
|
||
RecordingTestOutputHelper recordedOutput = new(output);
|
||
TestWorkerProcessFactory processFactory = new(recordedOutput);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
|
||
// Stream events so a regression that emitted an OperationComplete or
|
||
// OnWriteComplete with wrong handles would still be observable via the test
|
||
// output (we don't assert a specific event here — the docs note successful
|
||
// writes raise only OnWriteComplete, but WriteSecured against an unprotected
|
||
// item commonly fails with 0x80004021 in this provider, which raises no event).
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-write-secured",
|
||
ClientCorrelationId = "live-open-write-secured",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
int serverHandle = registerReply.Register.ServerHandle;
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, serverHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
int itemHandle = addItemReply.AddItem.ItemHandle;
|
||
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "Advise", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
// AuthenticateUser resolves an ArchestrA user id for the WriteSecured call.
|
||
// Credentials are env-overridable so the test honors the gateway's "do not
|
||
// log secrets" rule and works against either MXAccess's own user store or
|
||
// the LmxOpcUa-baseline GLAuth-bridged ArchestrA identity (admin/password —
|
||
// see glauth.md; the shared GLAuth's dev-user password is "password", not
|
||
// the old per-box "admin123").
|
||
MxCommandReply authReply = await fixture.Service.Invoke(
|
||
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
// AuthenticateUser is allowed to fail — MXAccess parity means a rejected
|
||
// credential is a legitimate reply, not a bug, so this must never be an
|
||
// Assert. But a silent fall-through to "operator only" (currentUserId 0)
|
||
// quietly demotes the rest of this test from an authenticated round-trip
|
||
// to a bare failure-reply-shape check, which a stale rig credential (e.g.
|
||
// one that has expired) would otherwise hide indefinitely in nightly runs. Log the
|
||
// outcome loudly — never the password (verifyUser only) — so that
|
||
// degradation is visible in the test output even though it never fails
|
||
// the test by itself.
|
||
bool authenticateUserSucceeded = authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok
|
||
&& authReply.AuthenticateUser is not null
|
||
&& authReply.AuthenticateUser.UserId != 0;
|
||
recordedOutput.WriteLine(
|
||
authenticateUserSucceeded
|
||
? $"AuthenticateUser SUCCEEDED for user={verifyUser} user_id={authReply.AuthenticateUser!.UserId}"
|
||
: $"AuthenticateUser FAILED (credential rejected or not recognized by the provider) "
|
||
+ $"for user={verifyUser} status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} "
|
||
+ $"diagnostic={authReply.DiagnosticMessage} -- WriteSecured below will run as an "
|
||
+ $"unauthenticated (currentUserId=0) failure-reply-shape check only, not an "
|
||
+ $"authenticated round-trip. If this is unexpected, verify "
|
||
+ $"MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER/_PASSWORD against the rig's ArchestrA "
|
||
+ $"identity (see glauth.md).");
|
||
|
||
int currentUserId = authenticateUserSucceeded ? authReply.AuthenticateUser!.UserId : 0;
|
||
|
||
MxCommandReply writeSecuredReply = await fixture.Service.Invoke(
|
||
CreateWriteSecuredRequest(
|
||
sessionId,
|
||
serverHandle,
|
||
itemHandle,
|
||
currentUserId,
|
||
verifierUserId: 0),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "WriteSecured", writeSecuredReply);
|
||
|
||
// Parity: the command itself completed its round-trip — the reply kind is
|
||
// WriteSecured and the gateway protocol status is set. The MXAccess outcome
|
||
// (Ok for an unprotected provider, MxaccessFailure with hresult 0x80004021
|
||
// when the item is not WriteSecured-eligible) lives in protocol_status +
|
||
// hresult, never as a transport fault. The diagnostic message must never
|
||
// contain the credential.
|
||
Assert.Equal(MxCommandKind.WriteSecured, writeSecuredReply.Kind);
|
||
Assert.True(
|
||
writeSecuredReply.ProtocolStatus.Code is ProtocolStatusCode.Ok
|
||
or ProtocolStatusCode.MxaccessFailure,
|
||
$"Unexpected WriteSecured protocol status {writeSecuredReply.ProtocolStatus.Code}.");
|
||
Assert.DoesNotContain(verifyPassword, writeSecuredReply.DiagnosticMessage ?? string.Empty, StringComparison.Ordinal);
|
||
}
|
||
finally
|
||
{
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
|
||
}
|
||
|
||
// CLAUDE.md credential contract: passwords and WriteSecured payloads must never
|
||
// reach logs. The buffered output covers the gateway ILogger sink, worker
|
||
// stdout/stderr, and every direct WriteLine the test body issued. A regression
|
||
// that dumped the request envelope, the AuthenticateUserCommand body, or any
|
||
// command-level WriteSecured payload would land here and trip this assertion.
|
||
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
|
||
}
|
||
|
||
/// <summary>
|
||
/// B8 live verification of the COM commands the B-bundle added against a fake
|
||
/// IMxAccessServer: <c>AuthenticateUser</c>, <c>ArchestrAUserToId</c>, <c>Suspend</c>,
|
||
/// and <c>Activate</c>. The contract being proven is that each command round-trips
|
||
/// to the worker and back carrying a real MXAccess outcome (Ok / an MxStatusProxy /
|
||
/// a non-zero HResult) and is NOT short-circuited to <c>INVALID_REQUEST</c> the way an
|
||
/// unimplemented command would be. MXAccess-level rejections (a wrong item class for
|
||
/// Suspend/Activate commonly returns 0x80070057) are parity, not test failures — we
|
||
/// assert the reply kind plus a non-INVALID_REQUEST protocol status, and log the
|
||
/// HResult for the record.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_NewComCommands_RoundTripWithRealReplies()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
// Credential-redaction: AuthenticateUser carries a password. Route every test
|
||
// log surface through the buffering helper so the post-run assertion proves the
|
||
// password never reached the gateway logger, worker stdout/stderr, or any
|
||
// WriteLine the test body issued (same pattern as the WriteSecured parity test).
|
||
RecordingTestOutputHelper recordedOutput = new(output);
|
||
TestWorkerProcessFactory processFactory = new(recordedOutput);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, recordedOutput);
|
||
|
||
string? sessionId = null;
|
||
(string verifyUser, string verifyPassword) = ResolveLiveMxAccessSecuredCredentials();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-new-com-commands",
|
||
ClientCorrelationId = "live-open-new-com",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
int serverHandle = registerReply.Register.ServerHandle;
|
||
|
||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||
CreateAddItemRequest(sessionId, serverHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "AddItem", addItemReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||
int itemHandle = addItemReply.AddItem.ItemHandle;
|
||
|
||
// AuthenticateUser — the B-bundle command under live verification. Before the
|
||
// B-bundle this command was unimplemented and the worker short-circuited it to
|
||
// INVALID_REQUEST. It must now produce a real reply (Ok with a user id when the
|
||
// provider accepts the credential, or a real MXAccess HResult when it does not).
|
||
MxCommandReply authReply = await fixture.Service.Invoke(
|
||
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
recordedOutput.WriteLine(
|
||
$"AuthenticateUser status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} user_id={authReply.AuthenticateUser?.UserId}");
|
||
Assert.Equal(MxCommandKind.AuthenticateUser, authReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, authReply.ProtocolStatus.Code);
|
||
Assert.True(
|
||
authReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
|
||
$"AuthenticateUser must surface a real MXAccess outcome, got {authReply.ProtocolStatus.Code}.");
|
||
int authenticatedUserId =
|
||
authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok && authReply.AuthenticateUser is not null
|
||
? authReply.AuthenticateUser.UserId
|
||
: 0;
|
||
if (authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
|
||
{
|
||
// On the dev rig AuthenticateUser("Administrator","") resolves to user id 1.
|
||
// Don't pin the exact value (provider/user-store dependent) — just prove a
|
||
// success carried a usable, non-zero ArchestrA user id through the reply.
|
||
Assert.NotEqual(0, authenticatedUserId);
|
||
}
|
||
|
||
// ArchestrAUserToId — resolves an ArchestrA user GUID to an integer user id.
|
||
// We feed an empty/placeholder GUID: the value is provider-dependent, so the
|
||
// assertion is the parity one (real reply, never INVALID_REQUEST). A non-zero
|
||
// HResult here is the expected MXAccess rejection of an unknown GUID.
|
||
MxCommandReply userToIdReply = await fixture.Service.Invoke(
|
||
CreateArchestrAUserToIdRequest(sessionId, serverHandle, userIdGuid: string.Empty),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "ArchestrAUserToId", userToIdReply);
|
||
recordedOutput.WriteLine($"ArchestrAUserToId user_id={userToIdReply.ArchestraUserToId?.UserId}");
|
||
Assert.Equal(MxCommandKind.ArchestraUserToId, userToIdReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, userToIdReply.ProtocolStatus.Code);
|
||
Assert.True(
|
||
userToIdReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
|
||
$"ArchestrAUserToId must surface a real MXAccess outcome, got {userToIdReply.ProtocolStatus.Code}.");
|
||
if (userToIdReply.ProtocolStatus.Code == ProtocolStatusCode.Ok)
|
||
{
|
||
// On the dev rig ArchestrAUserToId with a valid GUID resolves to user_id=1.
|
||
// Don't pin the exact id (provider-dependent) — just prove the Ok path carried
|
||
// a usable non-zero ArchestrA user id through the reply payload.
|
||
Assert.NotNull(userToIdReply.ArchestraUserToId);
|
||
Assert.NotEqual(0, userToIdReply.ArchestraUserToId.UserId);
|
||
}
|
||
|
||
// Suspend / Activate against the added-but-not-advised item (no Advise was issued
|
||
// between AddItem and this call). The dev-rig TestInt item class may not be
|
||
// suspendable (MXAccess returns 0x80070057 / E_INVALIDARG for a wrong item class
|
||
// — see B8 notes). That is MXAccess parity: assert the reply kind and a
|
||
// non-INVALID_REQUEST status, surface the HResult and MxStatusProxy for the
|
||
// record, and do NOT treat a provider-side rejection as a test failure.
|
||
MxCommandReply suspendReply = await fixture.Service.Invoke(
|
||
CreateSuspendRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "Suspend", suspendReply);
|
||
recordedOutput.WriteLine(
|
||
$"Suspend status_proxy success={suspendReply.Suspend?.Status?.Success} hresult=0x{(uint)suspendReply.Hresult:X8}");
|
||
Assert.Equal(MxCommandKind.Suspend, suspendReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, suspendReply.ProtocolStatus.Code);
|
||
Assert.True(
|
||
suspendReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
|
||
$"Suspend must surface a real MXAccess outcome, got {suspendReply.ProtocolStatus.Code}.");
|
||
|
||
MxCommandReply activateReply = await fixture.Service.Invoke(
|
||
CreateActivateRequest(sessionId, serverHandle, itemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReplyTo(recordedOutput, "Activate", activateReply);
|
||
recordedOutput.WriteLine(
|
||
$"Activate status_proxy success={activateReply.Activate?.Status?.Success} hresult=0x{(uint)activateReply.Hresult:X8}");
|
||
Assert.Equal(MxCommandKind.Activate, activateReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, activateReply.ProtocolStatus.Code);
|
||
Assert.True(
|
||
activateReply.ProtocolStatus.Code is ProtocolStatusCode.Ok or ProtocolStatusCode.MxaccessFailure,
|
||
$"Activate must surface a real MXAccess outcome, got {activateReply.ProtocolStatus.Code}.");
|
||
}
|
||
finally
|
||
{
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
|
||
}
|
||
|
||
// Credential contract: the AuthenticateUser password must never reach any log
|
||
// surface (gateway logger, worker stdout/stderr, or test WriteLine).
|
||
Assert.DoesNotContain(verifyPassword, recordedOutput.Captured, StringComparison.Ordinal);
|
||
}
|
||
|
||
/// <summary>
|
||
/// B8 §3.2 buffered-data path: adds a BUFFERED item (<c>AddBufferedItem</c>), sets the
|
||
/// buffered update interval (<c>SetBufferedUpdateInterval</c>), advises it, then attempts
|
||
/// to observe an <see cref="MxEventFamily.OnBufferedDataChange"/> event carrying multiple
|
||
/// samples so the worker's multi-sample conversion (VariantConverter →
|
||
/// OnBufferedDataChangeEvent quality/timestamp arrays) can be validated live.
|
||
/// <para>
|
||
/// The AddBufferedItem + SetBufferedUpdateInterval round-trips are asserted unconditionally
|
||
/// (they are the B-bundle commands under verification). The buffered EVENT capture is
|
||
/// best-effort: if the rig's object logic does not drive a buffered batch within the live
|
||
/// event timeout (the same environmental limitation seen with the externally-undrivable
|
||
/// alarm rig), the test records the buffered conversion as an unverified residual rather
|
||
/// than failing — the command path is proven, the live multi-sample conversion is not.
|
||
/// When a batch IS captured, the converted value and quality/timestamp arrays are asserted
|
||
/// to be non-empty and internally consistent (no crash, no dropped payload).
|
||
/// </para>
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_BufferedItem_AddsSetsIntervalAndAttemptsCapture()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
// AddBufferedItem takes (item_definition, item_context) like AddItem2. The dev rig
|
||
// exposes TestChildObject.TestInt; the buffered form is item="TestInt",
|
||
// context="TestChildObject" (per B8 notes). Split the configured live item so a
|
||
// custom MXGATEWAY_LIVE_MXACCESS_ITEM override still works.
|
||
(string bufferedItem, string bufferedContext) = SplitLiveItemForBuffered(IntegrationTestEnvironment.LiveMxAccessItem);
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-buffered",
|
||
ClientCorrelationId = "live-open-buffered",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||
CreateRegisterRequest(sessionId),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Register", registerReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||
int serverHandle = registerReply.Register.ServerHandle;
|
||
|
||
// SetBufferedUpdateInterval first so the buffered cadence is established before
|
||
// the item is added/advised. MXAccess rounds to 100ms units and rejects < 1.
|
||
MxCommandReply intervalReply = await fixture.Service.Invoke(
|
||
CreateSetBufferedUpdateIntervalRequest(sessionId, serverHandle, updateIntervalMilliseconds: 1000),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("SetBufferedUpdateInterval", intervalReply);
|
||
Assert.Equal(MxCommandKind.SetBufferedUpdateInterval, intervalReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, intervalReply.ProtocolStatus.Code);
|
||
Assert.Equal(ProtocolStatusCode.Ok, intervalReply.ProtocolStatus.Code);
|
||
|
||
// AddBufferedItem — must return a real item handle (the dev rig yields handle 1).
|
||
MxCommandReply addBufferedReply = await fixture.Service.Invoke(
|
||
CreateAddBufferedItemRequest(sessionId, serverHandle, bufferedItem, bufferedContext),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("AddBufferedItem", addBufferedReply);
|
||
Assert.Equal(MxCommandKind.AddBufferedItem, addBufferedReply.Kind);
|
||
Assert.NotEqual(ProtocolStatusCode.InvalidRequest, addBufferedReply.ProtocolStatus.Code);
|
||
Assert.Equal(ProtocolStatusCode.Ok, addBufferedReply.ProtocolStatus.Code);
|
||
Assert.NotNull(addBufferedReply.AddBufferedItem);
|
||
int bufferedItemHandle = addBufferedReply.AddBufferedItem.ItemHandle;
|
||
Assert.True(bufferedItemHandle > 0, "AddBufferedItem must yield a usable item handle.");
|
||
|
||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||
CreateAdviseRequest(sessionId, serverHandle, bufferedItemHandle),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Advise(buffered)", adviseReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||
|
||
// Best-effort capture of a SAMPLE-BEARING buffered batch.
|
||
//
|
||
// Live observation (B8, re-read 2026-11-13): immediately after Advise the provider
|
||
// delivers one OnBufferedDataChange with data_type=NoData and zero quality and
|
||
// timestamp samples. Its status row is ConfigurationError: detail 6
|
||
// (InvalidReference) for the (item, context) form used here, and detail 1002
|
||
// (InvalidPropertyId) for a full reference to an attribute without a buffer. It is
|
||
// a resolution failure, not a bootstrap. The DEV Galaxy has no buffered attribute,
|
||
// so no sample-bearing batch can arrive (docs/LiveCaptures-2026-11-13.md §5). The
|
||
// converter handling it without crashing is still worth keeping. We match only a
|
||
// batch that carries samples, so the test validates a multi-sample conversion if
|
||
// the item is pointed at a buffered attribute.
|
||
MxEvent? bufferedBatch = null;
|
||
try
|
||
{
|
||
bufferedBatch = await eventWriter
|
||
.WaitForMessageAsync(
|
||
candidate => candidate.Family == MxEventFamily.OnBufferedDataChange
|
||
&& candidate.ServerHandle == serverHandle
|
||
&& candidate.ItemHandle == bufferedItemHandle
|
||
&& candidate.OnBufferedDataChange is not null
|
||
&& (CountArrayElements(candidate.OnBufferedDataChange.QualityValues) > 0
|
||
|| CountArrayElements(candidate.OnBufferedDataChange.TimestampValues) > 0),
|
||
IntegrationTestEnvironment.LiveMxAccessEventTimeout,
|
||
streamCancellation.Token)
|
||
.ConfigureAwait(false);
|
||
}
|
||
catch (TimeoutException ex)
|
||
{
|
||
output.WriteLine($"B8: sample-bearing batch predicate timed out: {ex.Message}");
|
||
bufferedBatch = null;
|
||
}
|
||
|
||
// Whether or not a sample-bearing batch arrived, record what buffered events the
|
||
// rig DID deliver (typically just the empty NoData bootstrap) for the record.
|
||
int bootstrapBufferedEvents = CountMatchingEvents(
|
||
eventWriter,
|
||
e => e.Family == MxEventFamily.OnBufferedDataChange
|
||
&& e.ServerHandle == serverHandle
|
||
&& e.ItemHandle == bufferedItemHandle);
|
||
|
||
if (bufferedBatch is null)
|
||
{
|
||
// RESIDUAL (documented): the command path (AddBufferedItem +
|
||
// SetBufferedUpdateInterval + Advise) is proven and the buffered EVENT plumbing
|
||
// is live (the NoData resolution-failure event arrives and converts without
|
||
// crashing), but the rig has no buffered attribute to deliver a sample-bearing
|
||
// batch (docs/LiveCaptures-2026-11-13.md §5).
|
||
// The §3.2 OnBufferedDataChange MULTI-SAMPLE conversion therefore remains
|
||
// unverified live. This is environmental, not a defect — let the test pass.
|
||
output.WriteLine(
|
||
"B8 RESIDUAL: AddBufferedItem/SetBufferedUpdateInterval/Advise round-tripped and "
|
||
+ $"{bootstrapBufferedEvents} OnBufferedDataChange event(s) arrived (empty NoData "
|
||
+ "bootstrap, converted without crash/drop), but no sample-bearing buffered batch "
|
||
+ $"was observed within {IntegrationTestEnvironment.LiveMxAccessEventTimeout}. Live "
|
||
+ "§3.2 multi-sample conversion remains unverified (rig object logic may not drive "
|
||
+ "buffered samples on demand).");
|
||
|
||
// The residual claim ("empty NoData bootstrap arrives and converts without crashing")
|
||
// is only meaningful if at least one OnBufferedDataChange event arrived. Assert that
|
||
// the buffered subscription registered and the worker's event plumbing fired at all.
|
||
Assert.True(
|
||
bootstrapBufferedEvents > 0,
|
||
"No OnBufferedDataChange event arrived at all after Advise; the buffered subscription may not have registered.");
|
||
return;
|
||
}
|
||
|
||
// A SAMPLE-BEARING buffered batch was captured — validate the §3.2 conversion.
|
||
LogEvent(bufferedBatch);
|
||
OnBufferedDataChangeEvent body = bufferedBatch.OnBufferedDataChange;
|
||
Assert.NotNull(body);
|
||
|
||
int qualityCount = CountArrayElements(body.QualityValues);
|
||
int timestampCount = CountArrayElements(body.TimestampValues);
|
||
output.WriteLine(
|
||
$"B8 CAPTURED buffered batch: data_type={body.DataType} raw_data_type={body.RawDataType} "
|
||
+ $"quality_samples={qualityCount} timestamp_samples={timestampCount} "
|
||
+ $"value_kind={bufferedBatch.Value?.KindCase}");
|
||
|
||
// The predicate guaranteed at least one sample; the converted aggregate value
|
||
// must also exist (no crash, no dropped payload).
|
||
Assert.True(
|
||
qualityCount > 0 || timestampCount > 0,
|
||
"Sample-bearing OnBufferedDataChange lost its samples after the predicate matched.");
|
||
Assert.NotNull(bufferedBatch.Value);
|
||
|
||
// When MXAccess delivers parallel quality + timestamp arrays the converted
|
||
// arrays must agree in length; a mismatch is a real conversion defect (a sample
|
||
// was dropped on one side).
|
||
if (qualityCount > 0 && timestampCount > 0)
|
||
{
|
||
Assert.Equal(qualityCount, timestampCount);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that killing the worker process marks the session
|
||
/// <see cref="SessionState.Faulted"/> with a clean fault classification — the gateway
|
||
/// must observe the abnormal exit, transition the session, and surface a non-empty
|
||
/// fault description rather than hanging or crashing.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_AbnormalWorkerExit_MarksSessionFaulted()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-abnormal-exit",
|
||
ClientCorrelationId = "live-open-abnormal",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
// Kill the worker process directly. WorkerClient's read loop hits an
|
||
// end-of-stream on the named pipe and routes through SetFaulted; the
|
||
// session manager then marks the session Faulted. We avoid CloseSession
|
||
// so the transition is driven by the abnormal exit, not a graceful path.
|
||
processFactory.KillAllAndDetach();
|
||
|
||
DateTimeOffset waitDeadline = DateTimeOffset.UtcNow + StreamShutdownTimeout;
|
||
SessionState observedState = SessionState.Unspecified;
|
||
string? observedFault = null;
|
||
while (DateTimeOffset.UtcNow < waitDeadline)
|
||
{
|
||
if (fixture.TryGetSession(sessionId, out GatewaySession? session))
|
||
{
|
||
observedState = session.State;
|
||
observedFault = session.FinalFault;
|
||
if (observedState == SessionState.Faulted)
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
|
||
await Task.Delay(TimeSpan.FromMilliseconds(50)).ConfigureAwait(false);
|
||
}
|
||
|
||
output.WriteLine($"AbnormalExit observed_state={observedState} fault={observedFault}");
|
||
Assert.Equal(SessionState.Faulted, observedState);
|
||
Assert.False(string.IsNullOrWhiteSpace(observedFault), "Faulted session must carry a non-empty fault description.");
|
||
|
||
// The fault classification must come from a known worker-client error code so
|
||
// operators get an actionable cause string rather than an opaque exception
|
||
// trace. We accept the classifications WorkerClient actually drives on an
|
||
// abnormal exit (kill-the-process path): the read loop hits EndOfStream and
|
||
// calls SetFaulted with WorkerClientErrorCode.PipeDisconnected and the
|
||
// message "Worker pipe disconnected." (see WorkerClient.cs:378-381). The
|
||
// earlier broad list (including "worker") matched every WorkerClient fault
|
||
// message (they all begin with "Worker"); tighten to the pipe/disconnect/
|
||
// end-of-stream classifications that match THIS path, so a regression that
|
||
// routed an unrelated fault here would surface as a test failure rather
|
||
// than silently passing. "heartbeat" is dropped
|
||
// because HeartbeatGraceSeconds (15s) exceeds the StreamShutdownTimeout
|
||
// (10s) poll window, so a heartbeat-expired transition can never be
|
||
// observed inside this test.
|
||
Assert.True(
|
||
observedFault!.Contains("pipe disconnected", StringComparison.OrdinalIgnoreCase)
|
||
|| observedFault.Contains("end of stream", StringComparison.OrdinalIgnoreCase),
|
||
$"Fault description '{observedFault}' did not match a known abnormal-exit classification "
|
||
+ "(expected 'pipe disconnected' or 'end of stream' from WorkerClient's EndOfStream path).");
|
||
|
||
// Also assert the StreamEvents call observed the fault
|
||
// — the chain that puts the session into Faulted goes through ReadEventsAsync
|
||
// propagating a WorkerClientException into EventStreamService, which calls
|
||
// session.MarkFaulted. The gateway then maps the WorkerClientException to an
|
||
// RpcException at the public boundary (MxAccessGatewayService.MapException →
|
||
// MapWorkerClientException). Polling session.State alone would silently pass
|
||
// if a future refactor moved MarkFaulted off the stream-consumption path —
|
||
// assert the streamTask itself terminated with a fault so the test couples
|
||
// to the actual fault-propagation path. Compare to the inverse assertion in
|
||
// the Write parity test (line 217: Assert.False(streamTask.IsFaulted, ...)).
|
||
try
|
||
{
|
||
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
|
||
}
|
||
catch (Exception streamException)
|
||
{
|
||
output.WriteLine($"StreamEvents task terminated with: {streamException.GetType().Name}: {streamException.Message}");
|
||
}
|
||
|
||
Assert.True(
|
||
streamTask.IsCompleted,
|
||
"StreamEvents task did not complete within the shutdown timeout after the worker was killed.");
|
||
Assert.True(
|
||
streamTask.IsFaulted,
|
||
"StreamEvents task must fault on abnormal worker exit, not complete cleanly — "
|
||
+ "the fault-propagation path from WorkerClient.SetFaulted through ReadEventsAsync is the contract.");
|
||
}
|
||
finally
|
||
{
|
||
streamCancellation.Cancel();
|
||
// sessionId is intentionally null here — the session is already faulted and a
|
||
// CloseSession round-trip would just log a cleanup failure. We still wait for
|
||
// the worker process exit so the next test starts with a clean state.
|
||
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// TST-05: the real worker answers the four non-terminating control commands off-STA in
|
||
/// <c>WorkerPipeSession.HandleControlCommandAsync</c>. The default suite only ever sees the
|
||
/// FakeWorkerHarness canned replies, so this drives <c>Ping</c>, <c>GetSessionState</c>,
|
||
/// <c>GetWorkerInfo</c>, and <c>DrainEvents</c> through the gateway against a live worker and
|
||
/// asserts each reply carries real worker state rather than an <c>INVALID_REQUEST</c>.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_ControlCommands_ReturnRealWorkerState()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
|
||
string? sessionId = null;
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-control",
|
||
ClientCorrelationId = "live-open-control",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
Assert.True(openReply.WorkerProcessId > 0);
|
||
|
||
MxCommandReply pingReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-ping", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Ping,
|
||
Ping = new PingCommand { Message = "live-ping-echo" },
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("Ping", pingReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, pingReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.Ping, pingReply.Kind);
|
||
|
||
MxCommandReply stateReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-get-session-state", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.GetSessionState,
|
||
GetSessionState = new GetSessionStateCommand(),
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("GetSessionState", stateReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, stateReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.GetSessionState, stateReply.Kind);
|
||
Assert.NotNull(stateReply.SessionState);
|
||
Assert.Equal(SessionState.Ready, stateReply.SessionState.State);
|
||
|
||
MxCommandReply infoReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-get-worker-info", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.GetWorkerInfo,
|
||
GetWorkerInfo = new GetWorkerInfoCommand(),
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("GetWorkerInfo", infoReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, infoReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.GetWorkerInfo, infoReply.Kind);
|
||
Assert.NotNull(infoReply.WorkerInfo);
|
||
output.WriteLine(
|
||
$"GetWorkerInfo pid={infoReply.WorkerInfo.WorkerProcessId} version={infoReply.WorkerInfo.WorkerVersion} "
|
||
+ $"progid={infoReply.WorkerInfo.MxaccessProgid} clsid={infoReply.WorkerInfo.MxaccessClsid}");
|
||
Assert.Equal(openReply.WorkerProcessId, infoReply.WorkerInfo.WorkerProcessId);
|
||
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.WorkerVersion));
|
||
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessProgid));
|
||
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessClsid));
|
||
|
||
// No StreamEvents subscriber is attached, so whatever the worker queued since start
|
||
// is still drainable; the reply must carry the DrainEvents payload (possibly empty).
|
||
MxCommandReply drainReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-drain-events", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.DrainEvents,
|
||
DrainEvents = new DrainEventsCommand { MaxEvents = 16 },
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("DrainEvents", drainReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, drainReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.DrainEvents, drainReply.Kind);
|
||
Assert.Equal(MxCommandReply.PayloadOneofCase.DrainEvents, drainReply.PayloadCase);
|
||
Assert.True(drainReply.DrainEvents.Events.Count <= 16);
|
||
output.WriteLine($"DrainEvents drained={drainReply.DrainEvents.Events.Count}");
|
||
|
||
// A control command must not disturb the session: it is still Ready afterwards.
|
||
Assert.True(fixture.TryGetSession(sessionId, out GatewaySession? session));
|
||
Assert.Equal(SessionState.Ready, session!.State);
|
||
}
|
||
finally
|
||
{
|
||
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// TST-05: <c>ShutdownWorker</c> writes its OK reply before the worker shuts down, so the
|
||
/// reply reaches the caller (a reply written after the pipe closed would surface as a
|
||
/// pipe-disconnected failure instead), the worker process then exits on its own, and a later
|
||
/// command on the session does not succeed. Uses its own fixture and worker because it
|
||
/// terminates the worker.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[LiveMxAccessFact]
|
||
public async Task GatewaySession_WithLiveWorker_ShutdownWorker_RepliesOkThenWorkerExits()
|
||
{
|
||
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||
Assert.True(
|
||
File.Exists(workerExecutablePath),
|
||
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
|
||
|
||
TestWorkerProcessFactory processFactory = new(output);
|
||
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
|
||
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||
|
||
string? sessionId = null;
|
||
Task? streamTask = null;
|
||
using CancellationTokenSource streamCancellation = new();
|
||
|
||
try
|
||
{
|
||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||
new OpenSessionRequest
|
||
{
|
||
ClientSessionName = "live-mxaccess-shutdown-worker",
|
||
ClientCorrelationId = "live-open-shutdown-worker",
|
||
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
sessionId = openReply.SessionId;
|
||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||
|
||
// The gateway observes a worker's pipe closing through the event-stream read loop
|
||
// (the same path the abnormal-exit fact relies on); without a subscriber the session
|
||
// would sit in Ready until the heartbeat grace expired.
|
||
streamTask = fixture.Service.StreamEvents(
|
||
new StreamEventsRequest { SessionId = sessionId },
|
||
eventWriter,
|
||
new TestServerCallContext(streamCancellation.Token));
|
||
|
||
MxCommandReply shutdownReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-shutdown-worker", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.ShutdownWorker,
|
||
ShutdownWorker = new ShutdownWorkerCommand
|
||
{
|
||
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(5)),
|
||
},
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("ShutdownWorker", shutdownReply);
|
||
Assert.Equal(ProtocolStatusCode.Ok, shutdownReply.ProtocolStatus.Code);
|
||
Assert.Equal(MxCommandKind.ShutdownWorker, shutdownReply.Kind);
|
||
|
||
// The worker exits on its own — no CloseSession, no kill. WaitForProcessesAsync
|
||
// throws on timeout, which fails the test.
|
||
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
|
||
|
||
// The session can no longer serve commands once its worker is gone: either the
|
||
// gateway rejects the call outright or the reply is not Ok.
|
||
MxCommandReply? afterReply = null;
|
||
Exception? afterException = null;
|
||
try
|
||
{
|
||
afterReply = await fixture.Service.Invoke(
|
||
CreateControlRequest(sessionId, "live-ping-after-shutdown", new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Ping,
|
||
Ping = new PingCommand(),
|
||
}),
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
LogReply("PingAfterShutdown", afterReply);
|
||
}
|
||
catch (Exception exception)
|
||
{
|
||
afterException = exception;
|
||
output.WriteLine($"PingAfterShutdown rejected: {exception.GetType().Name}: {exception.Message}");
|
||
}
|
||
|
||
Assert.True(
|
||
afterException is not null || afterReply!.ProtocolStatus.Code != ProtocolStatusCode.Ok,
|
||
"A command sent after ShutdownWorker must not succeed against the exited worker.");
|
||
|
||
// Logged, not asserted: the gateway's WorkerClient moves to Closed on the shutdown
|
||
// ack, but GatewaySession.State is not driven from that transition today, so the
|
||
// public session state still reads Ready here (observed live on windev). That is a
|
||
// gateway-side gap, not a worker one — this fact pins the worker half of the
|
||
// contract: OK reply first, then a clean exit.
|
||
string observedState = fixture.TryGetSession(sessionId, out GatewaySession? session)
|
||
? session.State.ToString()
|
||
: "<removed>";
|
||
output.WriteLine($"ShutdownWorker observed_session_state={observedState}");
|
||
}
|
||
finally
|
||
{
|
||
// The worker is gone; a CloseSession round-trip would only log a cleanup failure.
|
||
streamCancellation.Cancel();
|
||
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
private static MxCommandRequest CreateControlRequest(
|
||
string sessionId,
|
||
string correlationId,
|
||
MxCommand command)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = correlationId,
|
||
Command = command,
|
||
};
|
||
}
|
||
|
||
/// <summary>
|
||
/// Closes the session and drains the event stream / worker processes without letting a
|
||
/// cleanup timeout mask the original failure from the test body.
|
||
/// </summary>
|
||
/// <param name="propagateStreamFaults">
|
||
/// When <see langword="true"/>, a faulted <paramref name="streamTask"/> is rethrown so the
|
||
/// test fails on a silent stream-task exception (the Write parity test relies on this so
|
||
/// stream-side defects in event delivery are visible). When <see langword="false"/>, all
|
||
/// cleanup exceptions are logged and swallowed so a real test-body assertion failure is not
|
||
/// masked by a shutdown timeout.
|
||
/// </param>
|
||
private async Task ShutDownAsync(
|
||
GatewayServiceFixture fixture,
|
||
TestWorkerProcessFactory processFactory,
|
||
string? sessionId,
|
||
Task? streamTask,
|
||
bool propagateStreamFaults = false)
|
||
{
|
||
Exception? streamFault = null;
|
||
|
||
try
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(sessionId))
|
||
{
|
||
await CloseSessionAsync(fixture, sessionId).ConfigureAwait(false);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
output.WriteLine($"Cleanup error during session close: {ex}");
|
||
}
|
||
|
||
if (streamTask is not null)
|
||
{
|
||
try
|
||
{
|
||
await streamTask.WaitAsync(StreamShutdownTimeout).ConfigureAwait(false);
|
||
}
|
||
catch (OperationCanceledException ex)
|
||
{
|
||
// A linked CancellationToken on the streaming TestServerCallContext is the
|
||
// intended way to stop StreamEvents promptly — treat the resulting
|
||
// OperationCanceledException as a clean shutdown, not a fault.
|
||
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Message}");
|
||
}
|
||
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled)
|
||
{
|
||
// MxAccessGatewayService.MapException intentionally converts the
|
||
// server-side OperationCanceledException into RpcException(Cancelled)
|
||
// so real gRPC clients see the standard cancellation status. Treat
|
||
// that as the same clean-shutdown signal as a raw OCE.
|
||
output.WriteLine($"Event stream task cancelled during shutdown: {ex.Status.Detail}");
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
// Cleanup runs in a finally block. By default a faulted StreamEvents task is
|
||
// logged and swallowed so a test-body assertion failure is not masked. When
|
||
// the caller opts into propagateStreamFaults (the Write parity test), we
|
||
// rethrow the fault after the worker-process wait so a silent stream-side
|
||
// defect actually fails the test.
|
||
output.WriteLine($"Event stream task faulted during shutdown: {ex}");
|
||
if (propagateStreamFaults)
|
||
{
|
||
streamFault = ex;
|
||
}
|
||
}
|
||
}
|
||
|
||
try
|
||
{
|
||
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
output.WriteLine($"Cleanup error while waiting for worker processes to exit: {ex}");
|
||
}
|
||
|
||
if (streamFault is not null)
|
||
{
|
||
throw streamFault;
|
||
}
|
||
}
|
||
|
||
private static MxCommandRequest CreateRegisterRequest(string sessionId)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-register",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Register,
|
||
Register = new RegisterCommand
|
||
{
|
||
ClientName = IntegrationTestEnvironment.LiveMxAccessClientName,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateAddItemRequest(
|
||
string sessionId,
|
||
int serverHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-add-item",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AddItem,
|
||
AddItem = new AddItemCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemDefinition = IntegrationTestEnvironment.LiveMxAccessItem,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateAdviseRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-advise",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Advise,
|
||
Advise = new AdviseCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateAdviseSupervisoryRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-advise-supervisory",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AdviseSupervisory,
|
||
AdviseSupervisory = new AdviseSupervisoryCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateWriteRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle,
|
||
MxValue value)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-write-restore",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Write,
|
||
Write = new WriteCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
Value = value,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateWriteRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-write",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Write,
|
||
Write = new WriteCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
Value = new MxValue
|
||
{
|
||
DataType = MxDataType.Integer,
|
||
Int32Value = 1,
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateWrite2Request(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle,
|
||
DateTime timestampUtc)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-write2",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Write2,
|
||
Write2 = new Write2Command
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
Value = new MxValue
|
||
{
|
||
DataType = MxDataType.Integer,
|
||
Int32Value = 2,
|
||
},
|
||
TimestampValue = new MxValue
|
||
{
|
||
DataType = MxDataType.Time,
|
||
VariantType = "VT_DATE",
|
||
TimestampValue = Timestamp.FromDateTime(timestampUtc),
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateUnAdviseRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-unadvise",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.UnAdvise,
|
||
UnAdvise = new UnAdviseCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateRemoveItemRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-remove-item",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.RemoveItem,
|
||
RemoveItem = new RemoveItemCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateUnregisterRequest(
|
||
string sessionId,
|
||
int serverHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-unregister",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Unregister,
|
||
Unregister = new UnregisterCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateAuthenticateUserRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
string verifyUser,
|
||
string verifyPassword)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-authenticate-user",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AuthenticateUser,
|
||
AuthenticateUser = new AuthenticateUserCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
VerifyUser = verifyUser,
|
||
VerifyUserPassword = verifyPassword,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateArchestrAUserToIdRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
string userIdGuid)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-archestra-user-to-id",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.ArchestraUserToId,
|
||
ArchestraUserToId = new ArchestrAUserToIdCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
UserIdGuid = userIdGuid,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateAddBufferedItemRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
string itemDefinition,
|
||
string itemContext)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-add-buffered-item",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.AddBufferedItem,
|
||
AddBufferedItem = new AddBufferedItemCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemDefinition = itemDefinition,
|
||
ItemContext = itemContext,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateSetBufferedUpdateIntervalRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int updateIntervalMilliseconds)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-set-buffered-update-interval",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.SetBufferedUpdateInterval,
|
||
SetBufferedUpdateInterval = new SetBufferedUpdateIntervalCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
UpdateIntervalMilliseconds = updateIntervalMilliseconds,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateSuspendRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-suspend",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Suspend,
|
||
Suspend = new SuspendCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateActivateRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-activate",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Activate,
|
||
Activate = new ActivateCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static MxCommandRequest CreateWriteSecuredRequest(
|
||
string sessionId,
|
||
int serverHandle,
|
||
int itemHandle,
|
||
int currentUserId,
|
||
int verifierUserId)
|
||
{
|
||
return new MxCommandRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-write-secured",
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.WriteSecured,
|
||
WriteSecured = new WriteSecuredCommand
|
||
{
|
||
ServerHandle = serverHandle,
|
||
ItemHandle = itemHandle,
|
||
CurrentUserId = currentUserId,
|
||
VerifierUserId = verifierUserId,
|
||
Value = new MxValue
|
||
{
|
||
DataType = MxDataType.Integer,
|
||
Int32Value = 2,
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private static (string VerifyUser, string VerifyPassword) ResolveLiveMxAccessSecuredCredentials()
|
||
{
|
||
string verifyUser = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER")
|
||
?? "admin";
|
||
string verifyPassword = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD")
|
||
?? "password";
|
||
return (verifyUser, verifyPassword);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Splits a dotted MXAccess reference (e.g. "TestChildObject.TestInt") into the
|
||
/// (item_definition, item_context) pair AddBufferedItem expects — attribute name and
|
||
/// owning object. An undotted reference is passed through with an empty context.
|
||
/// </summary>
|
||
private static (string Item, string Context) SplitLiveItemForBuffered(string liveItem)
|
||
{
|
||
int lastDot = liveItem.LastIndexOf('.');
|
||
if (lastDot < 0 || lastDot >= liveItem.Length - 1)
|
||
{
|
||
return (liveItem, string.Empty);
|
||
}
|
||
|
||
string context = liveItem[..lastDot];
|
||
string item = liveItem[(lastDot + 1)..];
|
||
return (item, context);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Counts the elements in a converted buffered <see cref="MxArray"/> across whichever
|
||
/// typed-array oneof case the VariantConverter populated, so the buffered-capture
|
||
/// assertions are independent of the rig item's element type.
|
||
/// </summary>
|
||
private static int CountArrayElements(MxArray? array)
|
||
{
|
||
if (array is null)
|
||
{
|
||
return 0;
|
||
}
|
||
|
||
return array.ValuesCase switch
|
||
{
|
||
MxArray.ValuesOneofCase.BoolValues => array.BoolValues.Values.Count,
|
||
MxArray.ValuesOneofCase.Int32Values => array.Int32Values.Values.Count,
|
||
MxArray.ValuesOneofCase.Int64Values => array.Int64Values.Values.Count,
|
||
MxArray.ValuesOneofCase.FloatValues => array.FloatValues.Values.Count,
|
||
MxArray.ValuesOneofCase.DoubleValues => array.DoubleValues.Values.Count,
|
||
MxArray.ValuesOneofCase.StringValues => array.StringValues.Values.Count,
|
||
MxArray.ValuesOneofCase.TimestampValues => array.TimestampValues.Values.Count,
|
||
MxArray.ValuesOneofCase.RawValues => array.RawValues.Values.Count,
|
||
_ => 0,
|
||
};
|
||
}
|
||
|
||
private static int CountMatchingEvents(
|
||
RecordingServerStreamWriter<MxEvent> writer,
|
||
Func<MxEvent, bool> predicate)
|
||
{
|
||
int count = 0;
|
||
foreach (MxEvent message in writer.Messages)
|
||
{
|
||
if (predicate(message))
|
||
{
|
||
count++;
|
||
}
|
||
}
|
||
|
||
return count;
|
||
}
|
||
|
||
private async Task CloseSessionAsync(
|
||
GatewayServiceFixture fixture,
|
||
string sessionId)
|
||
{
|
||
CloseSessionReply closeReply = await fixture.Service.CloseSession(
|
||
new CloseSessionRequest
|
||
{
|
||
SessionId = sessionId,
|
||
ClientCorrelationId = "live-close",
|
||
},
|
||
new TestServerCallContext()).ConfigureAwait(false);
|
||
|
||
output.WriteLine($"CloseSession status={closeReply.ProtocolStatus.Code} final_state={closeReply.FinalState}");
|
||
}
|
||
|
||
private void LogReply(
|
||
string method,
|
||
MxCommandReply reply)
|
||
{
|
||
LogReplyTo(output, method, reply);
|
||
}
|
||
|
||
private static void LogReplyTo(
|
||
ITestOutputHelper sink,
|
||
string method,
|
||
MxCommandReply reply)
|
||
{
|
||
sink.WriteLine(
|
||
$"{method} status={reply.ProtocolStatus.Code} hresult={reply.Hresult} diagnostic={reply.DiagnosticMessage}");
|
||
|
||
foreach (MxStatusProxy status in reply.Statuses)
|
||
{
|
||
sink.WriteLine(
|
||
$"{method} mxstatus success={status.Success} category={status.Category} detail={status.Detail} text={status.DiagnosticText}");
|
||
}
|
||
}
|
||
|
||
private void LogEvent(MxEvent dataChange)
|
||
{
|
||
output.WriteLine(
|
||
$"Event family={dataChange.Family} worker_sequence={dataChange.WorkerSequence} server_handle={dataChange.ServerHandle} item_handle={dataChange.ItemHandle} quality={dataChange.Quality}");
|
||
output.WriteLine(
|
||
$"Event value_type={dataChange.Value?.DataType} raw_status={dataChange.RawStatus}");
|
||
}
|
||
|
||
/// <summary>
|
||
/// Test fixture that assembles the gateway service with a worker process factory for live MXAccess testing.
|
||
/// </summary>
|
||
private sealed class GatewayServiceFixture : IAsyncDisposable
|
||
{
|
||
private readonly GatewayMetrics _metrics = new();
|
||
private readonly SessionRegistry _registry = new();
|
||
private readonly ILoggerFactory _loggerFactory;
|
||
|
||
/// <summary>
|
||
/// Initializes the fixture with worker executable path, factory, and test output helper.
|
||
/// </summary>
|
||
/// <param name="workerExecutablePath">Path to the worker process executable.</param>
|
||
/// <param name="processFactory">Factory for creating worker processes.</param>
|
||
/// <param name="output">Test output helper for logging.</param>
|
||
/// <param name="alarms">Optional configuration for the central alarm monitor.</param>
|
||
public GatewayServiceFixture(
|
||
string workerExecutablePath,
|
||
IWorkerProcessFactory processFactory,
|
||
ITestOutputHelper output,
|
||
AlarmsOptions? alarms = null)
|
||
{
|
||
IOptions<GatewayOptions> options = Options.Create(CreateOptions(workerExecutablePath, alarms));
|
||
_loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(new TestOutputLoggerProvider(output)));
|
||
WorkerProcessLauncher launcher = new(
|
||
options,
|
||
processFactory,
|
||
new WorkerProcessStartedProbe(),
|
||
_metrics);
|
||
SessionWorkerClientFactory workerClientFactory = new(
|
||
launcher,
|
||
options,
|
||
_metrics,
|
||
_loggerFactory);
|
||
SessionManager sessionManager = new(
|
||
_registry,
|
||
workerClientFactory,
|
||
options,
|
||
_metrics,
|
||
logger: _loggerFactory.CreateLogger<SessionManager>());
|
||
Manager = sessionManager;
|
||
MxAccessGrpcMapper mapper = new();
|
||
EventStreamService eventStreamService = new(
|
||
sessionManager,
|
||
options,
|
||
_metrics);
|
||
|
||
Monitor = new GatewayAlarmMonitor(
|
||
sessionManager, EmptyAlarmWatchListResolver.Instance, _metrics, options,
|
||
_loggerFactory.CreateLogger<GatewayAlarmMonitor>());
|
||
Service = new MxAccessGatewayService(
|
||
sessionManager,
|
||
new GatewayRequestIdentityAccessor(),
|
||
new AllowAllConstraintEnforcer(),
|
||
new MxAccessGrpcRequestValidator(),
|
||
mapper,
|
||
eventStreamService,
|
||
_metrics,
|
||
_loggerFactory.CreateLogger<MxAccessGatewayService>(),
|
||
Monitor);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The assembled gateway service instance.
|
||
/// </summary>
|
||
public MxAccessGatewayService Service { get; }
|
||
|
||
/// <summary>The central alarm monitor for tests of the public alarm RPCs.</summary>
|
||
public GatewayAlarmMonitor Monitor { get; }
|
||
|
||
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
|
||
public SessionManager Manager { get; }
|
||
|
||
/// <summary>
|
||
/// Looks up a session by id directly against the in-process registry. The abnormal
|
||
/// worker-exit test needs to observe the session's State / FinalFault as the gateway
|
||
/// transitions it to Faulted, which the public gRPC API only exposes indirectly via
|
||
/// CloseSession's reply (and not before a graceful close completes).
|
||
/// </summary>
|
||
/// <param name="sessionId">The session identifier.</param>
|
||
/// <param name="session">The session if found; otherwise null.</param>
|
||
/// <returns><see langword="true"/> if a session with the given id was found; otherwise <see langword="false"/>.</returns>
|
||
public bool TryGetSession(string sessionId, [MaybeNullWhen(false)] out GatewaySession session)
|
||
{
|
||
return _registry.TryGet(sessionId, out session);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Disposes the fixture resources and closes all sessions.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public async ValueTask DisposeAsync()
|
||
{
|
||
await Monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
|
||
foreach (GatewaySession session in _registry.Snapshot())
|
||
{
|
||
await session.DisposeAsync().ConfigureAwait(false);
|
||
}
|
||
|
||
Monitor.Dispose();
|
||
_loggerFactory.Dispose();
|
||
_metrics.Dispose();
|
||
}
|
||
|
||
private static GatewayOptions CreateOptions(string workerExecutablePath, AlarmsOptions? alarms)
|
||
{
|
||
return new GatewayOptions
|
||
{
|
||
Alarms = alarms ?? new AlarmsOptions(),
|
||
Worker = new WorkerOptions
|
||
{
|
||
ExecutablePath = workerExecutablePath,
|
||
StartupTimeoutSeconds = 30,
|
||
ShutdownTimeoutSeconds = 15,
|
||
HeartbeatCheckIntervalSeconds = 5,
|
||
HeartbeatGraceSeconds = 15,
|
||
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||
RequiredArchitecture = WorkerArchitecture.X86,
|
||
},
|
||
Sessions = new SessionOptions
|
||
{
|
||
DefaultCommandTimeoutSeconds = 15,
|
||
MaxSessions = alarms?.Enabled == true ? 2 : 1,
|
||
},
|
||
Events = new EventOptions
|
||
{
|
||
QueueCapacity = 32,
|
||
},
|
||
};
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gathers messages written to a server stream for test inspection.
|
||
/// </summary>
|
||
private sealed class RecordingServerStreamWriter<T> : IServerStreamWriter<T>, IDisposable
|
||
{
|
||
private readonly object syncRoot = new();
|
||
private readonly List<T> messages = [];
|
||
private readonly SemaphoreSlim messageArrived = new(0);
|
||
|
||
/// <summary>
|
||
/// All messages that have been written to the stream.
|
||
/// </summary>
|
||
public IReadOnlyList<T> Messages
|
||
{
|
||
get
|
||
{
|
||
lock (syncRoot)
|
||
{
|
||
return messages.ToArray();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Inherited write options.
|
||
/// </summary>
|
||
public WriteOptions? WriteOptions { get; set; }
|
||
|
||
/// <summary>
|
||
/// Records the message unless the token is already cancelled.
|
||
/// </summary>
|
||
/// <param name="message">The message to write.</param>
|
||
/// <param name="cancellationToken">Token that cancels the write.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public Task WriteAsync(T message, CancellationToken cancellationToken)
|
||
{
|
||
// The interface default throws NotSupportedException for a cancelable token; the
|
||
// gateway passes the call's token on every streamed write.
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
return WriteAsync(message);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Records the message and signals any pending waiter.
|
||
/// </summary>
|
||
/// <param name="message">The message to write.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public Task WriteAsync(T message)
|
||
{
|
||
lock (syncRoot)
|
||
{
|
||
messages.Add(message);
|
||
}
|
||
|
||
messageArrived.Release();
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Waits for the first recorded message that satisfies <paramref name="predicate"/>,
|
||
/// up to the specified timeout. Earlier non-matching messages (for example a
|
||
/// registration-state bootstrap event) are skipped rather than treated as the result.
|
||
/// </summary>
|
||
/// <param name="predicate">Filter the awaited message must satisfy.</param>
|
||
/// <param name="timeout">The maximum total time to wait.</param>
|
||
/// <param name="cancellationToken">
|
||
/// Token observed alongside the timeout so a per-test cancellation (for example the
|
||
/// gRPC call context's token) aborts the wait promptly instead of hanging until the
|
||
/// timeout elapses.
|
||
/// </param>
|
||
/// <returns>The first message that satisfies the predicate.</returns>
|
||
public async Task<T> WaitForMessageAsync(
|
||
Func<T, bool> predicate,
|
||
TimeSpan timeout,
|
||
CancellationToken cancellationToken = default)
|
||
{
|
||
using CancellationTokenSource timeoutCancellation = new(timeout);
|
||
using CancellationTokenSource linkedCancellation =
|
||
CancellationTokenSource.CreateLinkedTokenSource(timeoutCancellation.Token, cancellationToken);
|
||
int scanned = 0;
|
||
|
||
while (true)
|
||
{
|
||
T[] snapshot;
|
||
lock (syncRoot)
|
||
{
|
||
snapshot = messages.ToArray();
|
||
}
|
||
|
||
for (; scanned < snapshot.Length; scanned++)
|
||
{
|
||
if (predicate(snapshot[scanned]))
|
||
{
|
||
return snapshot[scanned];
|
||
}
|
||
}
|
||
|
||
try
|
||
{
|
||
await messageArrived.WaitAsync(linkedCancellation.Token).ConfigureAwait(false);
|
||
}
|
||
catch (OperationCanceledException) when (timeoutCancellation.IsCancellationRequested)
|
||
{
|
||
throw new TimeoutException(
|
||
$"No stream message satisfied the predicate within {timeout}. Recorded {scanned} message(s).");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Releases the wait handle backing <c>messageArrived</c>. The writer owns an
|
||
/// <see cref="IDisposable"/> field so it must be disposable itself; the leak
|
||
/// is otherwise bounded only by how many opt-in live tests run.
|
||
/// </summary>
|
||
public void Dispose()
|
||
{
|
||
messageArrived.Dispose();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Minimal <see cref="ServerCallContext"/> stub for invoking the gRPC service
|
||
/// in-process. It is a hand-written fake with no verification behavior — it
|
||
/// only supplies the context values the service reads during a call.
|
||
/// </summary>
|
||
private sealed class TestServerCallContext(CancellationToken cancellationToken = default) : ServerCallContext
|
||
{
|
||
private readonly Metadata requestHeaders = [];
|
||
private readonly Metadata responseTrailers = [];
|
||
private readonly Dictionary<object, object> userState = [];
|
||
private Status status;
|
||
private WriteOptions? writeOptions;
|
||
|
||
/// <inheritdoc />
|
||
protected override string MethodCore => "/mxaccess_gateway.v1.MxAccessGateway/Test";
|
||
|
||
/// <inheritdoc />
|
||
protected override string HostCore => "localhost";
|
||
|
||
/// <inheritdoc />
|
||
protected override string PeerCore => "ipv4:127.0.0.1:5000";
|
||
|
||
/// <inheritdoc />
|
||
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
|
||
|
||
/// <inheritdoc />
|
||
protected override Metadata RequestHeadersCore => requestHeaders;
|
||
|
||
/// <inheritdoc />
|
||
protected override CancellationToken CancellationTokenCore => cancellationToken;
|
||
|
||
/// <inheritdoc />
|
||
protected override Metadata ResponseTrailersCore => responseTrailers;
|
||
|
||
/// <inheritdoc />
|
||
protected override Status StatusCore
|
||
{
|
||
get => status;
|
||
set => status = value;
|
||
}
|
||
|
||
/// <inheritdoc />
|
||
protected override WriteOptions? WriteOptionsCore
|
||
{
|
||
get => writeOptions;
|
||
set => writeOptions = value;
|
||
}
|
||
|
||
/// <inheritdoc />
|
||
protected override AuthContext AuthContextCore { get; } = new(
|
||
string.Empty,
|
||
new Dictionary<string, List<AuthProperty>>(StringComparer.Ordinal));
|
||
|
||
/// <inheritdoc />
|
||
protected override IDictionary<object, object> UserStateCore => userState;
|
||
|
||
/// <inheritdoc />
|
||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
|
||
{
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
/// <inheritdoc />
|
||
protected override ContextPropagationToken CreatePropagationTokenCore(
|
||
ContextPropagationOptions? options)
|
||
{
|
||
throw new NotSupportedException();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Factory that launches worker processes and records their outputs for testing.
|
||
/// </summary>
|
||
private sealed class TestWorkerProcessFactory(ITestOutputHelper output) : IWorkerProcessFactory
|
||
{
|
||
private readonly ConcurrentBag<TestWorkerProcess> processes = [];
|
||
|
||
/// <inheritdoc />
|
||
public IWorkerProcess Start(ProcessStartInfo startInfo)
|
||
{
|
||
startInfo.RedirectStandardError = true;
|
||
startInfo.RedirectStandardOutput = true;
|
||
startInfo.UseShellExecute = false;
|
||
|
||
Process process = new()
|
||
{
|
||
StartInfo = startInfo,
|
||
EnableRaisingEvents = true,
|
||
};
|
||
|
||
process.OutputDataReceived += (_, args) => WriteWorkerOutput("stdout", args.Data);
|
||
process.ErrorDataReceived += (_, args) => WriteWorkerOutput("stderr", args.Data);
|
||
|
||
if (!process.Start())
|
||
{
|
||
process.Dispose();
|
||
throw new InvalidOperationException("Worker process failed to start.");
|
||
}
|
||
|
||
process.BeginOutputReadLine();
|
||
process.BeginErrorReadLine();
|
||
|
||
TestWorkerProcess workerProcess = new(process);
|
||
processes.Add(workerProcess);
|
||
output.WriteLine($"WorkerProcess started pid={workerProcess.Id} path={startInfo.FileName}");
|
||
|
||
return workerProcess;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Waits for every recorded worker process to exit, up to the specified timeout per process.
|
||
/// </summary>
|
||
/// <param name="timeout">The maximum time to wait for each process.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public async Task WaitForProcessesAsync(TimeSpan timeout)
|
||
{
|
||
foreach (TestWorkerProcess process in processes)
|
||
{
|
||
if (process.HasExited)
|
||
{
|
||
output.WriteLine($"WorkerProcess exited pid={process.Id} exit_code={process.ExitCode}");
|
||
continue;
|
||
}
|
||
|
||
using CancellationTokenSource timeoutCancellation = new(timeout);
|
||
await process.WaitForExitAsync(timeoutCancellation.Token).ConfigureAwait(false);
|
||
output.WriteLine($"WorkerProcess exited pid={process.Id} exit_code={process.ExitCode}");
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Kills every recorded worker process tree so the abnormal-exit test can simulate a
|
||
/// crashed worker without going through the graceful shutdown handshake. Failures to
|
||
/// kill an already-dead process are tolerated.
|
||
/// </summary>
|
||
public void KillAllAndDetach()
|
||
{
|
||
foreach (TestWorkerProcess process in processes)
|
||
{
|
||
if (process.HasExited)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
try
|
||
{
|
||
process.Kill(entireProcessTree: true);
|
||
output.WriteLine($"WorkerProcess killed pid={process.Id} (abnormal-exit simulation)");
|
||
}
|
||
catch (InvalidOperationException ex)
|
||
{
|
||
output.WriteLine($"WorkerProcess kill skipped pid={process.Id}: {ex.Message}");
|
||
}
|
||
}
|
||
}
|
||
|
||
private void WriteWorkerOutput(
|
||
string streamName,
|
||
string? line)
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(line))
|
||
{
|
||
output.WriteLine($"worker_{streamName}: {line}");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Adapter wrapping a System.Diagnostics.Process as IWorkerProcess for testing.
|
||
/// </summary>
|
||
private sealed class TestWorkerProcess(Process process) : IWorkerProcess
|
||
{
|
||
/// <inheritdoc />
|
||
public int Id => process.Id;
|
||
|
||
/// <inheritdoc />
|
||
public bool HasExited => process.HasExited;
|
||
|
||
/// <inheritdoc />
|
||
public int? ExitCode => process.HasExited ? process.ExitCode : null;
|
||
|
||
/// <inheritdoc />
|
||
public async ValueTask WaitForExitAsync(CancellationToken cancellationToken)
|
||
{
|
||
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
/// <inheritdoc />
|
||
public void Kill(bool entireProcessTree)
|
||
{
|
||
process.Kill(entireProcessTree);
|
||
}
|
||
|
||
/// <summary>Disposes the wrapped process.</summary>
|
||
public void Dispose()
|
||
{
|
||
process.Dispose();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Logger provider that writes all output to the test output helper.
|
||
/// </summary>
|
||
private sealed class TestOutputLoggerProvider(ITestOutputHelper output) : ILoggerProvider
|
||
{
|
||
/// <summary>Creates a logger that writes to the test output helper for the given category.</summary>
|
||
/// <param name="categoryName">Category name for the logger.</param>
|
||
/// <returns>The created logger.</returns>
|
||
public ILogger CreateLogger(string categoryName)
|
||
{
|
||
return new TestOutputLogger(output, categoryName);
|
||
}
|
||
|
||
/// <summary>No resources to release; provided to satisfy <see cref="ILoggerProvider"/>.</summary>
|
||
public void Dispose()
|
||
{
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Logger that writes messages to the test output helper.
|
||
/// </summary>
|
||
private sealed class TestOutputLogger(
|
||
ITestOutputHelper output,
|
||
string categoryName) : ILogger
|
||
{
|
||
/// <summary>Not supported; this test logger does not track scopes.</summary>
|
||
/// <typeparam name="TState">The type of the state to begin the scope for.</typeparam>
|
||
/// <param name="state">The identifier for the scope.</param>
|
||
/// <returns>Always <see langword="null"/>.</returns>
|
||
public IDisposable? BeginScope<TState>(TState state)
|
||
where TState : notnull
|
||
{
|
||
return null;
|
||
}
|
||
|
||
/// <summary>Determines whether the given log level is enabled.</summary>
|
||
/// <param name="logLevel">The log level to check.</param>
|
||
/// <returns><see langword="true"/> if <paramref name="logLevel"/> is at least <see cref="LogLevel.Information"/>; otherwise <see langword="false"/>.</returns>
|
||
public bool IsEnabled(LogLevel logLevel)
|
||
{
|
||
return logLevel >= LogLevel.Information;
|
||
}
|
||
|
||
/// <summary>Writes the formatted log message and any exception to the test output helper.</summary>
|
||
/// <typeparam name="TState">The type of the object to be logged.</typeparam>
|
||
/// <param name="logLevel">The severity of the log entry.</param>
|
||
/// <param name="eventId">The event id associated with the log entry.</param>
|
||
/// <param name="state">The entry to be logged, which can be an object or a message string.</param>
|
||
/// <param name="exception">The exception related to this entry, if any.</param>
|
||
/// <param name="formatter">Function that creates the log message from <paramref name="state"/> and <paramref name="exception"/>.</param>
|
||
public void Log<TState>(
|
||
LogLevel logLevel,
|
||
EventId eventId,
|
||
TState state,
|
||
Exception? exception,
|
||
Func<TState, Exception?, string> formatter)
|
||
{
|
||
if (!IsEnabled(logLevel))
|
||
{
|
||
return;
|
||
}
|
||
|
||
output.WriteLine($"{logLevel} {categoryName}: {formatter(state, exception)}");
|
||
if (exception is not null)
|
||
{
|
||
output.WriteLine(exception.ToString());
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Buffering wrapper around an <see cref="ITestOutputHelper"/> that mirrors every line
|
||
/// written through it into a <see cref="StringBuilder"/> the test owns. The WriteSecured
|
||
/// parity test uses this to make CLAUDE.md's "passwords and
|
||
/// <c>WriteSecured</c> payloads must never reach logs" rule a property of the entire
|
||
/// test output stream — gateway <see cref="ILogger"/> entries (echoed via
|
||
/// <see cref="TestOutputLoggerProvider"/>), worker stdout/stderr (echoed via
|
||
/// <see cref="TestWorkerProcessFactory.WriteWorkerOutput"/>), and direct
|
||
/// <c>output.WriteLine</c> calls all land in the same buffer, so a future maintenance
|
||
/// change that prints a credential through any of those channels is caught by the
|
||
/// assertion rather than slipping past the existing <c>DiagnosticMessage</c> check.
|
||
/// </summary>
|
||
private sealed class RecordingTestOutputHelper(ITestOutputHelper inner) : ITestOutputHelper
|
||
{
|
||
private readonly StringBuilder buffer = new();
|
||
private readonly object syncRoot = new();
|
||
|
||
/// <summary>Gets the accumulated output buffer contents.</summary>
|
||
public string Captured
|
||
{
|
||
get
|
||
{
|
||
lock (syncRoot)
|
||
{
|
||
return buffer.ToString();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Writes a line of text to the buffer and inner helper.</summary>
|
||
/// <param name="message">The message to write.</param>
|
||
public void WriteLine(string message)
|
||
{
|
||
lock (syncRoot)
|
||
{
|
||
buffer.AppendLine(message);
|
||
}
|
||
|
||
inner.WriteLine(message);
|
||
}
|
||
|
||
/// <summary>Writes a formatted line of text to the buffer and inner helper.</summary>
|
||
/// <param name="format">The message format string.</param>
|
||
/// <param name="args">The format arguments.</param>
|
||
public void WriteLine(string format, params object[] args)
|
||
{
|
||
string formatted = string.Format(System.Globalization.CultureInfo.InvariantCulture, format, args);
|
||
lock (syncRoot)
|
||
{
|
||
buffer.AppendLine(formatted);
|
||
}
|
||
|
||
inner.WriteLine(format, args);
|
||
}
|
||
}
|
||
|
||
private sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
|
||
{
|
||
/// <inheritdoc />
|
||
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
|
||
ApiKeyIdentity? identity,
|
||
string tagAddress,
|
||
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
|
||
|
||
/// <inheritdoc />
|
||
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
|
||
ApiKeyIdentity? identity,
|
||
GatewaySession session,
|
||
int serverHandle,
|
||
int itemHandle,
|
||
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
|
||
|
||
/// <inheritdoc />
|
||
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
|
||
ApiKeyIdentity? identity,
|
||
GatewaySession session,
|
||
int serverHandle,
|
||
int itemHandle,
|
||
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
|
||
|
||
/// <inheritdoc />
|
||
public Task RecordDenialAsync(
|
||
ApiKeyIdentity? identity,
|
||
string commandKind,
|
||
string target,
|
||
ConstraintFailure failure,
|
||
string? correlationId,
|
||
CancellationToken cancellationToken) => Task.CompletedTask;
|
||
|
||
/// <inheritdoc />
|
||
public Task RecordBulkDenialAsync(
|
||
ApiKeyIdentity? identity,
|
||
string commandKind,
|
||
ConstraintDenialSummary summary,
|
||
string? correlationId,
|
||
CancellationToken cancellationToken) => Task.CompletedTask;
|
||
}
|
||
|
||
}
|