using System.Diagnostics.Metrics;
using System.Runtime.CompilerServices;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
public sealed class MxAccessGatewayServiceTests
{
/// Verifies that OpenSession returns correct session details for a valid request.
[Fact]
public async Task OpenSession_WithValidRequest_ReturnsSessionDetails()
{
GatewayRequestIdentityAccessor identityAccessor = new();
FakeSessionManager sessionManager = new()
{
OpenSessionResult = CreateSession("session-1", processId: 4321),
};
MxAccessGatewayService service = CreateService(sessionManager, identityAccessor);
using IDisposable identityScope = identityAccessor.Push(CreateIdentity());
OpenSessionReply reply = await service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "operator-session",
CommandTimeout = Duration.FromTimeSpan(TimeSpan.FromSeconds(7)),
},
new TestServerCallContext());
Assert.Equal("session-1", reply.SessionId);
Assert.Equal(GatewayContractInfo.DefaultBackendName, reply.BackendName);
Assert.Equal(4321, reply.WorkerProcessId);
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, reply.WorkerProtocolVersion);
Assert.Equal(GatewayContractInfo.GatewayProtocolVersion, reply.GatewayProtocolVersion);
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
Assert.Contains("unary-invoke", reply.Capabilities);
Assert.Equal("Operator Key", sessionManager.LastClientIdentity);
Assert.Equal("operator01", sessionManager.LastOwnerKeyId);
Assert.Equal("operator-session", sessionManager.LastOpenRequest?.ClientSessionName);
}
///
/// Verifies that Invoke maps a genuinely missing session to NotFound via the
/// service's own ResolveSession lookup. No InvokeException is
/// injected — makes
/// TryGetSession return false, so this test fails if the service drops
/// its missing-session check rather than passing for the wrong reason.
///
[Fact]
public async Task Invoke_WhenSessionMissing_ThrowsNotFound()
{
FakeSessionManager sessionManager = new() { ResolveOnlySeededSessions = true };
MxAccessGatewayService service = CreateService(sessionManager);
RpcException exception = await Assert.ThrowsAsync(
async () => await service.Invoke(
CreatePingRequest("session-missing"),
new TestServerCallContext()));
Assert.Equal(StatusCode.NotFound, exception.StatusCode);
Assert.Contains("session-missing", exception.Status.Detail, StringComparison.Ordinal);
// The service must reject before delegating to the session manager.
Assert.Equal(0, sessionManager.InvokeCount);
}
///
/// Verifies that Invoke resolves a session that was seeded into the session
/// manager when is on,
/// confirming the missing-session test above is gated on a real lookup.
///
[Fact]
public async Task Invoke_WhenSessionSeeded_ResolvesAndInvokes()
{
FakeSessionManager sessionManager = new() { ResolveOnlySeededSessions = true };
sessionManager.SeedSession(CreateSession("session-1", processId: 1234));
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandReply reply = await service.Invoke(
CreatePingRequest("session-1"),
new TestServerCallContext());
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
Assert.Equal(1, sessionManager.InvokeCount);
}
/// Verifies that Invoke throws InvalidArgument and does not invoke the session manager when payload is mismatched.
[Fact]
public async Task Invoke_WithMismatchedPayload_ThrowsInvalidArgumentAndDoesNotCallSessionManager()
{
FakeSessionManager sessionManager = new();
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandRequest request = new()
{
SessionId = "session-1",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
Ping = new PingCommand { Message = "wrong-payload" },
},
};
RpcException exception = await Assert.ThrowsAsync(
async () => await service.Invoke(request, new TestServerCallContext()));
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
Assert.Equal(0, sessionManager.InvokeCount);
}
/// Verifies that Invoke returns HResult status and method payload from worker reply.
[Fact]
public async Task Invoke_WithWorkerReply_ReturnsHresultStatusAndMethodPayload()
{
const int hresult = unchecked((int)0x80004005);
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "worker-correlation",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
Hresult = hresult,
AddItem = new AddItemReply { ItemHandle = 42 },
DiagnosticMessage = "mxaccess diagnostic",
},
},
};
sessionManager.InvokeReply.Reply.Statuses.Add(new MxStatusProxy
{
Success = 0,
Category = MxStatusCategory.SoftwareError,
Detail = 1001,
DiagnosticText = "status detail",
});
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandRequest request = new()
{
SessionId = "session-1",
ClientCorrelationId = "client-correlation",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand
{
ServerHandle = 12,
ItemDefinition = "Galaxy.Tag.Value",
},
},
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(MxCommandKind.AddItem, sessionManager.LastWorkerCommand?.Command.Kind);
Assert.Equal("Galaxy.Tag.Value", sessionManager.LastWorkerCommand?.Command.AddItem.ItemDefinition);
Assert.NotNull(sessionManager.LastWorkerCommand?.EnqueueTimestamp);
Assert.Equal(hresult, reply.Hresult);
Assert.Equal(42, reply.AddItem.ItemHandle);
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
}
/// Verifies that StreamEvents writes only events after the specified worker sequence.
[Fact]
public async Task StreamEvents_WithAfterSequence_WritesOnlyLaterEvents()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 2));
MxAccessGatewayService service = CreateService(sessionManager);
RecordingServerStreamWriter writer = new();
await service.StreamEvents(
new StreamEventsRequest
{
SessionId = "session-1",
AfterWorkerSequence = 1,
},
writer,
new TestServerCallContext());
MxEvent writtenEvent = Assert.Single(writer.Messages);
Assert.Equal((ulong)2, writtenEvent.WorkerSequence);
Assert.Equal("session-1", sessionManager.LastReadEventsSessionId);
}
///
/// Verifies that StreamEvents records the send-duration histogram per event.
///
/// Tests-027 (concurrency flake): the listener must filter by the specific
/// instance owned by this test, not by the process-shared
/// . Otherwise a parallel test that constructs its own
/// and records mxgateway.events.stream_send.duration would
/// cross-contaminate families and break the equality assertion below. See the companion
///
/// regression for the cross-talk reproduction.
///
[Fact]
public async Task StreamEvents_WhenEventIsWritten_RecordsSendDuration()
{
using GatewayMetrics metrics = new();
using MeterListener listener = new();
List families = [];
listener.InstrumentPublished = (instrument, meterListener) =>
{
if (ReferenceEquals(instrument.Meter, metrics.Meter)
&& instrument.Name == "mxgateway.events.stream_send.duration")
{
meterListener.EnableMeasurementEvents(instrument);
}
};
listener.SetMeasurementEventCallback(
(instrument, measurement, tags, _) =>
{
if (!ReferenceEquals(instrument.Meter, metrics.Meter)
|| instrument.Name != "mxgateway.events.stream_send.duration")
{
return;
}
foreach (KeyValuePair tag in tags)
{
if (tag.Key == "family" && tag.Value is string family)
{
families.Add(family);
}
}
});
listener.Start();
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 2));
MxAccessGatewayService service = CreateService(sessionManager, metrics: metrics);
RecordingServerStreamWriter writer = new();
await service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext());
Assert.Equal([MxEventFamily.OnDataChange.ToString()], families);
}
///
/// Tests-027 regression: a that filters by the specific
/// instance (via )
/// must NOT observe measurements recorded on a different that shares
/// the same . This is the cross-talk vector that previously
/// caused StreamEvents_WhenEventIsWritten_RecordsSendDuration to fail intermittently when
/// run in parallel with another test recording the same histogram.
///
[Fact]
public async Task StreamEvents_RecordSendDurationListener_IgnoresMeasurementsFromOtherMetersWithSameName()
{
using GatewayMetrics metricsUnderTest = new();
using GatewayMetrics otherMetrics = new();
using MeterListener listener = new();
List families = [];
listener.InstrumentPublished = (instrument, meterListener) =>
{
// Subscribe to the stream_send histogram on BOTH meters so the listener
// would observe a cross-talk measurement if the callback did not filter.
if (instrument.Name == "mxgateway.events.stream_send.duration")
{
meterListener.EnableMeasurementEvents(instrument);
}
};
listener.SetMeasurementEventCallback(
(instrument, measurement, tags, _) =>
{
if (!ReferenceEquals(instrument.Meter, metricsUnderTest.Meter)
|| instrument.Name != "mxgateway.events.stream_send.duration")
{
return;
}
foreach (KeyValuePair tag in tags)
{
if (tag.Key == "family" && tag.Value is string family)
{
families.Add(family);
}
}
});
listener.Start();
// Simulate the cross-talk: another test's GatewayMetrics records a value
// before the test-under-test does its single event publish. The listener
// must filter this out by Meter reference.
otherMetrics.RecordEventStreamSend(MxEventFamily.OnWriteComplete.ToString(), TimeSpan.FromMilliseconds(123));
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 2));
MxAccessGatewayService service = CreateService(sessionManager, metrics: metricsUnderTest);
RecordingServerStreamWriter writer = new();
await service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext());
// Only the test-under-test's OnDataChange recording should be observed —
// the OnAlarm recording on the sibling meter must NOT leak through.
Assert.Equal([MxEventFamily.OnDataChange.ToString()], families);
}
/// Verifies that CloseSession throws InvalidArgument when session ID is blank.
[Fact]
public async Task CloseSession_WithBlankSessionId_ThrowsInvalidArgument()
{
MxAccessGatewayService service = CreateService(new FakeSessionManager());
RpcException exception = await Assert.ThrowsAsync(
async () => await service.CloseSession(
new CloseSessionRequest(),
new TestServerCallContext()));
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
}
// ===== AcknowledgeAlarm + StreamAlarms handler contract =====
//
// AcknowledgeAlarm validates alarm_full_reference then delegates to the
// session-less IGatewayAlarmService; StreamAlarms forwards the central
// alarm feed. CreateService injects FakeGatewayAlarmService.
/// Verifies AcknowledgeAlarm rejects an empty alarm_full_reference.
[Fact]
public async Task AcknowledgeAlarm_WithMissingAlarmReference_ThrowsInvalidArgument()
{
MxAccessGatewayService service = CreateService(new FakeSessionManager());
RpcException exception = await Assert.ThrowsAsync(
async () => await service.AcknowledgeAlarm(
new AcknowledgeAlarmRequest { OperatorUser = "alice" },
new TestServerCallContext()));
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
}
/// Verifies AcknowledgeAlarm delegates a valid request to the alarm service.
[Fact]
public async Task AcknowledgeAlarm_WithValidRequest_DelegatesToAlarmService()
{
MxAccessGatewayService service = CreateService(new FakeSessionManager());
AcknowledgeAlarmReply reply = await service.AcknowledgeAlarm(
new AcknowledgeAlarmRequest
{
ClientCorrelationId = "corr-1",
AlarmFullReference = "Galaxy!Area.Tank01.Level.HiHi",
Comment = "investigating",
OperatorUser = "alice",
},
new TestServerCallContext());
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
Assert.Equal("corr-1", reply.CorrelationId);
}
/// Verifies StreamAlarms forwards the central alarm feed, ending with snapshot_complete.
[Fact]
public async Task StreamAlarms_ForwardsTheCentralAlarmFeed()
{
MxAccessGatewayService service = CreateService(new FakeSessionManager());
RecordingServerStreamWriter sink = new();
await service.StreamAlarms(
new StreamAlarmsRequest(),
sink,
new TestServerCallContext());
Assert.Contains(
sink.Messages,
message => message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete);
}
/// Verifies OpenSession advertises the alarm RPC capability strings.
[Fact]
public async Task OpenSession_AdvertisesAlarmRpcCapabilities()
{
FakeSessionManager sessionManager = new();
GatewayRequestIdentityAccessor identityAccessor = new();
MxAccessGatewayService service = CreateService(sessionManager, identityAccessor);
using IDisposable identityScope = identityAccessor.Push(CreateIdentity());
OpenSessionReply reply = await service.OpenSession(
new OpenSessionRequest(),
new TestServerCallContext());
Assert.Contains("unary-acknowledge-alarm", reply.Capabilities);
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
}
private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null)
{
return new MxAccessGatewayService(
sessionManager,
identityAccessor ?? new GatewayRequestIdentityAccessor(),
constraintEnforcer ?? new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(),
new MxAccessGrpcMapper(),
new FakeEventStreamService(sessionManager),
metrics ?? new GatewayMetrics(),
NullLogger.Instance,
new FakeGatewayAlarmService());
}
private static ApiKeyIdentity CreateIdentity()
{
return new ApiKeyIdentity(
KeyId: "operator01",
KeyPrefix: "mxgw_operator01",
DisplayName: "Operator Key",
Scopes: new HashSet(StringComparer.Ordinal));
}
private static GatewaySession CreateSession(
string sessionId,
int processId)
{
GatewaySession session = new(
sessionId,
GatewayContractInfo.DefaultBackendName,
"pipe",
"nonce",
"Operator Key",
"operator-session",
"client-correlation",
TimeSpan.FromSeconds(7),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(10),
DateTimeOffset.UtcNow);
session.AttachWorkerClient(new FakeWorkerClient(processId));
session.MarkReady();
return session;
}
private static MxCommandRequest CreatePingRequest(string sessionId)
{
return new MxCommandRequest
{
SessionId = sessionId,
Command = new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand { Message = "ping" },
},
};
}
private static WorkerEvent CreateWorkerEvent(
string sessionId,
ulong workerSequence)
{
return new WorkerEvent
{
Event = new MxEvent
{
Family = MxEventFamily.OnDataChange,
SessionId = sessionId,
WorkerSequence = workerSequence,
OnDataChange = new OnDataChangeEvent(),
},
};
}
private sealed class FakeSessionManager : ISessionManager
{
private readonly Dictionary seededSessions = new(StringComparer.Ordinal);
/// The session to return from OpenSessionAsync.
public GatewaySession? OpenSessionResult { get; init; }
///
/// When true, only resolves sessions that have been
/// explicitly seeded via (or ),
/// and returns false for any other id. This exercises the gateway service's own
/// missing-session handling instead of masking it with a synthesized session.
///
public bool ResolveOnlySeededSessions { get; init; }
/// Registers a session so resolves its id.
/// Session to register by its .
public void SeedSession(GatewaySession session)
{
seededSessions[session.SessionId] = session;
}
/// The last OpenSessionAsync request captured.
public SessionOpenRequest? LastOpenRequest { get; private set; }
/// The last client identity passed to OpenSessionAsync.
public string? LastClientIdentity { get; private set; }
/// The last owner key id passed to OpenSessionAsync.
public string? LastOwnerKeyId { get; private set; }
/// The last session ID passed to ReadEventsAsync.
public string? LastReadEventsSessionId { get; private set; }
/// The last worker command passed to InvokeAsync.
public WorkerCommand? LastWorkerCommand { get; private set; }
/// The reply to return from InvokeAsync.
public WorkerCommandReply InvokeReply { get; init; } = new()
{
Reply = new MxCommandReply
{
SessionId = "session-1",
Kind = MxCommandKind.Ping,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
},
};
/// The exception to throw from InvokeAsync.
public Exception? InvokeException { get; init; }
/// The number of times InvokeAsync was called.
public int InvokeCount { get; private set; }
/// The events to return from ReadEventsAsync.
public List Events { get; } = [];
/// Records the session ID passed to ReadEventsAsync.
/// Identifier of the session.
public void RecordReadEventsSessionId(string sessionId)
{
LastReadEventsSessionId = sessionId;
}
///
public Task OpenSessionAsync(
SessionOpenRequest request,
string? clientIdentity,
string? ownerKeyId,
CancellationToken cancellationToken)
{
LastOpenRequest = request;
LastClientIdentity = clientIdentity;
LastOwnerKeyId = ownerKeyId;
return Task.FromResult(OpenSessionResult ?? CreateSession("session-1", processId: 1234));
}
///
public bool TryGetSession(
string sessionId,
out GatewaySession session)
{
if (seededSessions.TryGetValue(sessionId, out GatewaySession? seeded))
{
session = seeded;
return true;
}
if (ResolveOnlySeededSessions)
{
session = null!;
return false;
}
session = OpenSessionResult ?? CreateSession(sessionId, processId: 1234);
return true;
}
///
public Task InvokeAsync(
string sessionId,
WorkerCommand command,
CancellationToken cancellationToken)
{
InvokeCount++;
LastWorkerCommand = command;
if (InvokeException is not null)
{
throw InvokeException;
}
return Task.FromResult(InvokeReply);
}
///
public async IAsyncEnumerable ReadEventsAsync(
string sessionId,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
LastReadEventsSessionId = sessionId;
foreach (WorkerEvent workerEvent in Events)
{
cancellationToken.ThrowIfCancellationRequested();
await Task.Yield();
yield return workerEvent;
}
}
///
public Task CloseSessionAsync(
string sessionId,
CancellationToken cancellationToken)
{
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
}
///
public Task KillWorkerAsync(
string sessionId,
string reason,
CancellationToken cancellationToken)
{
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
}
///
public Task CloseExpiredLeasesAsync(
DateTimeOffset now,
CancellationToken cancellationToken)
{
return Task.FromResult(0);
}
///
public Task ShutdownAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
{
///
public async IAsyncEnumerable StreamEventsAsync(
StreamEventsRequest request,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
sessionManager.RecordReadEventsSessionId(request.SessionId);
foreach (WorkerEvent workerEvent in sessionManager.Events)
{
cancellationToken.ThrowIfCancellationRequested();
await Task.Yield();
if (workerEvent.Event.WorkerSequence <= request.AfterWorkerSequence)
{
continue;
}
yield return workerEvent.Event;
}
}
}
private sealed class FakeWorkerClient(int processId) : IWorkerClient
{
///
public string SessionId { get; } = "session-1";
///
public int? ProcessId { get; } = processId;
///
public WorkerClientState State { get; } = WorkerClientState.Ready;
///
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
///
public Task StartAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
///
public Task InvokeAsync(
WorkerCommand command,
TimeSpan timeout,
CancellationToken cancellationToken)
{
return Task.FromResult(new WorkerCommandReply());
}
///
public async IAsyncEnumerable ReadEventsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.CompletedTask;
yield break;
}
///
public Task ShutdownAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
///
public void Kill(string reason)
{
}
///
public ValueTask DisposeAsync()
{
return ValueTask.CompletedTask;
}
}
}