fix(grpc): typed alarm-feed overflow, memoized alarm snapshot, cancellable stream writes, uniform reply correlation id
- StreamAlarms: slow subscribers are completed with AlarmFeedSubscriberOverflowException and surface as ResourceExhausted (like a StreamEvents overflow) instead of warning-with-stack + Unavailable. - GatewayAlarmMonitor.StreamAsync reuses the memoized alarm-set projection instead of cloning the whole set under the broadcast lock per attach; the prefix filter runs outside the lock. - StreamEvents/StreamAlarms pass the call token to response writes; a write failure after client cancel maps to Cancelled without a warning. - Every MxCommandReply.correlation_id now echoes the request's client_correlation_id (previously only constraint-denied replies did; normal replies leaked the internal envelope id).
This commit is contained in:
Joseph Doherty
@@ -2,9 +2,11 @@ using System.Diagnostics.Metrics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
@@ -213,6 +215,8 @@ public sealed class MxAccessGatewayServiceTests
|
||||
Assert.Equal(42, reply.AddItem.ItemHandle);
|
||||
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
|
||||
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
|
||||
// The reply echoes the caller's client_correlation_id, not the internal worker envelope id.
|
||||
Assert.Equal("client-correlation", reply.CorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that StreamEvents writes only events after the specified worker sequence.</summary>
|
||||
@@ -463,11 +467,151 @@ public sealed class MxAccessGatewayServiceTests
|
||||
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every reply echoes the request's <c>client_correlation_id</c>, including when the caller
|
||||
/// sent none: the internal worker envelope id never reaches the client.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Invoke_WithoutClientCorrelationId_DoesNotLeakWorkerEnvelopeId()
|
||||
{
|
||||
FakeSessionManager sessionManager = new()
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "4f1d1b0e-internal-envelope-guid",
|
||||
Kind = MxCommandKind.AddItem,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
AddItem = new AddItemReply { ItemHandle = 42 },
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
MxCommandRequest request = new()
|
||||
{
|
||||
SessionId = "session-1",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
|
||||
},
|
||||
};
|
||||
|
||||
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
|
||||
|
||||
Assert.Equal(string.Empty, reply.CorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// StreamEvents hands the call's cancellation token to every response write, so a write
|
||||
/// blocked on a slow or vanished client is released when the call is cancelled.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEvents_PassesCallCancellationTokenToResponseWrites()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
RecordingServerStreamWriter<MxEvent> writer = new();
|
||||
using CancellationTokenSource callCts = new();
|
||||
|
||||
await service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = "session-1" },
|
||||
writer,
|
||||
new TestServerCallContext(cancellationToken: callCts.Token));
|
||||
|
||||
Assert.Single(writer.Messages);
|
||||
Assert.Equal(callCts.Token, writer.LastWriteCancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A response write that fails with an <see cref="IOException"/> because the client went
|
||||
/// away (the call token is cancelled) maps to Cancelled and logs no warning: a client
|
||||
/// disconnect is routine, not a gateway failure.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEvents_WhenWriteFailsAfterClientCancel_ReturnsCancelledWithoutWarning()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
|
||||
RecordingLogger<MxAccessGatewayService> logger = new();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
|
||||
using CancellationTokenSource callCts = new();
|
||||
ThrowingStreamWriter<MxEvent> writer = new(() =>
|
||||
{
|
||||
callCts.Cancel();
|
||||
return new IOException("The client reset the request stream.");
|
||||
});
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
() => service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = "session-1" },
|
||||
writer,
|
||||
new TestServerCallContext(cancellationToken: callCts.Token)));
|
||||
|
||||
Assert.Equal(StatusCode.Cancelled, exception.StatusCode);
|
||||
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The same IOException with a live call token is a genuine transport failure and keeps the
|
||||
/// existing Unavailable + warning mapping.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEvents_WhenWriteFailsWithoutCancel_KeepsUnavailableMapping()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
|
||||
RecordingLogger<MxAccessGatewayService> logger = new();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
|
||||
ThrowingStreamWriter<MxEvent> writer = new(() => new IOException("Transport failure."));
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
() => service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = "session-1" },
|
||||
writer,
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.Unavailable, exception.StatusCode);
|
||||
Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A StreamAlarms subscriber the monitor drops for falling behind receives ResourceExhausted
|
||||
/// (the same status a StreamEvents overflow carries), and the gateway logs no
|
||||
/// warning-with-stack for what is a client-side slowness condition.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamAlarms_WhenSubscriberOverflows_ReturnsResourceExhaustedWithoutWarning()
|
||||
{
|
||||
RecordingLogger<MxAccessGatewayService> logger = new();
|
||||
FakeGatewayAlarmService alarms = new() { StreamFault = new AlarmFeedSubscriberOverflowException() };
|
||||
MxAccessGatewayService service = CreateService(new FakeSessionManager(), alarmService: alarms, logger: logger);
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
() => service.StreamAlarms(
|
||||
new StreamAlarmsRequest(),
|
||||
new RecordingServerStreamWriter<AlarmFeedMessage>(),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
|
||||
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
|
||||
}
|
||||
|
||||
private static MxAccessGatewayService CreateService(
|
||||
FakeSessionManager sessionManager,
|
||||
IGatewayRequestIdentityAccessor? identityAccessor = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
IConstraintEnforcer? constraintEnforcer = null)
|
||||
IConstraintEnforcer? constraintEnforcer = null,
|
||||
IGatewayAlarmService? alarmService = null,
|
||||
ILogger<MxAccessGatewayService>? logger = null)
|
||||
{
|
||||
return new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
@@ -477,8 +621,8 @@ public sealed class MxAccessGatewayServiceTests
|
||||
new MxAccessGrpcMapper(),
|
||||
new FakeEventStreamService(sessionManager),
|
||||
metrics ?? new GatewayMetrics(),
|
||||
NullLogger<MxAccessGatewayService>.Instance,
|
||||
new FakeGatewayAlarmService());
|
||||
logger ?? NullLogger<MxAccessGatewayService>.Instance,
|
||||
alarmService ?? new FakeGatewayAlarmService());
|
||||
}
|
||||
|
||||
private static ApiKeyIdentity CreateIdentity()
|
||||
@@ -786,4 +930,18 @@ public sealed class MxAccessGatewayServiceTests
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>A stream writer whose token-bearing write throws the supplied exception.</summary>
|
||||
/// <typeparam name="T">The streamed message type.</typeparam>
|
||||
private sealed class ThrowingStreamWriter<T>(Func<Exception> createException) : IServerStreamWriter<T>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public WriteOptions? WriteOptions { get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task WriteAsync(T message) => throw createException();
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task WriteAsync(T message, CancellationToken cancellationToken) => throw createException();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user