fix(grpc): typed alarm-feed overflow, memoized alarm snapshot, cancellable stream writes, uniform reply correlation id

- StreamAlarms: slow subscribers are completed with
  AlarmFeedSubscriberOverflowException and surface as ResourceExhausted
  (like a StreamEvents overflow) instead of warning-with-stack + Unavailable.
- GatewayAlarmMonitor.StreamAsync reuses the memoized alarm-set projection
  instead of cloning the whole set under the broadcast lock per attach; the
  prefix filter runs outside the lock.
- StreamEvents/StreamAlarms pass the call token to response writes; a write
  failure after client cancel maps to Cancelled without a warning.
- Every MxCommandReply.correlation_id now echoes the request's
  client_correlation_id (previously only constraint-denied replies did;
  normal replies leaked the internal envelope id).
This commit is contained in:
Joseph Doherty
2026-10-22 06:27:36 -04:00
parent 19cc74f21a
commit 0b4a1682c8
11 changed files with 478 additions and 14 deletions
@@ -372,6 +372,105 @@ public sealed class AlarmTruncationSignalTests
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A subscriber that stops reading is dropped once its bounded queue fills, and the fault it
/// observes is the typed <see cref="AlarmFeedSubscriberOverflowException"/> — the signal the
/// gRPC layer maps to ResourceExhausted — rather than a bare InvalidOperationException.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberThatStopsReading_IsDroppedWithTypedOverflowFault()
{
// Comfortably past the monitor's per-subscriber queue capacity (2048).
const int ProbeCount = 2300;
using GatewayMetrics metrics = new();
StubSessionManager sessions = new();
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
// The slow subscriber: registered (first frame read) and then never read again.
using CancellationTokenSource slowCts = new(WaitTimeout);
await using IAsyncEnumerator<AlarmFeedMessage> slow =
monitor.StreamAsync(null, slowCts.Token).GetAsyncEnumerator(slowCts.Token);
Assert.True(await slow.MoveNextAsync());
// Raises on distinct references: each one is broadcast (no dedup) without forcing a
// reconcile, which keeps the burst cheap under a loaded parallel test run.
for (ulong sequence = 1; sequence <= ProbeCount; sequence++)
{
sessions.EmitEvent(UniqueRaise(sequence));
}
// The monitor caches each raise and broadcasts it under the same lock, so once the cache
// holds every raise, every broadcast has been offered to the slow subscriber. (A draining
// reader is no barrier here: under a loaded test run it can fall 2048 frames behind the
// burst and be dropped itself.)
await WaitUntilAsync(() => monitor.CurrentAlarms.Count >= ProbeCount, WaitTimeout);
// Drain what the slow subscriber had buffered; the terminal fault is the typed overflow.
await Assert.ThrowsAsync<AlarmFeedSubscriberOverflowException>(async () =>
{
while (await slow.MoveNextAsync())
{
}
});
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A StreamAsync attach reuses the memoized alarm-set projection that CurrentAlarms serves
/// (no per-attach clone of the whole set under the broadcast lock) and still applies the
/// subscriber's prefix filter to the snapshot it emits.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberAttach_ReusesMemoizedProjectionAndAppliesPrefixFilter()
{
using GatewayMetrics metrics = new();
StubSessionManager sessions = new()
{
Snapshots =
[
NewSnapshot("Galaxy!Area.Tank01.Level.HiHi", fromTruncatedSnapshot: false),
NewSnapshot("Galaxy!Area.Pump02.Flow.Lo", fromTruncatedSnapshot: false),
],
};
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
await WaitUntilAsync(() => monitor.CurrentAlarms.Count == 2, WaitTimeout);
IReadOnlyList<ActiveAlarmSnapshot> projection = monitor.CurrentAlarms;
List<ActiveAlarmSnapshot> streamed = [];
using CancellationTokenSource streamCts = new(WaitTimeout);
await foreach (AlarmFeedMessage message in monitor.StreamAsync("Galaxy!Area.Tank01", streamCts.Token))
{
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ActiveAlarm)
{
streamed.Add(message.ActiveAlarm);
}
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete)
{
break;
}
}
ActiveAlarmSnapshot only = Assert.Single(streamed);
Assert.Equal("Galaxy!Area.Tank01.Level.HiHi", only.AlarmFullReference);
Assert.Contains(projection, alarm => ReferenceEquals(alarm, only));
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
private static int CountOf(List<AlarmFeedMessage> received, AlarmFeedMessage.PayloadOneofCase payloadCase)
{
lock (received)
@@ -442,6 +541,23 @@ public sealed class AlarmTruncationSignalTests
},
};
// A Raise on a reference unique to the sequence, so the monitor never dedups it.
private static MxEvent UniqueRaise(ulong sequence) => new()
{
Family = MxEventFamily.OnAlarmTransition,
WorkerSequence = sequence,
OnAlarmTransition = new OnAlarmTransitionEvent
{
AlarmFullReference = $"Galaxy!Area.Tank{sequence}.Level.Hi",
SourceObjectReference = $"Tank{sequence}.Level",
AlarmTypeName = "AnalogLimitAlarm.Hi",
TransitionKind = AlarmTransitionKind.Raise,
Severity = 500,
SourceProvider = AlarmProviderMode.Alarmmgr,
TransitionTimestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
},
};
private static ActiveAlarmSnapshot NewSnapshot(string reference, bool fromTruncatedSnapshot)
{
return new ActiveAlarmSnapshot
@@ -2,9 +2,11 @@ using System.Diagnostics.Metrics;
using System.Runtime.CompilerServices;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
@@ -213,6 +215,8 @@ public sealed class MxAccessGatewayServiceTests
Assert.Equal(42, reply.AddItem.ItemHandle);
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
// The reply echoes the caller's client_correlation_id, not the internal worker envelope id.
Assert.Equal("client-correlation", reply.CorrelationId);
}
/// <summary>Verifies that StreamEvents writes only events after the specified worker sequence.</summary>
@@ -463,11 +467,151 @@ public sealed class MxAccessGatewayServiceTests
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
}
/// <summary>
/// Every reply echoes the request's <c>client_correlation_id</c>, including when the caller
/// sent none: the internal worker envelope id never reaches the client.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_WithoutClientCorrelationId_DoesNotLeakWorkerEnvelopeId()
{
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "4f1d1b0e-internal-envelope-guid",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
AddItem = new AddItemReply { ItemHandle = 42 },
},
},
};
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandRequest request = new()
{
SessionId = "session-1",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
},
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(string.Empty, reply.CorrelationId);
}
/// <summary>
/// StreamEvents hands the call's cancellation token to every response write, so a write
/// blocked on a slow or vanished client is released when the call is cancelled.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_PassesCallCancellationTokenToResponseWrites()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
MxAccessGatewayService service = CreateService(sessionManager);
RecordingServerStreamWriter<MxEvent> writer = new();
using CancellationTokenSource callCts = new();
await service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token));
Assert.Single(writer.Messages);
Assert.Equal(callCts.Token, writer.LastWriteCancellationToken);
}
/// <summary>
/// A response write that fails with an <see cref="IOException"/> because the client went
/// away (the call token is cancelled) maps to Cancelled and logs no warning: a client
/// disconnect is routine, not a gateway failure.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsAfterClientCancel_ReturnsCancelledWithoutWarning()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
using CancellationTokenSource callCts = new();
ThrowingStreamWriter<MxEvent> writer = new(() =>
{
callCts.Cancel();
return new IOException("The client reset the request stream.");
});
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token)));
Assert.Equal(StatusCode.Cancelled, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
/// <summary>
/// The same IOException with a live call token is a genuine transport failure and keeps the
/// existing Unavailable + warning mapping.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsWithoutCancel_KeepsUnavailableMapping()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
ThrowingStreamWriter<MxEvent> writer = new(() => new IOException("Transport failure."));
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext()));
Assert.Equal(StatusCode.Unavailable, exception.StatusCode);
Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning);
}
/// <summary>
/// A StreamAlarms subscriber the monitor drops for falling behind receives ResourceExhausted
/// (the same status a StreamEvents overflow carries), and the gateway logs no
/// warning-with-stack for what is a client-side slowness condition.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamAlarms_WhenSubscriberOverflows_ReturnsResourceExhaustedWithoutWarning()
{
RecordingLogger<MxAccessGatewayService> logger = new();
FakeGatewayAlarmService alarms = new() { StreamFault = new AlarmFeedSubscriberOverflowException() };
MxAccessGatewayService service = CreateService(new FakeSessionManager(), alarmService: alarms, logger: logger);
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamAlarms(
new StreamAlarmsRequest(),
new RecordingServerStreamWriter<AlarmFeedMessage>(),
new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null)
IConstraintEnforcer? constraintEnforcer = null,
IGatewayAlarmService? alarmService = null,
ILogger<MxAccessGatewayService>? logger = null)
{
return new MxAccessGatewayService(
sessionManager,
@@ -477,8 +621,8 @@ public sealed class MxAccessGatewayServiceTests
new MxAccessGrpcMapper(),
new FakeEventStreamService(sessionManager),
metrics ?? new GatewayMetrics(),
NullLogger<MxAccessGatewayService>.Instance,
new FakeGatewayAlarmService());
logger ?? NullLogger<MxAccessGatewayService>.Instance,
alarmService ?? new FakeGatewayAlarmService());
}
private static ApiKeyIdentity CreateIdentity()
@@ -786,4 +930,18 @@ public sealed class MxAccessGatewayServiceTests
}
}
/// <summary>A stream writer whose token-bearing write throws the supplied exception.</summary>
/// <typeparam name="T">The streamed message type.</typeparam>
private sealed class ThrowingStreamWriter<T>(Func<Exception> createException) : IServerStreamWriter<T>
{
/// <inheritdoc />
public WriteOptions? WriteOptions { get; set; }
/// <inheritdoc />
public Task WriteAsync(T message) => throw createException();
/// <inheritdoc />
public Task WriteAsync(T message, CancellationToken cancellationToken) => throw createException();
}
}
@@ -40,8 +40,18 @@ public sealed class FakeGatewayAlarmService : IGatewayAlarmService
yield return new AlarmFeedMessage { SnapshotComplete = true };
await Task.CompletedTask.ConfigureAwait(false);
if (StreamFault is not null)
{
throw StreamFault;
}
}
/// <summary>
/// Gets or sets an exception the feed throws after its snapshot, standing in for the monitor
/// completing a subscriber with a fault (for example a slow-subscriber overflow).
/// </summary>
public Exception? StreamFault { get; set; }
/// <inheritdoc />
public Task<AcknowledgeAlarmReply> AcknowledgeAsync(
AcknowledgeAlarmRequest request,
@@ -0,0 +1,48 @@
using Microsoft.Extensions.Logging;
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// <see cref="ILogger{TCategoryName}"/> that records every entry's level and formatted message,
/// so a test can assert that a path logs (or does not log) at a given level.
/// </summary>
/// <typeparam name="T">The logger category type.</typeparam>
public sealed class RecordingLogger<T> : ILogger<T>
{
private readonly object _syncRoot = new();
private readonly List<(LogLevel Level, string Message, Exception? Exception)> _entries = [];
/// <summary>Gets a snapshot of the recorded entries, in order.</summary>
public IReadOnlyList<(LogLevel Level, string Message, Exception? Exception)> Entries
{
get
{
lock (_syncRoot)
{
return _entries.ToArray();
}
}
}
/// <inheritdoc />
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull => null;
/// <inheritdoc />
public bool IsEnabled(LogLevel logLevel) => true;
/// <inheritdoc />
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
ArgumentNullException.ThrowIfNull(formatter);
lock (_syncRoot)
{
_entries.Add((logLevel, formatter(state, exception), exception));
}
}
}
@@ -29,6 +29,25 @@ public sealed class RecordingServerStreamWriter<T> : IServerStreamWriter<T>
/// <summary>Gets or sets options for writing messages to the stream.</summary>
public WriteOptions? WriteOptions { get; set; }
/// <summary>Gets the cancellation token passed to the most recent token-bearing write.</summary>
public CancellationToken LastWriteCancellationToken { get; private set; }
/// <summary>Records the supplied message unless the token is already cancelled.</summary>
/// <param name="message">The message to record.</param>
/// <param name="cancellationToken">Token that cancels the write.</param>
/// <returns>A completed task.</returns>
/// <remarks>
/// The interface default throws <see cref="NotSupportedException"/> for a cancelable token,
/// and the gateway passes the call's token on every streamed write, so a test writer must
/// override this overload the way the ASP.NET Core writer does.
/// </remarks>
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
LastWriteCancellationToken = cancellationToken;
return WriteAsync(message);
}
/// <summary>Records the supplied message.</summary>
/// <param name="message">The message to record.</param>
/// <returns>A completed task.</returns>