fix(grpc): record the real StreamEvents disconnect reason
StreamDisconnected always recorded "Detached". The reason tag is now a bounded EventStreamDisconnectReasons constant for how the stream ended: SessionClosed, Overflow, WorkerFault, ClientCancelled, Error, or Detached.
This commit is contained in:
Joseph Doherty
+1
-1
@@ -375,7 +375,7 @@ Under `FailFast` the session is faulted so subsequent commands return `FailedPre
|
||||
|
||||
### Cancellation and cleanup
|
||||
|
||||
The handler creates a linked cancellation token (`streamCts`) so that completing the consumer (client disconnect, error, or graceful end-of-stream) also cancels the producer. The `finally` block cancels the source, disposes the subscriber slot, awaits the producer (swallowing the expected cancellation), and emits `StreamDisconnected("Detached")` so dashboards see the disconnection regardless of cause.
|
||||
The handler creates a linked cancellation token (`streamCts`) so that completing the consumer (client disconnect, error, or graceful end-of-stream) also cancels the producer. The `finally` block cancels the source, disposes the subscriber slot, awaits the producer (swallowing the expected cancellation), and emits `StreamDisconnected(reason)` so dashboards see every disconnection. The reason is the bounded constant (`EventStreamDisconnectReasons`) for how the stream ended — `SessionClosed`, `Overflow`, `WorkerFault`, `ClientCancelled`, `Error`, or `Detached` — rather than one constant for every cause, so an operator can tell slow consumers from worker faults from routine client disconnects.
|
||||
|
||||
`WorkerClientException` thrown by the producer marks the session as faulted before completing the channel — the worker is presumed gone, and any subsequent command on that session must observe the fault rather than silently retry.
|
||||
|
||||
|
||||
+2
-2
@@ -45,7 +45,7 @@ All counters are `Counter<long>`. Tag values come from the call sites listed und
|
||||
| `mxgateway.workers.killed` | `reason` | Forced terminations of worker processes. |
|
||||
| `mxgateway.workers.exited` | `reason` | Clean or fault-driven worker exits. |
|
||||
| `mxgateway.heartbeats.failed` | *(none)* | Aggregate worker heartbeat misses. The exported counter carries **no** `session_id` tag — per-session attribution is unbounded cardinality (every session mints a new time series), so it is kept out of the exporter; use the dashboard snapshot or the log scope for per-session detail. |
|
||||
| `mxgateway.grpc.streams.disconnected` | `reason` | Detachments of the dashboard or client gRPC event stream. |
|
||||
| `mxgateway.grpc.streams.disconnected` | `reason` | Ends of a client `StreamEvents` stream. `reason` is one of a fixed set (`EventStreamDisconnectReasons`): `SessionClosed` (the session event stream completed), `Overflow` (subscriber fell behind its bounded queue), `WorkerFault` (the worker event stream faulted), `ClientCancelled` (the client cancelled or disconnected), `Error` (unclassified), `Detached` (the consumer stopped reading without cancelling). |
|
||||
| `mxgateway.retries.attempted` | `area` | Resilience retries executed by gateway components. |
|
||||
|
||||
### Histograms
|
||||
@@ -181,7 +181,7 @@ IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
|
||||
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
|
||||
...
|
||||
backlogRegistration.Dispose();
|
||||
metrics.StreamDisconnected("Detached");
|
||||
metrics.StreamDisconnected(disconnectReason); // an EventStreamDisconnectReasons constant
|
||||
...
|
||||
metrics.QueueOverflow("grpc-event-stream");
|
||||
metrics.Fault(SessionManagerErrorCode.EventQueueOverflow.ToString());
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// The bounded set of <c>reason</c> tag values recorded on
|
||||
/// <c>mxgateway.grpc.streams.disconnected</c> when a <c>StreamEvents</c> stream ends.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Constants rather than exception messages or free text so the metric's tag cardinality stays
|
||||
/// fixed: an exporter mints one time series per distinct tag value.
|
||||
/// </remarks>
|
||||
public static class EventStreamDisconnectReasons
|
||||
{
|
||||
/// <summary>The session's event stream completed cleanly (session closed or the worker stream ended).</summary>
|
||||
public const string SessionClosed = "SessionClosed";
|
||||
|
||||
/// <summary>The subscriber fell behind its bounded queue and was disconnected.</summary>
|
||||
public const string Overflow = "Overflow";
|
||||
|
||||
/// <summary>The worker event stream terminated with a fault.</summary>
|
||||
public const string WorkerFault = "WorkerFault";
|
||||
|
||||
/// <summary>The client cancelled the call or disconnected.</summary>
|
||||
public const string ClientCancelled = "ClientCancelled";
|
||||
|
||||
/// <summary>The stream ended on an unclassified error.</summary>
|
||||
public const string Error = "Error";
|
||||
|
||||
/// <summary>The consumer stopped reading without cancelling the call.</summary>
|
||||
public const string Detached = "Detached";
|
||||
}
|
||||
@@ -129,6 +129,10 @@ public sealed class EventStreamService(
|
||||
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
|
||||
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
|
||||
|
||||
// Why this stream ended, recorded as a bounded constant on the disconnect metric. Set at
|
||||
// the point the end is observed; a consumer that simply stops iterating keeps Detached.
|
||||
string disconnectReason = EventStreamDisconnectReasons.Detached;
|
||||
|
||||
try
|
||||
{
|
||||
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
|
||||
@@ -173,13 +177,31 @@ public sealed class EventStreamService(
|
||||
// here. Mirror the original ProduceEventsAsync behavior: fault the
|
||||
// session and record the metric, then propagate the terminal fault to the
|
||||
// gRPC client.
|
||||
disconnectReason = EventStreamDisconnectReasons.WorkerFault;
|
||||
session.MarkFaulted(workerException.Message);
|
||||
metrics.Fault(WorkerClientErrorCode.WorkerFaulted.ToString());
|
||||
throw;
|
||||
}
|
||||
catch (SessionManagerException sessionException)
|
||||
when (sessionException.ErrorCode == SessionManagerErrorCode.EventQueueOverflow)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.Overflow;
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
|
||||
throw;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.Error;
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!hasMore)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.SessionClosed;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -214,7 +236,15 @@ public sealed class EventStreamService(
|
||||
backlogRegistration.Dispose();
|
||||
subscriber.Dispose();
|
||||
|
||||
metrics.StreamDisconnected("Detached");
|
||||
// A consumer that stopped iterating mid-stream lands here with the Detached default;
|
||||
// when the call token has fired, that early stop was the client going away.
|
||||
if (disconnectReason == EventStreamDisconnectReasons.Detached
|
||||
&& cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
|
||||
}
|
||||
|
||||
metrics.StreamDisconnected(disconnectReason);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ public sealed class EventStreamServiceTests
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.QueueOverflows);
|
||||
Assert.Equal(1, snapshot.Faults);
|
||||
// The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the
|
||||
// The finally block in StreamEventsAsync records StreamDisconnected (reason Overflow) on the
|
||||
// overflow+fault path too; pin it here so a regression removing that call is caught.
|
||||
Assert.Equal(1, snapshot.StreamDisconnects);
|
||||
}
|
||||
@@ -572,6 +572,142 @@ public sealed class EventStreamServiceTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A stream that ends because the session's event stream completed records the
|
||||
/// <c>SessionClosed</c> disconnect reason rather than a blanket constant.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenEventStreamCompletes_RecordsSessionClosedReason()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
using DisconnectReasonRecorder reasons = new(metrics);
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
|
||||
await CollectEventsAsync(service, session.SessionId);
|
||||
|
||||
Assert.Equal([EventStreamDisconnectReasons.SessionClosed], reasons.Reasons);
|
||||
}
|
||||
|
||||
/// <summary>A per-subscriber overflow records the <c>Overflow</c> disconnect reason.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenStreamQueueOverflows_RecordsOverflowReason()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
using DisconnectReasonRecorder reasons = new(metrics);
|
||||
GatewaySession session = CreateReadySession(
|
||||
workerClient,
|
||||
queueCapacity: 1,
|
||||
metrics: metrics,
|
||||
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
|
||||
EventStreamService service = CreateService(
|
||||
new FakeSessionManager(session),
|
||||
metrics,
|
||||
queueCapacity: 1,
|
||||
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
|
||||
for (ulong sequence = 1; sequence <= 50; sequence++)
|
||||
{
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
|
||||
}
|
||||
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
await using (IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
|
||||
.GetAsyncEnumerator())
|
||||
{
|
||||
await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () =>
|
||||
{
|
||||
while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout))
|
||||
{
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Assert.Equal([EventStreamDisconnectReasons.Overflow], reasons.Reasons);
|
||||
}
|
||||
|
||||
/// <summary>A terminal worker fault records the <c>WorkerFault</c> disconnect reason.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_RecordsWorkerFaultReason()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
TerminalException = new WorkerClientException(
|
||||
WorkerClientErrorCode.WorkerFaulted,
|
||||
"worker terminal fault"),
|
||||
};
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
using DisconnectReasonRecorder reasons = new(metrics);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
|
||||
await using (IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
|
||||
.GetAsyncEnumerator())
|
||||
{
|
||||
await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
}
|
||||
|
||||
Assert.Equal([EventStreamDisconnectReasons.WorkerFault], reasons.Reasons);
|
||||
}
|
||||
|
||||
/// <summary>A client cancellation records the <c>ClientCancelled</c> disconnect reason.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenCanceled_RecordsClientCancelledReason()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
using DisconnectReasonRecorder reasons = new(metrics);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
|
||||
using CancellationTokenSource cancellationTokenSource = new();
|
||||
await using (IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, cancellationTokenSource.Token)
|
||||
.GetAsyncEnumerator(cancellationTokenSource.Token))
|
||||
{
|
||||
Task<bool> moveTask = subscriber.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
|
||||
await cancellationTokenSource.CancelAsync();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await moveTask.WaitAsync(TestTimeout));
|
||||
}
|
||||
|
||||
Assert.Equal([EventStreamDisconnectReasons.ClientCancelled], reasons.Reasons);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A consumer that stops reading without cancelling (the gRPC writer gave up) keeps the
|
||||
/// <c>Detached</c> reason.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenConsumerStopsEarly_RecordsDetachedReason()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
using DisconnectReasonRecorder reasons = new(metrics);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
|
||||
await using (IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
|
||||
.GetAsyncEnumerator())
|
||||
{
|
||||
await ReadNextAsync(subscriber);
|
||||
}
|
||||
|
||||
Assert.Equal([EventStreamDisconnectReasons.Detached], reasons.Reasons);
|
||||
}
|
||||
|
||||
private static async Task<MxEvent> ReadNextAsync(IAsyncEnumerator<MxEvent> enumerator)
|
||||
{
|
||||
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
@@ -896,4 +1032,63 @@ public sealed class EventStreamServiceTests
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Captures the <c>reason</c> tag of every <c>mxgateway.grpc.streams.disconnected</c>
|
||||
/// measurement recorded on one <see cref="GatewayMetrics"/> instance (filtered by meter
|
||||
/// reference, so parallel tests sharing the meter name cannot cross-talk).
|
||||
/// </summary>
|
||||
private sealed class DisconnectReasonRecorder : IDisposable
|
||||
{
|
||||
private readonly System.Diagnostics.Metrics.MeterListener _listener = new();
|
||||
private readonly List<string> _reasons = [];
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="DisconnectReasonRecorder"/> class.</summary>
|
||||
/// <param name="metrics">The metrics instance whose meter to observe.</param>
|
||||
public DisconnectReasonRecorder(GatewayMetrics metrics)
|
||||
{
|
||||
_listener.InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
||||
&& instrument.Name == "mxgateway.grpc.streams.disconnected")
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
_listener.SetMeasurementEventCallback<long>((instrument, _, tags, _) =>
|
||||
{
|
||||
if (!ReferenceEquals(instrument.Meter, metrics.Meter))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (KeyValuePair<string, object?> tag in tags)
|
||||
{
|
||||
if (tag.Key == "reason" && tag.Value is string reason)
|
||||
{
|
||||
lock (_reasons)
|
||||
{
|
||||
_reasons.Add(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
/// <summary>Gets the recorded reasons, in order.</summary>
|
||||
public string[] Reasons
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_reasons)
|
||||
{
|
||||
return [.. _reasons];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => _listener.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user