fix(grpc): record the real StreamEvents disconnect reason

StreamDisconnected always recorded "Detached". The reason tag is now a
bounded EventStreamDisconnectReasons constant for how the stream ended:
SessionClosed, Overflow, WorkerFault, ClientCancelled, Error, or Detached.
This commit is contained in:
Joseph Doherty
2026-10-22 06:29:31 -04:00
parent 0b4a1682c8
commit 6ae5ed905f
5 changed files with 260 additions and 5 deletions
@@ -0,0 +1,30 @@
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
/// <summary>
/// The bounded set of <c>reason</c> tag values recorded on
/// <c>mxgateway.grpc.streams.disconnected</c> when a <c>StreamEvents</c> stream ends.
/// </summary>
/// <remarks>
/// Constants rather than exception messages or free text so the metric's tag cardinality stays
/// fixed: an exporter mints one time series per distinct tag value.
/// </remarks>
public static class EventStreamDisconnectReasons
{
/// <summary>The session's event stream completed cleanly (session closed or the worker stream ended).</summary>
public const string SessionClosed = "SessionClosed";
/// <summary>The subscriber fell behind its bounded queue and was disconnected.</summary>
public const string Overflow = "Overflow";
/// <summary>The worker event stream terminated with a fault.</summary>
public const string WorkerFault = "WorkerFault";
/// <summary>The client cancelled the call or disconnected.</summary>
public const string ClientCancelled = "ClientCancelled";
/// <summary>The stream ended on an unclassified error.</summary>
public const string Error = "Error";
/// <summary>The consumer stopped reading without cancelling the call.</summary>
public const string Detached = "Detached";
}
@@ -129,6 +129,10 @@ public sealed class EventStreamService(
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
// Why this stream ended, recorded as a bounded constant on the disconnect metric. Set at
// the point the end is observed; a consumer that simply stops iterating keeps Detached.
string disconnectReason = EventStreamDisconnectReasons.Detached;
try
{
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
@@ -173,13 +177,31 @@ public sealed class EventStreamService(
// here. Mirror the original ProduceEventsAsync behavior: fault the
// session and record the metric, then propagate the terminal fault to the
// gRPC client.
disconnectReason = EventStreamDisconnectReasons.WorkerFault;
session.MarkFaulted(workerException.Message);
metrics.Fault(WorkerClientErrorCode.WorkerFaulted.ToString());
throw;
}
catch (SessionManagerException sessionException)
when (sessionException.ErrorCode == SessionManagerErrorCode.EventQueueOverflow)
{
disconnectReason = EventStreamDisconnectReasons.Overflow;
throw;
}
catch (OperationCanceledException)
{
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
throw;
}
catch (Exception)
{
disconnectReason = EventStreamDisconnectReasons.Error;
throw;
}
if (!hasMore)
{
disconnectReason = EventStreamDisconnectReasons.SessionClosed;
break;
}
@@ -214,7 +236,15 @@ public sealed class EventStreamService(
backlogRegistration.Dispose();
subscriber.Dispose();
metrics.StreamDisconnected("Detached");
// A consumer that stopped iterating mid-stream lands here with the Detached default;
// when the call token has fired, that early stop was the client going away.
if (disconnectReason == EventStreamDisconnectReasons.Detached
&& cancellationToken.IsCancellationRequested)
{
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
}
metrics.StreamDisconnected(disconnectReason);
}
}