fix(grpc): record the real StreamEvents disconnect reason
StreamDisconnected always recorded "Detached". The reason tag is now a bounded EventStreamDisconnectReasons constant for how the stream ended: SessionClosed, Overflow, WorkerFault, ClientCancelled, Error, or Detached.
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// The bounded set of <c>reason</c> tag values recorded on
|
||||
/// <c>mxgateway.grpc.streams.disconnected</c> when a <c>StreamEvents</c> stream ends.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Constants rather than exception messages or free text so the metric's tag cardinality stays
|
||||
/// fixed: an exporter mints one time series per distinct tag value.
|
||||
/// </remarks>
|
||||
public static class EventStreamDisconnectReasons
|
||||
{
|
||||
/// <summary>The session's event stream completed cleanly (session closed or the worker stream ended).</summary>
|
||||
public const string SessionClosed = "SessionClosed";
|
||||
|
||||
/// <summary>The subscriber fell behind its bounded queue and was disconnected.</summary>
|
||||
public const string Overflow = "Overflow";
|
||||
|
||||
/// <summary>The worker event stream terminated with a fault.</summary>
|
||||
public const string WorkerFault = "WorkerFault";
|
||||
|
||||
/// <summary>The client cancelled the call or disconnected.</summary>
|
||||
public const string ClientCancelled = "ClientCancelled";
|
||||
|
||||
/// <summary>The stream ended on an unclassified error.</summary>
|
||||
public const string Error = "Error";
|
||||
|
||||
/// <summary>The consumer stopped reading without cancelling the call.</summary>
|
||||
public const string Detached = "Detached";
|
||||
}
|
||||
@@ -129,6 +129,10 @@ public sealed class EventStreamService(
|
||||
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
|
||||
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
|
||||
|
||||
// Why this stream ended, recorded as a bounded constant on the disconnect metric. Set at
|
||||
// the point the end is observed; a consumer that simply stops iterating keeps Detached.
|
||||
string disconnectReason = EventStreamDisconnectReasons.Detached;
|
||||
|
||||
try
|
||||
{
|
||||
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
|
||||
@@ -173,13 +177,31 @@ public sealed class EventStreamService(
|
||||
// here. Mirror the original ProduceEventsAsync behavior: fault the
|
||||
// session and record the metric, then propagate the terminal fault to the
|
||||
// gRPC client.
|
||||
disconnectReason = EventStreamDisconnectReasons.WorkerFault;
|
||||
session.MarkFaulted(workerException.Message);
|
||||
metrics.Fault(WorkerClientErrorCode.WorkerFaulted.ToString());
|
||||
throw;
|
||||
}
|
||||
catch (SessionManagerException sessionException)
|
||||
when (sessionException.ErrorCode == SessionManagerErrorCode.EventQueueOverflow)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.Overflow;
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
|
||||
throw;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.Error;
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!hasMore)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.SessionClosed;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -214,7 +236,15 @@ public sealed class EventStreamService(
|
||||
backlogRegistration.Dispose();
|
||||
subscriber.Dispose();
|
||||
|
||||
metrics.StreamDisconnected("Detached");
|
||||
// A consumer that stopped iterating mid-stream lands here with the Detached default;
|
||||
// when the call token has fired, that early stop was the client going away.
|
||||
if (disconnectReason == EventStreamDisconnectReasons.Detached
|
||||
&& cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
|
||||
}
|
||||
|
||||
metrics.StreamDisconnected(disconnectReason);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user