fix(telemetry): 0.3.1 OTLP export calls are not traced as client spans (#56)
The Serilog OTLP log sink is not part of the OTel SDK and does not run
inside SuppressInstrumentationScope, so every log batch produced a root
opentelemetry.proto.collector.logs.v1.LogsService/Export gRPC client span.
AddZbTelemetry now drops telemetry's own transport from tracing:
- HttpClient instrumentation: FilterHttpRequestMessage rejects export calls
- gRPC client instrumentation (no Filter option): the start-time
EnrichWithHttpRequestMessage callback clears Recorded/IsAllDataRequested,
which also leaves the child HTTP activity unsampled.
An export call is any /opentelemetry.proto.collector.* path, or an
OTLP/HTTP /v1/{traces,metrics,logs} path on the configured endpoint's
host:port (ZbOtlpExportCallFilter). Every other outgoing call keeps its span.
Tests: predicate theory + an end-to-end run against a loopback OTLP
receiver with positive controls (app gRPC + HttpClient calls traced) and
negatives (the real log-sink export, /v1/logs, gRPC path over HttpClient);
mutation-checked three ways.
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Recognises the outgoing calls an OTLP exporter makes, so that the gRPC-client and HttpClient
|
||||
/// instrumentation do not trace telemetry's own transport (scadaproj#56).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The OpenTelemetry SDK's own OTLP exporters run inside <c>SuppressInstrumentationScope</c>, but
|
||||
/// <c>Serilog.Sinks.OpenTelemetry</c> is not part of the SDK and does not, so every log batch it sent
|
||||
/// produced a root <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The
|
||||
/// match is on the call itself, not on who made it: any OTLP/gRPC call (path
|
||||
/// <c>/opentelemetry.proto.collector.*</c>, a namespace reserved to the OTLP collector services) and
|
||||
/// any OTLP/HTTP call (<c>/v1/traces</c>, <c>/v1/metrics</c>, <c>/v1/logs</c>) to the configured
|
||||
/// <see cref="ZbTelemetryOptions.OtlpEndpoint"/>'s host and port. Every other outgoing call keeps its span.
|
||||
/// </remarks>
|
||||
public static class ZbOtlpExportCallFilter
|
||||
{
|
||||
/// <summary>The path prefix every OTLP/gRPC collector service shares.</summary>
|
||||
public const string GrpcPathPrefix = "/opentelemetry.proto.collector.";
|
||||
|
||||
private static readonly string[] HttpSignalPaths = ["/v1/traces", "/v1/metrics", "/v1/logs"];
|
||||
|
||||
/// <summary>
|
||||
/// <see langword="true"/> when <paramref name="request"/> is an OTLP export call: an OTLP/gRPC
|
||||
/// collector path on any host, or an OTLP/HTTP signal path on <paramref name="otlpEndpoint"/>'s
|
||||
/// host and port.
|
||||
/// </summary>
|
||||
/// <param name="request">The outgoing request.</param>
|
||||
/// <param name="otlpEndpoint">The configured OTLP endpoint, or <see langword="null"/>.</param>
|
||||
public static bool IsExportCall(HttpRequestMessage request, Uri? otlpEndpoint)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
var uri = request.RequestUri;
|
||||
if (uri is null || !uri.IsAbsoluteUri)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var path = uri.AbsolutePath;
|
||||
if (path.StartsWith(GrpcPathPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return otlpEndpoint is not null
|
||||
&& string.Equals(uri.Host, otlpEndpoint.Host, StringComparison.OrdinalIgnoreCase)
|
||||
&& uri.Port == otlpEndpoint.Port
|
||||
&& HttpSignalPaths.Any(p => path.EndsWith(p, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops <paramref name="activity"/> from being recorded or exported. Clearing the
|
||||
/// <see cref="ActivityTraceFlags.Recorded"/> flag at start also leaves any child activity (the
|
||||
/// HttpClient request under a gRPC call) unsampled under the default parent-based sampler.
|
||||
/// </summary>
|
||||
internal static void Unrecord(Activity activity)
|
||||
{
|
||||
activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded;
|
||||
activity.IsAllDataRequested = false;
|
||||
}
|
||||
|
||||
internal static Uri? ParseEndpoint(ZbTelemetryOptions options) =>
|
||||
Uri.TryCreate(options.OtlpEndpoint, UriKind.Absolute, out var uri) ? uri : null;
|
||||
}
|
||||
@@ -77,10 +77,25 @@ public static class ZbTelemetryExtensions
|
||||
tracing.AddSource(source);
|
||||
}
|
||||
|
||||
// Telemetry's own OTLP transport is not traced (scadaproj#56): the SDK exporters
|
||||
// suppress their own instrumentation, the Serilog OTLP log sink does not. The gRPC
|
||||
// client instrumentation has no Filter option, so its start-time enrich callback
|
||||
// un-records the activity instead; it runs before the HttpClient request starts,
|
||||
// so the child HTTP span is left unsampled too.
|
||||
var otlpEndpoint = ZbOtlpExportCallFilter.ParseEndpoint(options);
|
||||
tracing
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation()
|
||||
.AddGrpcClientInstrumentation();
|
||||
.AddHttpClientInstrumentation(o =>
|
||||
o.FilterHttpRequestMessage = request =>
|
||||
!ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
|
||||
.AddGrpcClientInstrumentation(o =>
|
||||
o.EnrichWithHttpRequestMessage = (activity, request) =>
|
||||
{
|
||||
if (ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
|
||||
{
|
||||
ZbOtlpExportCallFilter.Unrecord(activity);
|
||||
}
|
||||
});
|
||||
|
||||
ApplyTracingExporter(tracing, options);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user