fix(telemetry): 0.3.1 OTLP export calls are not traced as client spans (#56)

The Serilog OTLP log sink is not part of the OTel SDK and does not run
inside SuppressInstrumentationScope, so every log batch produced a root
opentelemetry.proto.collector.logs.v1.LogsService/Export gRPC client span.

AddZbTelemetry now drops telemetry's own transport from tracing:
- HttpClient instrumentation: FilterHttpRequestMessage rejects export calls
- gRPC client instrumentation (no Filter option): the start-time
  EnrichWithHttpRequestMessage callback clears Recorded/IsAllDataRequested,
  which also leaves the child HTTP activity unsampled.
An export call is any /opentelemetry.proto.collector.* path, or an
OTLP/HTTP /v1/{traces,metrics,logs} path on the configured endpoint's
host:port (ZbOtlpExportCallFilter). Every other outgoing call keeps its span.

Tests: predicate theory + an end-to-end run against a loopback OTLP
receiver with positive controls (app gRPC + HttpClient calls traced) and
negatives (the real log-sink export, /v1/logs, gRPC path over HttpClient);
mutation-checked three ways.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-12 22:09:24 -05:00
parent d93ca7b8c2
commit 85654d7b43
6 changed files with 235 additions and 4 deletions
@@ -0,0 +1,66 @@
using System.Diagnostics;
namespace ZB.MOM.WW.Telemetry;
/// <summary>
/// Recognises the outgoing calls an OTLP exporter makes, so that the gRPC-client and HttpClient
/// instrumentation do not trace telemetry's own transport (scadaproj#56).
/// </summary>
/// <remarks>
/// The OpenTelemetry SDK's own OTLP exporters run inside <c>SuppressInstrumentationScope</c>, but
/// <c>Serilog.Sinks.OpenTelemetry</c> is not part of the SDK and does not, so every log batch it sent
/// produced a root <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The
/// match is on the call itself, not on who made it: any OTLP/gRPC call (path
/// <c>/opentelemetry.proto.collector.*</c>, a namespace reserved to the OTLP collector services) and
/// any OTLP/HTTP call (<c>/v1/traces</c>, <c>/v1/metrics</c>, <c>/v1/logs</c>) to the configured
/// <see cref="ZbTelemetryOptions.OtlpEndpoint"/>'s host and port. Every other outgoing call keeps its span.
/// </remarks>
public static class ZbOtlpExportCallFilter
{
/// <summary>The path prefix every OTLP/gRPC collector service shares.</summary>
public const string GrpcPathPrefix = "/opentelemetry.proto.collector.";
private static readonly string[] HttpSignalPaths = ["/v1/traces", "/v1/metrics", "/v1/logs"];
/// <summary>
/// <see langword="true"/> when <paramref name="request"/> is an OTLP export call: an OTLP/gRPC
/// collector path on any host, or an OTLP/HTTP signal path on <paramref name="otlpEndpoint"/>'s
/// host and port.
/// </summary>
/// <param name="request">The outgoing request.</param>
/// <param name="otlpEndpoint">The configured OTLP endpoint, or <see langword="null"/>.</param>
public static bool IsExportCall(HttpRequestMessage request, Uri? otlpEndpoint)
{
ArgumentNullException.ThrowIfNull(request);
var uri = request.RequestUri;
if (uri is null || !uri.IsAbsoluteUri)
{
return false;
}
var path = uri.AbsolutePath;
if (path.StartsWith(GrpcPathPrefix, StringComparison.Ordinal))
{
return true;
}
return otlpEndpoint is not null
&& string.Equals(uri.Host, otlpEndpoint.Host, StringComparison.OrdinalIgnoreCase)
&& uri.Port == otlpEndpoint.Port
&& HttpSignalPaths.Any(p => path.EndsWith(p, StringComparison.Ordinal));
}
/// <summary>
/// Stops <paramref name="activity"/> from being recorded or exported. Clearing the
/// <see cref="ActivityTraceFlags.Recorded"/> flag at start also leaves any child activity (the
/// HttpClient request under a gRPC call) unsampled under the default parent-based sampler.
/// </summary>
internal static void Unrecord(Activity activity)
{
activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded;
activity.IsAllDataRequested = false;
}
internal static Uri? ParseEndpoint(ZbTelemetryOptions options) =>
Uri.TryCreate(options.OtlpEndpoint, UriKind.Absolute, out var uri) ? uri : null;
}
@@ -77,10 +77,25 @@ public static class ZbTelemetryExtensions
tracing.AddSource(source);
}
// Telemetry's own OTLP transport is not traced (scadaproj#56): the SDK exporters
// suppress their own instrumentation, the Serilog OTLP log sink does not. The gRPC
// client instrumentation has no Filter option, so its start-time enrich callback
// un-records the activity instead; it runs before the HttpClient request starts,
// so the child HTTP span is left unsampled too.
var otlpEndpoint = ZbOtlpExportCallFilter.ParseEndpoint(options);
tracing
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddGrpcClientInstrumentation();
.AddHttpClientInstrumentation(o =>
o.FilterHttpRequestMessage = request =>
!ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
.AddGrpcClientInstrumentation(o =>
o.EnrichWithHttpRequestMessage = (activity, request) =>
{
if (ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
{
ZbOtlpExportCallFilter.Unrecord(activity);
}
});
ApplyTracingExporter(tracing, options);
});