fix(telemetry): 0.3.1 OTLP export calls are not traced as client spans (#56)

The Serilog OTLP log sink is not part of the OTel SDK and does not run
inside SuppressInstrumentationScope, so every log batch produced a root
opentelemetry.proto.collector.logs.v1.LogsService/Export gRPC client span.

AddZbTelemetry now drops telemetry's own transport from tracing:
- HttpClient instrumentation: FilterHttpRequestMessage rejects export calls
- gRPC client instrumentation (no Filter option): the start-time
  EnrichWithHttpRequestMessage callback clears Recorded/IsAllDataRequested,
  which also leaves the child HTTP activity unsampled.
An export call is any /opentelemetry.proto.collector.* path, or an
OTLP/HTTP /v1/{traces,metrics,logs} path on the configured endpoint's
host:port (ZbOtlpExportCallFilter). Every other outgoing call keeps its span.

Tests: predicate theory + an end-to-end run against a loopback OTLP
receiver with positive controls (app gRPC + HttpClient calls traced) and
negatives (the real log-sink export, /v1/logs, gRPC path over HttpClient);
mutation-checked three ways.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-12 22:09:24 -05:00
parent d93ca7b8c2
commit 85654d7b43
6 changed files with 235 additions and 4 deletions
+1 -1
View File
@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<LangVersion>latest</LangVersion>
<Version>0.3.0</Version>
<Version>0.3.1</Version>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
@@ -0,0 +1,66 @@
using System.Diagnostics;
namespace ZB.MOM.WW.Telemetry;
/// <summary>
/// Recognises the outgoing calls an OTLP exporter makes, so that the gRPC-client and HttpClient
/// instrumentation do not trace telemetry's own transport (scadaproj#56).
/// </summary>
/// <remarks>
/// The OpenTelemetry SDK's own OTLP exporters run inside <c>SuppressInstrumentationScope</c>, but
/// <c>Serilog.Sinks.OpenTelemetry</c> is not part of the SDK and does not, so every log batch it sent
/// produced a root <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The
/// match is on the call itself, not on who made it: any OTLP/gRPC call (path
/// <c>/opentelemetry.proto.collector.*</c>, a namespace reserved to the OTLP collector services) and
/// any OTLP/HTTP call (<c>/v1/traces</c>, <c>/v1/metrics</c>, <c>/v1/logs</c>) to the configured
/// <see cref="ZbTelemetryOptions.OtlpEndpoint"/>'s host and port. Every other outgoing call keeps its span.
/// </remarks>
public static class ZbOtlpExportCallFilter
{
/// <summary>The path prefix every OTLP/gRPC collector service shares.</summary>
public const string GrpcPathPrefix = "/opentelemetry.proto.collector.";
private static readonly string[] HttpSignalPaths = ["/v1/traces", "/v1/metrics", "/v1/logs"];
/// <summary>
/// <see langword="true"/> when <paramref name="request"/> is an OTLP export call: an OTLP/gRPC
/// collector path on any host, or an OTLP/HTTP signal path on <paramref name="otlpEndpoint"/>'s
/// host and port.
/// </summary>
/// <param name="request">The outgoing request.</param>
/// <param name="otlpEndpoint">The configured OTLP endpoint, or <see langword="null"/>.</param>
public static bool IsExportCall(HttpRequestMessage request, Uri? otlpEndpoint)
{
ArgumentNullException.ThrowIfNull(request);
var uri = request.RequestUri;
if (uri is null || !uri.IsAbsoluteUri)
{
return false;
}
var path = uri.AbsolutePath;
if (path.StartsWith(GrpcPathPrefix, StringComparison.Ordinal))
{
return true;
}
return otlpEndpoint is not null
&& string.Equals(uri.Host, otlpEndpoint.Host, StringComparison.OrdinalIgnoreCase)
&& uri.Port == otlpEndpoint.Port
&& HttpSignalPaths.Any(p => path.EndsWith(p, StringComparison.Ordinal));
}
/// <summary>
/// Stops <paramref name="activity"/> from being recorded or exported. Clearing the
/// <see cref="ActivityTraceFlags.Recorded"/> flag at start also leaves any child activity (the
/// HttpClient request under a gRPC call) unsampled under the default parent-based sampler.
/// </summary>
internal static void Unrecord(Activity activity)
{
activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded;
activity.IsAllDataRequested = false;
}
internal static Uri? ParseEndpoint(ZbTelemetryOptions options) =>
Uri.TryCreate(options.OtlpEndpoint, UriKind.Absolute, out var uri) ? uri : null;
}
@@ -77,10 +77,25 @@ public static class ZbTelemetryExtensions
tracing.AddSource(source);
}
// Telemetry's own OTLP transport is not traced (scadaproj#56): the SDK exporters
// suppress their own instrumentation, the Serilog OTLP log sink does not. The gRPC
// client instrumentation has no Filter option, so its start-time enrich callback
// un-records the activity instead; it runs before the HttpClient request starts,
// so the child HTTP span is left unsampled too.
var otlpEndpoint = ZbOtlpExportCallFilter.ParseEndpoint(options);
tracing
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddGrpcClientInstrumentation();
.AddHttpClientInstrumentation(o =>
o.FilterHttpRequestMessage = request =>
!ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
.AddGrpcClientInstrumentation(o =>
o.EnrichWithHttpRequestMessage = (activity, request) =>
{
if (ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
{
ZbOtlpExportCallFilter.Unrecord(activity);
}
});
ApplyTracingExporter(tracing, options);
});
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Net;
using System.Text;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
@@ -7,6 +8,9 @@ using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Grpc.Core;
using Grpc.Net.Client;
using OpenTelemetry.Trace;
using Serilog;
using ZB.MOM.WW.Telemetry;
using ZB.MOM.WW.Telemetry.Serilog;
@@ -22,6 +26,7 @@ public sealed class OtlpLogExportTests
{
private const string LogsPath = "/opentelemetry.proto.collector.logs.v1.LogsService/Export";
private const string MetricsPath = "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export";
private const string TracesPath = "/opentelemetry.proto.collector.trace.v1.TraceService/Export";
// --- the decision ---------------------------------------------------------------------
@@ -196,4 +201,93 @@ public sealed class OtlpLogExportTests
Assert.Empty(receiver.Calls);
}
// --- scadaproj#56: telemetry's own OTLP transport is not traced --------------------------
private static async Task WaitForAsync(Func<bool> condition, int timeoutMs = 8000)
{
for (var waited = 0; waited < timeoutMs && !condition(); waited += 100)
{
await Task.Delay(100);
}
}
/// <summary>
/// Before 0.3.1 every log batch the Serilog OTLP sink sent produced a root
/// <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The app's own
/// outgoing gRPC and HTTP calls are the positive controls: they must still be traced, to the
/// same receiver, in the same trace export.
/// </summary>
[Fact]
public async Task Otlp_export_calls_are_not_traced_but_the_apps_own_calls_are()
{
await using var receiver = await FakeOtlpReceiver.StartAsync();
var builder = Host.CreateApplicationBuilder();
builder.AddZbSerilog(o => o.ServiceName = "span-filter-test");
builder.Services.AddZbTelemetry(o =>
{
o.ServiceName = "span-filter-test";
o.Exporter = ZbExporter.Otlp;
o.OtlpEndpoint = receiver.Endpoint;
});
using (var host = builder.Build())
{
_ = host.Services.GetRequiredService<TracerProvider>();
// Positive control 1: an ordinary gRPC client call (ScadaBridge→site, HG/mxgw clients).
using (var channel = GrpcChannel.ForAddress(receiver.Endpoint))
{
var bytes = Marshallers.Create(b => b, b => b);
var ping = new Method<byte[], byte[]>(MethodType.Unary, "zb.test.v1.AppService", "Ping", bytes, bytes);
await channel.CreateCallInvoker().AsyncUnaryCall(ping, null, new CallOptions(), []);
}
using var http = new HttpClient
{
DefaultRequestVersion = HttpVersion.Version20,
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact,
};
// Positive control 2: an ordinary HttpClient call.
(await http.PostAsync(receiver.Endpoint + "/zb-test/http-probe", new ByteArrayContent([]))).Dispose();
// HttpClient-side negatives: an OTLP/HTTP call to the configured endpoint, and an
// OTLP/gRPC path sent through a plain HttpClient.
(await http.PostAsync(receiver.Endpoint + "/v1/logs", new ByteArrayContent([]))).Dispose();
(await http.PostAsync(receiver.Endpoint + "/opentelemetry.proto.collector.zbprobe.v1.ProbeService/Export",
new ByteArrayContent([]))).Dispose();
// The gRPC-side negative: a real log batch from the Serilog OTLP sink, sent while the
// tracer provider is still live so its span (if any) is in the final trace export.
host.Services.GetRequiredService<global::Serilog.ILogger>().Information("span filter {Marker}", "log-line-56");
await WaitForAsync(() => receiver.Received(LogsPath));
Assert.True(receiver.Received(LogsPath), "the log sink never exported, so the negative proves nothing");
await Task.Delay(500); // let the export call's activity stop before the final trace flush
}
await WaitForAsync(() => receiver.BodiesOf(TracesPath).Contains("zb.test.v1.AppService/Ping"));
var spans = receiver.BodiesOf(TracesPath);
// The loopback receiver's own ASP.NET Core server spans land in the same export (the
// instrumentation listens process-wide) and carry url.path, so the assertions key on
// client-span markers only: a span *name* for gRPC client spans, url.full for HttpClient.
Assert.True(HasSpanNamed(spans, "zb.test.v1.AppService/Ping"), "positive control: the app's gRPC call was not traced");
Assert.Contains(receiver.Endpoint + "/zb-test/http-probe", spans);
Assert.False(HasSpanNamed(spans, "opentelemetry.proto.collector.logs.v1.LogsService/Export"),
"the Serilog OTLP sink's LogsService/Export call was traced");
Assert.DoesNotContain(receiver.Endpoint + "/v1/logs", spans);
Assert.DoesNotContain(receiver.Endpoint + "/opentelemetry.proto.collector.", spans);
}
/// <summary>
/// True when an OTLP trace export body carries a span with this exact name: protobuf
/// <c>Span.name</c> is field 5 (tag byte 0x2A) followed by the length, then the UTF-8 name.
/// </summary>
private static bool HasSpanNamed(string bodies, string name)
{
Assert.InRange(name.Length, 1, 127); // single-byte varint length
return bodies.Contains("\u002A" + (char)name.Length + name, StringComparison.Ordinal);
}
}
@@ -0,0 +1,54 @@
using ZB.MOM.WW.Telemetry;
namespace ZB.MOM.WW.Telemetry.Tests;
/// <summary>scadaproj#56: which outgoing calls count as telemetry's own OTLP transport.</summary>
public sealed class OtlpExportCallFilterTests
{
private static readonly Uri Endpoint = new("http://collector:4317");
private static bool IsExport(string url, Uri? endpoint = null) =>
ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(HttpMethod.Post, url), endpoint ?? Endpoint);
[Theory]
[InlineData("http://collector:4317/opentelemetry.proto.collector.logs.v1.LogsService/Export")]
[InlineData("http://collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export")]
[InlineData("http://collector:4317/opentelemetry.proto.collector.metrics.v1.MetricsService/Export")]
// OTLP/gRPC is recognised on any host: a sink pointed somewhere else is still transport.
[InlineData("http://other-collector:9999/opentelemetry.proto.collector.logs.v1.LogsService/Export")]
// OTLP/HTTP to the configured endpoint.
[InlineData("http://collector:4317/v1/logs")]
[InlineData("http://COLLECTOR:4317/v1/traces")]
[InlineData("http://collector:4317/otlp/v1/metrics")]
public void Otlp_export_calls_are_recognised(string url) => Assert.True(IsExport(url));
[Theory]
// The apps' own outgoing calls (ScadaBridge→site, HistorianGateway/mxgw clients, REST).
[InlineData("http://site-a:8083/scadabridge.centralcontrol.v1.CentralControlService/Heartbeat")]
[InlineData("http://historian:32565/HistoryService/GetInterfaceVersion")]
[InlineData("http://collector:4317/mxaccess.v1.MxAccessGateway/Read")]
[InlineData("http://rest-api/api/orders")]
// An /opentelemetry.proto.collector look-alike that is not the namespace.
[InlineData("http://collector:4317/opentelemetry.proto.collectorx/Export")]
// /v1/logs on any other host or port is an ordinary REST call.
[InlineData("http://other-host:4317/v1/logs")]
[InlineData("http://collector:4318/v1/logs")]
public void Ordinary_calls_are_not_export_calls(string url) => Assert.False(IsExport(url));
[Fact]
public void Without_an_endpoint_only_the_grpc_namespace_matches()
{
Assert.True(ZbOtlpExportCallFilter.IsExportCall(
new HttpRequestMessage(HttpMethod.Post, "http://x:1/opentelemetry.proto.collector.logs.v1.LogsService/Export"), null));
Assert.False(ZbOtlpExportCallFilter.IsExportCall(
new HttpRequestMessage(HttpMethod.Post, "http://collector:4317/v1/logs"), null));
}
[Fact]
public void A_request_without_an_absolute_uri_is_not_an_export_call()
{
Assert.False(ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(), Endpoint));
Assert.False(ZbOtlpExportCallFilter.IsExportCall(
new HttpRequestMessage(HttpMethod.Post, new Uri("/v1/logs", UriKind.Relative)), Endpoint));
}
}
@@ -35,7 +35,9 @@ public sealed class ResourceIdentityTests
var expected = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion;
Assert.Equal(expected, ZbResource.VersionOf(assembly));
Assert.StartsWith("0.3.0", ZbResource.VersionOf(assembly));
// The package version (Directory.Build.props), not a fixed literal that every release must edit.
var v = assembly.GetName().Version!;
Assert.StartsWith($"{v.Major}.{v.Minor}.{v.Build}+", ZbResource.VersionOf(assembly));
Assert.Null(ZbResource.VersionOf(null));
}