fix(telemetry): 0.3.1 OTLP export calls are not traced as client spans (#56)
The Serilog OTLP log sink is not part of the OTel SDK and does not run
inside SuppressInstrumentationScope, so every log batch produced a root
opentelemetry.proto.collector.logs.v1.LogsService/Export gRPC client span.
AddZbTelemetry now drops telemetry's own transport from tracing:
- HttpClient instrumentation: FilterHttpRequestMessage rejects export calls
- gRPC client instrumentation (no Filter option): the start-time
EnrichWithHttpRequestMessage callback clears Recorded/IsAllDataRequested,
which also leaves the child HTTP activity unsampled.
An export call is any /opentelemetry.proto.collector.* path, or an
OTLP/HTTP /v1/{traces,metrics,logs} path on the configured endpoint's
host:port (ZbOtlpExportCallFilter). Every other outgoing call keeps its span.
Tests: predicate theory + an end-to-end run against a loopback OTLP
receiver with positive controls (app gRPC + HttpClient calls traced) and
negatives (the real log-sink export, /v1/logs, gRPC path over HttpClient);
mutation-checked three ways.
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Version>0.3.0</Version>
|
||||
<Version>0.3.1</Version>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
</PropertyGroup>
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Recognises the outgoing calls an OTLP exporter makes, so that the gRPC-client and HttpClient
|
||||
/// instrumentation do not trace telemetry's own transport (scadaproj#56).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The OpenTelemetry SDK's own OTLP exporters run inside <c>SuppressInstrumentationScope</c>, but
|
||||
/// <c>Serilog.Sinks.OpenTelemetry</c> is not part of the SDK and does not, so every log batch it sent
|
||||
/// produced a root <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The
|
||||
/// match is on the call itself, not on who made it: any OTLP/gRPC call (path
|
||||
/// <c>/opentelemetry.proto.collector.*</c>, a namespace reserved to the OTLP collector services) and
|
||||
/// any OTLP/HTTP call (<c>/v1/traces</c>, <c>/v1/metrics</c>, <c>/v1/logs</c>) to the configured
|
||||
/// <see cref="ZbTelemetryOptions.OtlpEndpoint"/>'s host and port. Every other outgoing call keeps its span.
|
||||
/// </remarks>
|
||||
public static class ZbOtlpExportCallFilter
|
||||
{
|
||||
/// <summary>The path prefix every OTLP/gRPC collector service shares.</summary>
|
||||
public const string GrpcPathPrefix = "/opentelemetry.proto.collector.";
|
||||
|
||||
private static readonly string[] HttpSignalPaths = ["/v1/traces", "/v1/metrics", "/v1/logs"];
|
||||
|
||||
/// <summary>
|
||||
/// <see langword="true"/> when <paramref name="request"/> is an OTLP export call: an OTLP/gRPC
|
||||
/// collector path on any host, or an OTLP/HTTP signal path on <paramref name="otlpEndpoint"/>'s
|
||||
/// host and port.
|
||||
/// </summary>
|
||||
/// <param name="request">The outgoing request.</param>
|
||||
/// <param name="otlpEndpoint">The configured OTLP endpoint, or <see langword="null"/>.</param>
|
||||
public static bool IsExportCall(HttpRequestMessage request, Uri? otlpEndpoint)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
var uri = request.RequestUri;
|
||||
if (uri is null || !uri.IsAbsoluteUri)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var path = uri.AbsolutePath;
|
||||
if (path.StartsWith(GrpcPathPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return otlpEndpoint is not null
|
||||
&& string.Equals(uri.Host, otlpEndpoint.Host, StringComparison.OrdinalIgnoreCase)
|
||||
&& uri.Port == otlpEndpoint.Port
|
||||
&& HttpSignalPaths.Any(p => path.EndsWith(p, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops <paramref name="activity"/> from being recorded or exported. Clearing the
|
||||
/// <see cref="ActivityTraceFlags.Recorded"/> flag at start also leaves any child activity (the
|
||||
/// HttpClient request under a gRPC call) unsampled under the default parent-based sampler.
|
||||
/// </summary>
|
||||
internal static void Unrecord(Activity activity)
|
||||
{
|
||||
activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded;
|
||||
activity.IsAllDataRequested = false;
|
||||
}
|
||||
|
||||
internal static Uri? ParseEndpoint(ZbTelemetryOptions options) =>
|
||||
Uri.TryCreate(options.OtlpEndpoint, UriKind.Absolute, out var uri) ? uri : null;
|
||||
}
|
||||
@@ -77,10 +77,25 @@ public static class ZbTelemetryExtensions
|
||||
tracing.AddSource(source);
|
||||
}
|
||||
|
||||
// Telemetry's own OTLP transport is not traced (scadaproj#56): the SDK exporters
|
||||
// suppress their own instrumentation, the Serilog OTLP log sink does not. The gRPC
|
||||
// client instrumentation has no Filter option, so its start-time enrich callback
|
||||
// un-records the activity instead; it runs before the HttpClient request starts,
|
||||
// so the child HTTP span is left unsampled too.
|
||||
var otlpEndpoint = ZbOtlpExportCallFilter.ParseEndpoint(options);
|
||||
tracing
|
||||
.AddAspNetCoreInstrumentation()
|
||||
.AddHttpClientInstrumentation()
|
||||
.AddGrpcClientInstrumentation();
|
||||
.AddHttpClientInstrumentation(o =>
|
||||
o.FilterHttpRequestMessage = request =>
|
||||
!ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
|
||||
.AddGrpcClientInstrumentation(o =>
|
||||
o.EnrichWithHttpRequestMessage = (activity, request) =>
|
||||
{
|
||||
if (ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint))
|
||||
{
|
||||
ZbOtlpExportCallFilter.Unrecord(activity);
|
||||
}
|
||||
});
|
||||
|
||||
ApplyTracingExporter(tracing, options);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
@@ -7,6 +8,9 @@ using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using OpenTelemetry.Trace;
|
||||
using Serilog;
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
using ZB.MOM.WW.Telemetry.Serilog;
|
||||
@@ -22,6 +26,7 @@ public sealed class OtlpLogExportTests
|
||||
{
|
||||
private const string LogsPath = "/opentelemetry.proto.collector.logs.v1.LogsService/Export";
|
||||
private const string MetricsPath = "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export";
|
||||
private const string TracesPath = "/opentelemetry.proto.collector.trace.v1.TraceService/Export";
|
||||
|
||||
// --- the decision ---------------------------------------------------------------------
|
||||
|
||||
@@ -196,4 +201,93 @@ public sealed class OtlpLogExportTests
|
||||
|
||||
Assert.Empty(receiver.Calls);
|
||||
}
|
||||
|
||||
// --- scadaproj#56: telemetry's own OTLP transport is not traced --------------------------
|
||||
|
||||
private static async Task WaitForAsync(Func<bool> condition, int timeoutMs = 8000)
|
||||
{
|
||||
for (var waited = 0; waited < timeoutMs && !condition(); waited += 100)
|
||||
{
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Before 0.3.1 every log batch the Serilog OTLP sink sent produced a root
|
||||
/// <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The app's own
|
||||
/// outgoing gRPC and HTTP calls are the positive controls: they must still be traced, to the
|
||||
/// same receiver, in the same trace export.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Otlp_export_calls_are_not_traced_but_the_apps_own_calls_are()
|
||||
{
|
||||
await using var receiver = await FakeOtlpReceiver.StartAsync();
|
||||
|
||||
var builder = Host.CreateApplicationBuilder();
|
||||
builder.AddZbSerilog(o => o.ServiceName = "span-filter-test");
|
||||
builder.Services.AddZbTelemetry(o =>
|
||||
{
|
||||
o.ServiceName = "span-filter-test";
|
||||
o.Exporter = ZbExporter.Otlp;
|
||||
o.OtlpEndpoint = receiver.Endpoint;
|
||||
});
|
||||
|
||||
using (var host = builder.Build())
|
||||
{
|
||||
_ = host.Services.GetRequiredService<TracerProvider>();
|
||||
|
||||
// Positive control 1: an ordinary gRPC client call (ScadaBridge→site, HG/mxgw clients).
|
||||
using (var channel = GrpcChannel.ForAddress(receiver.Endpoint))
|
||||
{
|
||||
var bytes = Marshallers.Create(b => b, b => b);
|
||||
var ping = new Method<byte[], byte[]>(MethodType.Unary, "zb.test.v1.AppService", "Ping", bytes, bytes);
|
||||
await channel.CreateCallInvoker().AsyncUnaryCall(ping, null, new CallOptions(), []);
|
||||
}
|
||||
|
||||
using var http = new HttpClient
|
||||
{
|
||||
DefaultRequestVersion = HttpVersion.Version20,
|
||||
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact,
|
||||
};
|
||||
|
||||
// Positive control 2: an ordinary HttpClient call.
|
||||
(await http.PostAsync(receiver.Endpoint + "/zb-test/http-probe", new ByteArrayContent([]))).Dispose();
|
||||
|
||||
// HttpClient-side negatives: an OTLP/HTTP call to the configured endpoint, and an
|
||||
// OTLP/gRPC path sent through a plain HttpClient.
|
||||
(await http.PostAsync(receiver.Endpoint + "/v1/logs", new ByteArrayContent([]))).Dispose();
|
||||
(await http.PostAsync(receiver.Endpoint + "/opentelemetry.proto.collector.zbprobe.v1.ProbeService/Export",
|
||||
new ByteArrayContent([]))).Dispose();
|
||||
|
||||
// The gRPC-side negative: a real log batch from the Serilog OTLP sink, sent while the
|
||||
// tracer provider is still live so its span (if any) is in the final trace export.
|
||||
host.Services.GetRequiredService<global::Serilog.ILogger>().Information("span filter {Marker}", "log-line-56");
|
||||
await WaitForAsync(() => receiver.Received(LogsPath));
|
||||
Assert.True(receiver.Received(LogsPath), "the log sink never exported, so the negative proves nothing");
|
||||
await Task.Delay(500); // let the export call's activity stop before the final trace flush
|
||||
}
|
||||
|
||||
await WaitForAsync(() => receiver.BodiesOf(TracesPath).Contains("zb.test.v1.AppService/Ping"));
|
||||
var spans = receiver.BodiesOf(TracesPath);
|
||||
|
||||
// The loopback receiver's own ASP.NET Core server spans land in the same export (the
|
||||
// instrumentation listens process-wide) and carry url.path, so the assertions key on
|
||||
// client-span markers only: a span *name* for gRPC client spans, url.full for HttpClient.
|
||||
Assert.True(HasSpanNamed(spans, "zb.test.v1.AppService/Ping"), "positive control: the app's gRPC call was not traced");
|
||||
Assert.Contains(receiver.Endpoint + "/zb-test/http-probe", spans);
|
||||
Assert.False(HasSpanNamed(spans, "opentelemetry.proto.collector.logs.v1.LogsService/Export"),
|
||||
"the Serilog OTLP sink's LogsService/Export call was traced");
|
||||
Assert.DoesNotContain(receiver.Endpoint + "/v1/logs", spans);
|
||||
Assert.DoesNotContain(receiver.Endpoint + "/opentelemetry.proto.collector.", spans);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when an OTLP trace export body carries a span with this exact name: protobuf
|
||||
/// <c>Span.name</c> is field 5 (tag byte 0x2A) followed by the length, then the UTF-8 name.
|
||||
/// </summary>
|
||||
private static bool HasSpanNamed(string bodies, string name)
|
||||
{
|
||||
Assert.InRange(name.Length, 1, 127); // single-byte varint length
|
||||
return bodies.Contains("\u002A" + (char)name.Length + name, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.Telemetry.Tests;
|
||||
|
||||
/// <summary>scadaproj#56: which outgoing calls count as telemetry's own OTLP transport.</summary>
|
||||
public sealed class OtlpExportCallFilterTests
|
||||
{
|
||||
private static readonly Uri Endpoint = new("http://collector:4317");
|
||||
|
||||
private static bool IsExport(string url, Uri? endpoint = null) =>
|
||||
ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(HttpMethod.Post, url), endpoint ?? Endpoint);
|
||||
|
||||
[Theory]
|
||||
[InlineData("http://collector:4317/opentelemetry.proto.collector.logs.v1.LogsService/Export")]
|
||||
[InlineData("http://collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export")]
|
||||
[InlineData("http://collector:4317/opentelemetry.proto.collector.metrics.v1.MetricsService/Export")]
|
||||
// OTLP/gRPC is recognised on any host: a sink pointed somewhere else is still transport.
|
||||
[InlineData("http://other-collector:9999/opentelemetry.proto.collector.logs.v1.LogsService/Export")]
|
||||
// OTLP/HTTP to the configured endpoint.
|
||||
[InlineData("http://collector:4317/v1/logs")]
|
||||
[InlineData("http://COLLECTOR:4317/v1/traces")]
|
||||
[InlineData("http://collector:4317/otlp/v1/metrics")]
|
||||
public void Otlp_export_calls_are_recognised(string url) => Assert.True(IsExport(url));
|
||||
|
||||
[Theory]
|
||||
// The apps' own outgoing calls (ScadaBridge→site, HistorianGateway/mxgw clients, REST).
|
||||
[InlineData("http://site-a:8083/scadabridge.centralcontrol.v1.CentralControlService/Heartbeat")]
|
||||
[InlineData("http://historian:32565/HistoryService/GetInterfaceVersion")]
|
||||
[InlineData("http://collector:4317/mxaccess.v1.MxAccessGateway/Read")]
|
||||
[InlineData("http://rest-api/api/orders")]
|
||||
// An /opentelemetry.proto.collector look-alike that is not the namespace.
|
||||
[InlineData("http://collector:4317/opentelemetry.proto.collectorx/Export")]
|
||||
// /v1/logs on any other host or port is an ordinary REST call.
|
||||
[InlineData("http://other-host:4317/v1/logs")]
|
||||
[InlineData("http://collector:4318/v1/logs")]
|
||||
public void Ordinary_calls_are_not_export_calls(string url) => Assert.False(IsExport(url));
|
||||
|
||||
[Fact]
|
||||
public void Without_an_endpoint_only_the_grpc_namespace_matches()
|
||||
{
|
||||
Assert.True(ZbOtlpExportCallFilter.IsExportCall(
|
||||
new HttpRequestMessage(HttpMethod.Post, "http://x:1/opentelemetry.proto.collector.logs.v1.LogsService/Export"), null));
|
||||
Assert.False(ZbOtlpExportCallFilter.IsExportCall(
|
||||
new HttpRequestMessage(HttpMethod.Post, "http://collector:4317/v1/logs"), null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_request_without_an_absolute_uri_is_not_an_export_call()
|
||||
{
|
||||
Assert.False(ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(), Endpoint));
|
||||
Assert.False(ZbOtlpExportCallFilter.IsExportCall(
|
||||
new HttpRequestMessage(HttpMethod.Post, new Uri("/v1/logs", UriKind.Relative)), Endpoint));
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,9 @@ public sealed class ResourceIdentityTests
|
||||
var expected = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion;
|
||||
|
||||
Assert.Equal(expected, ZbResource.VersionOf(assembly));
|
||||
Assert.StartsWith("0.3.0", ZbResource.VersionOf(assembly));
|
||||
// The package version (Directory.Build.props), not a fixed literal that every release must edit.
|
||||
var v = assembly.GetName().Version!;
|
||||
Assert.StartsWith($"{v.Major}.{v.Minor}.{v.Build}+", ZbResource.VersionOf(assembly));
|
||||
Assert.Null(ZbResource.VersionOf(null));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user