diff --git a/ZB.MOM.WW.Telemetry/Directory.Build.props b/ZB.MOM.WW.Telemetry/Directory.Build.props index 3ce42a8..7a8443c 100644 --- a/ZB.MOM.WW.Telemetry/Directory.Build.props +++ b/ZB.MOM.WW.Telemetry/Directory.Build.props @@ -5,7 +5,7 @@ enable enable latest - 0.3.0 + 0.3.1 true diff --git a/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbOtlpExportCallFilter.cs b/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbOtlpExportCallFilter.cs new file mode 100644 index 0000000..6c8579d --- /dev/null +++ b/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbOtlpExportCallFilter.cs @@ -0,0 +1,66 @@ +using System.Diagnostics; + +namespace ZB.MOM.WW.Telemetry; + +/// +/// Recognises the outgoing calls an OTLP exporter makes, so that the gRPC-client and HttpClient +/// instrumentation do not trace telemetry's own transport (scadaproj#56). +/// +/// +/// The OpenTelemetry SDK's own OTLP exporters run inside SuppressInstrumentationScope, but +/// Serilog.Sinks.OpenTelemetry is not part of the SDK and does not, so every log batch it sent +/// produced a root opentelemetry.proto.collector.logs.v1.LogsService/Export client span. The +/// match is on the call itself, not on who made it: any OTLP/gRPC call (path +/// /opentelemetry.proto.collector.*, a namespace reserved to the OTLP collector services) and +/// any OTLP/HTTP call (/v1/traces, /v1/metrics, /v1/logs) to the configured +/// 's host and port. Every other outgoing call keeps its span. +/// +public static class ZbOtlpExportCallFilter +{ + /// The path prefix every OTLP/gRPC collector service shares. + public const string GrpcPathPrefix = "/opentelemetry.proto.collector."; + + private static readonly string[] HttpSignalPaths = ["/v1/traces", "/v1/metrics", "/v1/logs"]; + + /// + /// when is an OTLP export call: an OTLP/gRPC + /// collector path on any host, or an OTLP/HTTP signal path on 's + /// host and port. + /// + /// The outgoing request. + /// The configured OTLP endpoint, or . + public static bool IsExportCall(HttpRequestMessage request, Uri? otlpEndpoint) + { + ArgumentNullException.ThrowIfNull(request); + var uri = request.RequestUri; + if (uri is null || !uri.IsAbsoluteUri) + { + return false; + } + + var path = uri.AbsolutePath; + if (path.StartsWith(GrpcPathPrefix, StringComparison.Ordinal)) + { + return true; + } + + return otlpEndpoint is not null + && string.Equals(uri.Host, otlpEndpoint.Host, StringComparison.OrdinalIgnoreCase) + && uri.Port == otlpEndpoint.Port + && HttpSignalPaths.Any(p => path.EndsWith(p, StringComparison.Ordinal)); + } + + /// + /// Stops from being recorded or exported. Clearing the + /// flag at start also leaves any child activity (the + /// HttpClient request under a gRPC call) unsampled under the default parent-based sampler. + /// + internal static void Unrecord(Activity activity) + { + activity.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded; + activity.IsAllDataRequested = false; + } + + internal static Uri? ParseEndpoint(ZbTelemetryOptions options) => + Uri.TryCreate(options.OtlpEndpoint, UriKind.Absolute, out var uri) ? uri : null; +} diff --git a/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbTelemetryExtensions.cs b/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbTelemetryExtensions.cs index 997d241..da0a06d 100644 --- a/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbTelemetryExtensions.cs +++ b/ZB.MOM.WW.Telemetry/src/ZB.MOM.WW.Telemetry/ZbTelemetryExtensions.cs @@ -77,10 +77,25 @@ public static class ZbTelemetryExtensions tracing.AddSource(source); } + // Telemetry's own OTLP transport is not traced (scadaproj#56): the SDK exporters + // suppress their own instrumentation, the Serilog OTLP log sink does not. The gRPC + // client instrumentation has no Filter option, so its start-time enrich callback + // un-records the activity instead; it runs before the HttpClient request starts, + // so the child HTTP span is left unsampled too. + var otlpEndpoint = ZbOtlpExportCallFilter.ParseEndpoint(options); tracing .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation(); + .AddHttpClientInstrumentation(o => + o.FilterHttpRequestMessage = request => + !ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint)) + .AddGrpcClientInstrumentation(o => + o.EnrichWithHttpRequestMessage = (activity, request) => + { + if (ZbOtlpExportCallFilter.IsExportCall(request, otlpEndpoint)) + { + ZbOtlpExportCallFilter.Unrecord(activity); + } + }); ApplyTracingExporter(tracing, options); }); diff --git a/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Serilog.Tests/OtlpLogExportTests.cs b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Serilog.Tests/OtlpLogExportTests.cs index c7770d8..b81cf4c 100644 --- a/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Serilog.Tests/OtlpLogExportTests.cs +++ b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Serilog.Tests/OtlpLogExportTests.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Net; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -7,6 +8,9 @@ using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Grpc.Core; +using Grpc.Net.Client; +using OpenTelemetry.Trace; using Serilog; using ZB.MOM.WW.Telemetry; using ZB.MOM.WW.Telemetry.Serilog; @@ -22,6 +26,7 @@ public sealed class OtlpLogExportTests { private const string LogsPath = "/opentelemetry.proto.collector.logs.v1.LogsService/Export"; private const string MetricsPath = "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export"; + private const string TracesPath = "/opentelemetry.proto.collector.trace.v1.TraceService/Export"; // --- the decision --------------------------------------------------------------------- @@ -196,4 +201,93 @@ public sealed class OtlpLogExportTests Assert.Empty(receiver.Calls); } + + // --- scadaproj#56: telemetry's own OTLP transport is not traced -------------------------- + + private static async Task WaitForAsync(Func condition, int timeoutMs = 8000) + { + for (var waited = 0; waited < timeoutMs && !condition(); waited += 100) + { + await Task.Delay(100); + } + } + + /// + /// Before 0.3.1 every log batch the Serilog OTLP sink sent produced a root + /// opentelemetry.proto.collector.logs.v1.LogsService/Export client span. The app's own + /// outgoing gRPC and HTTP calls are the positive controls: they must still be traced, to the + /// same receiver, in the same trace export. + /// + [Fact] + public async Task Otlp_export_calls_are_not_traced_but_the_apps_own_calls_are() + { + await using var receiver = await FakeOtlpReceiver.StartAsync(); + + var builder = Host.CreateApplicationBuilder(); + builder.AddZbSerilog(o => o.ServiceName = "span-filter-test"); + builder.Services.AddZbTelemetry(o => + { + o.ServiceName = "span-filter-test"; + o.Exporter = ZbExporter.Otlp; + o.OtlpEndpoint = receiver.Endpoint; + }); + + using (var host = builder.Build()) + { + _ = host.Services.GetRequiredService(); + + // Positive control 1: an ordinary gRPC client call (ScadaBridge→site, HG/mxgw clients). + using (var channel = GrpcChannel.ForAddress(receiver.Endpoint)) + { + var bytes = Marshallers.Create(b => b, b => b); + var ping = new Method(MethodType.Unary, "zb.test.v1.AppService", "Ping", bytes, bytes); + await channel.CreateCallInvoker().AsyncUnaryCall(ping, null, new CallOptions(), []); + } + + using var http = new HttpClient + { + DefaultRequestVersion = HttpVersion.Version20, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + }; + + // Positive control 2: an ordinary HttpClient call. + (await http.PostAsync(receiver.Endpoint + "/zb-test/http-probe", new ByteArrayContent([]))).Dispose(); + + // HttpClient-side negatives: an OTLP/HTTP call to the configured endpoint, and an + // OTLP/gRPC path sent through a plain HttpClient. + (await http.PostAsync(receiver.Endpoint + "/v1/logs", new ByteArrayContent([]))).Dispose(); + (await http.PostAsync(receiver.Endpoint + "/opentelemetry.proto.collector.zbprobe.v1.ProbeService/Export", + new ByteArrayContent([]))).Dispose(); + + // The gRPC-side negative: a real log batch from the Serilog OTLP sink, sent while the + // tracer provider is still live so its span (if any) is in the final trace export. + host.Services.GetRequiredService().Information("span filter {Marker}", "log-line-56"); + await WaitForAsync(() => receiver.Received(LogsPath)); + Assert.True(receiver.Received(LogsPath), "the log sink never exported, so the negative proves nothing"); + await Task.Delay(500); // let the export call's activity stop before the final trace flush + } + + await WaitForAsync(() => receiver.BodiesOf(TracesPath).Contains("zb.test.v1.AppService/Ping")); + var spans = receiver.BodiesOf(TracesPath); + + // The loopback receiver's own ASP.NET Core server spans land in the same export (the + // instrumentation listens process-wide) and carry url.path, so the assertions key on + // client-span markers only: a span *name* for gRPC client spans, url.full for HttpClient. + Assert.True(HasSpanNamed(spans, "zb.test.v1.AppService/Ping"), "positive control: the app's gRPC call was not traced"); + Assert.Contains(receiver.Endpoint + "/zb-test/http-probe", spans); + Assert.False(HasSpanNamed(spans, "opentelemetry.proto.collector.logs.v1.LogsService/Export"), + "the Serilog OTLP sink's LogsService/Export call was traced"); + Assert.DoesNotContain(receiver.Endpoint + "/v1/logs", spans); + Assert.DoesNotContain(receiver.Endpoint + "/opentelemetry.proto.collector.", spans); + } + + /// + /// True when an OTLP trace export body carries a span with this exact name: protobuf + /// Span.name is field 5 (tag byte 0x2A) followed by the length, then the UTF-8 name. + /// + private static bool HasSpanNamed(string bodies, string name) + { + Assert.InRange(name.Length, 1, 127); // single-byte varint length + return bodies.Contains("\u002A" + (char)name.Length + name, StringComparison.Ordinal); + } } diff --git a/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/OtlpExportCallFilterTests.cs b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/OtlpExportCallFilterTests.cs new file mode 100644 index 0000000..d4daa60 --- /dev/null +++ b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/OtlpExportCallFilterTests.cs @@ -0,0 +1,54 @@ +using ZB.MOM.WW.Telemetry; + +namespace ZB.MOM.WW.Telemetry.Tests; + +/// scadaproj#56: which outgoing calls count as telemetry's own OTLP transport. +public sealed class OtlpExportCallFilterTests +{ + private static readonly Uri Endpoint = new("http://collector:4317"); + + private static bool IsExport(string url, Uri? endpoint = null) => + ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(HttpMethod.Post, url), endpoint ?? Endpoint); + + [Theory] + [InlineData("http://collector:4317/opentelemetry.proto.collector.logs.v1.LogsService/Export")] + [InlineData("http://collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export")] + [InlineData("http://collector:4317/opentelemetry.proto.collector.metrics.v1.MetricsService/Export")] + // OTLP/gRPC is recognised on any host: a sink pointed somewhere else is still transport. + [InlineData("http://other-collector:9999/opentelemetry.proto.collector.logs.v1.LogsService/Export")] + // OTLP/HTTP to the configured endpoint. + [InlineData("http://collector:4317/v1/logs")] + [InlineData("http://COLLECTOR:4317/v1/traces")] + [InlineData("http://collector:4317/otlp/v1/metrics")] + public void Otlp_export_calls_are_recognised(string url) => Assert.True(IsExport(url)); + + [Theory] + // The apps' own outgoing calls (ScadaBridge→site, HistorianGateway/mxgw clients, REST). + [InlineData("http://site-a:8083/scadabridge.centralcontrol.v1.CentralControlService/Heartbeat")] + [InlineData("http://historian:32565/HistoryService/GetInterfaceVersion")] + [InlineData("http://collector:4317/mxaccess.v1.MxAccessGateway/Read")] + [InlineData("http://rest-api/api/orders")] + // An /opentelemetry.proto.collector look-alike that is not the namespace. + [InlineData("http://collector:4317/opentelemetry.proto.collectorx/Export")] + // /v1/logs on any other host or port is an ordinary REST call. + [InlineData("http://other-host:4317/v1/logs")] + [InlineData("http://collector:4318/v1/logs")] + public void Ordinary_calls_are_not_export_calls(string url) => Assert.False(IsExport(url)); + + [Fact] + public void Without_an_endpoint_only_the_grpc_namespace_matches() + { + Assert.True(ZbOtlpExportCallFilter.IsExportCall( + new HttpRequestMessage(HttpMethod.Post, "http://x:1/opentelemetry.proto.collector.logs.v1.LogsService/Export"), null)); + Assert.False(ZbOtlpExportCallFilter.IsExportCall( + new HttpRequestMessage(HttpMethod.Post, "http://collector:4317/v1/logs"), null)); + } + + [Fact] + public void A_request_without_an_absolute_uri_is_not_an_export_call() + { + Assert.False(ZbOtlpExportCallFilter.IsExportCall(new HttpRequestMessage(), Endpoint)); + Assert.False(ZbOtlpExportCallFilter.IsExportCall( + new HttpRequestMessage(HttpMethod.Post, new Uri("/v1/logs", UriKind.Relative)), Endpoint)); + } +} diff --git a/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/ResourceIdentityTests.cs b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/ResourceIdentityTests.cs index c8cfd4a..ff0b323 100644 --- a/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/ResourceIdentityTests.cs +++ b/ZB.MOM.WW.Telemetry/tests/ZB.MOM.WW.Telemetry.Tests/ResourceIdentityTests.cs @@ -35,7 +35,9 @@ public sealed class ResourceIdentityTests var expected = assembly.GetCustomAttribute()!.InformationalVersion; Assert.Equal(expected, ZbResource.VersionOf(assembly)); - Assert.StartsWith("0.3.0", ZbResource.VersionOf(assembly)); + // The package version (Directory.Build.props), not a fixed literal that every release must edit. + var v = assembly.GetName().Version!; + Assert.StartsWith($"{v.Major}.{v.Minor}.{v.Build}+", ZbResource.VersionOf(assembly)); Assert.Null(ZbResource.VersionOf(null)); }