85654d7b43
The Serilog OTLP log sink is not part of the OTel SDK and does not run
inside SuppressInstrumentationScope, so every log batch produced a root
opentelemetry.proto.collector.logs.v1.LogsService/Export gRPC client span.
AddZbTelemetry now drops telemetry's own transport from tracing:
- HttpClient instrumentation: FilterHttpRequestMessage rejects export calls
- gRPC client instrumentation (no Filter option): the start-time
EnrichWithHttpRequestMessage callback clears Recorded/IsAllDataRequested,
which also leaves the child HTTP activity unsampled.
An export call is any /opentelemetry.proto.collector.* path, or an
OTLP/HTTP /v1/{traces,metrics,logs} path on the configured endpoint's
host:port (ZbOtlpExportCallFilter). Every other outgoing call keeps its span.
Tests: predicate theory + an end-to-end run against a loopback OTLP
receiver with positive controls (app gRPC + HttpClient calls traced) and
negatives (the real log-sink export, /v1/logs, gRPC path over HttpClient);
mutation-checked three ways.
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
294 lines
13 KiB
C#
294 lines
13 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Net;
|
|
using System.Text;
|
|
using Microsoft.AspNetCore.Builder;
|
|
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.Http;
|
|
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using Grpc.Core;
|
|
using Grpc.Net.Client;
|
|
using OpenTelemetry.Trace;
|
|
using Serilog;
|
|
using ZB.MOM.WW.Telemetry;
|
|
using ZB.MOM.WW.Telemetry.Serilog;
|
|
|
|
namespace ZB.MOM.WW.Telemetry.Serilog.Tests;
|
|
|
|
/// <summary>
|
|
/// scadaproj#55: logs follow <c>AddZbTelemetry</c>'s OTLP exporter. Before 0.3.0 the sink keyed only
|
|
/// on the options passed to <c>AddZbSerilog</c>, which every app filled with the service name alone,
|
|
/// so no app ever exported a log record.
|
|
/// </summary>
|
|
public sealed class OtlpLogExportTests
|
|
{
|
|
private const string LogsPath = "/opentelemetry.proto.collector.logs.v1.LogsService/Export";
|
|
private const string MetricsPath = "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export";
|
|
private const string TracesPath = "/opentelemetry.proto.collector.trace.v1.TraceService/Export";
|
|
|
|
// --- the decision ---------------------------------------------------------------------
|
|
|
|
private static ZbTelemetryOptions Otlp(string name, bool exportLogs = true) => new()
|
|
{
|
|
ServiceName = name,
|
|
Exporter = ZbExporter.Otlp,
|
|
OtlpEndpoint = "http://collector:4317",
|
|
ExportLogs = exportLogs,
|
|
};
|
|
|
|
[Fact]
|
|
public void Serilog_options_selecting_Otlp_win()
|
|
{
|
|
var own = Otlp("own");
|
|
Assert.Same(own, ZbSerilogConfig.ResolveLogExport(own, Otlp("registered")));
|
|
}
|
|
|
|
[Fact]
|
|
public void Registered_telemetry_options_are_followed_when_Serilog_options_do_not_select_Otlp()
|
|
{
|
|
var registered = Otlp("registered");
|
|
Assert.Same(registered, ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, registered));
|
|
}
|
|
|
|
[Fact]
|
|
public void No_sink_when_nothing_selects_Otlp()
|
|
{
|
|
Assert.Null(ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, null));
|
|
Assert.Null(ZbSerilogConfig.ResolveLogExport(
|
|
new ZbTelemetryOptions { ServiceName = "own" },
|
|
new ZbTelemetryOptions { ServiceName = "registered", OtlpEndpoint = "http://collector:4317" }));
|
|
}
|
|
|
|
[Fact]
|
|
public void ExportLogs_false_keeps_logs_local_on_either_path()
|
|
{
|
|
Assert.Null(ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, Otlp("registered", exportLogs: false)));
|
|
Assert.Null(ZbSerilogConfig.ResolveLogExport(Otlp("own", exportLogs: false), null));
|
|
}
|
|
|
|
// --- end to end against a loopback OTLP gRPC receiver -----------------------------------
|
|
|
|
/// <summary>Minimal OTLP/gRPC receiver: records each export call's path and body, answers OK.</summary>
|
|
private sealed class FakeOtlpReceiver : IAsyncDisposable
|
|
{
|
|
private readonly WebApplication _app;
|
|
|
|
public ConcurrentBag<(string Path, byte[] Body)> Calls { get; } = new();
|
|
|
|
public string Endpoint { get; }
|
|
|
|
private FakeOtlpReceiver(WebApplication app, string endpoint)
|
|
{
|
|
_app = app;
|
|
Endpoint = endpoint;
|
|
}
|
|
|
|
public static async Task<FakeOtlpReceiver> StartAsync()
|
|
{
|
|
var builder = WebApplication.CreateSlimBuilder();
|
|
builder.Logging.ClearProviders();
|
|
builder.WebHost.ConfigureKestrel(k => k.Listen(System.Net.IPAddress.Loopback, 0,
|
|
l => l.Protocols = HttpProtocols.Http2));
|
|
var app = builder.Build();
|
|
FakeOtlpReceiver? receiver = null;
|
|
app.Run(async context =>
|
|
{
|
|
using var buffer = new MemoryStream();
|
|
await context.Request.Body.CopyToAsync(buffer);
|
|
receiver!.Calls.Add((context.Request.Path.Value ?? "", buffer.ToArray()));
|
|
context.Response.ContentType = "application/grpc";
|
|
context.Response.AppendTrailer("grpc-status", "0");
|
|
// One empty, uncompressed gRPC message (the Export*ServiceResponse).
|
|
await context.Response.Body.WriteAsync(new byte[5]);
|
|
});
|
|
await app.StartAsync();
|
|
var address = app.Urls.Single().Replace("[::1]", "127.0.0.1");
|
|
receiver = new FakeOtlpReceiver(app, address);
|
|
return receiver;
|
|
}
|
|
|
|
public bool Received(string path) => Calls.Any(c => c.Path == path);
|
|
|
|
public string BodiesOf(string path) =>
|
|
string.Join("\n", Calls.Where(c => c.Path == path).Select(c => Encoding.UTF8.GetString(c.Body)));
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await _app.StopAsync();
|
|
await _app.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The app shape every family app uses: <c>AddZbSerilog</c> with the service name only, and the
|
|
/// exporter set on <c>AddZbTelemetry</c>. Disposing the host flushes the log batch and the
|
|
/// final metric export.
|
|
/// </summary>
|
|
private static async Task RunAppAsync(FakeOtlpReceiver receiver, ZbExporter exporter, bool exportLogs)
|
|
{
|
|
var builder = Host.CreateApplicationBuilder();
|
|
builder.AddZbSerilog(o => o.ServiceName = "log-export-test");
|
|
builder.Services.AddZbTelemetry(o =>
|
|
{
|
|
o.ServiceName = "log-export-test";
|
|
o.ServiceVersion = "9.8.7+e2e";
|
|
o.SiteId = "site-e2e";
|
|
o.NodeRole = "role-e2e";
|
|
o.ResourceAttributes["zb.test.marker"] = "marker-e2e";
|
|
o.Exporter = exporter;
|
|
o.OtlpEndpoint = receiver.Endpoint;
|
|
o.ExportLogs = exportLogs;
|
|
});
|
|
|
|
using (var host = builder.Build())
|
|
{
|
|
// Resolve the meter provider so a metrics export can serve as the positive control.
|
|
_ = host.Services.GetService<OpenTelemetry.Metrics.MeterProvider>();
|
|
host.Services.GetRequiredService<global::Serilog.ILogger>().Information("hello over otlp {Marker}", "log-line-e2e");
|
|
}
|
|
|
|
// The sink's batch and the metric reader flush on dispose; allow the h2c round-trips to land.
|
|
for (var i = 0; i < 50 && !receiver.Received(exporter == ZbExporter.Otlp ? MetricsPath : LogsPath); i++)
|
|
{
|
|
await Task.Delay(100);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Logs_follow_AddZbTelemetry_Otlp_with_the_same_Resource()
|
|
{
|
|
await using var receiver = await FakeOtlpReceiver.StartAsync();
|
|
|
|
await RunAppAsync(receiver, ZbExporter.Otlp, exportLogs: true);
|
|
for (var i = 0; i < 50 && !receiver.Received(LogsPath); i++)
|
|
{
|
|
await Task.Delay(100);
|
|
}
|
|
|
|
Assert.True(receiver.Received(LogsPath), "no OTLP logs export call arrived; paths: " +
|
|
string.Join(", ", receiver.Calls.Select(c => c.Path).Distinct()));
|
|
var body = receiver.BodiesOf(LogsPath);
|
|
Assert.Contains("log-line-e2e", body);
|
|
Assert.Contains("log-export-test", body);
|
|
Assert.Contains("9.8.7+e2e", body);
|
|
Assert.Contains("site-e2e", body);
|
|
Assert.Contains("role-e2e", body);
|
|
Assert.Contains("marker-e2e", body);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ExportLogs_false_sends_metrics_but_no_logs()
|
|
{
|
|
await using var receiver = await FakeOtlpReceiver.StartAsync();
|
|
|
|
await RunAppAsync(receiver, ZbExporter.Otlp, exportLogs: false);
|
|
await Task.Delay(500);
|
|
|
|
// Positive control: the same receiver, reachable, got the metrics export.
|
|
Assert.True(receiver.Received(MetricsPath), "positive control failed: no metrics export arrived");
|
|
Assert.False(receiver.Received(LogsPath));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Prometheus_exporter_sends_no_logs()
|
|
{
|
|
await using var receiver = await FakeOtlpReceiver.StartAsync();
|
|
|
|
await RunAppAsync(receiver, ZbExporter.Prometheus, exportLogs: true);
|
|
await Task.Delay(500);
|
|
|
|
Assert.Empty(receiver.Calls);
|
|
}
|
|
|
|
// --- scadaproj#56: telemetry's own OTLP transport is not traced --------------------------
|
|
|
|
private static async Task WaitForAsync(Func<bool> condition, int timeoutMs = 8000)
|
|
{
|
|
for (var waited = 0; waited < timeoutMs && !condition(); waited += 100)
|
|
{
|
|
await Task.Delay(100);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Before 0.3.1 every log batch the Serilog OTLP sink sent produced a root
|
|
/// <c>opentelemetry.proto.collector.logs.v1.LogsService/Export</c> client span. The app's own
|
|
/// outgoing gRPC and HTTP calls are the positive controls: they must still be traced, to the
|
|
/// same receiver, in the same trace export.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Otlp_export_calls_are_not_traced_but_the_apps_own_calls_are()
|
|
{
|
|
await using var receiver = await FakeOtlpReceiver.StartAsync();
|
|
|
|
var builder = Host.CreateApplicationBuilder();
|
|
builder.AddZbSerilog(o => o.ServiceName = "span-filter-test");
|
|
builder.Services.AddZbTelemetry(o =>
|
|
{
|
|
o.ServiceName = "span-filter-test";
|
|
o.Exporter = ZbExporter.Otlp;
|
|
o.OtlpEndpoint = receiver.Endpoint;
|
|
});
|
|
|
|
using (var host = builder.Build())
|
|
{
|
|
_ = host.Services.GetRequiredService<TracerProvider>();
|
|
|
|
// Positive control 1: an ordinary gRPC client call (ScadaBridge→site, HG/mxgw clients).
|
|
using (var channel = GrpcChannel.ForAddress(receiver.Endpoint))
|
|
{
|
|
var bytes = Marshallers.Create(b => b, b => b);
|
|
var ping = new Method<byte[], byte[]>(MethodType.Unary, "zb.test.v1.AppService", "Ping", bytes, bytes);
|
|
await channel.CreateCallInvoker().AsyncUnaryCall(ping, null, new CallOptions(), []);
|
|
}
|
|
|
|
using var http = new HttpClient
|
|
{
|
|
DefaultRequestVersion = HttpVersion.Version20,
|
|
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact,
|
|
};
|
|
|
|
// Positive control 2: an ordinary HttpClient call.
|
|
(await http.PostAsync(receiver.Endpoint + "/zb-test/http-probe", new ByteArrayContent([]))).Dispose();
|
|
|
|
// HttpClient-side negatives: an OTLP/HTTP call to the configured endpoint, and an
|
|
// OTLP/gRPC path sent through a plain HttpClient.
|
|
(await http.PostAsync(receiver.Endpoint + "/v1/logs", new ByteArrayContent([]))).Dispose();
|
|
(await http.PostAsync(receiver.Endpoint + "/opentelemetry.proto.collector.zbprobe.v1.ProbeService/Export",
|
|
new ByteArrayContent([]))).Dispose();
|
|
|
|
// The gRPC-side negative: a real log batch from the Serilog OTLP sink, sent while the
|
|
// tracer provider is still live so its span (if any) is in the final trace export.
|
|
host.Services.GetRequiredService<global::Serilog.ILogger>().Information("span filter {Marker}", "log-line-56");
|
|
await WaitForAsync(() => receiver.Received(LogsPath));
|
|
Assert.True(receiver.Received(LogsPath), "the log sink never exported, so the negative proves nothing");
|
|
await Task.Delay(500); // let the export call's activity stop before the final trace flush
|
|
}
|
|
|
|
await WaitForAsync(() => receiver.BodiesOf(TracesPath).Contains("zb.test.v1.AppService/Ping"));
|
|
var spans = receiver.BodiesOf(TracesPath);
|
|
|
|
// The loopback receiver's own ASP.NET Core server spans land in the same export (the
|
|
// instrumentation listens process-wide) and carry url.path, so the assertions key on
|
|
// client-span markers only: a span *name* for gRPC client spans, url.full for HttpClient.
|
|
Assert.True(HasSpanNamed(spans, "zb.test.v1.AppService/Ping"), "positive control: the app's gRPC call was not traced");
|
|
Assert.Contains(receiver.Endpoint + "/zb-test/http-probe", spans);
|
|
Assert.False(HasSpanNamed(spans, "opentelemetry.proto.collector.logs.v1.LogsService/Export"),
|
|
"the Serilog OTLP sink's LogsService/Export call was traced");
|
|
Assert.DoesNotContain(receiver.Endpoint + "/v1/logs", spans);
|
|
Assert.DoesNotContain(receiver.Endpoint + "/opentelemetry.proto.collector.", spans);
|
|
}
|
|
|
|
/// <summary>
|
|
/// True when an OTLP trace export body carries a span with this exact name: protobuf
|
|
/// <c>Span.name</c> is field 5 (tag byte 0x2A) followed by the length, then the UTF-8 name.
|
|
/// </summary>
|
|
private static bool HasSpanNamed(string bodies, string name)
|
|
{
|
|
Assert.InRange(name.Length, 1, 127); // single-byte varint length
|
|
return bodies.Contains("\u002A" + (char)name.Length + name, StringComparison.Ordinal);
|
|
}
|
|
}
|