using System.Collections.Concurrent; using System.Net; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Grpc.Core; using Grpc.Net.Client; using OpenTelemetry.Trace; using Serilog; using ZB.MOM.WW.Telemetry; using ZB.MOM.WW.Telemetry.Serilog; namespace ZB.MOM.WW.Telemetry.Serilog.Tests; /// /// scadaproj#55: logs follow AddZbTelemetry's OTLP exporter. Before 0.3.0 the sink keyed only /// on the options passed to AddZbSerilog, which every app filled with the service name alone, /// so no app ever exported a log record. /// public sealed class OtlpLogExportTests { private const string LogsPath = "/opentelemetry.proto.collector.logs.v1.LogsService/Export"; private const string MetricsPath = "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export"; private const string TracesPath = "/opentelemetry.proto.collector.trace.v1.TraceService/Export"; // --- the decision --------------------------------------------------------------------- private static ZbTelemetryOptions Otlp(string name, bool exportLogs = true) => new() { ServiceName = name, Exporter = ZbExporter.Otlp, OtlpEndpoint = "http://collector:4317", ExportLogs = exportLogs, }; [Fact] public void Serilog_options_selecting_Otlp_win() { var own = Otlp("own"); Assert.Same(own, ZbSerilogConfig.ResolveLogExport(own, Otlp("registered"))); } [Fact] public void Registered_telemetry_options_are_followed_when_Serilog_options_do_not_select_Otlp() { var registered = Otlp("registered"); Assert.Same(registered, ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, registered)); } [Fact] public void No_sink_when_nothing_selects_Otlp() { Assert.Null(ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, null)); Assert.Null(ZbSerilogConfig.ResolveLogExport( new ZbTelemetryOptions { ServiceName = "own" }, new ZbTelemetryOptions { ServiceName = "registered", OtlpEndpoint = "http://collector:4317" })); } [Fact] public void ExportLogs_false_keeps_logs_local_on_either_path() { Assert.Null(ZbSerilogConfig.ResolveLogExport(new ZbTelemetryOptions { ServiceName = "own" }, Otlp("registered", exportLogs: false))); Assert.Null(ZbSerilogConfig.ResolveLogExport(Otlp("own", exportLogs: false), null)); } // --- end to end against a loopback OTLP gRPC receiver ----------------------------------- /// Minimal OTLP/gRPC receiver: records each export call's path and body, answers OK. private sealed class FakeOtlpReceiver : IAsyncDisposable { private readonly WebApplication _app; public ConcurrentBag<(string Path, byte[] Body)> Calls { get; } = new(); public string Endpoint { get; } private FakeOtlpReceiver(WebApplication app, string endpoint) { _app = app; Endpoint = endpoint; } public static async Task StartAsync() { var builder = WebApplication.CreateSlimBuilder(); builder.Logging.ClearProviders(); builder.WebHost.ConfigureKestrel(k => k.Listen(System.Net.IPAddress.Loopback, 0, l => l.Protocols = HttpProtocols.Http2)); var app = builder.Build(); FakeOtlpReceiver? receiver = null; app.Run(async context => { using var buffer = new MemoryStream(); await context.Request.Body.CopyToAsync(buffer); receiver!.Calls.Add((context.Request.Path.Value ?? "", buffer.ToArray())); context.Response.ContentType = "application/grpc"; context.Response.AppendTrailer("grpc-status", "0"); // One empty, uncompressed gRPC message (the Export*ServiceResponse). await context.Response.Body.WriteAsync(new byte[5]); }); await app.StartAsync(); var address = app.Urls.Single().Replace("[::1]", "127.0.0.1"); receiver = new FakeOtlpReceiver(app, address); return receiver; } public bool Received(string path) => Calls.Any(c => c.Path == path); public string BodiesOf(string path) => string.Join("\n", Calls.Where(c => c.Path == path).Select(c => Encoding.UTF8.GetString(c.Body))); public async ValueTask DisposeAsync() { await _app.StopAsync(); await _app.DisposeAsync(); } } /// /// The app shape every family app uses: AddZbSerilog with the service name only, and the /// exporter set on AddZbTelemetry. Disposing the host flushes the log batch and the /// final metric export. /// private static async Task RunAppAsync(FakeOtlpReceiver receiver, ZbExporter exporter, bool exportLogs) { var builder = Host.CreateApplicationBuilder(); builder.AddZbSerilog(o => o.ServiceName = "log-export-test"); builder.Services.AddZbTelemetry(o => { o.ServiceName = "log-export-test"; o.ServiceVersion = "9.8.7+e2e"; o.SiteId = "site-e2e"; o.NodeRole = "role-e2e"; o.ResourceAttributes["zb.test.marker"] = "marker-e2e"; o.Exporter = exporter; o.OtlpEndpoint = receiver.Endpoint; o.ExportLogs = exportLogs; }); using (var host = builder.Build()) { // Resolve the meter provider so a metrics export can serve as the positive control. _ = host.Services.GetService(); host.Services.GetRequiredService().Information("hello over otlp {Marker}", "log-line-e2e"); } // The sink's batch and the metric reader flush on dispose; allow the h2c round-trips to land. for (var i = 0; i < 50 && !receiver.Received(exporter == ZbExporter.Otlp ? MetricsPath : LogsPath); i++) { await Task.Delay(100); } } [Fact] public async Task Logs_follow_AddZbTelemetry_Otlp_with_the_same_Resource() { await using var receiver = await FakeOtlpReceiver.StartAsync(); await RunAppAsync(receiver, ZbExporter.Otlp, exportLogs: true); for (var i = 0; i < 50 && !receiver.Received(LogsPath); i++) { await Task.Delay(100); } Assert.True(receiver.Received(LogsPath), "no OTLP logs export call arrived; paths: " + string.Join(", ", receiver.Calls.Select(c => c.Path).Distinct())); var body = receiver.BodiesOf(LogsPath); Assert.Contains("log-line-e2e", body); Assert.Contains("log-export-test", body); Assert.Contains("9.8.7+e2e", body); Assert.Contains("site-e2e", body); Assert.Contains("role-e2e", body); Assert.Contains("marker-e2e", body); } [Fact] public async Task ExportLogs_false_sends_metrics_but_no_logs() { await using var receiver = await FakeOtlpReceiver.StartAsync(); await RunAppAsync(receiver, ZbExporter.Otlp, exportLogs: false); await Task.Delay(500); // Positive control: the same receiver, reachable, got the metrics export. Assert.True(receiver.Received(MetricsPath), "positive control failed: no metrics export arrived"); Assert.False(receiver.Received(LogsPath)); } [Fact] public async Task Prometheus_exporter_sends_no_logs() { await using var receiver = await FakeOtlpReceiver.StartAsync(); await RunAppAsync(receiver, ZbExporter.Prometheus, exportLogs: true); await Task.Delay(500); Assert.Empty(receiver.Calls); } // --- scadaproj#56: telemetry's own OTLP transport is not traced -------------------------- private static async Task WaitForAsync(Func condition, int timeoutMs = 8000) { for (var waited = 0; waited < timeoutMs && !condition(); waited += 100) { await Task.Delay(100); } } /// /// Before 0.3.1 every log batch the Serilog OTLP sink sent produced a root /// opentelemetry.proto.collector.logs.v1.LogsService/Export client span. The app's own /// outgoing gRPC and HTTP calls are the positive controls: they must still be traced, to the /// same receiver, in the same trace export. /// [Fact] public async Task Otlp_export_calls_are_not_traced_but_the_apps_own_calls_are() { await using var receiver = await FakeOtlpReceiver.StartAsync(); var builder = Host.CreateApplicationBuilder(); builder.AddZbSerilog(o => o.ServiceName = "span-filter-test"); builder.Services.AddZbTelemetry(o => { o.ServiceName = "span-filter-test"; o.Exporter = ZbExporter.Otlp; o.OtlpEndpoint = receiver.Endpoint; }); using (var host = builder.Build()) { _ = host.Services.GetRequiredService(); // Positive control 1: an ordinary gRPC client call (ScadaBridge→site, HG/mxgw clients). using (var channel = GrpcChannel.ForAddress(receiver.Endpoint)) { var bytes = Marshallers.Create(b => b, b => b); var ping = new Method(MethodType.Unary, "zb.test.v1.AppService", "Ping", bytes, bytes); await channel.CreateCallInvoker().AsyncUnaryCall(ping, null, new CallOptions(), []); } using var http = new HttpClient { DefaultRequestVersion = HttpVersion.Version20, DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, }; // Positive control 2: an ordinary HttpClient call. (await http.PostAsync(receiver.Endpoint + "/zb-test/http-probe", new ByteArrayContent([]))).Dispose(); // HttpClient-side negatives: an OTLP/HTTP call to the configured endpoint, and an // OTLP/gRPC path sent through a plain HttpClient. (await http.PostAsync(receiver.Endpoint + "/v1/logs", new ByteArrayContent([]))).Dispose(); (await http.PostAsync(receiver.Endpoint + "/opentelemetry.proto.collector.zbprobe.v1.ProbeService/Export", new ByteArrayContent([]))).Dispose(); // The gRPC-side negative: a real log batch from the Serilog OTLP sink, sent while the // tracer provider is still live so its span (if any) is in the final trace export. host.Services.GetRequiredService().Information("span filter {Marker}", "log-line-56"); await WaitForAsync(() => receiver.Received(LogsPath)); Assert.True(receiver.Received(LogsPath), "the log sink never exported, so the negative proves nothing"); await Task.Delay(500); // let the export call's activity stop before the final trace flush } await WaitForAsync(() => receiver.BodiesOf(TracesPath).Contains("zb.test.v1.AppService/Ping")); var spans = receiver.BodiesOf(TracesPath); // The loopback receiver's own ASP.NET Core server spans land in the same export (the // instrumentation listens process-wide) and carry url.path, so the assertions key on // client-span markers only: a span *name* for gRPC client spans, url.full for HttpClient. Assert.True(HasSpanNamed(spans, "zb.test.v1.AppService/Ping"), "positive control: the app's gRPC call was not traced"); Assert.Contains(receiver.Endpoint + "/zb-test/http-probe", spans); Assert.False(HasSpanNamed(spans, "opentelemetry.proto.collector.logs.v1.LogsService/Export"), "the Serilog OTLP sink's LogsService/Export call was traced"); Assert.DoesNotContain(receiver.Endpoint + "/v1/logs", spans); Assert.DoesNotContain(receiver.Endpoint + "/opentelemetry.proto.collector.", spans); } /// /// True when an OTLP trace export body carries a span with this exact name: protobuf /// Span.name is field 5 (tag byte 0x2A) followed by the length, then the UTF-8 name. /// private static bool HasSpanNamed(string bodies, string name) { Assert.InRange(name.Length, 1, 127); // single-byte varint length return bodies.Contains("\u002A" + (char)name.Length + name, StringComparison.Ordinal); } }