test(opcua): add fixed candidate saturation diagnosis with observer barriers

2026-11-10 14:51:19 -05:00
parent ab35ddd3c2
commit 54a4ff2149
8 changed files with 833 additions and 13 deletions
@@ -0,0 +1,127 @@
+// Diagnostic-only observer. The target remains the unchanged canonical lifecycle test process.
+using System.Diagnostics.Tracing;
+using System.Text.Json;
+using Microsoft.Diagnostics.NETCore.Client;
+using Microsoft.Diagnostics.Tracing;
+
+const string protocol = "sdk2-saturation-v1";
+const string markerProvider = "OtOpcUa-Sdk2-Saturation";
+var control = Path.GetFullPath(args[0]);
+var runId = args[1];
+if (!Guid.TryParseExact(runId, "D", out _)) throw new InvalidDataException("Malformed run identity");
+using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(1980));
+var ct = lifetime.Token;
+try
+{
+    var request = await WaitJson("attach-request.json", ct);
+    using (request)
+    {
+        Identity(request.RootElement);
+        Require(request.RootElement.GetProperty("firstCycle").GetInt32() == 321
+            && request.RootElement.GetProperty("completedRestarts").GetInt32() == 80, "Invalid attach boundary");
+        var targetPid = request.RootElement.GetProperty("targetPid").GetInt32();
+        Require(targetPid > 0 && targetPid != Environment.ProcessId, "Invalid target process");
+        var providers = new[]
+        {
+            new EventPipeProvider("Microsoft-Windows-DotNETRuntime", EventLevel.Verbose, 0x18),
+            new EventPipeProvider(markerProvider, EventLevel.Informational)
+        };
+        var client = new DiagnosticsClient(targetPid);
+        var session = await Task.Run(() => client.StartEventPipeSession(providers, requestRundown: false, circularBufferMB: 16), ct)
+            .WaitAsync(TimeSpan.FromSeconds(10), ct);
+        var trace = Path.Combine(control, "readiness.nettrace");
+        await using (var output = new FileStream(trace, FileMode.CreateNew, FileAccess.Write, FileShare.Read))
+        {
+        var copy = session.EventStream.CopyToAsync(output, ct);
+        await Publish("active.json", new { protocol, runId, targetPid, observerPid = Environment.ProcessId, active = true });
+        using var stop = await WaitJson("detach-request.json", ct);
+        Identity(stop.RootElement);
+        Require(stop.RootElement.GetProperty("targetPid").GetInt32() == targetPid
+            && stop.RootElement.GetProperty("lastCycle").GetInt32() == 384
+            && stop.RootElement.GetProperty("completedRestarts").GetInt32() == 96, "Invalid detach boundary");
+        await Task.Run(session.Stop, ct).WaitAsync(TimeSpan.FromSeconds(10), ct);
+        await copy.WaitAsync(TimeSpan.FromSeconds(10), ct);
+        await output.FlushAsync(ct);
+        }
+        await Task.Run(session.Dispose, ct).WaitAsync(TimeSpan.FromSeconds(5), ct);
+        var summary = await Task.Run(() => Parse(trace, runId), ct).WaitAsync(TimeSpan.FromSeconds(10), ct);
+        // Published only after Stop, stream drain, disposal and complete parsing. Never just a file-exists ACK.
+        await Publish("drained.json", new
+        {
+            protocol, runId, targetPid, observerPid = Environment.ProcessId, active = false, drained = true,
+            summary.markersValid, summary.eventsLost, summary.sdkFactoryJit, summary.sdkFactoryLoads,
+            summary.sdkAssemblyLoads, summary.windows, summary.otherJit, summary.loadedNativeBytes,
+            configuration = new { runtimeProvider = "Microsoft-Windows-DotNETRuntime", level = "Verbose", keywords = "0x18",
+                markerProvider, circularBufferMiB = 16, requestRundown = false },
+            acceptance = false
+        });
+    }
+}
+catch (Exception error)
+{
+    await Publish("observer-failed.json", new { protocol, runId, exceptionType = error.GetType().FullName });
+    // An IPC/Stop worker can still be blocked. The parent also bounds and reaps this separate process.
+    Environment.Exit(1);
+}
+
+void Identity(JsonElement value) => Require(value.GetProperty("protocol").GetString() == protocol
+    && value.GetProperty("runId").GetString() == runId, "Mismatched observer identity");
+static void Require(bool condition, string message) { if (!condition) throw new InvalidDataException(message); }
+async Task<JsonDocument> WaitJson(string name, CancellationToken token)
+{
+    var path = Path.Combine(control, name);
+    while (!File.Exists(path)) await Task.Delay(25, token);
+    return JsonDocument.Parse(await File.ReadAllTextAsync(path, token));
+}
+async Task Publish(string name, object value)
+{
+    var temporary = Path.Combine(control, name + ".tmp");
+    await using (var stream = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write))
+        await JsonSerializer.SerializeAsync(stream, value);
+    File.Move(temporary, Path.Combine(control, name));
+}
+static TraceResult Parse(string path, string expectedRun)
+{
+    var markers = new List<Marker>();
+    var events = new List<Observed>();
+    using var source = new EventPipeEventSource(path);
+    source.Clr.MethodJittingStarted += data => events.Add(new(data.TimeStampRelativeMSec,
+        data.MethodNamespace == "Opc.Ua.OpcUaExtensions" ? "factory-jit" : "other-jit", 0));
+    source.Clr.MethodLoadVerbose += data => events.Add(new(data.TimeStampRelativeMSec,
+        data.MethodNamespace == "Opc.Ua.OpcUaExtensions" ? "factory-load" : "other-load", data.MethodSize));
+    source.Clr.LoaderAssemblyLoad += data =>
+    {
+        if (data.FullyQualifiedAssemblyName.StartsWith("Opc.Ua.", StringComparison.Ordinal))
+            events.Add(new(data.TimeStampRelativeMSec, "sdk-assembly", 0));
+    };
+    source.Dynamic.All += data =>
+    {
+        if (data.ProviderName == "OtOpcUa-Sdk2-Saturation" && data.EventName == "Boundary")
+            markers.Add(new(data.TimeStampRelativeMSec, (string)data.PayloadByName("runId"),
+                (string)data.PayloadByName("name"), (int)data.PayloadByName("cycle"), (int)data.PayloadByName("restarts")));
+    };
+    source.Process();
+    (string Name, int Cycle, int Restarts)[] expected =
+    [ ("window1-start", 321, 80), ("window1-end", 352, 88),
+      ("window2-start", 353, 88), ("window2-end", 384, 96) ];
+    var valid = markers.Count == 4 && markers.Select((marker, index) =>
+        marker.Run == expectedRun && (marker.Name, marker.Cycle, marker.Restarts) == expected[index]).All(x => x)
+        && markers.Zip(markers.Skip(1)).All(pair => pair.First.Ms < pair.Second.Ms);
+    var windows = Enumerable.Range(0, 2).Select(index =>
+    {
+        var selected = valid ? events.Where(x => x.Ms >= markers[index * 2].Ms && x.Ms <= markers[index * 2 + 1].Ms).ToArray() : [];
+        return new Window($"window{index + 1}", 321 + index * 32, 352 + index * 32, 80 + index * 8, 88 + index * 8,
+            selected.Count(x => x.Kind == "factory-jit"), selected.Count(x => x.Kind == "factory-load"),
+            selected.Count(x => x.Kind == "sdk-assembly"));
+    }).ToArray();
+    // Totals include the whole attached interval, including marker gaps, startup and stopping the collector.
+    return new TraceResult(valid, source.EventsLost, events.Count(x => x.Kind == "factory-jit"),
+        events.Count(x => x.Kind == "factory-load"), events.Count(x => x.Kind == "sdk-assembly"), windows,
+        events.Count(x => x.Kind == "other-jit"), events.Sum(x => x.Bytes));
+}
+record Marker(double Ms, string Run, string Name, int Cycle, int Restarts);
+record Observed(double Ms, string Kind, long Bytes);
+record Window(string name, int firstCycle, int lastCycle, int startRestarts, int endRestarts,
+    int sdkFactoryJit, int sdkFactoryLoads, int sdkAssemblyLoads);
+record TraceResult(bool markersValid, int eventsLost, int sdkFactoryJit, int sdkFactoryLoads,
+    int sdkAssemblyLoads, Window[] windows, int otherJit, long loadedNativeBytes);