From 6f793d2521087b7c91a2aa428dd430137f344d96 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 10 Nov 2026 14:16:57 -0500 Subject: [PATCH] test(opcua): package cross-platform lifecycle memory controls --- .../opcua-lifecycle-memory/.gitignore | 7 + .../CertificateControl.cs | 39 + .../EventPipeCapture.cs | 15 + .../LifecycleControl.cs | 208 ++++ .../opcua-lifecycle-memory/README.md | 286 +++++ .../TelemetryControl.cs | 22 + .../opcua-lifecycle-memory/TraceSummary.cs | 62 + .../build_trace_tools.py | 38 + .../opcua-lifecycle-memory/evidence.json | 1054 +++++++++++++++++ .../opcua-lifecycle-memory/linux_pair.py | 113 ++ .../validation/opcua-lifecycle-memory/run.py | 134 +++ .../opcua-lifecycle-memory/summarize.py | 45 + 12 files changed, 2023 insertions(+) create mode 100644 scripts/validation/opcua-lifecycle-memory/.gitignore create mode 100644 scripts/validation/opcua-lifecycle-memory/CertificateControl.cs create mode 100644 scripts/validation/opcua-lifecycle-memory/EventPipeCapture.cs create mode 100644 scripts/validation/opcua-lifecycle-memory/LifecycleControl.cs create mode 100644 scripts/validation/opcua-lifecycle-memory/README.md create mode 100644 scripts/validation/opcua-lifecycle-memory/TelemetryControl.cs create mode 100644 scripts/validation/opcua-lifecycle-memory/TraceSummary.cs create mode 100644 scripts/validation/opcua-lifecycle-memory/build_trace_tools.py create mode 100644 scripts/validation/opcua-lifecycle-memory/evidence.json create mode 100644 scripts/validation/opcua-lifecycle-memory/linux_pair.py create mode 100644 scripts/validation/opcua-lifecycle-memory/run.py create mode 100644 scripts/validation/opcua-lifecycle-memory/summarize.py diff --git a/scripts/validation/opcua-lifecycle-memory/.gitignore b/scripts/validation/opcua-lifecycle-memory/.gitignore new file mode 100644 index 00000000..bd6df73f --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +bin/ +obj/ +private-fixture/ +*.pfx +*.der +*.nettrace diff --git a/scripts/validation/opcua-lifecycle-memory/CertificateControl.cs b/scripts/validation/opcua-lifecycle-memory/CertificateControl.cs new file mode 100644 index 00000000..69a3485a --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/CertificateControl.cs @@ -0,0 +1,39 @@ +using System.Diagnostics; +using System.Runtime; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text.Json; +using Opc.Ua.Security.Certificates; +var mode = args[0]; +var directory = args[1]; +Directory.CreateDirectory(directory); +using var key = RSA.Create(2048); +var request = new CertificateRequest("CN=NativeMemoryControl", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); +var original = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1)); +var pfx = original.Export(X509ContentType.Pfx); +// Certificate.From takes ownership of original; only wrapped disposes that native certificate. +using var wrapped = Certificate.From(original); +void Sample(int i, string kind) +{ + var live = GC.GetTotalMemory(true); + GC.WaitForPendingFinalizers(); + using var p = Process.GetCurrentProcess(); + p.Refresh(); + Console.WriteLine(JsonSerializer.Serialize(new { kind, i, pid = Environment.ProcessId, mode, live, rss = p.WorkingSet64, methods = JitInfo.GetCompiledMethodCount(), gc = GC.GetGCMemoryInfo().TotalCommittedBytes })); + Console.Out.Flush(); +} +Sample(0, "ready"); +while (!File.Exists(Path.Combine(directory, "go"))) Thread.Sleep(50); +for (int i = 1; i <= 1000; i++) +{ + using X509Certificate2 copy = mode switch + { + "pfx" => System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12(pfx, null, X509KeyStorageFlags.Exportable), + "sdk-copy" => wrapped.AsX509Certificate2(), + "shared-copy" => (X509Certificate2)typeof(X509Certificate2).GetConstructor(new[] { typeof(X509Certificate) })!.Invoke(new object[] { original }), + _ => System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadCertificate(original.RawData) + }; + if (i % 100 == 0) Sample(i, "sample"); +} +Sample(1000, "done"); +while (!File.Exists(Path.Combine(directory, "stop"))) Thread.Sleep(50); diff --git a/scripts/validation/opcua-lifecycle-memory/EventPipeCapture.cs b/scripts/validation/opcua-lifecycle-memory/EventPipeCapture.cs new file mode 100644 index 00000000..8267ae91 --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/EventPipeCapture.cs @@ -0,0 +1,15 @@ +// Read-only temporary diagnostic attachment. Requires Microsoft.Diagnostics.NETCore.Client from dotnet-trace. +using System.Diagnostics.Tracing; +using Microsoft.Diagnostics.NETCore.Client; + +var seconds = int.Parse(args[1]); +if (seconds < 1 || seconds > 300) throw new ArgumentOutOfRangeException(nameof(seconds), "Use a bounded 1..300 second diagnostic attachment"); +var client = new DiagnosticsClient(int.Parse(args[0])); +var providers = new[] { new EventPipeProvider("Microsoft-Windows-DotNETRuntime", EventLevel.Verbose, 0x18) }; +// Match the recorded experiment's API default (256 MiB capacity, not measured resident allocation). +using var session = client.StartEventPipeSession(providers, requestRundown: false, circularBufferMB: 256); +await using var file = File.Create(args[2]); +var copy = session.EventStream.CopyToAsync(file); +await Task.Delay(TimeSpan.FromSeconds(seconds)); +session.Stop(); +await copy; diff --git a/scripts/validation/opcua-lifecycle-memory/LifecycleControl.cs b/scripts/validation/opcua-lifecycle-memory/LifecycleControl.cs new file mode 100644 index 00000000..4c3e24a6 --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/LifecycleControl.cs @@ -0,0 +1,208 @@ +using System.Diagnostics; +using System.Collections; +using System.Reflection; +using System.Net; +using System.Net.Sockets; +using System.Runtime; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Opc.Ua; +using Opc.Ua.Client; +using Opc.Ua.Configuration; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; +var root = Path.GetFullPath(args[0]); +Directory.CreateDirectory(root); +var seconds = int.Parse(args[1]); +using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(seconds + 120)); +var ct = timeout.Token; +var clientPki = Path.Combine(root, "client-pki"); +var telemetry = DefaultTelemetry.Create(_ => { }); +var config = new ApplicationConfiguration(telemetry); +config.ApplicationName = "NativeLifecycleControl"; +config.ApplicationUri = "urn:NativeLifecycleControl:" + Guid.NewGuid(); +config.ApplicationType = ApplicationType.Client; +config.SecurityConfiguration = new SecurityConfiguration +{ + ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientPki, "own"), SubjectName = "CN=NativeLifecycleControl" }, + TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientPki, "issuer") }, + TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientPki, "trusted") }, + RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientPki, "rejected") }, + AutoAcceptUntrustedCertificates = true +}; +config.ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 15000 }; +var app = new ApplicationInstance(telemetry) { ApplicationConfiguration = config }; +await config.ValidateAsync(ApplicationType.Client, ct); +await app.CheckApplicationInstanceCertificatesAsync(true, null, ct); +int port; +using (var listener = new TcpListener(IPAddress.Loopback, 0)) +{ + listener.Start(); + port = ((IPEndPoint)listener.LocalEndpoint).Port; +} +var options = new OpcUaApplicationHostOptions { ApplicationName = "NativeLifecycleServer", ApplicationUri = "urn:NativeLifecycleServer:" + Guid.NewGuid(), OpcUaPort = port, PublicHostname = "127.0.0.1", PkiStoreRoot = Path.Combine(root, "server-pki"), AllowAnonymous = false, AutoAcceptUntrustedClientCertificates = true, EnabledSecurityProfiles = [OpcUaSecurityProfile.Basic256Sha256SignAndEncrypt] }; +OpcUaApplicationHost host = null!; +OtOpcUaSdkServer server = null!; +async Task Boot() +{ + server = new OtOpcUaSdkServer(); + host = new OpcUaApplicationHost(options, NullLogger.Instance, new Auth()); + await host.StartAsync(server, ct); + var n = server.NodeManager!; + n.NodeWriteGateway = new Gateway(); + n.EnsureFolder("Control", null, "Control", AddressSpaceRealm.Raw); + n.EnsureFolder("Equipment", null, "Equipment", AddressSpaceRealm.Uns); + n.EnsureVariable("Control/Scalar", "Control", "Scalar", "Int32", true, AddressSpaceRealm.Raw); + n.EnsureVariable("Control/Array", "Control", "Array", "Int16", true, AddressSpaceRealm.Raw, isArray: true, arrayLength: 4); + n.EnsureVariable("Equipment/Scalar", "Equipment", "Scalar", "Int32", true, AddressSpaceRealm.Uns); + for (var i = 0; i < 16; i++) n.EnsureVariable("Control/Browse" + i, "Control", "Browse" + i, "Int32", false, AddressSpaceRealm.Raw); + n.MaterialiseAlarmCondition("Control/Alarm", "Control", "Alarm", "OffNormalAlarm", 500, AddressSpaceRealm.Raw, isNative: true); + n.WriteValue("Control/Scalar", 0, 0, DateTime.UtcNow, AddressSpaceRealm.Raw); + n.WriteValue("Control/Array", new short[] { 0, 1, 2, 3 }, 0, DateTime.UtcNow, AddressSpaceRealm.Raw); + n.WriteValue("Equipment/Scalar", 0, 0, DateTime.UtcNow, AddressSpaceRealm.Uns); +} +async Task Connect() +{ + var t = DefaultTelemetry.Create(_ => { }); + var endpoint = await CoreClientUtils.SelectEndpointAsync(config, $"opc.tcp://127.0.0.1:{port}/OtOpcUa", true, t, ct); + var factory = new DefaultSessionFactory(t); + var identity = new UserIdentity("control", Encoding.UTF8.GetBytes(FixtureCredential.Value)); + return await factory.CreateAsync(config, new ConfiguredEndpoint(null, endpoint ?? throw new Exception("Missing endpoint"), EndpointConfiguration.Create(config)), false, false, "NativeLifecycleControl", 15000, identity, default, ct); +} +static DataValue Value(object v) +{ +#if SDK2 + if (!VariantHelper.TryCastFrom(v, out var variant)) throw new Exception("Unsupported control value"); + return new DataValue(variant); +#else + return new DataValue(new Variant(v)); +#endif +} +static object? Clr(DataValue v) +{ +#if SDK2 + return v.WrappedValue.AsBoxedObject(Variant.BoxingBehavior.Legacy); +#else + return v.Value; +#endif +} +object? CertificateStats() +{ +#if SDK2 + return new { created = Opc.Ua.Security.Certificates.Certificate.InstancesCreated, disposed = Opc.Ua.Security.Certificates.Certificate.InstancesDisposed, outstanding = Opc.Ua.Security.Certificates.Certificate.InstancesLeaked }; +#else + return null; +#endif +} +void Sample(int cycle, string kind) +{ + var live = GC.GetTotalMemory(true); + using var p = Process.GetCurrentProcess(); + p.Refresh(); + var m = GC.GetGCMemoryInfo(); + Console.WriteLine(JsonSerializer.Serialize(new { kind, cycle, pid = Environment.ProcessId, live, rss = p.WorkingSet64, gcCommitted = m.TotalCommittedBytes, gcHeap = m.HeapSizeBytes, fragmented = m.FragmentedBytes, jit = JitInfo.GetCompiledMethodCount(), jitIl = JitInfo.GetCompiledILBytes(), sdk = typeof(NodeId).Assembly.GetName().Version?.ToString(), scope = "matched encrypted public lifecycle subset; no history/revocation/proxy", assemblies = AppDomain.CurrentDomain.GetAssemblies().Length, dynamicAssemblies = AppDomain.CurrentDomain.GetAssemblies().Count(a => a.IsDynamic), certificate = CertificateStats(), timers = Timer.ActiveCount, threads = p.Threads.Count, sessions = server.CurrentInstance.SessionManager.GetSessions().Count, subscriptions = server.CurrentInstance.SubscriptionManager.GetSubscriptions().Count })); + Console.Out.Flush(); +} +int cycles = 0, restarts = 0; +await Boot(); +var watch = Stopwatch.StartNew(); +Sample(0, "start"); +try +{ + while (watch.Elapsed.TotalSeconds < seconds) + { + using (var owner = await Connect()) using (var other = await Connect()) + { + var ns = (ushort)owner.NamespaceUris.GetIndex("https://zb.com/otopcua/raw"); + var scalar = new NodeId("Control/Scalar", ns); + var array = new NodeId("Control/Array", ns); + using var subscription = new Subscription(owner.DefaultSubscription) { PublishingInterval = 100 }; + owner.AddSubscription(subscription); + await subscription.CreateAsync(ct); + subscription.AddItem(new MonitoredItem(subscription.DefaultItem) { StartNodeId = scalar, AttributeId = Attributes.Value, SamplingInterval = 50 }); + subscription.AddItem(new MonitoredItem(subscription.DefaultItem) { StartNodeId = array, AttributeId = Attributes.Value, SamplingInterval = 50 }); + await subscription.ApplyChangesAsync(ct); + var response = await owner.WriteAsync(null, [new WriteValue { NodeId = scalar, AttributeId = Attributes.Value, Value = Value(++cycles) }, new WriteValue { NodeId = array, AttributeId = Attributes.Value, Value = Value(new short[] { 1, 2, 3, 4 }) }], ct); + if (response.Results.ToList().Any(s => StatusCode.IsBad(s))) throw new Exception("Control write denied"); + if (!Equals(Clr(await owner.ReadValueAsync(scalar, ct)), cycles)) throw new Exception("Scalar changed"); + if (!((short[])Clr(await owner.ReadValueAsync(array, ct))!).SequenceEqual(new short[] { 1, 2, 3, 4 })) throw new Exception("Array changed"); + var browse = await owner.BrowseAsync(null, null, 1, [new BrowseDescription { NodeId = new NodeId("Control", ns), BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All }], ct); + await Task.Delay(1000, ct); + await subscription.DeleteAsync(true, ct); + await owner.CloseAsync(ct); + await other.CloseAsync(ct); + } + if (server.CurrentInstance.SessionManager.GetSessions().Count != 0 || server.CurrentInstance.SubscriptionManager.GetSubscriptions().Count != 0) throw new Exception("Control retained sessions/subscriptions"); + Sample(cycles, "sample"); + if (cycles % 4 == 0) + { + using var parked = await Connect(); + using var sub = new Subscription(parked.DefaultSubscription) { PublishingInterval = 100 }; + parked.AddSubscription(sub); + await sub.CreateAsync(ct); + sub.AddItem(new MonitoredItem(sub.DefaultItem) { StartNodeId = new NodeId("Control/Scalar", (ushort)parked.NamespaceUris.GetIndex("https://zb.com/otopcua/raw")), AttributeId = Attributes.Value, SamplingInterval = 50 }); + await sub.ApplyChangesAsync(ct); + await host.DisposeAsync(); + await Boot(); + restarts++; + } + } + Console.WriteLine(JsonSerializer.Serialize(new { kind = "complete", cycles, restarts, seconds = watch.Elapsed.TotalSeconds, fullGate = false })); +} +finally +{ + await host.DisposeAsync(); +#if SDK2 + await app.DisposeAsync(); +#endif +} +#if SDK2 +GC.Collect(); +GC.WaitForPendingFinalizers(); +GC.Collect(); +var certificateType = typeof(Opc.Ua.Security.Certificates.Certificate); +var report = new Dictionary(); +var trackingEnabled = Environment.GetEnvironmentVariable("OPCUA_CERTIFICATE_LEAK_TRACKING") == "1"; +if (trackingEnabled) +{ + foreach (var name in new[] { "EnumerateLiveCertificates", "EnumerateUnreachableUndisposedCertificates" }) + { + var method = certificateType.GetMethod(name, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("SDK2 diagnostic method missing: " + name); + var values = method.Invoke(null, null) as IEnumerable + ?? throw new InvalidOperationException("SDK2 diagnostic method return type changed: " + name); + var stacks = new List(); + foreach (var value in values) + { + var field = value!.GetType().GetField(name.Contains("Unreachable") ? "Item2" : "Item4") + ?? throw new InvalidOperationException("SDK2 diagnostic tuple field missing: " + name); + if (field.FieldType != typeof(string)) + throw new InvalidOperationException("SDK2 diagnostic stack field type changed"); + // Deliberately omit tuple Item1 (certificate identity). + stacks.Add((string)field.GetValue(value)!); + } + report[name] = new + { + count = stacks.Count, + allocations = stacks.GroupBy(x => x).OrderByDescending(g => g.Count()).Take(15) + .Select(g => new { count = g.Count(), stack = g.Key }).ToArray() + }; + } +} +Console.WriteLine(JsonSerializer.Serialize(new { kind = "certificate-final-diagnostics", trackingEnabled, stats = CertificateStats(), report })); +#endif +class Auth : IOpcUaUserAuthenticator +{ + public Task AuthenticateUserNameAsync(string user, string password, CancellationToken ct) => Task.FromResult(user == "control" && password == FixtureCredential.Value ? OpcUaUserAuthResult.Allow("control", [OpcUaDataPlaneRoles.WriteOperate]) : OpcUaUserAuthResult.Deny("Control credential refused")); +} +class Gateway : IOpcUaNodeWriteGateway +{ + public Task WriteAsync(string n, object? v, AddressSpaceRealm r, CancellationToken ct) => Task.FromResult(new NodeWriteOutcome(true, null)); +} + +static class FixtureCredential +{ + internal static string Value { get; } = Environment.GetEnvironmentVariable("OPCUA_MEMORY_PASSWORD") ?? throw new InvalidOperationException("Set OPCUA_MEMORY_PASSWORD for the task-owned fixture."); +} diff --git a/scripts/validation/opcua-lifecycle-memory/README.md b/scripts/validation/opcua-lifecycle-memory/README.md new file mode 100644 index 00000000..39aa4afe --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/README.md @@ -0,0 +1,286 @@ +# SDK 2 lifecycle memory investigation + +The required two-hour server lifecycle gate failed on macOS and Linux. The RSS +limit remains **cycle-3 baseline + 128 MiB**; retained managed memory remains ++64 MiB and descriptors +32. No threshold, production code, certificate provider, +or session factory was changed by this investigation. These controls do not +replace the full gate. Issue #799 remains open. + +## Preserved full-workload failures + +| Run | Cycles / restarts | Observation time | RSS / limit (bytes) | Outcome | +|---|---:|---:|---:|---| +| macOS `fa823378` | 250 / 62 | 1045.53 s | 365,215,744 / 363,331,584 | Failed | +| macOS `e135b7c9` profile reproduction | 245 / 61 | 1024.14 s | 368,656,384 / 363,053,056 | Failed | +| Linux full harness, recorded build `38098ce142ab8c382825aa754e0758071dd3a46b` | 248 / 61 | 903.508 s | 353,931,264 / 351,850,496 | Failed | +| macOS `6b2d4b81` bounded profile | 432 / 108 | 1807.479 s | First rejection: 364,199,936 / 363,266,048 | Failed; first assertion rethrown | + +The last macOS row requested 1,800 seconds. It first rejected RSS at cycle 253 / +1055.482 seconds, continued recording explicitly rejected observations, then +threw the original assertion before any completion record. Its final observation +was RSS 380,059,648, live managed 41,364,096, GC committed 37,060,608, JIT +compilations 81,535, descriptors 200, and zero sessions, subscriptions, monitored +items, browse/history continuation points, and PKI descriptors. SDK cached stores +remained two during operation. The runner and skip guard returned exit 1 with one +executed test and zero skipped tests. There is no passing two-hour result here. + +The first macOS run's reported PKI descriptor zero is invalid: `/var` and +`/private/var` aliases were not canonicalized. Corrected positive-handle and timeout +controls passed before subsequent profiles. Global descriptor counts and the RSS +failure itself were independent of that defect. A Shouldly source-expression +message incorrectly names `Descriptors` in the RSS failure; the actual numeric +comparison is RSS, as retained before the assertion. + +## What the controls demonstrate + +`LifecycleControl.cs` uses unchanged published product DLLs and public APIs for +both SDK generations. It opens two real encrypted **opc.tcp** sessions with +Basic256Sha256 / SignAndEncrypt, resolves namespace URIs, reads/writes Int32 and +Int16[] values, creates/deletes two data monitored items, abandons a browse +continuation, closes sessions, and restarts a server with an active subscription +every four cycles against the same task-owned PKI. Public session/subscription +counts must return to zero. It has no historian, revocation, surgical-rebuild, +paused-network, or private continuation-store coverage. It is a diagnostic subset, +not TLS and not the full soak workload. + +The original DLLs are the immutable 1.5.378.182 baseline publication associated +with `83ed880e`; its assembly version alone is 1.5.378.0 and does not identify the +patch. The candidate uses SDK 2.0.1 and the server implementation from `fa823378`. +Neither baseline nor candidate product code was rebuilt for these controls. +The controls were compiled once for each reference set, then those exact control +artifacts were used on both operating systems. Selected DLL and input-artifact +hashes are in `evidence.json`. Neither reference set's principal SDK/server DLLs +has a managed native/ReadyToRun header. + +| Ten-minute matched subset | Cycles / restarts | Final RSS | Change from cycle 3 | Final managed / change | Final JIT compilations | +|---|---:|---:|---:|---:|---:| +| macOS original | 212 / 53 | 330,301,440 | +121,683,968 | 271,168,424 / +169,990,456 | 28,066 | +| macOS SDK2 | 200 / 50 | 302,170,112 | +95,862,784 | 44,569,584 / +295,120 | 62,608 | +| Linux original | 249 / 62 | 384,950,272 | +176,836,608 | 147,139,568 / +99,356,752 | 28,321 | +| Linux SDK2 | 259 / 64 | 390,844,416 | +194,183,168 | 19,561,120 / +736 | 71,529 | + +Both ten-minute controls exited zero because their lifecycle checks passed; +`fullGate=false` is explicit. They do **not** enforce the full harness memory +assertions. In particular, both Linux subset RSS deltas exceed 128 MiB and the +original retains managed memory/timers. These observations do not waive either +failed full gate. They are not proof of an SDK1 defect or an SDK2 improvement. +Both control branches explicitly delete subscriptions, close normal sessions, +and dispose sessions/subscriptions. Parked resources intentionally encounter +server shutdown and are then disposed. Each version calls its unchanged product +host's DisposeAsync. The baseline host only calls ApplicationInstance.StopAsync; +that original ApplicationInstance exposes no disposal interface. The SDK2 branch +also uses its new ApplicationInstance.DisposeAsync ownership API. Per-connection +DefaultTelemetry factories are left to the same ownership pattern on both +branches, separately probed above. Thus measured original growth belongs to this +published-product/fixture lifecycle combination; its allocation owner has not +been isolated. These are one matched run per platform, concurrent processes, +with different runtime versions; they are not performance distributions. + +The thirty-minute Linux comparison uses the same immutable artifacts and image, +fresh independent fixtures, and SDK2 certificate allocation tracking enabled +before the first certificate (the same environment is passed to both versions). +Both finished with exit zero and `fullGate=false`: + +| Thirty-minute Linux subset | Cycles / restarts | Final RSS | Final live managed | GC committed | JIT compilations | Active timers | +|---|---:|---:|---:|---:|---:|---:| +| Original | 736 / 184 | 540,057,600 | 342,626,200 | 408,399,872 | 29,177 | 926 | +| SDK2 | 768 / 192 | 334,180,352 | 19,248,984 | 42,983,424 | 72,087 | 19 | + +Observed runtime was 1802.542 seconds original / 1804.523 seconds SDK2. The SDK2 +RSS fell from 390,782,976 at the 632.7-second checkpoint to 334,180,352 at the last +observation; live managed stayed near 20 MB. After disposal and final collection, +SDK2 wrapper counters were 19,272 created = 19,272 disposed, with zero live or +unreachable-undisposed wrappers under enabled tracking. These finite results +support post-transition saturation of this subset, not a two-hour full-workload +pass or a guarantee of indefinite stability. + +### Native and runtime observations + +- During the macOS bounded profile, `Security::AppleDatabase` counts increased + from 278 at cycle 52 / 211 seconds to 1,953 at cycle 370 / 1546 seconds. These + native objects continued accumulating while managed memory stayed near 40 MiB. +- The single-owner certificate probe makes a self-signed RSA fixture in memory, + then disposes each of 1,000 copies/imports. On macOS, public-only and + shared-handle copies leave AppleDatabase at 2; PFX imports and SDK + `Certificate.AsX509Certificate2` private-key copies grow it from 2 to 1,002. + A separate PFX process retained that count through a 180-second idle hold. + `Certificate.From(original)` owns the original; the checked-in control has no + second owner. All four corrected single-owner modes were rerun successfully. +- The same isolated Linux probe ended at approximately 49 MB RSS for public or + shared copies, 61 MB for PFX imports, and 69 MB for the SDK copy. Those are + endpoint RSS observations including different JIT/GC costs, not evidence of a + Linux native-object leak or a complete attribution. +- SDK2 public certificate counters after the Linux ten-minute lifecycle run were + 6,495 created = 6,495 disposed. A separate stack-tracked macOS lifecycle run + ended 591 = 591, zero live and zero unreachable-undisposed wrappers. Allocation + tracking is opt-in; empty internal reports without that opt-in are not proof. +- The Linux ten-minute native snapshots show SDK2 JIT `doublemapper` resident + mappings increasing from about 65.4 to 106.7 MiB, anonymous writable mappings + from 84.6 to 121.3 MiB, and `[heap]` around 19.2 MiB. Original JIT mappings were + about 33.3 to 37.7 MiB. Mapping totals are descriptive categories, not exclusive + object allocations: anonymous memory also contains GC regions, stacks, and + native allocator arenas. RSS includes file/shared pages and can fall with + reclamation. JIT counters are cumulative compilations, not unique live methods. +- 10,000 disposed versus undisposed `DefaultTelemetry` factories ended at similar + RSS (~63.4 MB) and live managed memory (~0.43 MB), with no timer growth. This + does not justify changing host telemetry ownership to fix this failure. + +Exact SDK tag source shows private-key PFX copying in nonvirtual +`Certificate.AsX509Certificate2`. A 60-second macOS sampled-thread-time trace +instead found the dominant sampled import path through +`CertificateIdentifierResolver.LoadPrivateKeyCoreAsync` → +`DirectoryCertificateStore.LoadPrivateKeyAsync` → `X509CertificateLoader.LoadPkcs12`. +Inclusive sampled thread-time was about 4.79 seconds for store load and 2.99 seconds +for PKCS12, versus 8.5 milliseconds for AsX509. These are neither exclusive CPU +costs nor call counts. ApplicationInstance already normalizes its validation +input to public DER. The validation copy delegate is internal, the manager sealed, +and no safe narrow application copy-policy seam was identified. Changing session +factory, certificate exportability, or key storage would alter behavior and was +not used as a workaround. + +A major JIT/RSS step appeared near 60–64 restarts in several workloads. This is a +hypothesis about runtime/code-cache warmup, not a finding that all RSS growth is +benign. A separate read-only EventPipe attachment observed the thirty-minute +Linux candidate around that transition using `Microsoft-Windows-DotNETRuntime`, +Verbose, keywords `0x18` (Loader + JIT), no rundown, 180 seconds. The transition was UTC 18:51:02.049–18:54:02.062. Factory JIT events +occurred UTC 18:52:48.249–18:53:21.434: 6,690 distinct method tokens from one +module, each once. The module is Opc.Ua.Core.Types.dll; metadata declares 8,442 +static methods on OpcUaExtensions. The observed set is a subset, so this does +not prove every possible factory or the complete canonical workload is warmed. +Method-load events totalled 51,327,536 native code bytes, including 51,161,724 +OptimizedTier1 bytes and 49,141,484 bytes in generated factories. Only 50 +reflection InvokeStub JIT starts occurred; the transition trace reports zero lost events. MethodSize sums count loaded code +versions, not resident/PSS allocation bytes. An additional 180-second attachment +after the transition checks continuing factory compilation. The second attachment (UTC 19:01:44.116–19:04:44.127) recorded 85 JIT starts, +zero lost events, zero Opc.Ua starts, zero generated-factory starts, and zero generated-factory +loaded bytes. Only 18 ordinary Tier1 method loads (5,720 bytes) and 20 +instrumented Tier1 loads (5,800 bytes) occurred. This supports a finite generated +factory burst in the observed subset; it does not prove complete canonical +workload saturation. The capture API's default circular-buffer capacity was 256 MiB; that is a +capacity setting, not measured resident overhead. Attachments can allocate event +buffers and perturb execution. Both prior full failures and the ten-minute +control were uninstrumented by these JIT attachments, and the thirty-minute final +interval is unattached. No captured native-code byte total is claimed to equal an +RSS delta. Raw traces/heaps are not committed. + +## Reproduce the diagnostic subset + +Requires Python 3 and .NET 10. `run.py` references existing DLL directories, +compiles only the control, records SHA-256 provenance, and creates a fresh local +artifact directory. First reference directory wins duplicate filenames. Use the +actual baseline publications, rather than restoring a different package version. +No production project or package file is changed. + +```bash +python3 scripts/validation/opcua-lifecycle-memory/run.py lifecycle \ + --references "$baselineHostDlls" --references "$baselineClientDlls" \ + --output /tmp/opcua-memory-original --seconds 600 +python3 scripts/validation/opcua-lifecycle-memory/run.py lifecycle --sdk2 \ + --references "$candidateDlls" --output /tmp/opcua-memory-sdk2 --seconds 600 \ + --certificate-tracking +``` + +Fixture credentials are generated in the runner environment and not logged. +PKI material stays under each local `private-fixture` directory. Do not publish +those directories or unredacted process/native logs. These controls grant a +fixture user WriteOperate and auto-accept task-owned certificates; they are +local test fixtures, not deployment configurations. + +For a matched Linux pair, first build both with `--build-only` into fresh +`/tmp/opcua-memory-original` and `/tmp/opcua-memory-sdk2` output directories, then: + +```bash +python3 scripts/validation/opcua-lifecycle-memory/linux_pair.py \ + --baseline /tmp/opcua-memory-original/bin/Release/net10.0 \ + --candidate /tmp/opcua-memory-sdk2/bin/Release/net10.0 \ + --output /tmp/opcua-memory-linux-pair --seconds 1800 \ + --image sha256:b04611ee9e1b7d36797012ed921962c57189cb4dbb3dff12afb8e158110f293b +``` + +The runner uses separate task-owned `--network none` containers, fresh PKI, +read-only executable mounts, bounded shutdown, and periodic read-only `/proc` +snapshots. It records child and container exit statuses independently. The image +is Ubuntu 24.04 arm64, .NET SDK 10.0.103 / runtime 10.0.3. The Mac runs used +macOS 26.5.2 arm64, SDK 10.0.403 / runtime 10.0.14. Runtime and OS differences are +explicit; do not attribute all cross-platform differences to SDK code. + +On macOS, run each certificate mode with a fresh output directory: + +```bash +python3 scripts/validation/opcua-lifecycle-memory/run.py certificate --sdk2 \ + --references "$candidateDlls" --output /tmp/opcua-memory-pfx --mode pfx +``` + +Other modes are `sdk-copy`, `public`, and `shared-copy`. The last intentionally +uses the old copy constructor by reflection as a diagnostic control. It is not +proposed production code. The control exports no fixture key/certificate data. +`TelemetryControl.cs` is available through `run.py telemetry --sdk2` with modes +`dispose` or `undisposed`. `summarize.py` emits Linux mapping categories without +file paths from local `*-smaps.txt` files. All controls are opt-in and outside the +ordinary solution/test graph. + +Optional trace reproduction uses the installed dotnet-trace 9.0.661903 managed +libraries, without adding a product dependency: + +```bash +python3 scripts/validation/opcua-lifecycle-memory/build_trace_tools.py \ + --libraries "$diagnosticToolLibraries" --output /tmp/opcua-memory-trace-tools +# Run capture in the same process namespace / user as the task-owned target. +dotnet /tmp/opcua-memory-trace-tools/capture/bin/Release/net10.0/capture.dll \ + "$targetPid" 180 /tmp/opcua-memory-local.nettrace +dotnet /tmp/opcua-memory-trace-tools/summary/bin/Release/net10.0/summary.dll \ + /tmp/opcua-memory-local.nettrace > /tmp/opcua-memory-method-summary.json +``` + +Capture is read-only and opt-in; raw traces remain local. The summary exports +static method names and aggregate counts rather than certificate identities, +arguments, or process mappings. The original transition parser also retained +method tokens/modules locally to cross-check name-key uniqueness. + +## Prospective saturation experiment (not executed) + +The observed finite factory burst supports a separately labeled readiness +experiment; it does not authorize moving the acceptance baseline. A reviewed +candidate proposal is exactly 384 canonical lifecycle cycles / 96 every-four-cycle +restarts, then one fixed settling cycle, all within 1,800 seconds. No early finish, +extension, or RSS-selected endpoint is permitted. Both fixed readiness windows +321–352 and 353–384 must contain zero generated-factory JIT/method-load events, zero SDK assembly +loads, and zero lost events. Other JIT/native-code totals remain review evidence, +not a claim that every runtime path is warmed. Trace capture is bounded and identical for both versions, then +detached/drained before the settling cycle. Ownership, FD, PKI, store, and logical +resource checks remain active throughout; original cold budget rejections remain +recorded. Any later full gate requires at least 7,200 additional seconds after a +prospectively approved baseline. No such warmed full gate has been run here. + +The count is beyond the observed 60–64-restart transition and projects to about +1,606 seconds from the Mac canonical run, but completion within the fixed cutoff +is not guaranteed under load. Failure to complete or satisfy readiness is a failed +experiment, not permission to extend. The full canonical test currently depends +on SDK2 value APIs and read-only private SDK2 continuation/store instrumentation. +An original-SDK adapter proving identical history, revocation, surgical-edit, +and stalled-network operation counts is a prerequisite for a matched canonical +experiment; the public subset does not provide that proof. Continued Mac native +inventory growth remains an independent unresolved gate, even if JIT saturates. + +## Evidence and limits + +`evidence.json` contains sanitized numeric endpoints, native inventory counts, +and hashes of retained diagnostic inputs. Full artifacts remain in the originating +worktrees or `/tmp/opcua2-rss-investigation`; no PKI, certificate identifiers, +credentials, raw traces, unredacted heap/process dumps, or complete test logs are +checked in. Existing failures were preserved untouched. + +Supported progress is a reproduced RSS failure, bounded logical/managed +inventories in SDK2, demonstrated macOS PFX native retention, and measured +cross-platform JIT/anonymous-memory growth. A complete allocation attribution and +safe production correction remain unresolved. The server two-hour gate remains +failed; a separate long-running client-only process is a different scope. + +Package verification: baseline and candidate control builds, both optional trace +tools, and certificate/telemetry controls built with zero warnings/errors. Local +control smoke runs and the fresh Linux pair smoke exited zero. Python compilation +and whitespace checks passed. One reused-output smoke failed because a manual +incremental build used a different `/tmp` path alias and its runtimeconfig was +missing; fresh canonical outputs passed, and the Linux runner now rejects an +incomplete output directory before launch. These smoke checks validate tooling +only and do not substitute for either long run. diff --git a/scripts/validation/opcua-lifecycle-memory/TelemetryControl.cs b/scripts/validation/opcua-lifecycle-memory/TelemetryControl.cs new file mode 100644 index 00000000..7c16933d --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/TelemetryControl.cs @@ -0,0 +1,22 @@ +using System.Diagnostics; +using System.Runtime; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Opc.Ua; +var dispose = args[0] == "dispose"; +for (int i = 0; i <= 10000; i++) +{ + if (i > 0) + { + var t = DefaultTelemetry.Create(_ => { }); + t.LoggerFactory.CreateLogger("TelemetryControl").LogDebug("control"); + if (dispose) ((IDisposable)t).Dispose(); + } + if (i % 250 == 0) + { + var live = GC.GetTotalMemory(true); + using var p = Process.GetCurrentProcess(); + p.Refresh(); + Console.WriteLine(JsonSerializer.Serialize(new { i, dispose, live, rss = p.WorkingSet64, methods = JitInfo.GetCompiledMethodCount(), il = JitInfo.GetCompiledILBytes(), gc = GC.GetGCMemoryInfo().TotalCommittedBytes, threads = p.Threads.Count, timers = Timer.ActiveCount })); + } +} diff --git a/scripts/validation/opcua-lifecycle-memory/TraceSummary.cs b/scripts/validation/opcua-lifecycle-memory/TraceSummary.cs new file mode 100644 index 00000000..5c049606 --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/TraceSummary.cs @@ -0,0 +1,62 @@ +// Redacted aggregate summary. Keep raw traces local. +using Microsoft.Diagnostics.Tracing; +using System.Text.Json; + +var starts = new Dictionary(); +var tiers = new Dictionary(); +var nativeBytes = new Dictionary(); +var factoryNames = new Dictionary(); +var factoryTokens = new HashSet(); +var factoryModules = new HashSet(); +var firstFactoryMs = double.PositiveInfinity; +var lastFactoryMs = 0d; +var factoryNativeBytes = 0L; +var sdkAssemblyLoads = 0; +using var source = new EventPipeEventSource(args[0]); +source.Clr.MethodJittingStarted += data => +{ + var name = data.MethodNamespace + "." + data.MethodName; + starts[name] = starts.GetValueOrDefault(name) + 1; + if (!name.StartsWith("Opc.Ua.OpcUaExtensions.", StringComparison.Ordinal)) return; + factoryNames[name] = factoryNames.GetValueOrDefault(name) + 1; + factoryTokens.Add(data.MethodToken); + factoryModules.Add(data.ModuleID); + firstFactoryMs = Math.Min(firstFactoryMs, data.TimeStampRelativeMSec); + lastFactoryMs = Math.Max(lastFactoryMs, data.TimeStampRelativeMSec); +}; +source.Clr.MethodLoadVerbose += data => +{ + var tier = data.OptimizationTier.ToString(); + tiers[tier] = tiers.GetValueOrDefault(tier) + 1; + nativeBytes[tier] = nativeBytes.GetValueOrDefault(tier) + data.MethodSize; + if (data.MethodNamespace == "Opc.Ua.OpcUaExtensions") factoryNativeBytes += data.MethodSize; +}; +source.Clr.LoaderAssemblyLoad += data => +{ + if (data.FullyQualifiedAssemblyName.StartsWith("Opc.Ua.", StringComparison.Ordinal)) sdkAssemblyLoads++; +}; +source.Process(); +Console.WriteLine(JsonSerializer.Serialize(new +{ + traceStartUtc = source.SessionStartTime.ToUniversalTime(), + traceEndUtc = source.SessionEndTime.ToUniversalTime(), + eventsLost = source.EventsLost, + sdkAssemblyLoads, + jittingStarts = starts.Values.Sum(), + // Name keys omit generic signatures; token/module counts independently identify the factory set. + distinctNameKeys = starts.Count, + opcUaJittingStarts = starts.Where(x => x.Key.StartsWith("Opc.Ua.", StringComparison.Ordinal)).Sum(x => x.Value), + invokeStubStarts = starts.Where(x => x.Key.StartsWith("dynamicClass.InvokeStub", StringComparison.Ordinal)).Sum(x => x.Value), + tiers, + summedLoadedNativeMethodBytes = nativeBytes, + factory = new + { + starts = factoryNames.Values.Sum(), distinctNames = factoryNames.Count, + distinctTokens = factoryTokens.Count, moduleCount = factoryModules.Count, + everyNameOnce = factoryNames.Values.All(x => x == 1), + firstMs = double.IsFinite(firstFactoryMs) ? (double?)firstFactoryMs : null, + lastMs = factoryNames.Count == 0 ? (double?)null : lastFactoryMs, + summedLoadedNativeMethodBytes = factoryNativeBytes, + examples = factoryNames.Keys.Take(10).ToArray() + } +}, new JsonSerializerOptions { WriteIndented = true })); diff --git a/scripts/validation/opcua-lifecycle-memory/build_trace_tools.py b/scripts/validation/opcua-lifecycle-memory/build_trace_tools.py new file mode 100644 index 00000000..b5fc947b --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/build_trace_tools.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Build optional local capture/summary tools using the user's installed dotnet-trace libraries.""" +import argparse +from pathlib import Path +import subprocess +import xml.etree.ElementTree as ET + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--libraries", required=True, type=Path, help="dotnet-trace tools/net8.0/any directory") +parser.add_argument("--output", required=True, type=Path, help="Fresh local output directory") +args = parser.parse_args() +args.output = args.output.resolve() +args.output.mkdir(parents=True, exist_ok=False) +for name, source, references in [ + ("capture", "EventPipeCapture.cs", ["Microsoft.Diagnostics.NETCore.Client"]), + ("summary", "TraceSummary.cs", ["Microsoft.Diagnostics.Tracing.TraceEvent", "Microsoft.Diagnostics.FastSerialization", "Microsoft.Diagnostics.NETCore.Client"]) +]: + output = args.output / name + output.mkdir() + project = ET.Element("Project", Sdk="Microsoft.NET.Sdk") + properties = ET.SubElement(project, "PropertyGroup") + for key, value in {"OutputType": "Exe", "TargetFramework": "net10.0", "ImplicitUsings": "enable", + "EnableDefaultCompileItems": "false"}.items(): + ET.SubElement(properties, key).text = value + items = ET.SubElement(project, "ItemGroup") + ET.SubElement(items, "FrameworkReference", Include="Microsoft.AspNetCore.App") + ET.SubElement(items, "Compile", Include=str(Path(__file__).resolve().parent / source)) + for reference in references: + dll = (args.libraries / (reference + ".dll")).resolve(strict=True) + item = ET.SubElement(items, "Reference", Include=reference) + ET.SubElement(item, "HintPath").text = str(dll) + path = output / (name + ".csproj") + ET.indent(project) + ET.ElementTree(project).write(path, encoding="unicode") + with (output / "build.log").open("w") as log: + result = subprocess.run(["dotnet", "build", str(path), "-c", "Release", "-m:2"], stdout=log, stderr=subprocess.STDOUT) + if result.returncode: + raise SystemExit(result.returncode) diff --git a/scripts/validation/opcua-lifecycle-memory/evidence.json b/scripts/validation/opcua-lifecycle-memory/evidence.json new file mode 100644 index 00000000..7408c2a2 --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/evidence.json @@ -0,0 +1,1054 @@ +{ + "schemaVersion": 1, + "status": "server full gate failed; diagnostic controls are not substitutes", + "limitsUnchanged": { + "managedGrowth": 67108864, + "rssGrowth": 134217728, + "descriptorsGrowth": 32 + }, + "runtime": { + "macOS": "26.5.2 arm64; SDK10.0.403/runtime10.0.14", + "linux": "Ubuntu24.04 arm64; SDK10.0.103/runtime10.0.3", + "linuxImage": "sha256:b04611ee9e1b7d36797012ed921962c57189cb4dbb3dff12afb8e158110f293b" + }, + "matchedControls": { + "mac": { + "original": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 101177968, + "rss": 208617472, + "gcCommitted": 59588608, + "gcHeap": 54874432, + "fragmented": 6839552, + "jit": 14521, + "jitIl": 1129209, + "sdk": "1.5.378.0", + "assemblies": 59, + "dynamicAssemblies": 0, + "timers": 11, + "threads": 31, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 212, + "live": 271168424, + "rss": 330301440, + "gcCommitted": 149782528, + "gcHeap": 139647008, + "fragmented": 6394248, + "jit": 28066, + "jitIl": 2291934, + "sdk": "1.5.378.0", + "assemblies": 61, + "dynamicAssemblies": 0, + "timers": 271, + "threads": 22, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 212, + "restarts": 53, + "seconds": 603.5835221, + "fullGate": false + }, + "finalCertificateCounters": null, + "rssDelta": 121683968, + "managedDelta": 169990456, + "exitStatus": 0 + }, + "sdk2": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 44274464, + "rss": 206307328, + "gcCommitted": 30588928, + "gcHeap": 26272496, + "fragmented": 6476976, + "jit": 30382, + "jitIl": 4209289, + "sdk": "2.0.1.0", + "assemblies": 66, + "dynamicAssemblies": 0, + "timers": 16, + "threads": 30, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 200, + "live": 44569584, + "rss": 302170112, + "gcCommitted": 32489472, + "gcHeap": 27401272, + "fragmented": 7305128, + "jit": 62608, + "jitIl": 8979409, + "sdk": "2.0.1.0", + "assemblies": 68, + "dynamicAssemblies": 0, + "timers": 18, + "threads": 26, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 200, + "restarts": 50, + "seconds": 601.0864806, + "fullGate": false + }, + "finalCertificateCounters": { + "created": 5333, + "disposed": 5333, + "outstanding": 0 + }, + "rssDelta": 95862784, + "managedDelta": 295120, + "exitStatus": 0 + } + }, + "linux600": { + "original": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 47782816, + "rss": 208113664, + "gcCommitted": 89714688, + "gcHeap": 55442640, + "fragmented": 7659824, + "jit": 14331, + "jitIl": 1159548, + "sdk": "1.5.378.0", + "assemblies": 59, + "dynamicAssemblies": 0, + "timers": 11, + "threads": 24, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 249, + "live": 147139568, + "rss": 384950272, + "gcCommitted": 244244480, + "gcHeap": 149097376, + "fragmented": 1974256, + "jit": 28321, + "jitIl": 2340727, + "sdk": "1.5.378.0", + "assemblies": 61, + "dynamicAssemblies": 0, + "timers": 321, + "threads": 22, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 249, + "restarts": 62, + "seconds": 600.1972942, + "fullGate": false + }, + "finalCertificateCounters": null, + "rssDelta": 176836608, + "managedDelta": 99356752, + "exitStatus": 0 + }, + "sdk2": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 19560384, + "rss": 196661248, + "gcCommitted": 34783232, + "gcHeap": 23613456, + "fragmented": 4061296, + "jit": 29706, + "jitIl": 4190572, + "sdk": "2.0.1.0", + "assemblies": 66, + "dynamicAssemblies": 0, + "timers": 16, + "threads": 27, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 259, + "live": 19561120, + "rss": 390844416, + "gcCommitted": 57884672, + "gcHeap": 23593784, + "fragmented": 4040888, + "jit": 71529, + "jitIl": 11607757, + "sdk": "2.0.1.0", + "assemblies": 68, + "dynamicAssemblies": 0, + "timers": 17, + "threads": 20, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 259, + "restarts": 64, + "seconds": 600.4640042, + "fullGate": false + }, + "finalCertificateCounters": { + "created": 6495, + "disposed": 6495, + "outstanding": 0 + }, + "rssDelta": 194183168, + "managedDelta": 736, + "exitStatus": 0 + } + }, + "linux1800": { + "original": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 47941928, + "rss": 187817984, + "gcCommitted": 70205440, + "gcHeap": 52762984, + "fragmented": 4829280, + "jit": 14173, + "jitIl": 1142353, + "sdk": "1.5.378.0", + "assemblies": 59, + "dynamicAssemblies": 0, + "timers": 11, + "threads": 24, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 736, + "live": 342626200, + "rss": 540057600, + "gcCommitted": 408399872, + "gcHeap": 344260144, + "fragmented": 1633944, + "jit": 29177, + "jitIl": 2357956, + "sdk": "1.5.378.0", + "assemblies": 61, + "dynamicAssemblies": 0, + "timers": 926, + "threads": 21, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 736, + "restarts": 184, + "seconds": 1802.5423307, + "fullGate": false + }, + "finalCertificateCounters": null, + "rssDelta": 352239616, + "managedDelta": 294684272, + "exitStatus": 0, + "checkpoints": [ + { + "seconds": 572.6621778011322, + "cycle": 237, + "rss": 362446848, + "live": 143003136, + "jit": 27409, + "timers": 306 + }, + { + "seconds": 872.7052655220032, + "cycle": 360, + "rss": 398036992, + "live": 191473488, + "jit": 28557, + "timers": 455 + }, + { + "seconds": 1172.7514917850494, + "cycle": 484, + "rss": 501948416, + "live": 241448616, + "jit": 28758, + "timers": 611 + }, + { + "seconds": 1472.8238759040833, + "cycle": 604, + "rss": 533143552, + "live": 289394184, + "jit": 28973, + "timers": 761 + }, + { + "seconds": 1772.8926355838776, + "cycle": 724, + "rss": 539000832, + "live": 337868136, + "jit": 29159, + "timers": 909 + } + ] + }, + "sdk2": { + "baseline": { + "kind": "sample", + "cycle": 3, + "live": 19539376, + "rss": 201515008, + "gcCommitted": 37089280, + "gcHeap": 19912552, + "fragmented": 381400, + "jit": 29711, + "jitIl": 4188622, + "sdk": "2.0.1.0", + "assemblies": 68, + "dynamicAssemblies": 0, + "timers": 16, + "threads": 27, + "sessions": 0, + "subscriptions": 0 + }, + "last": { + "kind": "sample", + "cycle": 768, + "live": 19248984, + "rss": 334180352, + "gcCommitted": 42983424, + "gcHeap": 21126520, + "fragmented": 1885760, + "jit": 72087, + "jitIl": 11641768, + "sdk": "2.0.1.0", + "assemblies": 70, + "dynamicAssemblies": 0, + "timers": 19, + "threads": 24, + "sessions": 0, + "subscriptions": 0 + }, + "complete": { + "kind": "complete", + "cycles": 768, + "restarts": 192, + "seconds": 1804.5232117, + "fullGate": false + }, + "finalCertificateCounters": { + "created": 19272, + "disposed": 19272, + "outstanding": 0 + }, + "rssDelta": 132665344, + "managedDelta": -290392, + "exitStatus": 0, + "checkpoints": [ + { + "seconds": 572.6644396781921, + "cycle": 244, + "rss": 319848448, + "live": 20011096, + "jit": 63520, + "timers": 19 + }, + { + "seconds": 872.7072067260742, + "cycle": 372, + "rss": 367570944, + "live": 19829080, + "jit": 71814, + "timers": 19 + }, + { + "seconds": 1172.7554125785828, + "cycle": 500, + "rss": 371867648, + "live": 20093096, + "jit": 71931, + "timers": 19 + }, + { + "seconds": 1472.82679271698, + "cycle": 628, + "rss": 336289792, + "live": 19168056, + "jit": 72010, + "timers": 19 + }, + { + "seconds": 1772.8958134651184, + "cycle": 755, + "rss": 338808832, + "live": 19176744, + "jit": 72081, + "timers": 17 + } + ] + } + } + }, + "macNativeInventory": [ + { + "cycle": 52, + "seconds": 211.5059865, + "AppleDatabase": 278, + "rss": 266944512, + "managed": 43628048, + "jit": 57225 + }, + { + "cycle": 80, + "seconds": 327.9173383, + "AppleDatabase": 432, + "rss": 272138240, + "managed": 41062832, + "jit": 59127 + }, + { + "cycle": 109, + "seconds": 452.5689726, + "AppleDatabase": 585, + "rss": 275382272, + "managed": 40214352, + "jit": 59635 + }, + { + "cycle": 139, + "seconds": 574.7633382, + "AppleDatabase": 738, + "rss": 314769408, + "managed": 46524912, + "jit": 70477 + }, + { + "cycle": 168, + "seconds": 695.6731877, + "AppleDatabase": 887, + "rss": 282542080, + "managed": 41086264, + "jit": 70621 + }, + { + "cycle": 196, + "seconds": 812.5495777, + "AppleDatabase": 1041, + "rss": 284278784, + "managed": 41095016, + "jit": 70730 + }, + { + "cycle": 225, + "seconds": 938.4280813, + "AppleDatabase": 1193, + "rss": 279134208, + "managed": 39769736, + "jit": 70846 + }, + { + "cycle": 255, + "seconds": 1061.1608808, + "AppleDatabase": 1347, + "rss": 369016832, + "managed": 40596696, + "jit": 80997 + }, + { + "cycle": 284, + "seconds": 1180.6272768, + "AppleDatabase": 1496, + "rss": 371048448, + "managed": 41516048, + "jit": 81084 + }, + { + "cycle": 312, + "seconds": 1300.2894187, + "AppleDatabase": 1645, + "rss": 372473856, + "managed": 40947688, + "jit": 81167 + }, + { + "cycle": 341, + "seconds": 1426.1547829, + "AppleDatabase": 1801, + "rss": 374882304, + "managed": 40292440, + "jit": 81249 + }, + { + "cycle": 370, + "seconds": 1545.673147, + "AppleDatabase": 1953, + "rss": 376864768, + "managed": 43981184, + "jit": 81349 + }, + { + "cycle": 400, + "seconds": 1668.1859446, + "AppleDatabase": 2105, + "rss": 377946112, + "managed": 41122608, + "jit": 81443 + }, + { + "cycle": 428, + "seconds": 1785.0064517, + "AppleDatabase": 2259, + "rss": 379650048, + "managed": 41270424, + "jit": 81524 + } + ], + "singleOwnerCertificateControls": { + "public": { + "mac": { + "first": { + "kind": "ready", + "i": 0, + "mode": "public", + "live": 362656, + "rss": 51494912, + "methods": 133, + "gc": 360448 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "public", + "live": 626624, + "rss": 55443456, + "methods": 360, + "gc": 884736 + }, + "AppleDatabaseBefore": 2, + "AppleDatabaseAfter": 2 + }, + "linux": { + "first": { + "kind": "ready", + "i": 0, + "mode": "public", + "live": 141248, + "rss": 39936000, + "methods": 70, + "gc": 614400 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "public", + "live": 236528, + "rss": 48881664, + "methods": 184, + "gc": 950272 + } + } + }, + "shared-copy": { + "mac": { + "first": { + "kind": "ready", + "i": 0, + "mode": "shared-copy", + "live": 362704, + "rss": 51396608, + "methods": 133, + "gc": 360448 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "shared-copy", + "live": 622672, + "rss": 55427072, + "methods": 367, + "gc": 622592 + }, + "AppleDatabaseBefore": 2, + "AppleDatabaseAfter": 2 + }, + "linux": { + "first": { + "kind": "ready", + "i": 0, + "mode": "shared-copy", + "live": 141256, + "rss": 39907328, + "methods": 70, + "gc": 614400 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "shared-copy", + "live": 244952, + "rss": 48787456, + "methods": 185, + "gc": 884736 + } + } + }, + "pfx": { + "mac": { + "first": { + "kind": "ready", + "i": 0, + "mode": "pfx", + "live": 362696, + "rss": 51462144, + "methods": 133, + "gc": 360448 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "pfx", + "live": 557832, + "rss": 80773120, + "methods": 1625, + "gc": 2719744 + }, + "AppleDatabaseBefore": 2, + "AppleDatabaseAfter": 1002 + }, + "linux": { + "first": { + "kind": "ready", + "i": 0, + "mode": "pfx", + "live": 141240, + "rss": 39931904, + "methods": 70, + "gc": 614400 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "pfx", + "live": 284504, + "rss": 60588032, + "methods": 1159, + "gc": 1863680 + } + } + }, + "sdk-copy": { + "mac": { + "first": { + "kind": "ready", + "i": 0, + "mode": "sdk-copy", + "live": 362728, + "rss": 51560448, + "methods": 133, + "gc": 360448 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "sdk-copy", + "live": 559992, + "rss": 86032384, + "methods": 1944, + "gc": 6914048 + }, + "AppleDatabaseBefore": 2, + "AppleDatabaseAfter": 1002 + }, + "linux": { + "first": { + "kind": "ready", + "i": 0, + "mode": "sdk-copy", + "live": 141248, + "rss": 40091648, + "methods": 70, + "gc": 614400 + }, + "last": { + "kind": "done", + "i": 1000, + "mode": "sdk-copy", + "live": 285568, + "rss": 69271552, + "methods": 1549, + "gc": 9592832 + } + } + } + }, + "telemetryControls": { + "dispose": { + "first": { + "i": 0, + "dispose": true, + "live": 192656, + "rss": 42647552, + "methods": 17, + "il": 680, + "gc": 360448, + "threads": 8, + "timers": 0 + }, + "last": { + "i": 10000, + "dispose": true, + "live": 429440, + "rss": 63356928, + "methods": 456, + "il": 27078, + "gc": 6127616, + "threads": 9, + "timers": 0 + } + }, + "undisposed": { + "first": { + "i": 0, + "dispose": false, + "live": 192664, + "rss": 42549248, + "methods": 17, + "il": 680, + "gc": 360448, + "threads": 8, + "timers": 0 + }, + "last": { + "i": 10000, + "dispose": false, + "live": 428536, + "rss": 63406080, + "methods": 447, + "il": 26794, + "gc": 6127616, + "threads": 9, + "timers": 0 + } + } + }, + "jitEvidence": { + "jit-transition": { + "traceStartUtc": "2026-11-10T18:51:02.049Z", + "traceEndUtc": "2026-11-10T18:54:02.0624369Z", + "eventsLost": 0, + "sdkAssemblyLoads": 0, + "jittingStarts": 9854, + "distinctNameKeys": 9508, + "opcUaJittingStarts": 9063, + "invokeStubStarts": 50, + "tiers": { + "Optimized": 50, + "OptimizedTier1": 9772, + "OptimizedTier1Instrumented": 14, + "QuickJitted": 4, + "QuickJittedInstrumented": 13, + "MinOptJitted": 1 + }, + "summedLoadedNativeMethodBytes": { + "Optimized": 2920, + "OptimizedTier1": 51161724, + "OptimizedTier1Instrumented": 6740, + "QuickJitted": 996, + "QuickJittedInstrumented": 4028, + "MinOptJitted": 151128 + }, + "factory": { + "starts": 6690, + "distinctNames": 6690, + "distinctTokens": 6690, + "moduleCount": 1, + "everyNameOnce": true, + "firstMs": 106200.5198, + "lastMs": 139384.935912, + "summedLoadedNativeMethodBytes": 49141484, + "examples": [ + "Opc.Ua.OpcUaExtensions.AddOpcUa", + "Opc.Ua.OpcUaExtensions.CreateDefaultBinary", + "Opc.Ua.OpcUaExtensions.CreateDefaultXml", + "Opc.Ua.OpcUaExtensions.CreateBaseDataType", + "Opc.Ua.OpcUaExtensions.CreateNumber", + "Opc.Ua.OpcUaExtensions.CreateInteger", + "Opc.Ua.OpcUaExtensions.CreateUInteger", + "Opc.Ua.OpcUaExtensions.CreateEnumeration", + "Opc.Ua.OpcUaExtensions.CreateBoolean", + "Opc.Ua.OpcUaExtensions.CreateSByte" + ] + } + }, + "jit-post-transition": { + "traceStartUtc": "2026-11-10T19:01:44.116Z", + "traceEndUtc": "2026-11-10T19:04:44.1266407Z", + "eventsLost": 0, + "sdkAssemblyLoads": 0, + "jittingStarts": 85, + "distinctNameKeys": 47, + "opcUaJittingStarts": 0, + "invokeStubStarts": 38, + "tiers": { + "OptimizedTier1": 18, + "OptimizedTier1Instrumented": 20, + "Optimized": 38, + "QuickJittedInstrumented": 5, + "QuickJitted": 4 + }, + "summedLoadedNativeMethodBytes": { + "OptimizedTier1": 5720, + "OptimizedTier1Instrumented": 5800, + "Optimized": 2204, + "QuickJittedInstrumented": 1344, + "QuickJitted": 604 + }, + "factory": { + "starts": 0, + "distinctNames": 0, + "distinctTokens": 0, + "moduleCount": 0, + "everyNameOnce": true, + "firstMs": null, + "lastMs": null, + "summedLoadedNativeMethodBytes": 0, + "examples": [] + } + } + }, + "macLatchedProfile": { + "header": { + "kind": "start", + "utc": "2026-11-10T18:16:51.500331+00:00", + "processId": 97149, + "mode": "profile", + "requestedSeconds": 1800, + "minimumFullSeconds": 7200, + "build": "6b2d4b81", + "sdk": "2.0.1+07e62f72500cc0923a2bab940653c8c1116fa286", + "diagnosticInstrumentation": "SDK2.0.1 CP lists and certificate-validation cached stores; read-only", + "limits": { + "retainedManagedGrowthBytes": 67108864, + "rssGrowthBytes": 134217728, + "fdGrowth": 32 + }, + "coverage": "encrypted sessions; scalar/Int16[] Raw+UNS; subscription removal; browse abandonment; history release/cross-session/close; surgical changes/rebuild; revocation; paused TCP read/recovery; same-PKI restart" + }, + "firstRejection": { + "kind": "budget-rejected", + "cycle": 253, + "seconds": 1055.577868, + "resources": { + "Cycle": 253, + "Seconds": 1055.4823127, + "Sessions": 0, + "Subscriptions": 0, + "Items": 0, + "BrowsePoints": 0, + "HistoryPoints": 0, + "LiveDataItems": 0, + "LiveEventItems": 0, + "MonitoredNodes": 0, + "Variables": 19, + "Folders": 4, + "Conditions": 1, + "CachedCertificateStores": 2, + "Descriptors": 191, + "PkiDescriptors": 0, + "ManagedBytes": 39759304, + "RssBytes": 364199936, + "AllocatedBytes": 3522198624, + "Gen0": 639, + "Gen1": 458, + "Gen2": 380, + "GcCommittedBytes": 37224448, + "GcHeapBytes": 22139504, + "GcFragmentedBytes": 1567656, + "PinnedObjects": 1, + "PendingFinalizers": 26, + "JitMethods": 80639, + "JitIlBytes": 12230333, + "ActiveTimers": 13, + "Threads": 28 + }, + "baseline": { + "Cycle": 3, + "Seconds": 8.4936211, + "Sessions": 0, + "Subscriptions": 0, + "Items": 0, + "BrowsePoints": 0, + "HistoryPoints": 0, + "LiveDataItems": 0, + "LiveEventItems": 0, + "MonitoredNodes": 0, + "Variables": 19, + "Folders": 4, + "Conditions": 1, + "CachedCertificateStores": 2, + "Descriptors": 189, + "PkiDescriptors": 0, + "ManagedBytes": 40609072, + "RssBytes": 229048320, + "AllocatedBytes": 55465160, + "Gen0": 10, + "Gen1": 6, + "Gen2": 5, + "GcCommittedBytes": 32358400, + "GcHeapBytes": 21413392, + "GcFragmentedBytes": 399944, + "PinnedObjects": 1, + "PendingFinalizers": 16, + "JitMethods": 36805, + "JitIlBytes": 4668701, + "ActiveTimers": 15, + "Threads": 33 + } + }, + "lastObservation": { + "Cycle": 432, + "Seconds": 1801.8864247, + "Sessions": 0, + "Subscriptions": 0, + "Items": 0, + "BrowsePoints": 0, + "HistoryPoints": 0, + "LiveDataItems": 0, + "LiveEventItems": 0, + "MonitoredNodes": 0, + "Variables": 19, + "Folders": 4, + "Conditions": 1, + "CachedCertificateStores": 2, + "Descriptors": 200, + "PkiDescriptors": 0, + "ManagedBytes": 41364096, + "RssBytes": 380059648, + "AllocatedBytes": 5970802824, + "Gen0": 1083, + "Gen1": 771, + "Gen2": 648, + "GcCommittedBytes": 37060608, + "GcHeapBytes": 22923624, + "GcFragmentedBytes": 1531104, + "PinnedObjects": 1, + "PendingFinalizers": 14, + "JitMethods": 81535, + "JitIlBytes": 12263314, + "ActiveTimers": 15, + "Threads": 27 + }, + "outcome": { + "kind": "failed", + "cycles": 432, + "restarts": 108, + "seconds": 1807.4785953, + "exceptionType": "Shouldly.ShouldAssertException" + } + }, + "referenceHashes": { + "baseline": { + "Opc.Ua.Core.dll": "602e85316943ad7ebae33a23d60a69a6ea50d91250ad0556906b816435daa37c", + "Opc.Ua.Types.dll": "c5c5e36cf6381444b408fe1b51726b17deca74173764cd62ccee954b3c8c2718", + "Opc.Ua.Server.dll": "d8d9706dea577858325730cd89896826ed82f7bfcf9503ccd0260311663a69f4", + "Opc.Ua.Client.dll": "fd000b186f4bc5dbd9947e876f96c178e8c4259ee3e8717d672a4ac2f4c3a058", + "ZB.MOM.WW.OtOpcUa.OpcUaServer.dll": "ee1f704fa1574b77bf56cef415c47c33a4428c6852db52bb1443d6765d092cb1", + "control.dll": "a3c0ec38276aab6e281013619e6e5979263dce0dbc88ba6b837ea3f846aea009" + }, + "candidate": { + "Opc.Ua.Core.dll": "9d0857aa6b7f3c49037d4ea557b4da8edf07e146069464481783e66c5c9cd35a", + "Opc.Ua.Types.dll": "c542f69882f4e8a4ebd5ccc28a3881d6967c7ffb6dcc3c56dc6c776f39c2ede1", + "Opc.Ua.Server.dll": "930e714a2f6e15d393840bcdc95e1afa8012bc740c3738a655f958142721864e", + "Opc.Ua.Client.dll": "620da406fabdbc75d11f9827ea7e39d2d3c762471bd2264b0d11f1d6cd9247cd", + "Opc.Ua.Core.Types.dll": "a7dd141b8dabe8848ff28c2fb8e16a85a0508e6a49e6b1a1c1e267175b99af96", + "ZB.MOM.WW.OtOpcUa.OpcUaServer.dll": "8768914278dd8b1fea02644cd21e4fb5456db698798b257276c85ae0572b2cb5", + "control.dll": "ad22c92b097a9fc5ceb112424938bb7e7bc30d7460e242d7ebf4d36eba6928f0" + } + }, + "diagnosticInputSha256": { + "mac.original.samples": "be2928b2a8eda377dc0d3230275e8eb7824b5b763b3053005fcb40700e48a6db", + "mac.sdk2.samples": "a3907010c07be0f9848cdcbe73b8292487166fc84af4b15cf260093c32ad872f", + "linux600.original.samples": "d73d1bcf4d1b5db67b45f681a2a2f5448cb0204b11af3a54153cbf576984f3f5", + "linux600.sdk2.samples": "6d945ff8d3a6f0cc15d76d0afec4f4ccff467e842be6a29b73825be63690391f", + "linux1800.original.samples": "40ca999e9e07b141a39eaa224432d9044cfde9806b51cb91a1e100f6dcf72947", + "linux1800.original.observations": "7c7ce340c14f22264025cbf0ec9af67eba64a236067c5d42a6e0b0daf1947470", + "linux1800.sdk2.samples": "fe9314c8aa2d916d9f311164ce93f366f70631c30a11e2b2fdc65b11a1440d9b", + "linux1800.sdk2.observations": "ff2455630ac57058e1d5675943c9489ded38b7b745c5d110c1c61fa10e396a0e", + "mac.native.heap-00": "f94e6eee654a80de3b7206d65c274659311ea865fc55213796a0058421863099", + "mac.native.heap-01": "a3ab7b95e844df1437de8c0ed93f5a15290ef09625f701d76617c1c322b842f9", + "mac.native.heap-02": "deb34716d2830ded8132a7f0b668c6c459352fa68496e6999e4a300d3f6824d1", + "mac.native.heap-03": "497eb60d9021ca4c510309649ba614733a5ec6f956e36fc65fbc3cdb1a2881e1", + "mac.native.heap-04": "7217d9705a417ce8b06565004964fb266c0f159555108b5b59f44b79c6917a4c", + "mac.native.heap-05": "16e962b121875422dc88cee77f29324718d4c2c745812a8d6fb112829c2cb108", + "mac.native.heap-06": "c24b6e51f15646c26cc0f895f94dd02e6f930d2939dd6619a765f01ba90bfaac", + "mac.native.heap-07": "c8db73aa5a3f2eb187d49d95c13b5c83104ad22e68c2fd7d9d49bfca695878dc", + "mac.native.heap-08": "e40e9a13e63ab89e159ace922666fa111cee18350efba3522d47452178db4e49", + "mac.native.heap-09": "08a9b56e2d7f0b7d28ae87e2065f279b642d02c7829a3c174727a71c64206208", + "mac.native.heap-10": "c73563e0ce35ddef6747d8e1ae643376db741b04ab0e450b62b5b5eaaf411eba", + "mac.native.heap-11": "8253524360288a8b695aa0895df8138d540f7f8dc89db3c81fb1286e2da5e8de", + "mac.native.heap-12": "a3c634dce2a1e9eaa1c2cd5b959115b65ec8eb5af7ae456434f19c9402ba090d", + "mac.native.heap-13": "2775fbd0058e5eb7784080ab9ea780d409ea0442e86be463acca96b75a870a64", + "certificate.mac.public": "b7b475faf2f97a3409f641e8fc79154aa19eadda49153f0979e0f12c28363499", + "certificate.mac.public.heap-before": "7ff2064816fd80f01a9b856fbda2aeef6c3e5329c5ce0e10c25c270055fbbe1e", + "certificate.mac.public.heap-after": "b76893f95f2daaee5420c9a5392fad794a133bdfe7f5432069889bbd8cd29956", + "certificate.linux.public": "f1e25d28463cedc0e1561957b554106c87070399d2e707079511c92024aa5f1f", + "certificate.mac.shared-copy": "ff4a00a388f8e2562cae3ab8084bfce93cf53372d5208934ab1d0fca116c9942", + "certificate.mac.shared-copy.heap-before": "e70412fa24a698492d5bed5da12cec062979b559f7b193f106143f447bc72e43", + "certificate.mac.shared-copy.heap-after": "19f62da47a291dfa1bbc93fe470cf400dc6b8a6e7825ada0fb6bde5bf0525c4a", + "certificate.linux.shared-copy": "1ce5f15a55c2e209a2dbbc5848bfd24a39229d1013c873bf2ed41854470276c5", + "certificate.mac.pfx": "241c5a327dfc72105746cb79804900a06e71a0c72847256cc61fd7f2cecbfee9", + "certificate.mac.pfx.heap-before": "15d8f324e85baf5b7bbb07ffe5a477cab10cc3cf2ec8dcec69f73217be8932b3", + "certificate.mac.pfx.heap-after": "451d4d6b3da76f768d087fd1eeaaa0ee750b452f4c8b1b067dc553db315cb877", + "certificate.linux.pfx": "fdcb5a9b638beebc66937eafac2ff35bb6cda6d5d4a52628c3d10ca26f8507ce", + "certificate.mac.sdk-copy": "3192fa040840b0e783b2321ca334be13cdbf6ddb1be2cea26c4e00c319348dd7", + "certificate.mac.sdk-copy.heap-before": "1d255a94b3454a8b91e8481a4f8ec299fcb039ca70467260a6f095f9f14b2e88", + "certificate.mac.sdk-copy.heap-after": "f084293d04192b34825cc320f44af508d63bbea768c0fe9ce0d0121593340b99", + "certificate.linux.sdk-copy": "ef4db302de590a4a29b551948534f7dbb7ad4fdb9f70e36f49ef23994ee9b102", + "telemetry.dispose": "b1d146a2c7b964141e2dcbf65ff4983aea589fbc6d7bc508e8bb6fcaacf444b2", + "telemetry.undisposed": "563a4e4167069c01ff6f2723f867c166809956b4cd19fbeb93672fd999f7e2f6", + "jit-transition.trace": "7f560d048ba723dfde13b663543ad09a0abc5212d524ef5c527c63edd6b97ddc", + "jit-post-transition.trace": "fe09c47ba1d74f9ca5426b7d3464be2cbac44e588c75261ce394457126e9a528", + "mac.latched30.resources": "e99c647eb5ef9abd918e9c9991973a9a96da3455f6a0aaee79e1604f21da0658", + "baseline.publication934Files.manifest": "beb9442eb010716a4d31e595f7521bbbd09f5a748516e7c6197aa3c59158ad16", + "sdk.tag.source.Certificate.cs": "6f0be22e80e05607d1c3e35ce0fe218273446c397c18df909124be61ee437f56", + "sdk.tag.source.DefaultTelemetry.cs": "72adca708b0649e822510061896405cf07dea3be411c4f221f8be13e156a0313", + "sdk.tag.source.DiagnosticsNodeManager.cs": "de1f89915459a997268b80c444a9b85bdf1831f2771ae3de240941071b2570d2", + "mac.full.fa.jsonl.0": "504f22cb34d966ee768b496a2f9de5b1ae9fc429ece0de98a2555413ac36449b", + "mac.full.fa.trx.0": "910216d49246b94ac4e86d288e3c65a3457ec7dc7ac83abf0a3b75ed623db6fe", + "mac.profile.e135.jsonl.0": "0faef406b2a0749e84e3c4f60702c3059fb4d0b9e8f3379a0405305b8a596263", + "mac.profile.e135.trx.0": "2f39f4dcf801e58b4b4466c811c98563767b5e49ca00950ef9aafdc89df0a963", + "linux.full.jsonl.0": "1d24d32382b5be6ebca2615389929818dd6c9874b7787cb91482ceaa5d3b9cd3", + "linux.full.trx.0": "99ca030ff7542e2dc43959d8cf9a2f129352db9c6a44d5d011c6e10fa9327f01", + "linux.full.trx.1": "798c97c3158ec35d9dbae754e98f57a7bf403ae10b81473f98a52a89c2401265", + "reproducer.source.build_trace_tools.py": "bc92dd5d4ba44d9dcade23abb2f771a0770c5a8ded3fc16c9c3fb0d66527d65f", + "reproducer.source.run.py": "c02b704a716205b24312916377c361ba9f20ffc1841f6092b0ce41d894ce726a", + "reproducer.source.TraceSummary.cs": "ae33c3487cf9919a7b0f743265890a514a3a3bc83fbc374d53dcc3f2f95b5cfc", + "reproducer.source.EventPipeCapture.cs": "b8fe7309d84756a55383b737eccb879ca32df58375ee16c274ecba70435fa383", + "reproducer.source.linux_pair.py": "1e1ef351461a5f93022e8d470a1afc6016df6dd0d87a1722167f403092c76beb", + "reproducer.source.summarize.py": "6ca78dd656af0d9a8a19a788634f6f90c03df49e1880612a1cb2f2df3f66c882", + "reproducer.source.TelemetryControl.cs": "8e7ee56cf9be6bf5fbf9e6297e3370a5d94efc7922800fc4c829091d866692ec", + "reproducer.source.LifecycleControl.cs": "7d8fff335c7eea763d988e03f7cd1c5b3663cad387d9ee14a86893ed012ee221", + "reproducer.source.CertificateControl.cs": "ac36cf2c01685105e295be30064c6bf62c4a9716ec6ae1de999c25b61c8a0106", + "controls.original-shared-source": "06f0b708042f628feb91f61e92ddf6811d1476d8d177ab5137091d28df1090d5" + } +} diff --git a/scripts/validation/opcua-lifecycle-memory/linux_pair.py b/scripts/validation/opcua-lifecycle-memory/linux_pair.py new file mode 100644 index 00000000..fb0caeba --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/linux_pair.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Run two already-built public lifecycle controls in isolated, offline Linux containers.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import secrets +import subprocess +import time +import uuid + + +def latest_sample(path): + # A concurrently-written final line may be incomplete. All complete rows remain in the source log. + for row in reversed(path.read_text().splitlines()): + try: + value = json.loads(row) + except json.JSONDecodeError: + continue + if value.get("kind") == "sample": + return value + return None + + +def run(options): + options.output = options.output.resolve() + options.output.mkdir(parents=True, exist_ok=False) + processes = [] + environment = os.environ.copy() + environment["OPCUA_MEMORY_PASSWORD"] = secrets.token_urlsafe(32) + # Both receive the same environment. SDK1 ignores the SDK2 diagnostic opt-in. + environment["OPCUA_CERTIFICATE_LEAK_TRACKING"] = "1" + started = time.monotonic() + try: + for label, directory in [("baseline", options.baseline), ("candidate", options.candidate)]: + directory = directory.resolve(strict=True) + for required in ["control.dll", "control.deps.json", "control.runtimeconfig.json"]: + if not (directory / required).is_file(): + raise FileNotFoundError("Use a complete, canonically-built control output: missing " + required) + output = options.output / label + output.mkdir() + container = "opcua-memory-" + label + "-" + uuid.uuid4().hex[:10] + shell = ('dotnet /app/control.dll /task/private-fixture ' + str(options.seconds) + + ' > /task/samples.jsonl 2>&1 & child=$!; echo "$child" > /task/dotnet.pid; ' + 'wait "$child"; status=$?; echo "$status" > /task/exit-status.txt; exit "$status"') + command = ["docker", "run", "--rm", "--name", container, "--network", "none", + "--env", "OPCUA_MEMORY_PASSWORD", "--env", "OPCUA_CERTIFICATE_LEAK_TRACKING", + "-v", str(directory) + ":/app:ro", "-v", str(output) + ":/task", + options.image, "bash", "-c", shell] + manifest = {"argv": command, "fullGate": False, "seconds": options.seconds, + "environmentNamesOnly": ["OPCUA_MEMORY_PASSWORD", "OPCUA_CERTIFICATE_LEAK_TRACKING"], + "startedUtc": datetime.now(timezone.utc).isoformat(), + "runtimeFiles": {p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(directory.iterdir()) if p.is_file()}} + (output / "command.json").write_text(json.dumps(manifest, indent=2) + "\n") + stream = (output / "container.log").open("w") + process = subprocess.Popen(command, env=environment, stdout=stream, stderr=subprocess.STDOUT) + processes.append((label, container, output, stream, process)) + next_sample = started + 60 + while any(process.poll() is None for _, _, _, _, process in processes): + if time.monotonic() - started > options.seconds + 240: + raise TimeoutError("Diagnostic controls exceeded bounded shutdown allowance") + if time.monotonic() >= next_sample: + for label, container, output, _, process in processes: + pid_file = output / "dotnet.pid" + if process.poll() is not None or not pid_file.exists(): + continue + pid = pid_file.read_text().strip() + if not pid.isdecimal(): + raise RuntimeError("Unexpected task-owned process id") + stamp = str(int(time.time())) + latest = latest_sample(output / "samples.jsonl") + with (output / "observations.jsonl").open("a") as observed: + observed.write(json.dumps({"utc": datetime.now(timezone.utc).isoformat(), + "seconds": time.monotonic() - started, + "sample": latest}) + "\n") + for name in ["smaps", "smaps_rollup", "status"]: + with (output / (stamp + "-" + name + ".txt")).open("w") as destination: + result = subprocess.run(["docker", "exec", container, "cat", "/proc/" + pid + "/" + name], + stdout=destination, stderr=subprocess.STDOUT, timeout=15) + if result.returncode and process.poll() is None: + raise RuntimeError(label + " live process diagnostic read failed") + next_sample += 60 + time.sleep(.5) + statuses = [] + for label, _, output, _, process in processes: + status = process.wait() + (output / "docker-exit-status.txt").write_text(str(status) + "\n") + print(label + " exit " + str(status), flush=True) + statuses.append(status) + return 0 if all(status == 0 for status in statuses) else 1 + finally: + for _, container, _, stream, process in processes: + if process.poll() is None: + subprocess.run(["docker", "stop", "--time", "10", container], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=20) + process.wait(timeout=20) + stream.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True, help="Already-built control output directory") + parser.add_argument("--candidate", type=Path, required=True, help="Already-built control output directory") + parser.add_argument("--output", type=Path, required=True, help="New local diagnostic directory") + parser.add_argument("--image", required=True, help="Immutable Linux .NET10 image digest") + parser.add_argument("--seconds", type=int, default=1800) + options = parser.parse_args() + if not 1 <= options.seconds <= 1800 or not options.image.startswith("sha256:"): + parser.error("Use an immutable image digest and duration 1..1800; never a full-soak gate") + raise SystemExit(run(options)) diff --git a/scripts/validation/opcua-lifecycle-memory/run.py b/scripts/validation/opcua-lifecycle-memory/run.py new file mode 100644 index 00000000..d92356da --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/run.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Build opt-in controls against existing published DLLs; never rebuild product code.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("kind", choices=["lifecycle", "certificate", "telemetry"]) + parser.add_argument("--references", type=Path, action="append", required=True, + help="Existing published DLL directory; first directory wins duplicate names") + parser.add_argument("--output", type=Path, required=True, help="New local artifact directory") + parser.add_argument("--sdk2", action="store_true") + parser.add_argument("--seconds", type=int, default=600) + parser.add_argument("--mode", help="Certificate: public/pfx/shared-copy/sdk-copy; telemetry: dispose/undisposed") + parser.add_argument("--build-only", action="store_true") + parser.add_argument("--certificate-tracking", action="store_true", help="SDK2 opt-in allocation-stack diagnostics") + options = parser.parse_args() + if options.kind != "lifecycle" and not options.sdk2: + parser.error("certificate and telemetry controls require --sdk2") + if not 1 <= options.seconds <= 1800: + parser.error("Diagnostic duration must be 1..1800 seconds; this is never a full soak") + options.output = options.output.resolve() + options.output.mkdir(parents=True, exist_ok=False) + names = {"lifecycle": "LifecycleControl.cs", "certificate": "CertificateControl.cs", "telemetry": "TelemetryControl.cs"} + source = Path(__file__).resolve().parent / names[options.kind] + references = {} + for directory in options.references: + directory = directory.resolve(strict=True) + for dll in sorted(directory.glob("*.dll")): + references.setdefault(dll.name, dll) + if "Opc.Ua.Core.dll" not in references: + parser.error("References must contain actual OPC UA managed assemblies") + project = ET.Element("Project", Sdk="Microsoft.NET.Sdk") + properties = ET.SubElement(project, "PropertyGroup") + for key, value in {"OutputType": "Exe", "TargetFramework": "net10.0", "ImplicitUsings": "enable", + "Nullable": "enable", "EnableDefaultCompileItems": "false", + "DefineConstants": "SDK2" if options.sdk2 else ""}.items(): + ET.SubElement(properties, key).text = value + items = ET.SubElement(project, "ItemGroup") + ET.SubElement(items, "FrameworkReference", Include="Microsoft.AspNetCore.App") + ET.SubElement(items, "Compile", Include=str(source)) + for name, path in references.items(): + reference = ET.SubElement(items, "Reference", Include=path.stem) + ET.SubElement(reference, "HintPath").text = str(path) + project_path = options.output / "control.csproj" + ET.indent(project) + ET.ElementTree(project).write(project_path, encoding="unicode") + manifest = {"kind": options.kind, "sdk2": options.sdk2, "seconds": options.seconds, + "fullSoakGate": False, "source": {source.name: digest(source)}, + "referencePrecedence": "first directory wins duplicate names", + "references": {name: digest(path) for name, path in references.items()}} + (options.output / "provenance.json").write_text(json.dumps(manifest, indent=2) + "\n") + with (options.output / "build.log").open("w") as log: + result = subprocess.run(["dotnet", "build", str(project_path), "-c", "Release", "-m:2"], + stdout=log, stderr=subprocess.STDOUT) + if result.returncode: + return result.returncode + if options.build_only: + return 0 + executable = options.output / "bin/Release/net10.0/control.dll" + environment = os.environ.copy() + environment["OPCUA_MEMORY_PASSWORD"] = secrets.token_urlsafe(32) + if options.certificate_tracking: + environment["OPCUA_CERTIFICATE_LEAK_TRACKING"] = "1" + if options.kind == "lifecycle": + command = ["dotnet", str(executable), str(options.output / "private-fixture"), str(options.seconds)] + with (options.output / "samples.jsonl").open("w") as log: + result = subprocess.run(command, env=environment, stdout=log, stderr=subprocess.STDOUT, + timeout=options.seconds + 180) + status = result.returncode + elif options.kind == "telemetry": + if options.mode not in ["dispose", "undisposed"]: + parser.error("Telemetry mode must be dispose or undisposed") + with (options.output / "samples.jsonl").open("w") as log: + result = subprocess.run(["dotnet", str(executable), options.mode], env=environment, + stdout=log, stderr=subprocess.STDOUT, timeout=180) + status = result.returncode + else: + if sys.platform != "darwin": + parser.error("Native certificate control currently requires macOS heap/vmmap") + if options.mode not in ["public", "pfx", "shared-copy", "sdk-copy"]: + parser.error("Certificate mode must be public, pfx, shared-copy or sdk-copy") + status = certificate_control(executable, options.output, options.mode, environment) + (options.output / "exit-status.txt").write_text(str(status) + "\n") + return status + + +def certificate_control(executable, output, mode, environment): + # Native snapshots contain process metadata; retain locally, do not publish them unredacted. + with (output / "samples.jsonl").open("w") as log: + process = subprocess.Popen(["dotnet", str(executable), mode, str(output)], env=environment, + stdout=log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + 180 + while not (output / "samples.jsonl").stat().st_size: + if process.poll() is not None or time.monotonic() > deadline: + raise RuntimeError("Certificate control failed before ready") + time.sleep(.05) + for label in ["before", "after"]: + if label == "after": + (output / "go").touch() + while True: + rows = (output / "samples.jsonl").read_text().splitlines() + if rows and json.loads(rows[-1]).get("kind") == "done": + break + if process.poll() is not None or time.monotonic() > deadline: + raise RuntimeError("Certificate control did not finish") + time.sleep(.1) + with (output / ("heap-" + label + ".txt")).open("w") as log_heap: + subprocess.run(["/usr/bin/heap", "--noContent", "-s", str(process.pid)], + stdout=log_heap, stderr=subprocess.STDOUT, timeout=20, check=True) + (output / "stop").touch() + return process.wait(timeout=10) + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=10) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/opcua-lifecycle-memory/summarize.py b/scripts/validation/opcua-lifecycle-memory/summarize.py new file mode 100644 index 00000000..dee1ec2a --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/summarize.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Emit mapping-category totals without path names from local Linux /proc/PID/smaps.""" +import argparse +from collections import defaultdict +import json +from pathlib import Path +import re + +HEADER = re.compile(r"^[0-9a-f]+-[0-9a-f]+\s+(\S+)\s+\S+\s+\S+\s+\S+\s*(.*)$") +METRIC = re.compile(r"^(Size|Rss|Pss|Private_Dirty|Private_Clean|Anonymous):\s+(\d+) kB$") + + +def summarize(path): + groups = defaultdict(lambda: defaultdict(int)) + group = None + for line in path.read_text().splitlines(): + header = HEADER.match(line) + if header: + permissions, name = header.groups() + if "doublemapper" in name: + group = "jit-doublemapper" + elif name == "[heap]": + group = "native-heap" + elif name.startswith("[stack"): + group = "stack" + elif not name: + group = "anonymous-exec" if "x" in permissions else "anonymous-write" if "w" in permissions else "anonymous-read" + elif name.endswith(".dll"): + group = "managed-assembly-files" + elif ".so" in name: + group = "native-library-files" + elif name.startswith("["): + group = "other-kernel-mappings" + else: + group = "other-files" + elif group and (metric := METRIC.match(line)): + groups[group][metric[1]] += int(metric[2]) * 1024 + return dict(groups) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="+", type=Path) + args = parser.parse_args() + print(json.dumps([{"sample": index, "groups": summarize(path)} for index, path in enumerate(args.files)], indent=2))