diff --git a/scripts/validation/opcua-lifecycle-memory/RESOLUTION.md b/scripts/validation/opcua-lifecycle-memory/RESOLUTION.md new file mode 100644 index 00000000..db9a9bab --- /dev/null +++ b/scripts/validation/opcua-lifecycle-memory/RESOLUTION.md @@ -0,0 +1,96 @@ +# Issue #799 resolution gates + +Status: prospective procedure under review, **not approved for launch**. The +retained cold failures remain failures. This document does not change the plan's +resource budgets, establish a replacement baseline, or authorize release. + +## Separate the evidence + +| Question | Evidence required | Current boundary | +|---|---|---| +| Finite generated-code startup | Fixed canonical operation sequence and bounded, lossless trace windows | Candidate saturation met the selected factory criterion. Summed native MethodSize does not explain all RSS. | +| Cross-version lifecycle ownership and memory | Equivalent operations, token invalidation, dropped harness roots, resource-owner checks and matched executable hosts | Original closed objects retain continuation lists while held by the harness; usable steady-state comparison is not established. | +| Continuing macOS native retention | Private-key import attribution, bounded native inventory, supported ownership correction and verification | Balanced managed disposal alone has not stopped Apple Security object accumulation. Linux results cannot close this component. | + +The coordinator owns integration and long executions. Two independent Sol agent +lanes own `baseline-canonical/` and `native-retention/` respectively; use high +effort for control implementation and ownership analysis. Astra at xhigh reviews +each control design, its result, any production correction, and the complete +acceptance procedure before execution. No worker changes the other lane's files, +production factory architecture, immutable baseline, or acceptance denominator. + +## Short controls before a new long run + +The original-SDK lane must demonstrate removal from SessionManager, rejection of +the old authentication token, and rejection of old browse/history continuation +tokens from a fresh session. A helper boundary must drop all strong Session, +continuation and harness references before bounded GC/weak-reference observations. +Observe closed-session/point weak references while the same host remains live, +then separately observe application/validator/store roots after normal host stop; +teardown must not hide retention of closed sessions. Inventory separately +identifies which resource owner releases each object. +Retain the earlier held-object failure. Do not explicitly dispose abandoned +objects or clear caches merely to make the baseline pass. Unsupported original +ownership probes remain unsupported, rather than being reported as zero. + +The native lane must attribute configured-store private-key loads separately +from public certificate and shared-handle operations using fixed counts and +identical certificates. Include a separate bounded store-update semantic control +so registry reuse cannot silently bypass configured-store rotation. Review exact +SDK 2.0.1 public ownership seams before any +application correction. A supported correction needs a failing-before/passing- +after ownership control and preserved trust, private-key availability, reconnect +and shutdown behavior. Custom crypto, copied transport negotiation, SDK forks, +factory substitution and certificate-policy changes require a separate design; +they are not fixes authorized by this procedure. Keep private PKI out of Git. + +## Prerequisites for reviewing warmed acceptance + +1. Establish operation/status/order parity and semantically valid close/ownership + checks across versions. Match executable host, runtime, architecture and + instrumentation. A growing original process is not an accepted denominator. +2. Freeze source commits, all artifact hashes, runtime/image identities, private + PKI identity, fixtures and the serial same-host schedule before launch. +3. Submit the actual harness change, negative controls and immutable artifact + receipts to Astra. Approval of diagnostics is not approval of acceptance. +4. Resolve continuing macOS native growth independently. A Linux acceptance run + cannot waive it, even if the Linux resource guard passes. + +Before this draft becomes executable, define version-appropriate standard-node +loading/factory trace coverage for the original binary. The absence of SDK2's +`Opc.Ua.OpcUaExtensions` type must not let the original pass an irrelevant zero +counter. Also predeclare identical retained-managed/GC and RSS sampling cadence +and statistics, including exactly which samples feed the relative 15% comparison. +These details remain pending; observed results cannot select them retroactively. + +## Proposed fixed warmup and measurement + +This section is a review proposal only. If the prerequisites and Astra gate pass, +qualification uses exactly 384 canonical cycles / 96 restarts plus one settling +cycle within 1,800 seconds. The fixed trace windows are cycles 321–352 and +353–384, with the continuous veto across their boundary. Require zero generated- +factory JIT/load events, zero new SDK assembly loads and zero lost events. Reject +missing markers, wrong identity or observer faults. Do not extend or retry the +qualification until a favorable sample appears. + +Prospectively select exactly the post-cycle-385 sample as the measurement +baseline; do not choose a minimum, median or later sample from observed RSS. +Retain all cold, peak and rejected rows. Measurement must then last at least +7,200 actual seconds **after** warmup, retaining the +64 MiB managed, +128 MiB +RSS and +32 descriptor caps, all canonical operations and semantic/native/ +ownership guards. Latch the first failure permanently. No resource resets, +skipped operations or completion claim on a rejected run. +Setup, warmup and final cleanup do not count toward measurement duration. Final +shutdown and ownership checks are mandatory after that duration completes. + +For uninstrumented acceptance, first perform an identical fresh paired +qualification rehearsal, then use the fixed warmup without EventPipe or +allocation StackTrace tracking. The earlier tracked candidate run cannot be +relabelled as this evidence. Predefine sampling and statistics for at least three +matched representative runs to assess the separate plan-relative throughput, +latency, allocation and RSS budgets. A two-hour absolute-cap pass alone does not +satisfy those comparisons. + +Only verified results meeting these distinct requirements can support closing +#799. Keep #782 G4 and draft PR #802 on hold until their remaining gates also +pass; Windows/runtime and licensed interoperability evidence remain separate. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventHistoryPagingOverTheWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventHistoryPagingOverTheWireTests.cs index 43447090..3e4c8886 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventHistoryPagingOverTheWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventHistoryPagingOverTheWireTests.cs @@ -122,12 +122,15 @@ public sealed class EventHistoryPagingOverTheWireTests /// (time, id) order, [start, end), cap + truncation flag — the SQL backends' contract. internal sealed class SqlLikeEventSource(List events) : IHistorianDataSource { + private int _reads; + internal int ReadCount => Volatile.Read(ref _reads); private readonly List _all = events.OrderBy(e => e.EventTimeUtc).ThenBy(e => e.EventId, StringComparer.Ordinal).ToList(); public async Task ReadEventsAsync( string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken ct) { + Interlocked.Increment(ref _reads); await Task.Delay(1, ct).ConfigureAwait(false); // complete on a pool thread, like a real backend var window = _all.Where(e => e.EventTimeUtc >= startUtc && e.EventTimeUtc < endUtc).ToList(); var cap = maxEvents <= 0 ? 1000 : maxEvents; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleCloseOwnershipTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleCloseOwnershipTests.cs new file mode 100644 index 00000000..7a42d89f --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleCloseOwnershipTests.cs @@ -0,0 +1,214 @@ +using System.Collections; +using System.Runtime.CompilerServices; +using System.Reflection; +using System.Text.Json; +using Opc.Ua; +using Opc.Ua.Client; +using Opc.Ua.Configuration; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; + +public sealed partial class Sdk2LifecycleSoakTests +{ + public static bool CloseControlEnabled => + Environment.GetEnvironmentVariable("OTOPCUA_SDK2_CLOSE_CONTROL") == "1"; + + [Fact(SkipUnless = nameof(CloseControlEnabled), Skip = "Opt in to the bounded SDK2 close/weak-root control.")] + public async Task Closed_tokens_are_rejected_and_ownership_is_observed_without_harness_roots() + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(120)); + var ct = deadline.Token; + // Positive and negative controls for the observer itself, independent of SDK objects. + var alive = new object(); + var aliveReference = new WeakReference(alive); + var collectible = CollectibleReference(); + var observer = await ObserveWeakAsync([new("retained-control", aliveReference), new("collectible-control", collectible)], ct); + observer.Last().Alive.ShouldBe([true, false]); + GC.KeepAlive(alive); + WriteCloseControl("observer-controls", observer); + + // The helper returns no rig, application, config, Session, point or store objects. + var stoppedTargets = await Task.Run(() => RunCloseFixture(ct), ct).WaitAsync(TimeSpan.FromSeconds(120), ct); + var stopped = await ObserveWeakAsync(stoppedTargets, ct); + WriteCloseControl("after-normal-host-stop", stopped); + stopped.Last().Alive.ShouldAllBe(value => !value); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakReference CollectibleReference() => new(new object()); + + private sealed record WeakTarget(string Kind, WeakReference Reference); + private sealed record WeakObservation(int Collection, string[] Kinds, bool[] Alive); + + private static async Task ObserveWeakAsync(WeakTarget[] targets, CancellationToken ct) + { + var observations = new List(); + // Fixed count/delay; no extension or retrieving/resurrecting targets during collection. + var delays = new[] { 50, 100, 250, 500, 1000 }; + for (var i = 0; i < delays.Length; i++) + { + ct.ThrowIfCancellationRequested(); + await Task.Run(() => + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); + }, ct).WaitAsync(TimeSpan.FromSeconds(5), ct); + await Task.Delay(delays[i], ct); + observations.Add(new(i + 1, targets.Select(t => t.Kind).ToArray(), targets.Select(t => t.Reference.IsAlive).ToArray())); + } + return observations.ToArray(); + } + + private static void WriteCloseControl(string phase, object observation) => + TestContext.Current.TestOutputHelper!.WriteLine(JsonSerializer.Serialize(new { phase, observation, acceptance = false })); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakTarget[] RunCloseFixture(CancellationToken ct) => ExerciseCloseOwnershipAsync(ct).GetAwaiter().GetResult(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakTarget[] RunClosedSession(ServerEventRig rig, ApplicationConfiguration config, CancellationToken ct) => + CloseAndDenyOldTokensAsync(rig, config, ct).GetAwaiter().GetResult(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task ExerciseCloseOwnershipAsync(CancellationToken ct) + { + var clientRoot = Path.Combine(Path.GetTempPath(), $"otua-close-client-{Guid.NewGuid():N}"); + var telemetry = DefaultTelemetry.Create(_ => { }); + var security = TestClientSecurity.Build(clientRoot); + security.ApplicationCertificate!.SubjectName = "CN=SDK2LifecycleSoak"; + var config = new ApplicationConfiguration(telemetry) + { + ApplicationName = "SDK2LifecycleSoak", ApplicationUri = $"urn:SDK2LifecycleSoak:{Guid.NewGuid():N}", + ApplicationType = ApplicationType.Client, SecurityConfiguration = security, + ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 15_000 }, + TransportQuotas = new TransportQuotas { OperationTimeout = 5000 }, + }; + ApplicationInstance? application = new(telemetry) { ApplicationConfiguration = config }; + ServerEventRig? rig = null; + try + { + await config.ValidateAsync(ApplicationType.Client, ct); + await application.CheckApplicationInstanceCertificatesAsync(true, null, ct); + rig = await ServerEventRig.StartAsync("SDK2CloseOwnership", encrypted: true, ct); + Configure(rig); + var closedTargets = RunClosedSession(rig, config, ct); + var closed = await ObserveWeakAsync(closedTargets, ct); + WriteCloseControl("closed-objects-same-host-live", closed); + closed.Last().Alive.ShouldAllBe(value => !value); + GC.KeepAlive(rig); + + var clientManager = config.CertificateManager; + var serverManager = rig.Server.CertificateManager!; + var serverApplication = RequiredField(rig, "_host").ApplicationInstance!; + var cores = CertificateCores(clientManager).Concat(CertificateCores(serverManager)).ToArray(); + var stoppedTargets = new List + { + new("client-application", new(application)), new("client-config", new(config)), + new("client-manager", new(clientManager)), new("server-manager", new(serverManager)), + new("server-application", new(serverApplication)), + new("server-rig", new(rig)), new("server", new(rig.Server)), + new("node-manager", new(rig.Nm)), new("event-source", new(rig.Nm.HistorianDataSource!)), + }; + foreach (var core in cores) + { + stoppedTargets.Add(new("certificate-validation-core", new(core))); + foreach (var store in RequiredField(core, "m_stores").Values) + if (store is not null) stoppedTargets.Add(new("cached-store", new(store))); + } + await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct); + await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct); + application = null; + RequiredField(clientManager, "m_disposed").ShouldBeTrue(); + RequiredField(serverManager, "m_disposed").ShouldBeTrue(); + cores.Sum(CachedStores).ShouldBe(0); + (await DescriptorCountsAsync(ct, clientRoot, RequiredField(rig, "_pkiRoot"))).Pki.ShouldBe(0); + rig = null; + WriteCloseControl("sdk2-disposal-proofs", new { managersDisposed = 2, cachedStores = 0, pkiDescriptors = 0 }); + return stoppedTargets.ToArray(); + } + finally + { + if (rig is not null) await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct); + if (application is not null) await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct); + if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task CloseAndDenyOldTokensAsync(ServerEventRig rig, + ApplicationConfiguration config, CancellationToken ct) + { + using var old = await ConnectAsync(rig, config, null, ct); + var authenticationProperty = typeof(ClientBase).GetProperty("AuthenticationToken", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Exact SDK2 authentication diagnostic is unavailable."); + authenticationProperty.PropertyType.ShouldBe(typeof(NodeId)); + var oldAuthentication = authenticationProperty.GetValue(old).ShouldBeOfType(); + oldAuthentication.IsNull.ShouldBeFalse(); + var browse = await BrowseAsync(old, ct); + var history = (await PageAsync(old, default, false, ct)).ContinuationPoint; + browse.IsEmpty.ShouldBeFalse(); history.IsEmpty.ShouldBeFalse(); + var serverSession = rig.Server.CurrentInstance.SessionManager.GetSessions().Single(); + var points = serverSession.ContinuationPoints; + var weak = new List + { + new("closed-client-session", new(old)), new("closed-server-session", new(serverSession)), + new("closed-point-owner", new(points)), + }; + using (RequiredField(points, "m_lock").EnterScope()) + foreach (var field in new[] { "m_browse", "m_history" }) + foreach (var point in RequiredField(points, field)) + { + weak.Add(new(field, new(point))); + if (point is Opc.Ua.Server.ContinuationPoint browsePoint && browsePoint.Data is { } data) + weak.Add(new("browse-data", new(data))); + if (field == "m_history") + { + var value = point.GetType().GetField("Value")!.GetValue(point)!; + weak.Add(new("history-value", new(value))); + weak.Add(new("history-state", new(value.GetType().GetProperty("State")!.GetValue(value)!))); + } + } + await old.CloseAsync(ct).WaitAsync(TimeSpan.FromSeconds(10), ct); + rig.Server.CurrentInstance.SessionManager.GetSessions().ShouldBeEmpty(); + ContinuationCounts(serverSession).ShouldBe((0, 0)); + + using var fresh = await ConnectAsync(rig, config, null, ct); + await OwnPointsRoundTripAsync(fresh, ct); + (await fresh.ReadValueAsync(Raw(fresh, Scalar), ct)).StatusCode.ShouldBe(StatusCodes.Good); + var requestHeader = new RequestHeader { AuthenticationToken = oldAuthentication }; + var failure = await Should.ThrowAsync(async () => + await fresh.ReadAsync(requestHeader, 0, TimestampsToReturn.Neither, + [new ReadValueId { NodeId = Raw(fresh, Scalar), AttributeId = Attributes.Value }], ct)); + // Exact outcome is recorded and narrowed at the reviewed execution gate. + (failure.StatusCode == StatusCodes.BadSessionIdInvalid || failure.StatusCode == StatusCodes.BadSessionClosed).ShouldBeTrue(); + requestHeader.AuthenticationToken.ShouldBe(oldAuthentication); + (await fresh.BrowseNextAsync(null, false, [browse], ct)).Results[0].StatusCode.ShouldBe(StatusCodes.BadContinuationPointInvalid); + var source = rig.Nm.HistorianDataSource.ShouldBeOfType(); + var readsBefore = source.ReadCount; + (await PageAsync(fresh, history, false, ct)).StatusCode.ShouldBe(StatusCodes.BadContinuationPointInvalid); + source.ReadCount.ShouldBe(readsBefore, "invalid old tokens must not reach the backend"); + (await fresh.ReadValueAsync(Raw(fresh, Scalar), ct)).StatusCode.ShouldBe(StatusCodes.Good); + await OwnPointsRoundTripAsync(fresh, ct); + await fresh.CloseAsync(ct).WaitAsync(TimeSpan.FromSeconds(10), ct); + rig.Server.CurrentInstance.SessionManager.GetSessions().ShouldBeEmpty(); + WriteCloseControl("wire-contract", new { oldAuthStatus = failure.StatusCode.Code, oldBrowseRejected = true, oldHistoryRejected = true, freshPositiveRounds = 2 }); + return weak.ToArray(); + } + + private static async Task OwnPointsRoundTripAsync(Opc.Ua.Client.ISession session, CancellationToken ct) + { + var browse = await BrowseAsync(session, ct); + browse.IsEmpty.ShouldBeFalse(); + (await session.BrowseNextAsync(null, false, [browse], ct)).Results[0].StatusCode.ShouldBe(StatusCodes.Good); + var history = (await PageAsync(session, default, false, ct)).ContinuationPoint; + history.IsEmpty.ShouldBeFalse(); + var resumed = await PageAsync(session, history, false, ct); + resumed.StatusCode.ShouldBe(StatusCodes.Good); + resumed.ContinuationPoint.IsEmpty.ShouldBeFalse(); + (await PageAsync(session, resumed.ContinuationPoint, true, ct)).StatusCode.ShouldBe(StatusCodes.Good); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs index 2e62fe75..a3d172ba 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs @@ -32,7 +32,7 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// No key, certificate, store path, token, or OPC UA payload is written to the JSONL evidence. /// [Trait("Category", "E2E")] -public sealed class Sdk2LifecycleSoakTests +public sealed partial class Sdk2LifecycleSoakTests { private const string Scalar = ServerEventRig.RawDeviceFolder + "/SoakScalar"; private const string ArrayTag = ServerEventRig.RawDeviceFolder + "/SoakArray";