using System.Collections.Concurrent; using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Reflection; using System.Runtime.InteropServices; using System.Text.Json; using Opc.Ua; using Opc.Ua.Configuration; using Opc.Ua.Server; using Shouldly; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using CoreHistoryReadResult = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult; return await Canonical.ComparatorAsync(args); internal static partial class Canonical { private const string Scalar = BaselineRig.RawDeviceFolder + "/SoakScalar"; private const string ArrayTag = BaselineRig.RawDeviceFolder + "/SoakArray"; private const string UnsScalar = "SoakEquipment/Scalar"; private static readonly DateTime Epoch = new(2026,10,1,0,0,0,DateTimeKind.Utc); internal static readonly OperationLedger Ledger = new(); private static StreamWriter CurrentEvidence = null!; /// /// lmxopcua#799 SDK 1.5.378.182 comparator: the canonical late-slope loop (same cycle body, same every-fourth-cycle /// active restart, same quiescent sample after each cycle, same cycle-3 cold baseline and caps with a latched first /// rejection, same 30 s cycle / 25 s restart / 120 s startup bounds, same 7,200 s phase) driven against the immutable /// efb9197a SDK1 publication. Comparator only: acceptance=false. Differences from the SDK2 test are listed in /// README.md (standalone process instead of the VSTest testhost; no SDK2 certificate-manager disposal observables; /// closed-session continuation lists recorded rather than asserted; managed-monotonic guard latched, not fatal). /// internal static async Task ComparatorAsync(string[] args) { if (args.Length != 3 || args[2] != "--comparator") throw new InvalidOperationException("Usage: 7200 --comparator"); var root = Path.GetFullPath(args[0]); var seconds = int.Parse(args[1]); // The measured phase matches the 7200 s late-slope run; a 60 s smoke exists only to qualify the tooling (never a result). if (seconds != 7200 && !(seconds == 60 && Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SMOKE") == "1")) throw new InvalidOperationException("The comparator phase matches the 7200 s late-slope run"); if (Directory.Exists(root)) throw new InvalidOperationException("Use a fresh private artifact directory"); Directory.CreateDirectory(root); await using var output = new StreamWriter(new FileStream(Path.Combine(root, "resources.jsonl"), FileMode.CreateNew, FileAccess.Write, FileShare.Read)); CurrentEvidence = output; var sdk = typeof(NodeId).Assembly.GetCustomAttribute()?.InformationalVersion; if (sdk is null || !sdk.StartsWith("1.5.378.182+", StringComparison.Ordinal)) throw new InvalidOperationException("The immutable .182 SDK is required"); var clientRoot = Path.Combine(root, "private-fixture", "client-pki"); var startup = Stopwatch.StartNew(); using var startupDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(120)); var telemetry = DefaultTelemetry.Create(_ => { }); var config = new ApplicationConfiguration(telemetry) { ApplicationName = "SDK2LifecycleSoak", ApplicationUri = $"urn:SDK2LifecycleSoak:{Guid.NewGuid():N}", ApplicationType = ApplicationType.Client, SecurityConfiguration = new SecurityConfiguration { ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "own"), SubjectName = "CN=SDK2LifecycleSoak" }, TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "issuer") }, TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "trusted") }, RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "rejected") }, AutoAcceptUntrustedCertificates = true }, ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 15000 }, TransportQuotas = new TransportQuotas { OperationTimeout = 5000 } }; var application = new ApplicationInstance(telemetry) { ApplicationConfiguration = config }; await config.ValidateAsync(ApplicationType.Client, startupDeadline.Token); await application.CheckApplicationInstanceCertificatesAsync(true, null, startupDeadline.Token); var rig = new BaselineRig(Path.Combine(root, "private-fixture")); await rig.BootAsync(startupDeadline.Token); Configure(rig); var startupSeconds = startup.Elapsed.TotalSeconds; var watch = Stopwatch.StartNew(); await LogAsync(output, new { kind = "start", utc = DateTimeOffset.UtcNow, processId = Environment.ProcessId, mode = "sdk1-comparator", requestedSeconds = seconds, build = Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SOURCE"), sourcePublication = "efb9197a", sdk, runtime = Environment.Version.ToString(), architecture = RuntimeInformation.ProcessArchitecture.ToString(), startupSeconds, limits = new { retainedManagedGrowthBytes = 64L << 20, rssGrowthBytes = 128L << 20, fdGrowth = 32 }, acceptance = false, unsupportedObservables = new[] { "SDK2 CertificateManager disposed flag", "SDK2 manager-owned cache-clear proof", "ApplicationInstance.DisposeAsync" }, }); var cycles = 0; var restarts = 0; var lastCompleted = 0d; Dictionary? baseline = null; object? firstRejection = null; object? monotonicRejection = null; var retained = new Queue(); var failures = new List(); try { while (lastCompleted < seconds) { var cycle = Stopwatch.StartNew(); using var cycleDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var ct = cycleDeadline.Token; await ExerciseAsync(rig, config, ++cycles, output, ct); await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct); var sample = await CaptureAsync(rig, config, cycles, watch.Elapsed.TotalSeconds, clientRoot, ct); await LogAsync(output, new { kind = "resource-observation", resources = sample }); foreach (var zero in new[] { "Sessions", "Subscriptions", "Items", "BrowsePoints", "HistoryPoints", "LiveDataItems", "LiveEventItems", "MonitoredNodes", "PkiDescriptors" }) Convert.ToInt64(sample[zero]).ShouldBe(0, $"quiescent {zero} must be zero"); if (cycles >= 3) { baseline ??= sample; foreach (var steady in new[] { "Variables", "Folders", "Conditions", "ServerCachedStoreReferences", "ClientCachedStoreReferences" }) sample[steady].ShouldBe(baseline[steady], $"quiescent {steady} must stay at its cycle-3 value"); var exceeded = new List(); if ((int)sample["Descriptors"] > (int)baseline["Descriptors"] + 32) exceeded.Add("Descriptors"); if ((long)sample["ManagedBytes"] > (long)baseline["ManagedBytes"] + (64L << 20)) exceeded.Add("ManagedBytes"); if ((long)sample["RssBytes"] > (long)baseline["RssBytes"] + (128L << 20)) exceeded.Add("RssBytes"); if (exceeded.Count > 0 && firstRejection is null) { firstRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline }; await LogAsync(output, new { kind = "budget-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline }); } retained.Enqueue((long)sample["ManagedBytes"]); if (retained.Count > 24) retained.Dequeue(); var window = retained.ToArray(); if (window.Length == 24 && monotonicRejection is null && window.Zip(window.Skip(1)).All(p => p.Second > p.First) && window[^1] - window[0] > (16L << 20)) { monotonicRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] }; await LogAsync(output, new { kind = "managed-monotonic-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] }); } } await LogAsync(output, new { kind = firstRejection is null ? "quiescent" : "quiescent-budget-rejected", cycleSeconds = cycle.Elapsed.TotalSeconds, closedSessionPointsWhileHeld = new { browse = ClosedListObservation.Browse, history = ClosedListObservation.History }, resources = sample }); if (cycles % 4 == 0) { var restart = Stopwatch.StartNew(); using var restartDeadline = CancellationTokenSource.CreateLinkedTokenSource(ct); restartDeadline.CancelAfter(TimeSpan.FromSeconds(25)); await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token); restartDeadline.Token.ThrowIfCancellationRequested(); var bounded = restart.Elapsed.TotalSeconds; bounded.ShouldBeLessThanOrEqualTo(25); Configure(rig); restarts++; await LogAsync(output, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds, boundedRestartSeconds = bounded }); } ct.ThrowIfCancellationRequested(); cycle.Elapsed.TotalSeconds.ShouldBeLessThanOrEqualTo(30); lastCompleted = watch.Elapsed.TotalSeconds; await LogAsync(output, new { kind = "comparator-cycle", cycle = cycles, restarts, elapsedSeconds = lastCompleted, cycleSeconds = cycle.Elapsed.TotalSeconds }); } } catch (Exception error) { failures.Add(error.GetType().FullName!); await LogAsync(output, new { kind = "failed", cycles, restarts, seconds = watch.Elapsed.TotalSeconds, exceptionType = error.GetType().FullName, message = error.Message.Length > 600 ? error.Message[..600] : error.Message, uaStatus = error is ServiceResultException ua ? ua.StatusCode.ToString() : null, ledger = Ledger.Snapshot() }); Console.Error.WriteLine(error); } var phaseSeconds = watch.Elapsed.TotalSeconds; var shutdown = Stopwatch.StartNew(); var shutdownVerified = false; try { await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20)); await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }).WaitAsync(TimeSpan.FromSeconds(5)); using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10)); (await BaselineMetrics.DescriptorsAsync(probe.Token, clientRoot, rig.PkiRoot)).Pki.ShouldBe(0); Directory.Exists(clientRoot).ShouldBeFalse(); Directory.Exists(rig.PkiRoot).ShouldBeFalse(); shutdownVerified = shutdown.Elapsed.TotalSeconds <= 60; } catch (Exception error) { failures.Add("shutdown:" + error.GetType().FullName); Console.Error.WriteLine(error); } await LogAsync(output, new { kind = "comparator-result", mode = "sdk1-comparator", cycles, restarts, cycleSeconds = phaseSeconds, shutdownSeconds = shutdown.Elapsed.TotalSeconds, durationCompleted = lastCompleted >= seconds && failures.Count == 0, shutdownVerified, coldBudgetRejected = firstRejection is not null, firstRejection, managedMonotonicRejected = monotonicRejection, fatalFailure = failures.Count != 0, failureTypes = failures, applicationInstanceDisposal = "unsupported on SDK 1.5 (no DisposeAsync)", ledger = Ledger.Snapshot(), acceptance = false, }); return failures.Count != 0 ? 1 : firstRejection is not null || monotonicRejection is not null ? 3 : 0; } private static async Task> CaptureAsync(BaselineRig rig, ApplicationConfiguration config, int cycle, double seconds, string clientRoot, CancellationToken ct) { var counts = PublicCounts(rig); var points = rig.Server.CurrentInstance.SessionManager.GetSessions().Select(ContinuationCounts).ToArray(); var descriptors = await BaselineMetrics.DescriptorsAsync(ct, clientRoot, rig.PkiRoot); var retainedBytes = GC.GetTotalMemory(forceFullCollection: true); var memory = GC.GetGCMemoryInfo(); using var process = Process.GetCurrentProcess(); process.Refresh(); return new Dictionary { ["Cycle"] = cycle, ["Seconds"] = seconds, ["Sessions"] = counts.Sessions, ["Subscriptions"] = counts.Subscriptions, ["Items"] = counts.Items, ["BrowsePoints"] = points.Sum(x => x.Browse), ["HistoryPoints"] = points.Sum(x => x.History), ["LiveDataItems"] = BaselineMetrics.Counter(rig.Nm, "LiveItemCount"), ["LiveEventItems"] = BaselineMetrics.Counter(rig.Nm, "LiveEventItemCount"), ["MonitoredNodes"] = BaselineMetrics.Counter(rig.Nm, "MonitoredNodeCount"), ["Variables"] = OptionalCounter(rig.Nm, "VariableCount"), ["Folders"] = OptionalCounter(rig.Nm, "FolderCount"), ["Conditions"] = OptionalCounter(rig.Nm, "AlarmConditionCount"), ["ServerCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(rig.Validator), ["ClientCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(config.CertificateValidator), ["Descriptors"] = descriptors.Total, ["PkiDescriptors"] = descriptors.Pki, ["ManagedBytes"] = retainedBytes, ["RssBytes"] = process.WorkingSet64, ["PrivateBytes"] = process.PrivateMemorySize64, ["AllocatedBytes"] = GC.GetTotalAllocatedBytes(), ["Gen0"] = GC.CollectionCount(0), ["Gen1"] = GC.CollectionCount(1), ["Gen2"] = GC.CollectionCount(2), ["GcCommittedBytes"] = memory.TotalCommittedBytes, ["GcHeapBytes"] = memory.HeapSizeBytes, ["GcFragmentedBytes"] = memory.FragmentedBytes, ["PinnedObjects"] = memory.PinnedObjectsCount, ["PendingFinalizers"] = memory.FinalizationPendingCount, ["JitMethods"] = System.Runtime.JitInfo.GetCompiledMethodCount(), ["JitIlBytes"] = System.Runtime.JitInfo.GetCompiledILBytes(), ["ActiveTimers"] = Timer.ActiveCount, ["Threads"] = process.Threads.Count, }; } private static int OptionalCounter(object manager, string name) => manager.GetType().GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)?.GetValue(manager) is int value ? value : -1; private static int ScalarValue(Variant value)=>value.Value is int result?result:throw new InvalidOperationException("Expected scalar Int32 CLR shape"); private static void RequirePoint(byte[]? point)=>(point is {Length:>0}).ShouldBeTrue(); private static (int Browse,int History) ContinuationCounts(ISession session)=>BaselineMetrics.Points(session); private static (int Sessions,int Subscriptions,int Items) PublicCounts(BaselineRig rig) {var subscriptions=rig.Server.CurrentInstance.SubscriptionManager.GetSubscriptions();return(rig.Server.CurrentInstance.SessionManager.GetSessions().Count,subscriptions.Count,subscriptions.Sum(s=>s.MonitoredItemCount));} private static async Task EventuallyAsync(Func condition,CancellationToken ct) {var elapsed=Stopwatch.StartNew();while(!condition()&&elapsed.Elapsed WriteAsync(string id,object? value,AddressSpaceRealm realm,CancellationToken ct)=>Task.FromResult(new NodeWriteOutcome(true,null));} internal sealed class SqlLikeEventSource(List events) : IHistorianDataSource { private int _eventReads; internal int EventReads => Volatile.Read(ref _eventReads); private readonly List _all = events.OrderBy(e => e.EventTimeUtc).ThenBy(e => e.EventId, StringComparer.Ordinal).ToList(); public async Task ReadEventsAsync( string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken ct) { Interlocked.Increment(ref _eventReads); await Task.Delay(1, ct).ConfigureAwait(false); // complete on a pool thread, like a real backend var window = _all.Where(e => e.EventTimeUtc >= startUtc && e.EventTimeUtc < endUtc).ToList(); var cap = maxEvents <= 0 ? 1000 : maxEvents; return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null); } public bool SupportsReverseEvents => true; /// The #675 overload: reverse is (lo, hi] by (time, id) descending. public async Task ReadEventsAsync( string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken ct) { if (!reverse) return await ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, ct).ConfigureAwait(false); await Task.Delay(1, ct).ConfigureAwait(false); var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc); var window = _all.Where(e => e.EventTimeUtc > lo && e.EventTimeUtc <= hi).Reverse().ToList(); var cap = maxEvents <= 0 ? 1000 : maxEvents; return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null); } public Task ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) => NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct); public Task ReadProcessedAsync( string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) => NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct); public Task ReadAtTimeAsync(string r, IReadOnlyList t, CancellationToken ct) => NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct); public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot(); public void Dispose() { } } /// Task-owned TCP relay. Pause blocks forwarding in both directions without closing either socket. private sealed class PauseProxy : IAsyncDisposable { private readonly TcpListener _listener = new(IPAddress.Loopback, 0); private readonly CancellationTokenSource _stop = new(); private readonly ConcurrentBag _sockets = []; private readonly ConcurrentBag _pumps = []; private readonly Task _accept; private TaskCompletionSource? _pause; internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port; internal PauseProxy(int targetPort) { _listener.Start(); _accept = AcceptAsync(targetPort); } internal void Pause() => Volatile.Write(ref _pause, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); internal void Resume() => Interlocked.Exchange(ref _pause, null)?.TrySetResult(); private async Task AcceptAsync(int port) { try { while (!_stop.IsCancellationRequested) { var incoming = await _listener.AcceptTcpClientAsync(_stop.Token); _sockets.Add(incoming); var outgoing = new TcpClient(); _sockets.Add(outgoing); await outgoing.ConnectAsync(IPAddress.Loopback, port, _stop.Token); _pumps.Add(PumpAsync(incoming.GetStream(), outgoing.GetStream())); _pumps.Add(PumpAsync(outgoing.GetStream(), incoming.GetStream())); } } catch (OperationCanceledException) when (_stop.IsCancellationRequested) { } } private async Task PumpAsync(NetworkStream source, NetworkStream destination) { var buffer = new byte[16 * 1024]; try { while (true) { var length = await source.ReadAsync(buffer, _stop.Token); if (length == 0) return; if (Volatile.Read(ref _pause) is { } paused) await paused.Task.WaitAsync(_stop.Token); await destination.WriteAsync(buffer.AsMemory(0, length), _stop.Token); } } catch (OperationCanceledException) when (_stop.IsCancellationRequested) { } catch (IOException) when (_stop.IsCancellationRequested) { } } public async ValueTask DisposeAsync() { Resume(); _stop.Cancel(); _listener.Stop(); foreach (var socket in _sockets) socket.Dispose(); try { await _accept; await Task.WhenAll(_pumps); } finally { _stop.Dispose(); } } } } internal sealed class OperationLedger { private readonly ConcurrentDictionary _counts=new(); internal void Record(string name)=>_counts.AddOrUpdate(name,1,(_,count)=>count+1); internal async Task Call(string name,Func> operation) {Record(name+".attempted");try{var value=await operation();Record(name+".returned");return value;}catch{Record(name+".faulted");throw;}} internal async Task Call(string name,Func operation) {Record(name+".attempted");try{await operation();Record(name+".returned");}catch{Record(name+".faulted");throw;}} internal IReadOnlyDictionary Snapshot()=>new SortedDictionary(_counts); }