lmxopcua/scripts/validation/opcua-lifecycle-memory/sdk1-comparator/Program.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text.Json;
using Opc.Ua;
using Opc.Ua.Configuration;
using Opc.Ua.Server;
using Shouldly;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using CoreHistoryReadResult = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
return await Canonical.ComparatorAsync(args);
internal static partial class Canonical
{
    private const string Scalar = BaselineRig.RawDeviceFolder + "/SoakScalar";
    private const string ArrayTag = BaselineRig.RawDeviceFolder + "/SoakArray";
    private const string UnsScalar = "SoakEquipment/Scalar";
    private static readonly DateTime Epoch = new(2026,10,1,0,0,0,DateTimeKind.Utc);
    internal static readonly OperationLedger Ledger = new();
    private static StreamWriter CurrentEvidence = null!;

    /// <summary>
    ///     lmxopcua#799 SDK 1.5.378.182 comparator: the canonical late-slope loop (same cycle body, same every-fourth-cycle
    ///     active restart, same quiescent sample after each cycle, same cycle-3 cold baseline and caps with a latched first
    ///     rejection, same 30 s cycle / 25 s restart / 120 s startup bounds, same 7,200 s phase) driven against the immutable
    ///     efb9197a SDK1 publication. Comparator only: acceptance=false. Differences from the SDK2 test are listed in
    ///     README.md (standalone process instead of the VSTest testhost; no SDK2 certificate-manager disposal observables;
    ///     closed-session continuation lists recorded rather than asserted; managed-monotonic guard latched, not fatal).
    /// </summary>
    internal static async Task<int> ComparatorAsync(string[] args)
    {
        if (args.Length != 3 || args[2] != "--comparator") throw new InvalidOperationException("Usage: <fresh-output> 7200 --comparator");
        var root = Path.GetFullPath(args[0]); var seconds = int.Parse(args[1]);
        // The measured phase matches the 7200 s late-slope run; a 60 s smoke exists only to qualify the tooling (never a result).
        if (seconds != 7200 && !(seconds == 60 && Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SMOKE") == "1"))
            throw new InvalidOperationException("The comparator phase matches the 7200 s late-slope run");
        if (Directory.Exists(root)) throw new InvalidOperationException("Use a fresh private artifact directory");
        Directory.CreateDirectory(root);
        await using var output = new StreamWriter(new FileStream(Path.Combine(root, "resources.jsonl"), FileMode.CreateNew, FileAccess.Write, FileShare.Read));
        CurrentEvidence = output;
        var sdk = typeof(NodeId).Assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
        if (sdk is null || !sdk.StartsWith("1.5.378.182+", StringComparison.Ordinal)) throw new InvalidOperationException("The immutable .182 SDK is required");
        var clientRoot = Path.Combine(root, "private-fixture", "client-pki");
        var startup = Stopwatch.StartNew();
        using var startupDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(120));
        var telemetry = DefaultTelemetry.Create(_ => { });
        var config = new ApplicationConfiguration(telemetry)
        {
            ApplicationName = "SDK2LifecycleSoak", ApplicationUri = $"urn:SDK2LifecycleSoak:{Guid.NewGuid():N}", ApplicationType = ApplicationType.Client,
            SecurityConfiguration = new SecurityConfiguration
            {
                ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "own"), SubjectName = "CN=SDK2LifecycleSoak" },
                TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "issuer") },
                TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "trusted") },
                RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "rejected") }, AutoAcceptUntrustedCertificates = true
            },
            ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 15000 }, TransportQuotas = new TransportQuotas { OperationTimeout = 5000 }
        };
        var application = new ApplicationInstance(telemetry) { ApplicationConfiguration = config };
        await config.ValidateAsync(ApplicationType.Client, startupDeadline.Token);
        await application.CheckApplicationInstanceCertificatesAsync(true, null, startupDeadline.Token);
        var rig = new BaselineRig(Path.Combine(root, "private-fixture"));
        await rig.BootAsync(startupDeadline.Token); Configure(rig);
        var startupSeconds = startup.Elapsed.TotalSeconds;
        var watch = Stopwatch.StartNew();
        await LogAsync(output, new
        {
            kind = "start", utc = DateTimeOffset.UtcNow, processId = Environment.ProcessId, mode = "sdk1-comparator", requestedSeconds = seconds,
            build = Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SOURCE"), sourcePublication = "efb9197a", sdk,
            runtime = Environment.Version.ToString(), architecture = RuntimeInformation.ProcessArchitecture.ToString(), startupSeconds,
            limits = new { retainedManagedGrowthBytes = 64L << 20, rssGrowthBytes = 128L << 20, fdGrowth = 32 }, acceptance = false,
            unsupportedObservables = new[] { "SDK2 CertificateManager disposed flag", "SDK2 manager-owned cache-clear proof", "ApplicationInstance.DisposeAsync" },
        });
        var cycles = 0; var restarts = 0; var lastCompleted = 0d;
        Dictionary<string, object>? baseline = null; object? firstRejection = null; object? monotonicRejection = null;
        var retained = new Queue<long>();
        var failures = new List<string>();
        try
        {
            while (lastCompleted < seconds)
            {
                var cycle = Stopwatch.StartNew();
                using var cycleDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(30));
                var ct = cycleDeadline.Token;
                await ExerciseAsync(rig, config, ++cycles, output, ct);
                await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct);
                var sample = await CaptureAsync(rig, config, cycles, watch.Elapsed.TotalSeconds, clientRoot, ct);
                await LogAsync(output, new { kind = "resource-observation", resources = sample });
                foreach (var zero in new[] { "Sessions", "Subscriptions", "Items", "BrowsePoints", "HistoryPoints", "LiveDataItems", "LiveEventItems", "MonitoredNodes", "PkiDescriptors" })
                    Convert.ToInt64(sample[zero]).ShouldBe(0, $"quiescent {zero} must be zero");
                if (cycles >= 3)
                {
                    baseline ??= sample;
                    foreach (var steady in new[] { "Variables", "Folders", "Conditions", "ServerCachedStoreReferences", "ClientCachedStoreReferences" })
                        sample[steady].ShouldBe(baseline[steady], $"quiescent {steady} must stay at its cycle-3 value");
                    var exceeded = new List<string>();
                    if ((int)sample["Descriptors"] > (int)baseline["Descriptors"] + 32) exceeded.Add("Descriptors");
                    if ((long)sample["ManagedBytes"] > (long)baseline["ManagedBytes"] + (64L << 20)) exceeded.Add("ManagedBytes");
                    if ((long)sample["RssBytes"] > (long)baseline["RssBytes"] + (128L << 20)) exceeded.Add("RssBytes");
                    if (exceeded.Count > 0 && firstRejection is null)
                    {
                        firstRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline };
                        await LogAsync(output, new { kind = "budget-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline });
                    }
                    retained.Enqueue((long)sample["ManagedBytes"]);
                    if (retained.Count > 24) retained.Dequeue();
                    var window = retained.ToArray();
                    if (window.Length == 24 && monotonicRejection is null && window.Zip(window.Skip(1)).All(p => p.Second > p.First) && window[^1] - window[0] > (16L << 20))
                    {
                        monotonicRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] };
                        await LogAsync(output, new { kind = "managed-monotonic-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] });
                    }
                }
                await LogAsync(output, new { kind = firstRejection is null ? "quiescent" : "quiescent-budget-rejected", cycleSeconds = cycle.Elapsed.TotalSeconds,
                    closedSessionPointsWhileHeld = new { browse = ClosedListObservation.Browse, history = ClosedListObservation.History }, resources = sample });
                if (cycles % 4 == 0)
                {
                    var restart = Stopwatch.StartNew();
                    using var restartDeadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
                    restartDeadline.CancelAfter(TimeSpan.FromSeconds(25));
                    await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token);
                    restartDeadline.Token.ThrowIfCancellationRequested();
                    var bounded = restart.Elapsed.TotalSeconds;
                    bounded.ShouldBeLessThanOrEqualTo(25);
                    Configure(rig); restarts++;
                    await LogAsync(output, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds, boundedRestartSeconds = bounded });
                }
                ct.ThrowIfCancellationRequested();
                cycle.Elapsed.TotalSeconds.ShouldBeLessThanOrEqualTo(30);
                lastCompleted = watch.Elapsed.TotalSeconds;
                await LogAsync(output, new { kind = "comparator-cycle", cycle = cycles, restarts, elapsedSeconds = lastCompleted, cycleSeconds = cycle.Elapsed.TotalSeconds });
            }
        }
        catch (Exception error)
        {
            failures.Add(error.GetType().FullName!);
            await LogAsync(output, new { kind = "failed", cycles, restarts, seconds = watch.Elapsed.TotalSeconds, exceptionType = error.GetType().FullName,
                message = error.Message.Length > 600 ? error.Message[..600] : error.Message,
                uaStatus = error is ServiceResultException ua ? ua.StatusCode.ToString() : null, ledger = Ledger.Snapshot() });
            Console.Error.WriteLine(error);
        }
        var phaseSeconds = watch.Elapsed.TotalSeconds;
        var shutdown = Stopwatch.StartNew();
        var shutdownVerified = false;
        try
        {
            await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20));
            await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }).WaitAsync(TimeSpan.FromSeconds(5));
            using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10));
            (await BaselineMetrics.DescriptorsAsync(probe.Token, clientRoot, rig.PkiRoot)).Pki.ShouldBe(0);
            Directory.Exists(clientRoot).ShouldBeFalse(); Directory.Exists(rig.PkiRoot).ShouldBeFalse();
            shutdownVerified = shutdown.Elapsed.TotalSeconds <= 60;
        }
        catch (Exception error) { failures.Add("shutdown:" + error.GetType().FullName); Console.Error.WriteLine(error); }
        await LogAsync(output, new
        {
            kind = "comparator-result", mode = "sdk1-comparator", cycles, restarts, cycleSeconds = phaseSeconds, shutdownSeconds = shutdown.Elapsed.TotalSeconds,
            durationCompleted = lastCompleted >= seconds && failures.Count == 0, shutdownVerified, coldBudgetRejected = firstRejection is not null,
            firstRejection, managedMonotonicRejected = monotonicRejection, fatalFailure = failures.Count != 0, failureTypes = failures,
            applicationInstanceDisposal = "unsupported on SDK 1.5 (no DisposeAsync)", ledger = Ledger.Snapshot(), acceptance = false,
        });
        return failures.Count != 0 ? 1 : firstRejection is not null || monotonicRejection is not null ? 3 : 0;
    }

    private static async Task<Dictionary<string, object>> CaptureAsync(BaselineRig rig, ApplicationConfiguration config, int cycle, double seconds,
        string clientRoot, CancellationToken ct)
    {
        var counts = PublicCounts(rig);
        var points = rig.Server.CurrentInstance.SessionManager.GetSessions().Select(ContinuationCounts).ToArray();
        var descriptors = await BaselineMetrics.DescriptorsAsync(ct, clientRoot, rig.PkiRoot);
        var retainedBytes = GC.GetTotalMemory(forceFullCollection: true);
        var memory = GC.GetGCMemoryInfo();
        using var process = Process.GetCurrentProcess();
        process.Refresh();
        return new Dictionary<string, object>
        {
            ["Cycle"] = cycle, ["Seconds"] = seconds, ["Sessions"] = counts.Sessions, ["Subscriptions"] = counts.Subscriptions, ["Items"] = counts.Items,
            ["BrowsePoints"] = points.Sum(x => x.Browse), ["HistoryPoints"] = points.Sum(x => x.History),
            ["LiveDataItems"] = BaselineMetrics.Counter(rig.Nm, "LiveItemCount"), ["LiveEventItems"] = BaselineMetrics.Counter(rig.Nm, "LiveEventItemCount"),
            ["MonitoredNodes"] = BaselineMetrics.Counter(rig.Nm, "MonitoredNodeCount"), ["Variables"] = OptionalCounter(rig.Nm, "VariableCount"),
            ["Folders"] = OptionalCounter(rig.Nm, "FolderCount"), ["Conditions"] = OptionalCounter(rig.Nm, "AlarmConditionCount"),
            ["ServerCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(rig.Validator),
            ["ClientCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(config.CertificateValidator),
            ["Descriptors"] = descriptors.Total, ["PkiDescriptors"] = descriptors.Pki, ["ManagedBytes"] = retainedBytes, ["RssBytes"] = process.WorkingSet64,
            ["PrivateBytes"] = process.PrivateMemorySize64, ["AllocatedBytes"] = GC.GetTotalAllocatedBytes(),
            ["Gen0"] = GC.CollectionCount(0), ["Gen1"] = GC.CollectionCount(1), ["Gen2"] = GC.CollectionCount(2), ["GcCommittedBytes"] = memory.TotalCommittedBytes,
            ["GcHeapBytes"] = memory.HeapSizeBytes, ["GcFragmentedBytes"] = memory.FragmentedBytes, ["PinnedObjects"] = memory.PinnedObjectsCount,
            ["PendingFinalizers"] = memory.FinalizationPendingCount, ["JitMethods"] = System.Runtime.JitInfo.GetCompiledMethodCount(),
            ["JitIlBytes"] = System.Runtime.JitInfo.GetCompiledILBytes(), ["ActiveTimers"] = Timer.ActiveCount, ["Threads"] = process.Threads.Count,
        };
    }

    private static int OptionalCounter(object manager, string name) =>
        manager.GetType().GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)?.GetValue(manager) is int value ? value : -1;

    private static int ScalarValue(Variant value)=>value.Value is int result?result:throw new InvalidOperationException("Expected scalar Int32 CLR shape");
    private static void RequirePoint(byte[]? point)=>(point is {Length:>0}).ShouldBeTrue();
    private static (int Browse,int History) ContinuationCounts(ISession session)=>BaselineMetrics.Points(session);
    private static (int Sessions,int Subscriptions,int Items) PublicCounts(BaselineRig rig)
    {var subscriptions=rig.Server.CurrentInstance.SubscriptionManager.GetSubscriptions();return(rig.Server.CurrentInstance.SessionManager.GetSessions().Count,subscriptions.Count,subscriptions.Sum(s=>s.MonitoredItemCount));}
    private static async Task EventuallyAsync(Func<bool> condition,CancellationToken ct)
    {var elapsed=Stopwatch.StartNew();while(!condition()&&elapsed.Elapsed<TimeSpan.FromSeconds(10))await Task.Delay(25,ct);condition().ShouldBeTrue();}
    private static async Task LogAsync(StreamWriter output,object value){await output.WriteLineAsync(JsonSerializer.Serialize(value));await output.FlushAsync();}
    private sealed class AcceptWrites:IOpcUaNodeWriteGateway
    {public Task<NodeWriteOutcome> WriteAsync(string id,object? value,AddressSpaceRealm realm,CancellationToken ct)=>Task.FromResult(new NodeWriteOutcome(true,null));}
    internal sealed class SqlLikeEventSource(List<HistoricalEvent> events) : IHistorianDataSource
    {
        private int _eventReads;
        internal int EventReads => Volatile.Read(ref _eventReads);
        private readonly List<HistoricalEvent> _all =
            events.OrderBy(e => e.EventTimeUtc).ThenBy(e => e.EventId, StringComparer.Ordinal).ToList();

        public async Task<HistoricalEventsResult> ReadEventsAsync(
            string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken ct)
        {
            Interlocked.Increment(ref _eventReads);
            await Task.Delay(1, ct).ConfigureAwait(false); // complete on a pool thread, like a real backend
            var window = _all.Where(e => e.EventTimeUtc >= startUtc && e.EventTimeUtc < endUtc).ToList();
            var cap = maxEvents <= 0 ? 1000 : maxEvents;
            return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
        }

        public bool SupportsReverseEvents => true;

        /// <summary>The #675 overload: reverse is <c>(lo, hi]</c> by (time, id) descending.</summary>
        public async Task<HistoricalEventsResult> ReadEventsAsync(
            string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken ct)
        {
            if (!reverse) return await ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, ct).ConfigureAwait(false);
            await Task.Delay(1, ct).ConfigureAwait(false);
            var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
            var window = _all.Where(e => e.EventTimeUtc > lo && e.EventTimeUtc <= hi).Reverse().ToList();
            var cap = maxEvents <= 0 ? 1000 : maxEvents;
            return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
        }

        public Task<CoreHistoryReadResult> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
            NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
        public Task<CoreHistoryReadResult> ReadProcessedAsync(
            string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) =>
            NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct);
        public Task<CoreHistoryReadResult> ReadAtTimeAsync(string r, IReadOnlyList<DateTime> t, CancellationToken ct) =>
            NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct);
        public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
        public void Dispose() { }
    }
    /// <summary>Task-owned TCP relay. Pause blocks forwarding in both directions without closing either socket.</summary>
    private sealed class PauseProxy : IAsyncDisposable
    {
        private readonly TcpListener _listener = new(IPAddress.Loopback, 0);
        private readonly CancellationTokenSource _stop = new();
        private readonly ConcurrentBag<TcpClient> _sockets = [];
        private readonly ConcurrentBag<Task> _pumps = [];
        private readonly Task _accept;
        private TaskCompletionSource? _pause;
        internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port;
        internal PauseProxy(int targetPort)
        {
            _listener.Start();
            _accept = AcceptAsync(targetPort);
        }
        internal void Pause() => Volatile.Write(ref _pause, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
        internal void Resume() => Interlocked.Exchange(ref _pause, null)?.TrySetResult();
        private async Task AcceptAsync(int port)
        {
            try
            {
                while (!_stop.IsCancellationRequested)
                {
                    var incoming = await _listener.AcceptTcpClientAsync(_stop.Token);
                    _sockets.Add(incoming);
                    var outgoing = new TcpClient(); _sockets.Add(outgoing);
                    await outgoing.ConnectAsync(IPAddress.Loopback, port, _stop.Token);
                    _pumps.Add(PumpAsync(incoming.GetStream(), outgoing.GetStream()));
                    _pumps.Add(PumpAsync(outgoing.GetStream(), incoming.GetStream()));
                }
            }
            catch (OperationCanceledException) when (_stop.IsCancellationRequested) { }
        }
        private async Task PumpAsync(NetworkStream source, NetworkStream destination)
        {
            var buffer = new byte[16 * 1024];
            try
            {
                while (true)
                {
                    var length = await source.ReadAsync(buffer, _stop.Token);
                    if (length == 0) return;
                    if (Volatile.Read(ref _pause) is { } paused) await paused.Task.WaitAsync(_stop.Token);
                    await destination.WriteAsync(buffer.AsMemory(0, length), _stop.Token);
                }
            }
            catch (OperationCanceledException) when (_stop.IsCancellationRequested) { }
            catch (IOException) when (_stop.IsCancellationRequested) { }
        }
        public async ValueTask DisposeAsync()
        {
            Resume(); _stop.Cancel(); _listener.Stop();
            foreach (var socket in _sockets) socket.Dispose();
            try { await _accept; await Task.WhenAll(_pumps); }
            finally { _stop.Dispose(); }
        }
    }
}

internal sealed class OperationLedger
{
    private readonly ConcurrentDictionary<string,long> _counts=new();
    internal void Record(string name)=>_counts.AddOrUpdate(name,1,(_,count)=>count+1);
    internal async Task<T> Call<T>(string name,Func<Task<T>> operation)
    {Record(name+".attempted");try{var value=await operation();Record(name+".returned");return value;}catch{Record(name+".faulted");throw;}}
    internal async Task Call(string name,Func<Task> operation)
    {Record(name+".attempted");try{await operation();Record(name+".returned");}catch{Record(name+".faulted");throw;}}
    internal IReadOnlyDictionary<string,long> Snapshot()=>new SortedDictionary<string,long>(_counts);
}