using System.Collections.Concurrent;
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text.Json;
using Opc.Ua;
using Opc.Ua.Configuration;
using Opc.Ua.Server;
using Shouldly;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using CoreHistoryReadResult = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
return await Canonical.ComparatorAsync(args);
internal static partial class Canonical
{
private const string Scalar = BaselineRig.RawDeviceFolder + "/SoakScalar";
private const string ArrayTag = BaselineRig.RawDeviceFolder + "/SoakArray";
private const string UnsScalar = "SoakEquipment/Scalar";
private static readonly DateTime Epoch = new(2026,10,1,0,0,0,DateTimeKind.Utc);
internal static readonly OperationLedger Ledger = new();
private static StreamWriter CurrentEvidence = null!;
///
/// lmxopcua#799 SDK 1.5.378.182 comparator: the canonical late-slope loop (same cycle body, same every-fourth-cycle
/// active restart, same quiescent sample after each cycle, same cycle-3 cold baseline and caps with a latched first
/// rejection, same 30 s cycle / 25 s restart / 120 s startup bounds, same 7,200 s phase) driven against the immutable
/// efb9197a SDK1 publication. Comparator only: acceptance=false. Differences from the SDK2 test are listed in
/// README.md (standalone process instead of the VSTest testhost; no SDK2 certificate-manager disposal observables;
/// closed-session continuation lists recorded rather than asserted; managed-monotonic guard latched, not fatal).
///
internal static async Task ComparatorAsync(string[] args)
{
if (args.Length != 3 || args[2] != "--comparator") throw new InvalidOperationException("Usage: 7200 --comparator");
var root = Path.GetFullPath(args[0]); var seconds = int.Parse(args[1]);
// The measured phase matches the 7200 s late-slope run; a 60 s smoke exists only to qualify the tooling (never a result).
if (seconds != 7200 && !(seconds == 60 && Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SMOKE") == "1"))
throw new InvalidOperationException("The comparator phase matches the 7200 s late-slope run");
if (Directory.Exists(root)) throw new InvalidOperationException("Use a fresh private artifact directory");
Directory.CreateDirectory(root);
await using var output = new StreamWriter(new FileStream(Path.Combine(root, "resources.jsonl"), FileMode.CreateNew, FileAccess.Write, FileShare.Read));
CurrentEvidence = output;
var sdk = typeof(NodeId).Assembly.GetCustomAttribute()?.InformationalVersion;
if (sdk is null || !sdk.StartsWith("1.5.378.182+", StringComparison.Ordinal)) throw new InvalidOperationException("The immutable .182 SDK is required");
var clientRoot = Path.Combine(root, "private-fixture", "client-pki");
var startup = Stopwatch.StartNew();
using var startupDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(120));
var telemetry = DefaultTelemetry.Create(_ => { });
var config = new ApplicationConfiguration(telemetry)
{
ApplicationName = "SDK2LifecycleSoak", ApplicationUri = $"urn:SDK2LifecycleSoak:{Guid.NewGuid():N}", ApplicationType = ApplicationType.Client,
SecurityConfiguration = new SecurityConfiguration
{
ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "own"), SubjectName = "CN=SDK2LifecycleSoak" },
TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "issuer") },
TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "trusted") },
RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(clientRoot, "rejected") }, AutoAcceptUntrustedCertificates = true
},
ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 15000 }, TransportQuotas = new TransportQuotas { OperationTimeout = 5000 }
};
var application = new ApplicationInstance(telemetry) { ApplicationConfiguration = config };
await config.ValidateAsync(ApplicationType.Client, startupDeadline.Token);
await application.CheckApplicationInstanceCertificatesAsync(true, null, startupDeadline.Token);
var rig = new BaselineRig(Path.Combine(root, "private-fixture"));
await rig.BootAsync(startupDeadline.Token); Configure(rig);
var startupSeconds = startup.Elapsed.TotalSeconds;
var watch = Stopwatch.StartNew();
await LogAsync(output, new
{
kind = "start", utc = DateTimeOffset.UtcNow, processId = Environment.ProcessId, mode = "sdk1-comparator", requestedSeconds = seconds,
build = Environment.GetEnvironmentVariable("OTOPCUA_COMPARATOR_SOURCE"), sourcePublication = "efb9197a", sdk,
runtime = Environment.Version.ToString(), architecture = RuntimeInformation.ProcessArchitecture.ToString(), startupSeconds,
limits = new { retainedManagedGrowthBytes = 64L << 20, rssGrowthBytes = 128L << 20, fdGrowth = 32 }, acceptance = false,
unsupportedObservables = new[] { "SDK2 CertificateManager disposed flag", "SDK2 manager-owned cache-clear proof", "ApplicationInstance.DisposeAsync" },
});
var cycles = 0; var restarts = 0; var lastCompleted = 0d;
Dictionary? baseline = null; object? firstRejection = null; object? monotonicRejection = null;
var retained = new Queue();
var failures = new List();
try
{
while (lastCompleted < seconds)
{
var cycle = Stopwatch.StartNew();
using var cycleDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var ct = cycleDeadline.Token;
await ExerciseAsync(rig, config, ++cycles, output, ct);
await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct);
var sample = await CaptureAsync(rig, config, cycles, watch.Elapsed.TotalSeconds, clientRoot, ct);
await LogAsync(output, new { kind = "resource-observation", resources = sample });
foreach (var zero in new[] { "Sessions", "Subscriptions", "Items", "BrowsePoints", "HistoryPoints", "LiveDataItems", "LiveEventItems", "MonitoredNodes", "PkiDescriptors" })
Convert.ToInt64(sample[zero]).ShouldBe(0, $"quiescent {zero} must be zero");
if (cycles >= 3)
{
baseline ??= sample;
foreach (var steady in new[] { "Variables", "Folders", "Conditions", "ServerCachedStoreReferences", "ClientCachedStoreReferences" })
sample[steady].ShouldBe(baseline[steady], $"quiescent {steady} must stay at its cycle-3 value");
var exceeded = new List();
if ((int)sample["Descriptors"] > (int)baseline["Descriptors"] + 32) exceeded.Add("Descriptors");
if ((long)sample["ManagedBytes"] > (long)baseline["ManagedBytes"] + (64L << 20)) exceeded.Add("ManagedBytes");
if ((long)sample["RssBytes"] > (long)baseline["RssBytes"] + (128L << 20)) exceeded.Add("RssBytes");
if (exceeded.Count > 0 && firstRejection is null)
{
firstRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline };
await LogAsync(output, new { kind = "budget-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, exceededFields = exceeded, resources = sample, baseline });
}
retained.Enqueue((long)sample["ManagedBytes"]);
if (retained.Count > 24) retained.Dequeue();
var window = retained.ToArray();
if (window.Length == 24 && monotonicRejection is null && window.Zip(window.Skip(1)).All(p => p.Second > p.First) && window[^1] - window[0] > (16L << 20))
{
monotonicRejection = new { cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] };
await LogAsync(output, new { kind = "managed-monotonic-rejected", cycle = cycles, seconds = watch.Elapsed.TotalSeconds, first = window[0], last = window[^1] });
}
}
await LogAsync(output, new { kind = firstRejection is null ? "quiescent" : "quiescent-budget-rejected", cycleSeconds = cycle.Elapsed.TotalSeconds,
closedSessionPointsWhileHeld = new { browse = ClosedListObservation.Browse, history = ClosedListObservation.History }, resources = sample });
if (cycles % 4 == 0)
{
var restart = Stopwatch.StartNew();
using var restartDeadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
restartDeadline.CancelAfter(TimeSpan.FromSeconds(25));
await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token);
restartDeadline.Token.ThrowIfCancellationRequested();
var bounded = restart.Elapsed.TotalSeconds;
bounded.ShouldBeLessThanOrEqualTo(25);
Configure(rig); restarts++;
await LogAsync(output, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds, boundedRestartSeconds = bounded });
}
ct.ThrowIfCancellationRequested();
cycle.Elapsed.TotalSeconds.ShouldBeLessThanOrEqualTo(30);
lastCompleted = watch.Elapsed.TotalSeconds;
await LogAsync(output, new { kind = "comparator-cycle", cycle = cycles, restarts, elapsedSeconds = lastCompleted, cycleSeconds = cycle.Elapsed.TotalSeconds });
}
}
catch (Exception error)
{
failures.Add(error.GetType().FullName!);
await LogAsync(output, new { kind = "failed", cycles, restarts, seconds = watch.Elapsed.TotalSeconds, exceptionType = error.GetType().FullName,
message = error.Message.Length > 600 ? error.Message[..600] : error.Message,
uaStatus = error is ServiceResultException ua ? ua.StatusCode.ToString() : null, ledger = Ledger.Snapshot() });
Console.Error.WriteLine(error);
}
var phaseSeconds = watch.Elapsed.TotalSeconds;
var shutdown = Stopwatch.StartNew();
var shutdownVerified = false;
try
{
await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20));
await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }).WaitAsync(TimeSpan.FromSeconds(5));
using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10));
(await BaselineMetrics.DescriptorsAsync(probe.Token, clientRoot, rig.PkiRoot)).Pki.ShouldBe(0);
Directory.Exists(clientRoot).ShouldBeFalse(); Directory.Exists(rig.PkiRoot).ShouldBeFalse();
shutdownVerified = shutdown.Elapsed.TotalSeconds <= 60;
}
catch (Exception error) { failures.Add("shutdown:" + error.GetType().FullName); Console.Error.WriteLine(error); }
await LogAsync(output, new
{
kind = "comparator-result", mode = "sdk1-comparator", cycles, restarts, cycleSeconds = phaseSeconds, shutdownSeconds = shutdown.Elapsed.TotalSeconds,
durationCompleted = lastCompleted >= seconds && failures.Count == 0, shutdownVerified, coldBudgetRejected = firstRejection is not null,
firstRejection, managedMonotonicRejected = monotonicRejection, fatalFailure = failures.Count != 0, failureTypes = failures,
applicationInstanceDisposal = "unsupported on SDK 1.5 (no DisposeAsync)", ledger = Ledger.Snapshot(), acceptance = false,
});
return failures.Count != 0 ? 1 : firstRejection is not null || monotonicRejection is not null ? 3 : 0;
}
private static async Task> CaptureAsync(BaselineRig rig, ApplicationConfiguration config, int cycle, double seconds,
string clientRoot, CancellationToken ct)
{
var counts = PublicCounts(rig);
var points = rig.Server.CurrentInstance.SessionManager.GetSessions().Select(ContinuationCounts).ToArray();
var descriptors = await BaselineMetrics.DescriptorsAsync(ct, clientRoot, rig.PkiRoot);
var retainedBytes = GC.GetTotalMemory(forceFullCollection: true);
var memory = GC.GetGCMemoryInfo();
using var process = Process.GetCurrentProcess();
process.Refresh();
return new Dictionary
{
["Cycle"] = cycle, ["Seconds"] = seconds, ["Sessions"] = counts.Sessions, ["Subscriptions"] = counts.Subscriptions, ["Items"] = counts.Items,
["BrowsePoints"] = points.Sum(x => x.Browse), ["HistoryPoints"] = points.Sum(x => x.History),
["LiveDataItems"] = BaselineMetrics.Counter(rig.Nm, "LiveItemCount"), ["LiveEventItems"] = BaselineMetrics.Counter(rig.Nm, "LiveEventItemCount"),
["MonitoredNodes"] = BaselineMetrics.Counter(rig.Nm, "MonitoredNodeCount"), ["Variables"] = OptionalCounter(rig.Nm, "VariableCount"),
["Folders"] = OptionalCounter(rig.Nm, "FolderCount"), ["Conditions"] = OptionalCounter(rig.Nm, "AlarmConditionCount"),
["ServerCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(rig.Validator),
["ClientCachedStoreReferences"] = BaselineMetrics.CachedStoreReferences(config.CertificateValidator),
["Descriptors"] = descriptors.Total, ["PkiDescriptors"] = descriptors.Pki, ["ManagedBytes"] = retainedBytes, ["RssBytes"] = process.WorkingSet64,
["PrivateBytes"] = process.PrivateMemorySize64, ["AllocatedBytes"] = GC.GetTotalAllocatedBytes(),
["Gen0"] = GC.CollectionCount(0), ["Gen1"] = GC.CollectionCount(1), ["Gen2"] = GC.CollectionCount(2), ["GcCommittedBytes"] = memory.TotalCommittedBytes,
["GcHeapBytes"] = memory.HeapSizeBytes, ["GcFragmentedBytes"] = memory.FragmentedBytes, ["PinnedObjects"] = memory.PinnedObjectsCount,
["PendingFinalizers"] = memory.FinalizationPendingCount, ["JitMethods"] = System.Runtime.JitInfo.GetCompiledMethodCount(),
["JitIlBytes"] = System.Runtime.JitInfo.GetCompiledILBytes(), ["ActiveTimers"] = Timer.ActiveCount, ["Threads"] = process.Threads.Count,
};
}
private static int OptionalCounter(object manager, string name) =>
manager.GetType().GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)?.GetValue(manager) is int value ? value : -1;
private static int ScalarValue(Variant value)=>value.Value is int result?result:throw new InvalidOperationException("Expected scalar Int32 CLR shape");
private static void RequirePoint(byte[]? point)=>(point is {Length:>0}).ShouldBeTrue();
private static (int Browse,int History) ContinuationCounts(ISession session)=>BaselineMetrics.Points(session);
private static (int Sessions,int Subscriptions,int Items) PublicCounts(BaselineRig rig)
{var subscriptions=rig.Server.CurrentInstance.SubscriptionManager.GetSubscriptions();return(rig.Server.CurrentInstance.SessionManager.GetSessions().Count,subscriptions.Count,subscriptions.Sum(s=>s.MonitoredItemCount));}
private static async Task EventuallyAsync(Func condition,CancellationToken ct)
{var elapsed=Stopwatch.StartNew();while(!condition()&&elapsed.Elapsed WriteAsync(string id,object? value,AddressSpaceRealm realm,CancellationToken ct)=>Task.FromResult(new NodeWriteOutcome(true,null));}
internal sealed class SqlLikeEventSource(List events) : IHistorianDataSource
{
private int _eventReads;
internal int EventReads => Volatile.Read(ref _eventReads);
private readonly List _all =
events.OrderBy(e => e.EventTimeUtc).ThenBy(e => e.EventId, StringComparer.Ordinal).ToList();
public async Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken ct)
{
Interlocked.Increment(ref _eventReads);
await Task.Delay(1, ct).ConfigureAwait(false); // complete on a pool thread, like a real backend
var window = _all.Where(e => e.EventTimeUtc >= startUtc && e.EventTimeUtc < endUtc).ToList();
var cap = maxEvents <= 0 ? 1000 : maxEvents;
return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
}
public bool SupportsReverseEvents => true;
/// The #675 overload: reverse is (lo, hi] by (time, id) descending.
public async Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken ct)
{
if (!reverse) return await ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, ct).ConfigureAwait(false);
await Task.Delay(1, ct).ConfigureAwait(false);
var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
var window = _all.Where(e => e.EventTimeUtc > lo && e.EventTimeUtc <= hi).Reverse().ToList();
var cap = maxEvents <= 0 ? 1000 : maxEvents;
return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
}
public Task ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
public Task ReadProcessedAsync(
string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) =>
NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct);
public Task ReadAtTimeAsync(string r, IReadOnlyList t, CancellationToken ct) =>
NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct);
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
/// Task-owned TCP relay. Pause blocks forwarding in both directions without closing either socket.
private sealed class PauseProxy : IAsyncDisposable
{
private readonly TcpListener _listener = new(IPAddress.Loopback, 0);
private readonly CancellationTokenSource _stop = new();
private readonly ConcurrentBag _sockets = [];
private readonly ConcurrentBag _pumps = [];
private readonly Task _accept;
private TaskCompletionSource? _pause;
internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port;
internal PauseProxy(int targetPort)
{
_listener.Start();
_accept = AcceptAsync(targetPort);
}
internal void Pause() => Volatile.Write(ref _pause, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
internal void Resume() => Interlocked.Exchange(ref _pause, null)?.TrySetResult();
private async Task AcceptAsync(int port)
{
try
{
while (!_stop.IsCancellationRequested)
{
var incoming = await _listener.AcceptTcpClientAsync(_stop.Token);
_sockets.Add(incoming);
var outgoing = new TcpClient(); _sockets.Add(outgoing);
await outgoing.ConnectAsync(IPAddress.Loopback, port, _stop.Token);
_pumps.Add(PumpAsync(incoming.GetStream(), outgoing.GetStream()));
_pumps.Add(PumpAsync(outgoing.GetStream(), incoming.GetStream()));
}
}
catch (OperationCanceledException) when (_stop.IsCancellationRequested) { }
}
private async Task PumpAsync(NetworkStream source, NetworkStream destination)
{
var buffer = new byte[16 * 1024];
try
{
while (true)
{
var length = await source.ReadAsync(buffer, _stop.Token);
if (length == 0) return;
if (Volatile.Read(ref _pause) is { } paused) await paused.Task.WaitAsync(_stop.Token);
await destination.WriteAsync(buffer.AsMemory(0, length), _stop.Token);
}
}
catch (OperationCanceledException) when (_stop.IsCancellationRequested) { }
catch (IOException) when (_stop.IsCancellationRequested) { }
}
public async ValueTask DisposeAsync()
{
Resume(); _stop.Cancel(); _listener.Stop();
foreach (var socket in _sockets) socket.Dispose();
try { await _accept; await Task.WhenAll(_pumps); }
finally { _stop.Dispose(); }
}
}
}
internal sealed class OperationLedger
{
private readonly ConcurrentDictionary _counts=new();
internal void Record(string name)=>_counts.AddOrUpdate(name,1,(_,count)=>count+1);
internal async Task Call(string name,Func> operation)
{Record(name+".attempted");try{var value=await operation();Record(name+".returned");return value;}catch{Record(name+".faulted");throw;}}
internal async Task Call(string name,Func operation)
{Record(name+".attempted");try{await operation();Record(name+".returned");}catch{Record(name+".faulted");throw;}}
internal IReadOnlyDictionary Snapshot()=>new SortedDictionary(_counts);
}