using System.Diagnostics; using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using Opc.Ua.Configuration; using Shouldly; using Xunit; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; public sealed partial class Sdk2LifecycleSoakTests { private sealed class LateSlopeRun { internal const string Mode = "late-slope"; internal const int Duration = 7200; internal const int StartupDeadline = 120; internal const int PhaseDeadline = 7230; internal const int CycleDeadline = 30; internal const int ShutdownDeadline = 60; private readonly List _failures = []; private double _shutdownSeconds; private bool _shutdownVerified; internal object? ClientManager { get; set; } internal bool PhaseStarted { get; set; } internal bool DurationCompleted { get; set; } internal double LastCompletedCycleSeconds { get; set; } internal static void ValidateRequest(int seconds) { seconds.ShouldBe(Duration, "late-slope diagnostic duration is fixed at 7200 seconds"); OperatingSystem.IsLinux().ShouldBeTrue("this diagnostic is qualified only for Linux"); RuntimeInformation.ProcessArchitecture.ShouldBe(Architecture.Arm64); Environment.Version.ToString().ShouldBe("10.0.14"); Environment.GetEnvironmentVariable("OPCUA_CERTIFICATE_LEAK_TRACKING").ShouldBe("1"); string.IsNullOrEmpty(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_OBSERVER")).ShouldBeTrue( "this diagnostic does not attach the saturation observer"); } internal static CancellationTokenSource StartCycleDeadline(CancellationToken token) => StartLinkedDeadline(token, TimeSpan.FromSeconds(CycleDeadline)); internal static CancellationTokenSource StartLinkedDeadline(CancellationToken token, TimeSpan duration) { var deadline = CancellationTokenSource.CreateLinkedTokenSource(token); deadline.CancelAfter(duration); return deadline; } internal static Task LogStartAsync(StreamWriter evidence, object start, double startupSeconds) { var row = JsonSerializer.SerializeToNode(start)!.AsObject(); row["runtime"] = Environment.Version.ToString(); row["architecture"] = RuntimeInformation.ProcessArchitecture.ToString(); row["startupSeconds"] = startupSeconds; row["acceptance"] = false; return LogAsync(evidence, row); } internal void Record(Exception error) => _failures.Add(ExceptionDispatchInfo.Capture(error)); internal async Task ReleaseOwnersAsync(Func? server, Func? client) { // The canceled operation token cannot skip either independently bounded owner. await AttemptAsync(server, "server host"); await AttemptAsync(client, "client application"); async Task AttemptAsync(Func? dispose, string owner) { if (dispose is null) return; try { await Task.Run(dispose, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); } catch (Exception error) { Record(new InvalidOperationException($"Late-slope {owner} cleanup failed.", error)); } } } internal async Task ShutdownAsync(ServerEventRig? rig, ApplicationInstance? application, string clientRoot, string? serverRoot) { var watch = Stopwatch.StartNew(); var failuresBeforeShutdown = _failures.Count; object? serverManager = null; object[]? cores = null; if (PhaseStarted) { try { var captured = await Task.Run(() => { var manager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager."); return (Manager: manager, Cores: CertificateCores(ClientManager!).Concat(CertificateCores(manager)).ToArray()); }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None); serverManager = captured.Manager; cores = captured.Cores; } catch (Exception error) { Record(error); } } await ReleaseOwnersAsync(rig is null ? null : () => rig.DisposeAsync().AsTask(), application is null ? null : () => application.DisposeAsync().AsTask()); try { await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }, CancellationToken.None) .WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None); } catch (Exception error) { Record(error); } if (PhaseStarted && cores is not null && _failures.Count == failuresBeforeShutdown) { try { using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var probeToken = probe.Token; await Task.Run(async () => { RequiredField(ClientManager!, "m_disposed").ShouldBeTrue(); RequiredField(serverManager!, "m_disposed").ShouldBeTrue(); cores.Sum(CachedStores).ShouldBe(0); (await DescriptorCountsAsync(probeToken, clientRoot, serverRoot!)).Pki.ShouldBe(0); Directory.Exists(clientRoot).ShouldBeFalse(); Directory.Exists(serverRoot!).ShouldBeFalse(); }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); _shutdownVerified = true; } catch (Exception error) { Record(error); } } _shutdownSeconds = watch.Elapsed.TotalSeconds; if (_shutdownSeconds > ShutdownDeadline) { _shutdownVerified = false; Record(new TimeoutException("Late-slope shutdown exceeded its fixed 60-second ceiling.")); } } internal async Task EmitAndPreserveFailuresAsync(StreamWriter evidence, int cycles, int restarts, double cycleSeconds, ExceptionDispatchInfo? firstRejectedBudget) { try { await LogAsync(evidence, new { kind = "late-slope-result", mode = Mode, cycles, restarts, cycleSeconds = PhaseStarted ? cycleSeconds : 0, shutdownSeconds = _shutdownSeconds, durationCompleted = DurationCompleted, shutdownVerified = _shutdownVerified, coldBudgetRejected = firstRejectedBudget is not null, fatalFailure = _failures.Count != 0, failureTypes = _failures.Select(x => x.SourceException.GetType().FullName).ToArray(), acceptance = false }); } catch (Exception error) { Record(error); } ThrowFailures(firstRejectedBudget); } internal void ThrowFailures(ExceptionDispatchInfo? firstRejectedBudget) { if (_failures.Count == 0) { firstRejectedBudget?.Throw(); return; } if (firstRejectedBudget is null && _failures.Count == 1) _failures[0].Throw(); var exceptions = _failures.Select(x => x.SourceException).ToList(); if (firstRejectedBudget is not null) exceptions.Insert(0, firstRejectedBudget.SourceException); throw new AggregateException("Late-slope diagnostic failed; original resource rejection retained when present.", exceptions); } } /// The cycle deadline cancels the token used by an in-flight asynchronous operation. [Fact] public async Task Late_slope_cycle_deadline_cancels_in_flight_operation() { var ct = TestContext.Current.CancellationToken; using var deadline = LateSlopeRun.StartLinkedDeadline(ct, TimeSpan.FromMilliseconds(75)); var operation = Task.Delay(Timeout.InfiniteTimeSpan, deadline.Token); await Should.ThrowAsync(() => operation.WaitAsync(TimeSpan.FromSeconds(5), ct)); operation.IsCanceled.ShouldBeTrue(); ct.IsCancellationRequested.ShouldBeFalse(); } /// A failing server cleanup cannot skip the client or erase the original budget failure. [Fact] public async Task Late_slope_cleanup_preserves_first_rejection_and_attempts_both_owners() { var run = new LateSlopeRun { PhaseStarted = true }; var original = new ShouldAssertException("first cold RSS rejection"); var later = new InvalidOperationException("server cleanup fault"); var clientDisposed = false; await run.ReleaseOwnersAsync(() => Task.FromException(later), () => { clientDisposed = true; return Task.CompletedTask; }); clientDisposed.ShouldBeTrue(); using var buffer = new MemoryStream(); await using var evidence = new StreamWriter(buffer, leaveOpen: true); var error = await Should.ThrowAsync(() => run.EmitAndPreserveFailuresAsync(evidence, 4, 1, 16, ExceptionDispatchInfo.Capture(original))); error.InnerExceptions.Count.ShouldBe(2); error.InnerExceptions[0].ShouldBeSameAs(original); error.InnerExceptions[1].InnerException.ShouldBeSameAs(later); using var row = ReadSingleLateSlopeResult(buffer); row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeTrue(); row.RootElement.GetProperty("shutdownVerified").GetBoolean().ShouldBeFalse(); } /// The real emitter retains the first resource failure and cannot write ordinary completion. [Fact] public async Task Late_slope_emits_only_diagnostic_result_before_rethrowing_original_budget_failure() { var run = new LateSlopeRun { PhaseStarted = true, DurationCompleted = true }; var original = new ShouldAssertException("original cold RSS rejection"); using var buffer = new MemoryStream(); await using var evidence = new StreamWriter(buffer, leaveOpen: true); ShouldAssertException? observed = null; try { await run.EmitAndPreserveFailuresAsync(evidence, 1800, 450, 7201, ExceptionDispatchInfo.Capture(original)); } catch (ShouldAssertException error) { observed = error; } observed.ShouldBeSameAs(original); using var row = ReadSingleLateSlopeResult(buffer); row.RootElement.GetProperty("durationCompleted").GetBoolean().ShouldBeTrue(); row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeFalse(); } private static JsonDocument ReadSingleLateSlopeResult(MemoryStream buffer) { var line = Encoding.UTF8.GetString(buffer.ToArray()).Split('\n', StringSplitOptions.RemoveEmptyEntries).ShouldHaveSingleItem(); var row = JsonDocument.Parse(line); row.RootElement.GetProperty("kind").GetString().ShouldBe("late-slope-result"); row.RootElement.GetProperty("acceptance").GetBoolean().ShouldBeFalse(); row.RootElement.GetProperty("coldBudgetRejected").GetBoolean().ShouldBeTrue(); return row; } }