lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs

using System.Diagnostics;
using System.Runtime.ExceptionServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using Opc.Ua.Configuration;
using Shouldly;
using Xunit;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

public sealed partial class Sdk2LifecycleSoakTests
{
    private sealed class LateSlopeRun
    {
        internal const string Mode = "late-slope";
        internal const int Duration = 7200;
        internal const int StartupDeadline = 120;
        internal const int PhaseDeadline = 7230;
        internal const int CycleDeadline = 30;
        internal const int ShutdownDeadline = 60;

        private readonly List<ExceptionDispatchInfo> _failures = [];
        private double _shutdownSeconds;
        private bool _shutdownVerified;

        internal object? ClientManager { get; set; }
        internal bool PhaseStarted { get; set; }
        internal bool DurationCompleted { get; set; }
        internal double LastCompletedCycleSeconds { get; set; }

        internal static void ValidateRequest(int seconds)
        {
            seconds.ShouldBe(Duration, "late-slope diagnostic duration is fixed at 7200 seconds");
            // lmxopcua#799 owner scope 2026-11-12: the deployment platforms, Linux (qualified on ARM64) and Windows x64.
            (OperatingSystem.IsLinux() || OperatingSystem.IsWindows()).ShouldBeTrue("this run is qualified only for Linux and Windows");
            RuntimeInformation.ProcessArchitecture.ShouldBe(OperatingSystem.IsWindows() ? Architecture.X64 : Architecture.Arm64);
            Environment.Version.ToString().ShouldBe("10.0.14");
            Environment.GetEnvironmentVariable("OPCUA_CERTIFICATE_LEAK_TRACKING").ShouldBe("1");
            string.IsNullOrEmpty(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_OBSERVER")).ShouldBeTrue(
                "this diagnostic does not attach the saturation observer");
        }

        internal static CancellationTokenSource StartCycleDeadline(CancellationToken token) =>
            StartLinkedDeadline(token, TimeSpan.FromSeconds(CycleDeadline));

        internal static CancellationTokenSource StartLinkedDeadline(CancellationToken token, TimeSpan duration)
        {
            var deadline = CancellationTokenSource.CreateLinkedTokenSource(token);
            deadline.CancelAfter(duration);
            return deadline;
        }

        internal static Task LogStartAsync(StreamWriter evidence, object start, double startupSeconds)
        {
            var row = JsonSerializer.SerializeToNode(start)!.AsObject();
            row["runtime"] = Environment.Version.ToString();
            row["architecture"] = RuntimeInformation.ProcessArchitecture.ToString();
            row["startupSeconds"] = startupSeconds;
            row["acceptance"] = false;
            return LogAsync(evidence, row);
        }

        internal void Record(Exception error) => _failures.Add(ExceptionDispatchInfo.Capture(error));

        internal async Task ReleaseOwnersAsync(Func<Task>? server, Func<Task>? client)
        {
            // The canceled operation token cannot skip either independently bounded owner.
            await AttemptAsync(server, "server host");
            await AttemptAsync(client, "client application");

            async Task AttemptAsync(Func<Task>? dispose, string owner)
            {
                if (dispose is null) return;
                try { await Task.Run(dispose, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); }
                catch (Exception error) { Record(new InvalidOperationException($"Late-slope {owner} cleanup failed.", error)); }
            }
        }

        internal async Task ShutdownAsync(ServerEventRig? rig, ApplicationInstance? application,
            string clientRoot, string? serverRoot)
        {
            var watch = Stopwatch.StartNew();
            var failuresBeforeShutdown = _failures.Count;
            object? serverManager = null;
            object[]? cores = null;
            if (PhaseStarted)
            {
                try
                {
                    var captured = await Task.Run(() =>
                    {
                        var manager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager.");
                        return (Manager: manager, Cores: CertificateCores(ClientManager!).Concat(CertificateCores(manager)).ToArray());
                    }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
                    serverManager = captured.Manager;
                    cores = captured.Cores;
                }
                catch (Exception error) { Record(error); }
            }

            await ReleaseOwnersAsync(rig is null ? null : () => rig.DisposeAsync().AsTask(),
                application is null ? null : () => application.DisposeAsync().AsTask());
            try
            {
                await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }, CancellationToken.None)
                    .WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
            }
            catch (Exception error) { Record(error); }

            if (PhaseStarted && cores is not null && _failures.Count == failuresBeforeShutdown)
            {
                try
                {
                    using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10));
                    var probeToken = probe.Token;
                    await Task.Run(async () =>
                    {
                        RequiredField<bool>(ClientManager!, "m_disposed").ShouldBeTrue();
                        RequiredField<bool>(serverManager!, "m_disposed").ShouldBeTrue();
                        cores.Sum(CachedStores).ShouldBe(0);
                        (await DescriptorCountsAsync(probeToken, clientRoot, serverRoot!)).Pki.ShouldBe(0);
                        Directory.Exists(clientRoot).ShouldBeFalse();
                        Directory.Exists(serverRoot!).ShouldBeFalse();
                    }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
                    _shutdownVerified = true;
                }
                catch (Exception error) { Record(error); }
            }
            _shutdownSeconds = watch.Elapsed.TotalSeconds;
            if (_shutdownSeconds > ShutdownDeadline)
            {
                _shutdownVerified = false;
                Record(new TimeoutException("Late-slope shutdown exceeded its fixed 60-second ceiling."));
            }
        }

        internal async Task EmitAndPreserveFailuresAsync(StreamWriter evidence, int cycles, int restarts,
            double cycleSeconds, ExceptionDispatchInfo? firstRejectedBudget)
        {
            try
            {
                await LogAsync(evidence, new { kind = "late-slope-result", mode = Mode, cycles, restarts,
                    cycleSeconds = PhaseStarted ? cycleSeconds : 0, shutdownSeconds = _shutdownSeconds,
                    durationCompleted = DurationCompleted, shutdownVerified = _shutdownVerified,
                    coldBudgetRejected = firstRejectedBudget is not null, fatalFailure = _failures.Count != 0,
                    failureTypes = _failures.Select(x => x.SourceException.GetType().FullName).ToArray(), acceptance = false });
            }
            catch (Exception error) { Record(error); }
            ThrowFailures(firstRejectedBudget);
        }

        internal void ThrowFailures(ExceptionDispatchInfo? firstRejectedBudget)
        {
            if (_failures.Count == 0) { firstRejectedBudget?.Throw(); return; }
            if (firstRejectedBudget is null && _failures.Count == 1) _failures[0].Throw();
            var exceptions = _failures.Select(x => x.SourceException).ToList();
            if (firstRejectedBudget is not null) exceptions.Insert(0, firstRejectedBudget.SourceException);
            throw new AggregateException("Late-slope diagnostic failed; original resource rejection retained when present.", exceptions);
        }
    }

    /// <summary>The cycle deadline cancels the token used by an in-flight asynchronous operation.</summary>
    [Fact]
    public async Task Late_slope_cycle_deadline_cancels_in_flight_operation()
    {
        var ct = TestContext.Current.CancellationToken;
        using var deadline = LateSlopeRun.StartLinkedDeadline(ct, TimeSpan.FromMilliseconds(75));
        var operation = Task.Delay(Timeout.InfiniteTimeSpan, deadline.Token);
        await Should.ThrowAsync<OperationCanceledException>(() => operation.WaitAsync(TimeSpan.FromSeconds(5), ct));
        operation.IsCanceled.ShouldBeTrue();
        ct.IsCancellationRequested.ShouldBeFalse();
    }

    /// <summary>A failing server cleanup cannot skip the client or erase the original budget failure.</summary>
    [Fact]
    public async Task Late_slope_cleanup_preserves_first_rejection_and_attempts_both_owners()
    {
        var run = new LateSlopeRun { PhaseStarted = true };
        var original = new ShouldAssertException("first cold RSS rejection");
        var later = new InvalidOperationException("server cleanup fault");
        var clientDisposed = false;
        await run.ReleaseOwnersAsync(() => Task.FromException(later), () =>
        {
            clientDisposed = true;
            return Task.CompletedTask;
        });
        clientDisposed.ShouldBeTrue();
        using var buffer = new MemoryStream();
        await using var evidence = new StreamWriter(buffer, leaveOpen: true);
        var error = await Should.ThrowAsync<AggregateException>(() =>
            run.EmitAndPreserveFailuresAsync(evidence, 4, 1, 16, ExceptionDispatchInfo.Capture(original)));
        error.InnerExceptions.Count.ShouldBe(2);
        error.InnerExceptions[0].ShouldBeSameAs(original);
        error.InnerExceptions[1].InnerException.ShouldBeSameAs(later);
        using var row = ReadSingleLateSlopeResult(buffer);
        row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeTrue();
        row.RootElement.GetProperty("shutdownVerified").GetBoolean().ShouldBeFalse();
    }

    /// <summary>The real emitter retains the first resource failure and cannot write ordinary completion.</summary>
    [Fact]
    public async Task Late_slope_emits_only_diagnostic_result_before_rethrowing_original_budget_failure()
    {
        var run = new LateSlopeRun { PhaseStarted = true, DurationCompleted = true };
        var original = new ShouldAssertException("original cold RSS rejection");
        using var buffer = new MemoryStream();
        await using var evidence = new StreamWriter(buffer, leaveOpen: true);
        ShouldAssertException? observed = null;
        try { await run.EmitAndPreserveFailuresAsync(evidence, 1800, 450, 7201, ExceptionDispatchInfo.Capture(original)); }
        catch (ShouldAssertException error) { observed = error; }
        observed.ShouldBeSameAs(original);
        using var row = ReadSingleLateSlopeResult(buffer);
        row.RootElement.GetProperty("durationCompleted").GetBoolean().ShouldBeTrue();
        row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeFalse();
    }

    private static JsonDocument ReadSingleLateSlopeResult(MemoryStream buffer)
    {
        var line = Encoding.UTF8.GetString(buffer.ToArray()).Split('\n', StringSplitOptions.RemoveEmptyEntries).ShouldHaveSingleItem();
        var row = JsonDocument.Parse(line);
        row.RootElement.GetProperty("kind").GetString().ShouldBe("late-slope-result");
        row.RootElement.GetProperty("acceptance").GetBoolean().ShouldBeFalse();
        row.RootElement.GetProperty("coldBudgetRejected").GetBoolean().ShouldBeTrue();
        return row;
    }
}